1 /*
   2  * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc_interface/collectedHeap.inline.hpp"
  29 #include "interpreter/interpreter.hpp"
  30 #include "memory/cardTableModRefBS.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "utilities/macros.hpp"
  40 #if INCLUDE_ALL_GCS
  41 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
  42 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
  43 #include "gc_implementation/g1/heapRegion.hpp"
  44 #endif // INCLUDE_ALL_GCS
  45 
  46 #ifdef PRODUCT
  47 #define BLOCK_COMMENT(str) /* nothing */
  48 #define STOP(error) stop(error)
  49 #else
  50 #define BLOCK_COMMENT(str) block_comment(str)
  51 #define STOP(error) block_comment(error); stop(error)
  52 #endif
  53 
  54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  55 // Implementation of AddressLiteral
  56 
  57 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  58   _is_lval = false;
  59   _target = target;
  60   switch (rtype) {
  61   case relocInfo::oop_type:
  62   case relocInfo::metadata_type:
  63     // Oops are a special case. Normally they would be their own section
  64     // but in cases like icBuffer they are literals in the code stream that
  65     // we don't have a section for. We use none so that we get a literal address
  66     // which is always patchable.
  67     break;
  68   case relocInfo::external_word_type:
  69     _rspec = external_word_Relocation::spec(target);
  70     break;
  71   case relocInfo::internal_word_type:
  72     _rspec = internal_word_Relocation::spec(target);
  73     break;
  74   case relocInfo::opt_virtual_call_type:
  75     _rspec = opt_virtual_call_Relocation::spec();
  76     break;
  77   case relocInfo::static_call_type:
  78     _rspec = static_call_Relocation::spec();
  79     break;
  80   case relocInfo::runtime_call_type:
  81     _rspec = runtime_call_Relocation::spec();
  82     break;
  83   case relocInfo::poll_type:
  84   case relocInfo::poll_return_type:
  85     _rspec = Relocation::spec_simple(rtype);
  86     break;
  87   case relocInfo::none:
  88     break;
  89   default:
  90     ShouldNotReachHere();
  91     break;
  92   }
  93 }
  94 
  95 // Implementation of Address
  96 
  97 #ifdef _LP64
  98 
  99 Address Address::make_array(ArrayAddress adr) {
 100   // Not implementable on 64bit machines
 101   // Should have been handled higher up the call chain.
 102   ShouldNotReachHere();
 103   return Address();
 104 }
 105 
 106 // exceedingly dangerous constructor
 107 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 108   _base  = noreg;
 109   _index = noreg;
 110   _scale = no_scale;
 111   _disp  = disp;
 112   switch (rtype) {
 113     case relocInfo::external_word_type:
 114       _rspec = external_word_Relocation::spec(loc);
 115       break;
 116     case relocInfo::internal_word_type:
 117       _rspec = internal_word_Relocation::spec(loc);
 118       break;
 119     case relocInfo::runtime_call_type:
 120       // HMM
 121       _rspec = runtime_call_Relocation::spec();
 122       break;
 123     case relocInfo::poll_type:
 124     case relocInfo::poll_return_type:
 125       _rspec = Relocation::spec_simple(rtype);
 126       break;
 127     case relocInfo::none:
 128       break;
 129     default:
 130       ShouldNotReachHere();
 131   }
 132 }
 133 #else // LP64
 134 
 135 Address Address::make_array(ArrayAddress adr) {
 136   AddressLiteral base = adr.base();
 137   Address index = adr.index();
 138   assert(index._disp == 0, "must not have disp"); // maybe it can?
 139   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 140   array._rspec = base._rspec;
 141   return array;
 142 }
 143 
 144 // exceedingly dangerous constructor
 145 Address::Address(address loc, RelocationHolder spec) {
 146   _base  = noreg;
 147   _index = noreg;
 148   _scale = no_scale;
 149   _disp  = (intptr_t) loc;
 150   _rspec = spec;
 151 }
 152 
 153 #endif // _LP64
 154 
 155 
 156 
 157 // Convert the raw encoding form into the form expected by the constructor for
 158 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 159 // that to noreg for the Address constructor.
 160 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 161   RelocationHolder rspec;
 162   if (disp_reloc != relocInfo::none) {
 163     rspec = Relocation::spec_simple(disp_reloc);
 164   }
 165   bool valid_index = index != rsp->encoding();
 166   if (valid_index) {
 167     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 168     madr._rspec = rspec;
 169     return madr;
 170   } else {
 171     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 172     madr._rspec = rspec;
 173     return madr;
 174   }
 175 }
 176 
 177 // Implementation of Assembler
 178 
 179 int AbstractAssembler::code_fill_byte() {
 180   return (u_char)'\xF4'; // hlt
 181 }
 182 
 183 // make this go away someday
 184 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 185   if (rtype == relocInfo::none)
 186         emit_int32(data);
 187   else  emit_data(data, Relocation::spec_simple(rtype), format);
 188 }
 189 
 190 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 191   assert(imm_operand == 0, "default format must be immediate in this file");
 192   assert(inst_mark() != NULL, "must be inside InstructionMark");
 193   if (rspec.type() !=  relocInfo::none) {
 194     #ifdef ASSERT
 195       check_relocation(rspec, format);
 196     #endif
 197     // Do not use AbstractAssembler::relocate, which is not intended for
 198     // embedded words.  Instead, relocate to the enclosing instruction.
 199 
 200     // hack. call32 is too wide for mask so use disp32
 201     if (format == call32_operand)
 202       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 203     else
 204       code_section()->relocate(inst_mark(), rspec, format);
 205   }
 206   emit_int32(data);
 207 }
 208 
 209 static int encode(Register r) {
 210   int enc = r->encoding();
 211   if (enc >= 8) {
 212     enc -= 8;
 213   }
 214   return enc;
 215 }
 216 
 217 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 218   assert(dst->has_byte_register(), "must have byte register");
 219   assert(isByte(op1) && isByte(op2), "wrong opcode");
 220   assert(isByte(imm8), "not a byte");
 221   assert((op1 & 0x01) == 0, "should be 8bit operation");
 222   emit_int8(op1);
 223   emit_int8(op2 | encode(dst));
 224   emit_int8(imm8);
 225 }
 226 
 227 
 228 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 229   assert(isByte(op1) && isByte(op2), "wrong opcode");
 230   assert((op1 & 0x01) == 1, "should be 32bit operation");
 231   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 232   if (is8bit(imm32)) {
 233     emit_int8(op1 | 0x02); // set sign bit
 234     emit_int8(op2 | encode(dst));
 235     emit_int8(imm32 & 0xFF);
 236   } else {
 237     emit_int8(op1);
 238     emit_int8(op2 | encode(dst));
 239     emit_int32(imm32);
 240   }
 241 }
 242 
 243 // Force generation of a 4 byte immediate value even if it fits into 8bit
 244 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 245   assert(isByte(op1) && isByte(op2), "wrong opcode");
 246   assert((op1 & 0x01) == 1, "should be 32bit operation");
 247   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 248   emit_int8(op1);
 249   emit_int8(op2 | encode(dst));
 250   emit_int32(imm32);
 251 }
 252 
 253 // immediate-to-memory forms
 254 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 255   assert((op1 & 0x01) == 1, "should be 32bit operation");
 256   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 257   if (is8bit(imm32)) {
 258     emit_int8(op1 | 0x02); // set sign bit
 259     emit_operand(rm, adr, 1);
 260     emit_int8(imm32 & 0xFF);
 261   } else {
 262     emit_int8(op1);
 263     emit_operand(rm, adr, 4);
 264     emit_int32(imm32);
 265   }
 266 }
 267 
 268 
 269 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 270   assert(isByte(op1) && isByte(op2), "wrong opcode");
 271   emit_int8(op1);
 272   emit_int8(op2 | encode(dst) << 3 | encode(src));
 273 }
 274 
 275 
 276 void Assembler::emit_operand(Register reg, Register base, Register index,
 277                              Address::ScaleFactor scale, int disp,
 278                              RelocationHolder const& rspec,
 279                              int rip_relative_correction) {
 280   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 281 
 282   // Encode the registers as needed in the fields they are used in
 283 
 284   int regenc = encode(reg) << 3;
 285   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 286   int baseenc = base->is_valid() ? encode(base) : 0;
 287 
 288   if (base->is_valid()) {
 289     if (index->is_valid()) {
 290       assert(scale != Address::no_scale, "inconsistent address");
 291       // [base + index*scale + disp]
 292       if (disp == 0 && rtype == relocInfo::none  &&
 293           base != rbp LP64_ONLY(&& base != r13)) {
 294         // [base + index*scale]
 295         // [00 reg 100][ss index base]
 296         assert(index != rsp, "illegal addressing mode");
 297         emit_int8(0x04 | regenc);
 298         emit_int8(scale << 6 | indexenc | baseenc);
 299       } else if (is8bit(disp) && rtype == relocInfo::none) {
 300         // [base + index*scale + imm8]
 301         // [01 reg 100][ss index base] imm8
 302         assert(index != rsp, "illegal addressing mode");
 303         emit_int8(0x44 | regenc);
 304         emit_int8(scale << 6 | indexenc | baseenc);
 305         emit_int8(disp & 0xFF);
 306       } else {
 307         // [base + index*scale + disp32]
 308         // [10 reg 100][ss index base] disp32
 309         assert(index != rsp, "illegal addressing mode");
 310         emit_int8(0x84 | regenc);
 311         emit_int8(scale << 6 | indexenc | baseenc);
 312         emit_data(disp, rspec, disp32_operand);
 313       }
 314     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 315       // [rsp + disp]
 316       if (disp == 0 && rtype == relocInfo::none) {
 317         // [rsp]
 318         // [00 reg 100][00 100 100]
 319         emit_int8(0x04 | regenc);
 320         emit_int8(0x24);
 321       } else if (is8bit(disp) && rtype == relocInfo::none) {
 322         // [rsp + imm8]
 323         // [01 reg 100][00 100 100] disp8
 324         emit_int8(0x44 | regenc);
 325         emit_int8(0x24);
 326         emit_int8(disp & 0xFF);
 327       } else {
 328         // [rsp + imm32]
 329         // [10 reg 100][00 100 100] disp32
 330         emit_int8(0x84 | regenc);
 331         emit_int8(0x24);
 332         emit_data(disp, rspec, disp32_operand);
 333       }
 334     } else {
 335       // [base + disp]
 336       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 337       if (disp == 0 && rtype == relocInfo::none &&
 338           base != rbp LP64_ONLY(&& base != r13)) {
 339         // [base]
 340         // [00 reg base]
 341         emit_int8(0x00 | regenc | baseenc);
 342       } else if (is8bit(disp) && rtype == relocInfo::none) {
 343         // [base + disp8]
 344         // [01 reg base] disp8
 345         emit_int8(0x40 | regenc | baseenc);
 346         emit_int8(disp & 0xFF);
 347       } else {
 348         // [base + disp32]
 349         // [10 reg base] disp32
 350         emit_int8(0x80 | regenc | baseenc);
 351         emit_data(disp, rspec, disp32_operand);
 352       }
 353     }
 354   } else {
 355     if (index->is_valid()) {
 356       assert(scale != Address::no_scale, "inconsistent address");
 357       // [index*scale + disp]
 358       // [00 reg 100][ss index 101] disp32
 359       assert(index != rsp, "illegal addressing mode");
 360       emit_int8(0x04 | regenc);
 361       emit_int8(scale << 6 | indexenc | 0x05);
 362       emit_data(disp, rspec, disp32_operand);
 363     } else if (rtype != relocInfo::none ) {
 364       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 365       // [00 000 101] disp32
 366 
 367       emit_int8(0x05 | regenc);
 368       // Note that the RIP-rel. correction applies to the generated
 369       // disp field, but _not_ to the target address in the rspec.
 370 
 371       // disp was created by converting the target address minus the pc
 372       // at the start of the instruction. That needs more correction here.
 373       // intptr_t disp = target - next_ip;
 374       assert(inst_mark() != NULL, "must be inside InstructionMark");
 375       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 376       int64_t adjusted = disp;
 377       // Do rip-rel adjustment for 64bit
 378       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 379       assert(is_simm32(adjusted),
 380              "must be 32bit offset (RIP relative address)");
 381       emit_data((int32_t) adjusted, rspec, disp32_operand);
 382 
 383     } else {
 384       // 32bit never did this, did everything as the rip-rel/disp code above
 385       // [disp] ABSOLUTE
 386       // [00 reg 100][00 100 101] disp32
 387       emit_int8(0x04 | regenc);
 388       emit_int8(0x25);
 389       emit_data(disp, rspec, disp32_operand);
 390     }
 391   }
 392 }
 393 
 394 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 395                              Address::ScaleFactor scale, int disp,
 396                              RelocationHolder const& rspec) {
 397   emit_operand((Register)reg, base, index, scale, disp, rspec);
 398 }
 399 
 400 // Secret local extension to Assembler::WhichOperand:
 401 #define end_pc_operand (_WhichOperand_limit)
 402 
 403 address Assembler::locate_operand(address inst, WhichOperand which) {
 404   // Decode the given instruction, and return the address of
 405   // an embedded 32-bit operand word.
 406 
 407   // If "which" is disp32_operand, selects the displacement portion
 408   // of an effective address specifier.
 409   // If "which" is imm64_operand, selects the trailing immediate constant.
 410   // If "which" is call32_operand, selects the displacement of a call or jump.
 411   // Caller is responsible for ensuring that there is such an operand,
 412   // and that it is 32/64 bits wide.
 413 
 414   // If "which" is end_pc_operand, find the end of the instruction.
 415 
 416   address ip = inst;
 417   bool is_64bit = false;
 418 
 419   debug_only(bool has_disp32 = false);
 420   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 421 
 422   again_after_prefix:
 423   switch (0xFF & *ip++) {
 424 
 425   // These convenience macros generate groups of "case" labels for the switch.
 426 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 427 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 428              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 429 #define REP16(x) REP8((x)+0): \
 430               case REP8((x)+8)
 431 
 432   case CS_segment:
 433   case SS_segment:
 434   case DS_segment:
 435   case ES_segment:
 436   case FS_segment:
 437   case GS_segment:
 438     // Seems dubious
 439     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 440     assert(ip == inst+1, "only one prefix allowed");
 441     goto again_after_prefix;
 442 
 443   case 0x67:
 444   case REX:
 445   case REX_B:
 446   case REX_X:
 447   case REX_XB:
 448   case REX_R:
 449   case REX_RB:
 450   case REX_RX:
 451   case REX_RXB:
 452     NOT_LP64(assert(false, "64bit prefixes"));
 453     goto again_after_prefix;
 454 
 455   case REX_W:
 456   case REX_WB:
 457   case REX_WX:
 458   case REX_WXB:
 459   case REX_WR:
 460   case REX_WRB:
 461   case REX_WRX:
 462   case REX_WRXB:
 463     NOT_LP64(assert(false, "64bit prefixes"));
 464     is_64bit = true;
 465     goto again_after_prefix;
 466 
 467   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 468   case 0x88: // movb a, r
 469   case 0x89: // movl a, r
 470   case 0x8A: // movb r, a
 471   case 0x8B: // movl r, a
 472   case 0x8F: // popl a
 473     debug_only(has_disp32 = true);
 474     break;
 475 
 476   case 0x68: // pushq #32
 477     if (which == end_pc_operand) {
 478       return ip + 4;
 479     }
 480     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 481     return ip;                  // not produced by emit_operand
 482 
 483   case 0x66: // movw ... (size prefix)
 484     again_after_size_prefix2:
 485     switch (0xFF & *ip++) {
 486     case REX:
 487     case REX_B:
 488     case REX_X:
 489     case REX_XB:
 490     case REX_R:
 491     case REX_RB:
 492     case REX_RX:
 493     case REX_RXB:
 494     case REX_W:
 495     case REX_WB:
 496     case REX_WX:
 497     case REX_WXB:
 498     case REX_WR:
 499     case REX_WRB:
 500     case REX_WRX:
 501     case REX_WRXB:
 502       NOT_LP64(assert(false, "64bit prefix found"));
 503       goto again_after_size_prefix2;
 504     case 0x8B: // movw r, a
 505     case 0x89: // movw a, r
 506       debug_only(has_disp32 = true);
 507       break;
 508     case 0xC7: // movw a, #16
 509       debug_only(has_disp32 = true);
 510       tail_size = 2;  // the imm16
 511       break;
 512     case 0x0F: // several SSE/SSE2 variants
 513       ip--;    // reparse the 0x0F
 514       goto again_after_prefix;
 515     default:
 516       ShouldNotReachHere();
 517     }
 518     break;
 519 
 520   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 521     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 522     // these asserts are somewhat nonsensical
 523 #ifndef _LP64
 524     assert(which == imm_operand || which == disp32_operand,
 525            err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, ip));
 526 #else
 527     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 528            which == narrow_oop_operand && !is_64bit,
 529            err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, ip));
 530 #endif // _LP64
 531     return ip;
 532 
 533   case 0x69: // imul r, a, #32
 534   case 0xC7: // movl a, #32(oop?)
 535     tail_size = 4;
 536     debug_only(has_disp32 = true); // has both kinds of operands!
 537     break;
 538 
 539   case 0x0F: // movx..., etc.
 540     switch (0xFF & *ip++) {
 541     case 0x3A: // pcmpestri
 542       tail_size = 1;
 543     case 0x38: // ptest, pmovzxbw
 544       ip++; // skip opcode
 545       debug_only(has_disp32 = true); // has both kinds of operands!
 546       break;
 547 
 548     case 0x70: // pshufd r, r/a, #8
 549       debug_only(has_disp32 = true); // has both kinds of operands!
 550     case 0x73: // psrldq r, #8
 551       tail_size = 1;
 552       break;
 553 
 554     case 0x12: // movlps
 555     case 0x28: // movaps
 556     case 0x2E: // ucomiss
 557     case 0x2F: // comiss
 558     case 0x54: // andps
 559     case 0x55: // andnps
 560     case 0x56: // orps
 561     case 0x57: // xorps
 562     case 0x6E: // movd
 563     case 0x7E: // movd
 564     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 565       debug_only(has_disp32 = true);
 566       break;
 567 
 568     case 0xAD: // shrd r, a, %cl
 569     case 0xAF: // imul r, a
 570     case 0xBE: // movsbl r, a (movsxb)
 571     case 0xBF: // movswl r, a (movsxw)
 572     case 0xB6: // movzbl r, a (movzxb)
 573     case 0xB7: // movzwl r, a (movzxw)
 574     case REP16(0x40): // cmovl cc, r, a
 575     case 0xB0: // cmpxchgb
 576     case 0xB1: // cmpxchg
 577     case 0xC1: // xaddl
 578     case 0xC7: // cmpxchg8
 579     case REP16(0x90): // setcc a
 580       debug_only(has_disp32 = true);
 581       // fall out of the switch to decode the address
 582       break;
 583 
 584     case 0xC4: // pinsrw r, a, #8
 585       debug_only(has_disp32 = true);
 586     case 0xC5: // pextrw r, r, #8
 587       tail_size = 1;  // the imm8
 588       break;
 589 
 590     case 0xAC: // shrd r, a, #8
 591       debug_only(has_disp32 = true);
 592       tail_size = 1;  // the imm8
 593       break;
 594 
 595     case REP16(0x80): // jcc rdisp32
 596       if (which == end_pc_operand)  return ip + 4;
 597       assert(which == call32_operand, "jcc has no disp32 or imm");
 598       return ip;
 599     default:
 600       ShouldNotReachHere();
 601     }
 602     break;
 603 
 604   case 0x81: // addl a, #32; addl r, #32
 605     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 606     // on 32bit in the case of cmpl, the imm might be an oop
 607     tail_size = 4;
 608     debug_only(has_disp32 = true); // has both kinds of operands!
 609     break;
 610 
 611   case 0x83: // addl a, #8; addl r, #8
 612     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 613     debug_only(has_disp32 = true); // has both kinds of operands!
 614     tail_size = 1;
 615     break;
 616 
 617   case 0x9B:
 618     switch (0xFF & *ip++) {
 619     case 0xD9: // fnstcw a
 620       debug_only(has_disp32 = true);
 621       break;
 622     default:
 623       ShouldNotReachHere();
 624     }
 625     break;
 626 
 627   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 628   case REP4(0x10): // adc...
 629   case REP4(0x20): // and...
 630   case REP4(0x30): // xor...
 631   case REP4(0x08): // or...
 632   case REP4(0x18): // sbb...
 633   case REP4(0x28): // sub...
 634   case 0xF7: // mull a
 635   case 0x8D: // lea r, a
 636   case 0x87: // xchg r, a
 637   case REP4(0x38): // cmp...
 638   case 0x85: // test r, a
 639     debug_only(has_disp32 = true); // has both kinds of operands!
 640     break;
 641 
 642   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 643   case 0xC6: // movb a, #8
 644   case 0x80: // cmpb a, #8
 645   case 0x6B: // imul r, a, #8
 646     debug_only(has_disp32 = true); // has both kinds of operands!
 647     tail_size = 1; // the imm8
 648     break;
 649 
 650   case 0xC4: // VEX_3bytes
 651   case 0xC5: // VEX_2bytes
 652     assert((UseAVX > 0), "shouldn't have VEX prefix");
 653     assert(ip == inst+1, "no prefixes allowed");
 654     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 655     // but they have prefix 0x0F and processed when 0x0F processed above.
 656     //
 657     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 658     // instructions (these instructions are not supported in 64-bit mode).
 659     // To distinguish them bits [7:6] are set in the VEX second byte since
 660     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 661     // those VEX bits REX and vvvv bits are inverted.
 662     //
 663     // Fortunately C2 doesn't generate these instructions so we don't need
 664     // to check for them in product version.
 665 
 666     // Check second byte
 667     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 668 
 669     // First byte
 670     if ((0xFF & *inst) == VEX_3bytes) {
 671       ip++; // third byte
 672       is_64bit = ((VEX_W & *ip) == VEX_W);
 673     }
 674     ip++; // opcode
 675     // To find the end of instruction (which == end_pc_operand).
 676     switch (0xFF & *ip) {
 677     case 0x61: // pcmpestri r, r/a, #8
 678     case 0x70: // pshufd r, r/a, #8
 679     case 0x73: // psrldq r, #8
 680       tail_size = 1;  // the imm8
 681       break;
 682     default:
 683       break;
 684     }
 685     ip++; // skip opcode
 686     debug_only(has_disp32 = true); // has both kinds of operands!
 687     break;
 688 
 689   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 690   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 691   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 692   case 0xDD: // fld_d a; fst_d a; fstp_d a
 693   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 694   case 0xDF: // fild_d a; fistp_d a
 695   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 696   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 697   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 698     debug_only(has_disp32 = true);
 699     break;
 700 
 701   case 0xE8: // call rdisp32
 702   case 0xE9: // jmp  rdisp32
 703     if (which == end_pc_operand)  return ip + 4;
 704     assert(which == call32_operand, "call has no disp32 or imm");
 705     return ip;
 706 
 707   case 0xF0:                    // Lock
 708     assert(os::is_MP(), "only on MP");
 709     goto again_after_prefix;
 710 
 711   case 0xF3:                    // For SSE
 712   case 0xF2:                    // For SSE2
 713     switch (0xFF & *ip++) {
 714     case REX:
 715     case REX_B:
 716     case REX_X:
 717     case REX_XB:
 718     case REX_R:
 719     case REX_RB:
 720     case REX_RX:
 721     case REX_RXB:
 722     case REX_W:
 723     case REX_WB:
 724     case REX_WX:
 725     case REX_WXB:
 726     case REX_WR:
 727     case REX_WRB:
 728     case REX_WRX:
 729     case REX_WRXB:
 730       NOT_LP64(assert(false, "found 64bit prefix"));
 731       ip++;
 732     default:
 733       ip++;
 734     }
 735     debug_only(has_disp32 = true); // has both kinds of operands!
 736     break;
 737 
 738   default:
 739     ShouldNotReachHere();
 740 
 741 #undef REP8
 742 #undef REP16
 743   }
 744 
 745   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 746 #ifdef _LP64
 747   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 748 #else
 749   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 750   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 751 #endif // LP64
 752   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 753 
 754   // parse the output of emit_operand
 755   int op2 = 0xFF & *ip++;
 756   int base = op2 & 0x07;
 757   int op3 = -1;
 758   const int b100 = 4;
 759   const int b101 = 5;
 760   if (base == b100 && (op2 >> 6) != 3) {
 761     op3 = 0xFF & *ip++;
 762     base = op3 & 0x07;   // refetch the base
 763   }
 764   // now ip points at the disp (if any)
 765 
 766   switch (op2 >> 6) {
 767   case 0:
 768     // [00 reg  100][ss index base]
 769     // [00 reg  100][00   100  esp]
 770     // [00 reg base]
 771     // [00 reg  100][ss index  101][disp32]
 772     // [00 reg  101]               [disp32]
 773 
 774     if (base == b101) {
 775       if (which == disp32_operand)
 776         return ip;              // caller wants the disp32
 777       ip += 4;                  // skip the disp32
 778     }
 779     break;
 780 
 781   case 1:
 782     // [01 reg  100][ss index base][disp8]
 783     // [01 reg  100][00   100  esp][disp8]
 784     // [01 reg base]               [disp8]
 785     ip += 1;                    // skip the disp8
 786     break;
 787 
 788   case 2:
 789     // [10 reg  100][ss index base][disp32]
 790     // [10 reg  100][00   100  esp][disp32]
 791     // [10 reg base]               [disp32]
 792     if (which == disp32_operand)
 793       return ip;                // caller wants the disp32
 794     ip += 4;                    // skip the disp32
 795     break;
 796 
 797   case 3:
 798     // [11 reg base]  (not a memory addressing mode)
 799     break;
 800   }
 801 
 802   if (which == end_pc_operand) {
 803     return ip + tail_size;
 804   }
 805 
 806 #ifdef _LP64
 807   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 808 #else
 809   assert(which == imm_operand, "instruction has only an imm field");
 810 #endif // LP64
 811   return ip;
 812 }
 813 
 814 address Assembler::locate_next_instruction(address inst) {
 815   // Secretly share code with locate_operand:
 816   return locate_operand(inst, end_pc_operand);
 817 }
 818 
 819 
 820 #ifdef ASSERT
 821 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 822   address inst = inst_mark();
 823   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 824   address opnd;
 825 
 826   Relocation* r = rspec.reloc();
 827   if (r->type() == relocInfo::none) {
 828     return;
 829   } else if (r->is_call() || format == call32_operand) {
 830     // assert(format == imm32_operand, "cannot specify a nonzero format");
 831     opnd = locate_operand(inst, call32_operand);
 832   } else if (r->is_data()) {
 833     assert(format == imm_operand || format == disp32_operand
 834            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 835     opnd = locate_operand(inst, (WhichOperand)format);
 836   } else {
 837     assert(format == imm_operand, "cannot specify a format");
 838     return;
 839   }
 840   assert(opnd == pc(), "must put operand where relocs can find it");
 841 }
 842 #endif // ASSERT
 843 
 844 void Assembler::emit_operand32(Register reg, Address adr) {
 845   assert(reg->encoding() < 8, "no extended registers");
 846   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
 847   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 848                adr._rspec);
 849 }
 850 
 851 void Assembler::emit_operand(Register reg, Address adr,
 852                              int rip_relative_correction) {
 853   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 854                adr._rspec,
 855                rip_relative_correction);
 856 }
 857 
 858 void Assembler::emit_operand(XMMRegister reg, Address adr) {
 859   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 860                adr._rspec);
 861 }
 862 
 863 // MMX operations
 864 void Assembler::emit_operand(MMXRegister reg, Address adr) {
 865   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
 866   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
 867 }
 868 
 869 // work around gcc (3.2.1-7a) bug
 870 void Assembler::emit_operand(Address adr, MMXRegister reg) {
 871   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
 872   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
 873 }
 874 
 875 
 876 void Assembler::emit_farith(int b1, int b2, int i) {
 877   assert(isByte(b1) && isByte(b2), "wrong opcode");
 878   assert(0 <= i &&  i < 8, "illegal stack offset");
 879   emit_int8(b1);
 880   emit_int8(b2 + i);
 881 }
 882 
 883 
 884 // Now the Assembler instructions (identical for 32/64 bits)
 885 
 886 void Assembler::adcl(Address dst, int32_t imm32) {
 887   InstructionMark im(this);
 888   prefix(dst);
 889   emit_arith_operand(0x81, rdx, dst, imm32);
 890 }
 891 
 892 void Assembler::adcl(Address dst, Register src) {
 893   InstructionMark im(this);
 894   prefix(dst, src);
 895   emit_int8(0x11);
 896   emit_operand(src, dst);
 897 }
 898 
 899 void Assembler::adcl(Register dst, int32_t imm32) {
 900   prefix(dst);
 901   emit_arith(0x81, 0xD0, dst, imm32);
 902 }
 903 
 904 void Assembler::adcl(Register dst, Address src) {
 905   InstructionMark im(this);
 906   prefix(src, dst);
 907   emit_int8(0x13);
 908   emit_operand(dst, src);
 909 }
 910 
 911 void Assembler::adcl(Register dst, Register src) {
 912   (void) prefix_and_encode(dst->encoding(), src->encoding());
 913   emit_arith(0x13, 0xC0, dst, src);
 914 }
 915 
 916 void Assembler::addl(Address dst, int32_t imm32) {
 917   InstructionMark im(this);
 918   prefix(dst);
 919   emit_arith_operand(0x81, rax, dst, imm32);
 920 }
 921 
 922 void Assembler::addl(Address dst, Register src) {
 923   InstructionMark im(this);
 924   prefix(dst, src);
 925   emit_int8(0x01);
 926   emit_operand(src, dst);
 927 }
 928 
 929 void Assembler::addl(Register dst, int32_t imm32) {
 930   prefix(dst);
 931   emit_arith(0x81, 0xC0, dst, imm32);
 932 }
 933 
 934 void Assembler::addl(Register dst, Address src) {
 935   InstructionMark im(this);
 936   prefix(src, dst);
 937   emit_int8(0x03);
 938   emit_operand(dst, src);
 939 }
 940 
 941 void Assembler::addl(Register dst, Register src) {
 942   (void) prefix_and_encode(dst->encoding(), src->encoding());
 943   emit_arith(0x03, 0xC0, dst, src);
 944 }
 945 
 946 void Assembler::addr_nop_4() {
 947   assert(UseAddressNop, "no CPU support");
 948   // 4 bytes: NOP DWORD PTR [EAX+0]
 949   emit_int8(0x0F);
 950   emit_int8(0x1F);
 951   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 952   emit_int8(0);    // 8-bits offset (1 byte)
 953 }
 954 
 955 void Assembler::addr_nop_5() {
 956   assert(UseAddressNop, "no CPU support");
 957   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 958   emit_int8(0x0F);
 959   emit_int8(0x1F);
 960   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 961   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 962   emit_int8(0);    // 8-bits offset (1 byte)
 963 }
 964 
 965 void Assembler::addr_nop_7() {
 966   assert(UseAddressNop, "no CPU support");
 967   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 968   emit_int8(0x0F);
 969   emit_int8(0x1F);
 970   emit_int8((unsigned char)0x80);
 971                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 972   emit_int32(0);   // 32-bits offset (4 bytes)
 973 }
 974 
 975 void Assembler::addr_nop_8() {
 976   assert(UseAddressNop, "no CPU support");
 977   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 978   emit_int8(0x0F);
 979   emit_int8(0x1F);
 980   emit_int8((unsigned char)0x84);
 981                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 982   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 983   emit_int32(0);   // 32-bits offset (4 bytes)
 984 }
 985 
 986 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 988   emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
 989 }
 990 
 991 void Assembler::addsd(XMMRegister dst, Address src) {
 992   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 993   emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
 994 }
 995 
 996 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 997   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 998   emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
 999 }
1000 
1001 void Assembler::addss(XMMRegister dst, Address src) {
1002   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1003   emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
1004 }
1005 
1006 void Assembler::aesdec(XMMRegister dst, Address src) {
1007   assert(VM_Version::supports_aes(), "");
1008   InstructionMark im(this);
1009   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1010   emit_int8((unsigned char)0xDE);
1011   emit_operand(dst, src);
1012 }
1013 
1014 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1015   assert(VM_Version::supports_aes(), "");
1016   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1017   emit_int8((unsigned char)0xDE);
1018   emit_int8(0xC0 | encode);
1019 }
1020 
1021 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1022   assert(VM_Version::supports_aes(), "");
1023   InstructionMark im(this);
1024   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1025   emit_int8((unsigned char)0xDF);
1026   emit_operand(dst, src);
1027 }
1028 
1029 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1030   assert(VM_Version::supports_aes(), "");
1031   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1032   emit_int8((unsigned char)0xDF);
1033   emit_int8((unsigned char)(0xC0 | encode));
1034 }
1035 
1036 void Assembler::aesenc(XMMRegister dst, Address src) {
1037   assert(VM_Version::supports_aes(), "");
1038   InstructionMark im(this);
1039   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1040   emit_int8((unsigned char)0xDC);
1041   emit_operand(dst, src);
1042 }
1043 
1044 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1045   assert(VM_Version::supports_aes(), "");
1046   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1047   emit_int8((unsigned char)0xDC);
1048   emit_int8(0xC0 | encode);
1049 }
1050 
1051 void Assembler::aesenclast(XMMRegister dst, Address src) {
1052   assert(VM_Version::supports_aes(), "");
1053   InstructionMark im(this);
1054   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1055   emit_int8((unsigned char)0xDD);
1056   emit_operand(dst, src);
1057 }
1058 
1059 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1060   assert(VM_Version::supports_aes(), "");
1061   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1062   emit_int8((unsigned char)0xDD);
1063   emit_int8((unsigned char)(0xC0 | encode));
1064 }
1065 
1066 
1067 void Assembler::andl(Address dst, int32_t imm32) {
1068   InstructionMark im(this);
1069   prefix(dst);
1070   emit_int8((unsigned char)0x81);
1071   emit_operand(rsp, dst, 4);
1072   emit_int32(imm32);
1073 }
1074 
1075 void Assembler::andl(Register dst, int32_t imm32) {
1076   prefix(dst);
1077   emit_arith(0x81, 0xE0, dst, imm32);
1078 }
1079 
1080 void Assembler::andl(Register dst, Address src) {
1081   InstructionMark im(this);
1082   prefix(src, dst);
1083   emit_int8(0x23);
1084   emit_operand(dst, src);
1085 }
1086 
1087 void Assembler::andl(Register dst, Register src) {
1088   (void) prefix_and_encode(dst->encoding(), src->encoding());
1089   emit_arith(0x23, 0xC0, dst, src);
1090 }
1091 
1092 void Assembler::andnl(Register dst, Register src1, Register src2) {
1093   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1094   int encode = vex_prefix_0F38_and_encode(dst, src1, src2);
1095   emit_int8((unsigned char)0xF2);
1096   emit_int8((unsigned char)(0xC0 | encode));
1097 }
1098 
1099 void Assembler::andnl(Register dst, Register src1, Address src2) {
1100   InstructionMark im(this);
1101   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1102   vex_prefix_0F38(dst, src1, src2);
1103   emit_int8((unsigned char)0xF2);
1104   emit_operand(dst, src2);
1105 }
1106 
1107 void Assembler::bsfl(Register dst, Register src) {
1108   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1109   emit_int8(0x0F);
1110   emit_int8((unsigned char)0xBC);
1111   emit_int8((unsigned char)(0xC0 | encode));
1112 }
1113 
1114 void Assembler::bsrl(Register dst, Register src) {
1115   assert(!VM_Version::supports_lzcnt(), "encoding is treated as LZCNT");
1116   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1117   emit_int8(0x0F);
1118   emit_int8((unsigned char)0xBD);
1119   emit_int8((unsigned char)(0xC0 | encode));
1120 }
1121 
1122 void Assembler::bswapl(Register reg) { // bswap
1123   int encode = prefix_and_encode(reg->encoding());
1124   emit_int8(0x0F);
1125   emit_int8((unsigned char)(0xC8 | encode));
1126 }
1127 
1128 void Assembler::blsil(Register dst, Register src) {
1129   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1130   int encode = vex_prefix_0F38_and_encode(rbx, dst, src);
1131   emit_int8((unsigned char)0xF3);
1132   emit_int8((unsigned char)(0xC0 | encode));
1133 }
1134 
1135 void Assembler::blsil(Register dst, Address src) {
1136   InstructionMark im(this);
1137   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1138   vex_prefix_0F38(rbx, dst, src);
1139   emit_int8((unsigned char)0xF3);
1140   emit_operand(rbx, src);
1141 }
1142 
1143 void Assembler::blsmskl(Register dst, Register src) {
1144   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1145   int encode = vex_prefix_0F38_and_encode(rdx, dst, src);
1146   emit_int8((unsigned char)0xF3);
1147   emit_int8((unsigned char)(0xC0 | encode));
1148 }
1149 
1150 void Assembler::blsmskl(Register dst, Address src) {
1151   InstructionMark im(this);
1152   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1153   vex_prefix_0F38(rdx, dst, src);
1154   emit_int8((unsigned char)0xF3);
1155   emit_operand(rdx, src);
1156 }
1157 
1158 void Assembler::blsrl(Register dst, Register src) {
1159   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1160   int encode = vex_prefix_0F38_and_encode(rcx, dst, src);
1161   emit_int8((unsigned char)0xF3);
1162   emit_int8((unsigned char)(0xC0 | encode));
1163 }
1164 
1165 void Assembler::blsrl(Register dst, Address src) {
1166   InstructionMark im(this);
1167   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
1168   vex_prefix_0F38(rcx, dst, src);
1169   emit_int8((unsigned char)0xF3);
1170   emit_operand(rcx, src);
1171 }
1172 
1173 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1174   // suspect disp32 is always good
1175   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1176 
1177   if (L.is_bound()) {
1178     const int long_size = 5;
1179     int offs = (int)( target(L) - pc() );
1180     assert(offs <= 0, "assembler error");
1181     InstructionMark im(this);
1182     // 1110 1000 #32-bit disp
1183     emit_int8((unsigned char)0xE8);
1184     emit_data(offs - long_size, rtype, operand);
1185   } else {
1186     InstructionMark im(this);
1187     // 1110 1000 #32-bit disp
1188     L.add_patch_at(code(), locator());
1189 
1190     emit_int8((unsigned char)0xE8);
1191     emit_data(int(0), rtype, operand);
1192   }
1193 }
1194 
1195 void Assembler::call(Register dst) {
1196   int encode = prefix_and_encode(dst->encoding());
1197   emit_int8((unsigned char)0xFF);
1198   emit_int8((unsigned char)(0xD0 | encode));
1199 }
1200 
1201 
1202 void Assembler::call(Address adr) {
1203   InstructionMark im(this);
1204   prefix(adr);
1205   emit_int8((unsigned char)0xFF);
1206   emit_operand(rdx, adr);
1207 }
1208 
1209 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1210   assert(entry != NULL, "call most probably wrong");
1211   InstructionMark im(this);
1212   emit_int8((unsigned char)0xE8);
1213   intptr_t disp = entry - (pc() + sizeof(int32_t));
1214   assert(is_simm32(disp), "must be 32bit offset (call2)");
1215   // Technically, should use call32_operand, but this format is
1216   // implied by the fact that we're emitting a call instruction.
1217 
1218   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1219   emit_data((int) disp, rspec, operand);
1220 }
1221 
1222 void Assembler::cdql() {
1223   emit_int8((unsigned char)0x99);
1224 }
1225 
1226 void Assembler::cld() {
1227   emit_int8((unsigned char)0xFC);
1228 }
1229 
1230 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1231   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1232   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1233   emit_int8(0x0F);
1234   emit_int8(0x40 | cc);
1235   emit_int8((unsigned char)(0xC0 | encode));
1236 }
1237 
1238 
1239 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1240   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1241   prefix(src, dst);
1242   emit_int8(0x0F);
1243   emit_int8(0x40 | cc);
1244   emit_operand(dst, src);
1245 }
1246 
1247 void Assembler::cmpb(Address dst, int imm8) {
1248   InstructionMark im(this);
1249   prefix(dst);
1250   emit_int8((unsigned char)0x80);
1251   emit_operand(rdi, dst, 1);
1252   emit_int8(imm8);
1253 }
1254 
1255 void Assembler::cmpl(Address dst, int32_t imm32) {
1256   InstructionMark im(this);
1257   prefix(dst);
1258   emit_int8((unsigned char)0x81);
1259   emit_operand(rdi, dst, 4);
1260   emit_int32(imm32);
1261 }
1262 
1263 void Assembler::cmpl(Register dst, int32_t imm32) {
1264   prefix(dst);
1265   emit_arith(0x81, 0xF8, dst, imm32);
1266 }
1267 
1268 void Assembler::cmpl(Register dst, Register src) {
1269   (void) prefix_and_encode(dst->encoding(), src->encoding());
1270   emit_arith(0x3B, 0xC0, dst, src);
1271 }
1272 
1273 
1274 void Assembler::cmpl(Register dst, Address  src) {
1275   InstructionMark im(this);
1276   prefix(src, dst);
1277   emit_int8((unsigned char)0x3B);
1278   emit_operand(dst, src);
1279 }
1280 
1281 void Assembler::cmpw(Address dst, int imm16) {
1282   InstructionMark im(this);
1283   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1284   emit_int8(0x66);
1285   emit_int8((unsigned char)0x81);
1286   emit_operand(rdi, dst, 2);
1287   emit_int16(imm16);
1288 }
1289 
1290 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1291 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1292 // The ZF is set if the compared values were equal, and cleared otherwise.
1293 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1294   InstructionMark im(this);
1295   prefix(adr, reg);
1296   emit_int8(0x0F);
1297   emit_int8((unsigned char)0xB1);
1298   emit_operand(reg, adr);
1299 }
1300 
1301 void Assembler::comisd(XMMRegister dst, Address src) {
1302   // NOTE: dbx seems to decode this as comiss even though the
1303   // 0x66 is there. Strangly ucomisd comes out correct
1304   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1305   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1306 }
1307 
1308 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1309   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1310   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1311 }
1312 
1313 void Assembler::comiss(XMMRegister dst, Address src) {
1314   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1315   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE);
1316 }
1317 
1318 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1319   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1320   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE);
1321 }
1322 
1323 void Assembler::cpuid() {
1324   emit_int8(0x0F);
1325   emit_int8((unsigned char)0xA2);
1326 }
1327 
1328 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1329   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1330   emit_simd_arith_nonds(0xE6, dst, src, VEX_SIMD_F3);
1331 }
1332 
1333 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1334   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1335   emit_simd_arith_nonds(0x5B, dst, src, VEX_SIMD_NONE);
1336 }
1337 
1338 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1339   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1340   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1341 }
1342 
1343 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1344   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1345   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1346 }
1347 
1348 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1349   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1350   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
1351   emit_int8(0x2A);
1352   emit_int8((unsigned char)(0xC0 | encode));
1353 }
1354 
1355 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1356   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1357   emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2);
1358 }
1359 
1360 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1361   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1362   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
1363   emit_int8(0x2A);
1364   emit_int8((unsigned char)(0xC0 | encode));
1365 }
1366 
1367 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1368   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1369   emit_simd_arith(0x2A, dst, src, VEX_SIMD_F3);
1370 }
1371 
1372 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1373   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1374   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
1375 }
1376 
1377 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1378   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1379   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
1380 }
1381 
1382 
1383 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1384   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1385   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2);
1386   emit_int8(0x2C);
1387   emit_int8((unsigned char)(0xC0 | encode));
1388 }
1389 
1390 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1391   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1392   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);
1393   emit_int8(0x2C);
1394   emit_int8((unsigned char)(0xC0 | encode));
1395 }
1396 
1397 void Assembler::decl(Address dst) {
1398   // Don't use it directly. Use MacroAssembler::decrement() instead.
1399   InstructionMark im(this);
1400   prefix(dst);
1401   emit_int8((unsigned char)0xFF);
1402   emit_operand(rcx, dst);
1403 }
1404 
1405 void Assembler::divsd(XMMRegister dst, Address src) {
1406   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1407   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1408 }
1409 
1410 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1411   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1412   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1413 }
1414 
1415 void Assembler::divss(XMMRegister dst, Address src) {
1416   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1417   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
1418 }
1419 
1420 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1421   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1422   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
1423 }
1424 
1425 void Assembler::emms() {
1426   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1427   emit_int8(0x0F);
1428   emit_int8(0x77);
1429 }
1430 
1431 void Assembler::hlt() {
1432   emit_int8((unsigned char)0xF4);
1433 }
1434 
1435 void Assembler::idivl(Register src) {
1436   int encode = prefix_and_encode(src->encoding());
1437   emit_int8((unsigned char)0xF7);
1438   emit_int8((unsigned char)(0xF8 | encode));
1439 }
1440 
1441 void Assembler::divl(Register src) { // Unsigned
1442   int encode = prefix_and_encode(src->encoding());
1443   emit_int8((unsigned char)0xF7);
1444   emit_int8((unsigned char)(0xF0 | encode));
1445 }
1446 
1447 void Assembler::imull(Register dst, Register src) {
1448   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1449   emit_int8(0x0F);
1450   emit_int8((unsigned char)0xAF);
1451   emit_int8((unsigned char)(0xC0 | encode));
1452 }
1453 
1454 
1455 void Assembler::imull(Register dst, Register src, int value) {
1456   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1457   if (is8bit(value)) {
1458     emit_int8(0x6B);
1459     emit_int8((unsigned char)(0xC0 | encode));
1460     emit_int8(value & 0xFF);
1461   } else {
1462     emit_int8(0x69);
1463     emit_int8((unsigned char)(0xC0 | encode));
1464     emit_int32(value);
1465   }
1466 }
1467 
1468 void Assembler::imull(Register dst, Address src) {
1469   InstructionMark im(this);
1470   prefix(src, dst);
1471   emit_int8(0x0F);
1472   emit_int8((unsigned char) 0xAF);
1473   emit_operand(dst, src);
1474 }
1475 
1476 
1477 void Assembler::incl(Address dst) {
1478   // Don't use it directly. Use MacroAssembler::increment() instead.
1479   InstructionMark im(this);
1480   prefix(dst);
1481   emit_int8((unsigned char)0xFF);
1482   emit_operand(rax, dst);
1483 }
1484 
1485 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1486   InstructionMark im(this);
1487   assert((0 <= cc) && (cc < 16), "illegal cc");
1488   if (L.is_bound()) {
1489     address dst = target(L);
1490     assert(dst != NULL, "jcc most probably wrong");
1491 
1492     const int short_size = 2;
1493     const int long_size = 6;
1494     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1495     if (maybe_short && is8bit(offs - short_size)) {
1496       // 0111 tttn #8-bit disp
1497       emit_int8(0x70 | cc);
1498       emit_int8((offs - short_size) & 0xFF);
1499     } else {
1500       // 0000 1111 1000 tttn #32-bit disp
1501       assert(is_simm32(offs - long_size),
1502              "must be 32bit offset (call4)");
1503       emit_int8(0x0F);
1504       emit_int8((unsigned char)(0x80 | cc));
1505       emit_int32(offs - long_size);
1506     }
1507   } else {
1508     // Note: could eliminate cond. jumps to this jump if condition
1509     //       is the same however, seems to be rather unlikely case.
1510     // Note: use jccb() if label to be bound is very close to get
1511     //       an 8-bit displacement
1512     L.add_patch_at(code(), locator());
1513     emit_int8(0x0F);
1514     emit_int8((unsigned char)(0x80 | cc));
1515     emit_int32(0);
1516   }
1517 }
1518 
1519 void Assembler::jccb(Condition cc, Label& L) {
1520   if (L.is_bound()) {
1521     const int short_size = 2;
1522     address entry = target(L);
1523 #ifdef ASSERT
1524     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1525     intptr_t delta = short_branch_delta();
1526     if (delta != 0) {
1527       dist += (dist < 0 ? (-delta) :delta);
1528     }
1529     assert(is8bit(dist), "Dispacement too large for a short jmp");
1530 #endif
1531     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
1532     // 0111 tttn #8-bit disp
1533     emit_int8(0x70 | cc);
1534     emit_int8((offs - short_size) & 0xFF);
1535   } else {
1536     InstructionMark im(this);
1537     L.add_patch_at(code(), locator());
1538     emit_int8(0x70 | cc);
1539     emit_int8(0);
1540   }
1541 }
1542 
1543 void Assembler::jmp(Address adr) {
1544   InstructionMark im(this);
1545   prefix(adr);
1546   emit_int8((unsigned char)0xFF);
1547   emit_operand(rsp, adr);
1548 }
1549 
1550 void Assembler::jmp(Label& L, bool maybe_short) {
1551   if (L.is_bound()) {
1552     address entry = target(L);
1553     assert(entry != NULL, "jmp most probably wrong");
1554     InstructionMark im(this);
1555     const int short_size = 2;
1556     const int long_size = 5;
1557     intptr_t offs = entry - pc();
1558     if (maybe_short && is8bit(offs - short_size)) {
1559       emit_int8((unsigned char)0xEB);
1560       emit_int8((offs - short_size) & 0xFF);
1561     } else {
1562       emit_int8((unsigned char)0xE9);
1563       emit_int32(offs - long_size);
1564     }
1565   } else {
1566     // By default, forward jumps are always 32-bit displacements, since
1567     // we can't yet know where the label will be bound.  If you're sure that
1568     // the forward jump will not run beyond 256 bytes, use jmpb to
1569     // force an 8-bit displacement.
1570     InstructionMark im(this);
1571     L.add_patch_at(code(), locator());
1572     emit_int8((unsigned char)0xE9);
1573     emit_int32(0);
1574   }
1575 }
1576 
1577 void Assembler::jmp(Register entry) {
1578   int encode = prefix_and_encode(entry->encoding());
1579   emit_int8((unsigned char)0xFF);
1580   emit_int8((unsigned char)(0xE0 | encode));
1581 }
1582 
1583 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
1584   InstructionMark im(this);
1585   emit_int8((unsigned char)0xE9);
1586   assert(dest != NULL, "must have a target");
1587   intptr_t disp = dest - (pc() + sizeof(int32_t));
1588   assert(is_simm32(disp), "must be 32bit offset (jmp)");
1589   emit_data(disp, rspec.reloc(), call32_operand);
1590 }
1591 
1592 void Assembler::jmpb(Label& L) {
1593   if (L.is_bound()) {
1594     const int short_size = 2;
1595     address entry = target(L);
1596     assert(entry != NULL, "jmp most probably wrong");
1597 #ifdef ASSERT
1598     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1599     intptr_t delta = short_branch_delta();
1600     if (delta != 0) {
1601       dist += (dist < 0 ? (-delta) :delta);
1602     }
1603     assert(is8bit(dist), "Dispacement too large for a short jmp");
1604 #endif
1605     intptr_t offs = entry - pc();
1606     emit_int8((unsigned char)0xEB);
1607     emit_int8((offs - short_size) & 0xFF);
1608   } else {
1609     InstructionMark im(this);
1610     L.add_patch_at(code(), locator());
1611     emit_int8((unsigned char)0xEB);
1612     emit_int8(0);
1613   }
1614 }
1615 
1616 void Assembler::ldmxcsr( Address src) {
1617   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1618   InstructionMark im(this);
1619   prefix(src);
1620   emit_int8(0x0F);
1621   emit_int8((unsigned char)0xAE);
1622   emit_operand(as_Register(2), src);
1623 }
1624 
1625 void Assembler::leal(Register dst, Address src) {
1626   InstructionMark im(this);
1627 #ifdef _LP64
1628   emit_int8(0x67); // addr32
1629   prefix(src, dst);
1630 #endif // LP64
1631   emit_int8((unsigned char)0x8D);
1632   emit_operand(dst, src);
1633 }
1634 
1635 void Assembler::lfence() {
1636   emit_int8(0x0F);
1637   emit_int8((unsigned char)0xAE);
1638   emit_int8((unsigned char)0xE8);
1639 }
1640 
1641 void Assembler::lock() {
1642   emit_int8((unsigned char)0xF0);
1643 }
1644 
1645 void Assembler::lzcntl(Register dst, Register src) {
1646   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
1647   emit_int8((unsigned char)0xF3);
1648   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1649   emit_int8(0x0F);
1650   emit_int8((unsigned char)0xBD);
1651   emit_int8((unsigned char)(0xC0 | encode));
1652 }
1653 
1654 // Emit mfence instruction
1655 void Assembler::mfence() {
1656   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
1657   emit_int8(0x0F);
1658   emit_int8((unsigned char)0xAE);
1659   emit_int8((unsigned char)0xF0);
1660 }
1661 
1662 void Assembler::mov(Register dst, Register src) {
1663   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
1664 }
1665 
1666 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
1667   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1668   emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_66);
1669 }
1670 
1671 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
1672   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1673   emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_NONE);
1674 }
1675 
1676 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
1677   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1678   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE);
1679   emit_int8(0x16);
1680   emit_int8((unsigned char)(0xC0 | encode));
1681 }
1682 
1683 void Assembler::movb(Register dst, Address src) {
1684   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
1685   InstructionMark im(this);
1686   prefix(src, dst, true);
1687   emit_int8((unsigned char)0x8A);
1688   emit_operand(dst, src);
1689 }
1690 
1691 
1692 void Assembler::movb(Address dst, int imm8) {
1693   InstructionMark im(this);
1694    prefix(dst);
1695   emit_int8((unsigned char)0xC6);
1696   emit_operand(rax, dst, 1);
1697   emit_int8(imm8);
1698 }
1699 
1700 
1701 void Assembler::movb(Address dst, Register src) {
1702   assert(src->has_byte_register(), "must have byte register");
1703   InstructionMark im(this);
1704   prefix(dst, src, true);
1705   emit_int8((unsigned char)0x88);
1706   emit_operand(src, dst);
1707 }
1708 
1709 void Assembler::movdl(XMMRegister dst, Register src) {
1710   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1711   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
1712   emit_int8(0x6E);
1713   emit_int8((unsigned char)(0xC0 | encode));
1714 }
1715 
1716 void Assembler::movdl(Register dst, XMMRegister src) {
1717   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1718   // swap src/dst to get correct prefix
1719   int encode = simd_prefix_and_encode(src, dst, VEX_SIMD_66);
1720   emit_int8(0x7E);
1721   emit_int8((unsigned char)(0xC0 | encode));
1722 }
1723 
1724 void Assembler::movdl(XMMRegister dst, Address src) {
1725   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1726   InstructionMark im(this);
1727   simd_prefix(dst, src, VEX_SIMD_66);
1728   emit_int8(0x6E);
1729   emit_operand(dst, src);
1730 }
1731 
1732 void Assembler::movdl(Address dst, XMMRegister src) {
1733   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1734   InstructionMark im(this);
1735   simd_prefix(dst, src, VEX_SIMD_66);
1736   emit_int8(0x7E);
1737   emit_operand(src, dst);
1738 }
1739 
1740 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
1741   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1742   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
1743 }
1744 
1745 void Assembler::movdqa(XMMRegister dst, Address src) {
1746   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1747   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
1748 }
1749 
1750 void Assembler::movdqu(XMMRegister dst, Address src) {
1751   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1752   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
1753 }
1754 
1755 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
1756   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1757   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
1758 }
1759 
1760 void Assembler::movdqu(Address dst, XMMRegister src) {
1761   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1762   InstructionMark im(this);
1763   simd_prefix(dst, src, VEX_SIMD_F3);
1764   emit_int8(0x7F);
1765   emit_operand(src, dst);
1766 }
1767 
1768 // Move Unaligned 256bit Vector
1769 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
1770   assert(UseAVX, "");
1771   bool vector256 = true;
1772   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, vector256);
1773   emit_int8(0x6F);
1774   emit_int8((unsigned char)(0xC0 | encode));
1775 }
1776 
1777 void Assembler::vmovdqu(XMMRegister dst, Address src) {
1778   assert(UseAVX, "");
1779   InstructionMark im(this);
1780   bool vector256 = true;
1781   vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector256);
1782   emit_int8(0x6F);
1783   emit_operand(dst, src);
1784 }
1785 
1786 void Assembler::vmovdqu(Address dst, XMMRegister src) {
1787   assert(UseAVX, "");
1788   InstructionMark im(this);
1789   bool vector256 = true;
1790   // swap src<->dst for encoding
1791   assert(src != xnoreg, "sanity");
1792   vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector256);
1793   emit_int8(0x7F);
1794   emit_operand(src, dst);
1795 }
1796 
1797 // Uses zero extension on 64bit
1798 
1799 void Assembler::movl(Register dst, int32_t imm32) {
1800   int encode = prefix_and_encode(dst->encoding());
1801   emit_int8((unsigned char)(0xB8 | encode));
1802   emit_int32(imm32);
1803 }
1804 
1805 void Assembler::movl(Register dst, Register src) {
1806   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1807   emit_int8((unsigned char)0x8B);
1808   emit_int8((unsigned char)(0xC0 | encode));
1809 }
1810 
1811 void Assembler::movl(Register dst, Address src) {
1812   InstructionMark im(this);
1813   prefix(src, dst);
1814   emit_int8((unsigned char)0x8B);
1815   emit_operand(dst, src);
1816 }
1817 
1818 void Assembler::movl(Address dst, int32_t imm32) {
1819   InstructionMark im(this);
1820   prefix(dst);
1821   emit_int8((unsigned char)0xC7);
1822   emit_operand(rax, dst, 4);
1823   emit_int32(imm32);
1824 }
1825 
1826 void Assembler::movl(Address dst, Register src) {
1827   InstructionMark im(this);
1828   prefix(dst, src);
1829   emit_int8((unsigned char)0x89);
1830   emit_operand(src, dst);
1831 }
1832 
1833 // New cpus require to use movsd and movss to avoid partial register stall
1834 // when loading from memory. But for old Opteron use movlpd instead of movsd.
1835 // The selection is done in MacroAssembler::movdbl() and movflt().
1836 void Assembler::movlpd(XMMRegister dst, Address src) {
1837   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1838   emit_simd_arith(0x12, dst, src, VEX_SIMD_66);
1839 }
1840 
1841 void Assembler::movq( MMXRegister dst, Address src ) {
1842   assert( VM_Version::supports_mmx(), "" );
1843   emit_int8(0x0F);
1844   emit_int8(0x6F);
1845   emit_operand(dst, src);
1846 }
1847 
1848 void Assembler::movq( Address dst, MMXRegister src ) {
1849   assert( VM_Version::supports_mmx(), "" );
1850   emit_int8(0x0F);
1851   emit_int8(0x7F);
1852   // workaround gcc (3.2.1-7a) bug
1853   // In that version of gcc with only an emit_operand(MMX, Address)
1854   // gcc will tail jump and try and reverse the parameters completely
1855   // obliterating dst in the process. By having a version available
1856   // that doesn't need to swap the args at the tail jump the bug is
1857   // avoided.
1858   emit_operand(dst, src);
1859 }
1860 
1861 void Assembler::movq(XMMRegister dst, Address src) {
1862   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1863   InstructionMark im(this);
1864   simd_prefix(dst, src, VEX_SIMD_F3);
1865   emit_int8(0x7E);
1866   emit_operand(dst, src);
1867 }
1868 
1869 void Assembler::movq(Address dst, XMMRegister src) {
1870   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1871   InstructionMark im(this);
1872   simd_prefix(dst, src, VEX_SIMD_66);
1873   emit_int8((unsigned char)0xD6);
1874   emit_operand(src, dst);
1875 }
1876 
1877 void Assembler::movsbl(Register dst, Address src) { // movsxb
1878   InstructionMark im(this);
1879   prefix(src, dst);
1880   emit_int8(0x0F);
1881   emit_int8((unsigned char)0xBE);
1882   emit_operand(dst, src);
1883 }
1884 
1885 void Assembler::movsbl(Register dst, Register src) { // movsxb
1886   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
1887   int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
1888   emit_int8(0x0F);
1889   emit_int8((unsigned char)0xBE);
1890   emit_int8((unsigned char)(0xC0 | encode));
1891 }
1892 
1893 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
1894   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1895   emit_simd_arith(0x10, dst, src, VEX_SIMD_F2);
1896 }
1897 
1898 void Assembler::movsd(XMMRegister dst, Address src) {
1899   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1900   emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F2);
1901 }
1902 
1903 void Assembler::movsd(Address dst, XMMRegister src) {
1904   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1905   InstructionMark im(this);
1906   simd_prefix(dst, src, VEX_SIMD_F2);
1907   emit_int8(0x11);
1908   emit_operand(src, dst);
1909 }
1910 
1911 void Assembler::movss(XMMRegister dst, XMMRegister src) {
1912   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1913   emit_simd_arith(0x10, dst, src, VEX_SIMD_F3);
1914 }
1915 
1916 void Assembler::movss(XMMRegister dst, Address src) {
1917   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1918   emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F3);
1919 }
1920 
1921 void Assembler::movss(Address dst, XMMRegister src) {
1922   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1923   InstructionMark im(this);
1924   simd_prefix(dst, src, VEX_SIMD_F3);
1925   emit_int8(0x11);
1926   emit_operand(src, dst);
1927 }
1928 
1929 void Assembler::movswl(Register dst, Address src) { // movsxw
1930   InstructionMark im(this);
1931   prefix(src, dst);
1932   emit_int8(0x0F);
1933   emit_int8((unsigned char)0xBF);
1934   emit_operand(dst, src);
1935 }
1936 
1937 void Assembler::movswl(Register dst, Register src) { // movsxw
1938   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1939   emit_int8(0x0F);
1940   emit_int8((unsigned char)0xBF);
1941   emit_int8((unsigned char)(0xC0 | encode));
1942 }
1943 
1944 void Assembler::movw(Address dst, int imm16) {
1945   InstructionMark im(this);
1946 
1947   emit_int8(0x66); // switch to 16-bit mode
1948   prefix(dst);
1949   emit_int8((unsigned char)0xC7);
1950   emit_operand(rax, dst, 2);
1951   emit_int16(imm16);
1952 }
1953 
1954 void Assembler::movw(Register dst, Address src) {
1955   InstructionMark im(this);
1956   emit_int8(0x66);
1957   prefix(src, dst);
1958   emit_int8((unsigned char)0x8B);
1959   emit_operand(dst, src);
1960 }
1961 
1962 void Assembler::movw(Address dst, Register src) {
1963   InstructionMark im(this);
1964   emit_int8(0x66);
1965   prefix(dst, src);
1966   emit_int8((unsigned char)0x89);
1967   emit_operand(src, dst);
1968 }
1969 
1970 void Assembler::movzbl(Register dst, Address src) { // movzxb
1971   InstructionMark im(this);
1972   prefix(src, dst);
1973   emit_int8(0x0F);
1974   emit_int8((unsigned char)0xB6);
1975   emit_operand(dst, src);
1976 }
1977 
1978 void Assembler::movzbl(Register dst, Register src) { // movzxb
1979   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
1980   int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
1981   emit_int8(0x0F);
1982   emit_int8((unsigned char)0xB6);
1983   emit_int8(0xC0 | encode);
1984 }
1985 
1986 void Assembler::movzwl(Register dst, Address src) { // movzxw
1987   InstructionMark im(this);
1988   prefix(src, dst);
1989   emit_int8(0x0F);
1990   emit_int8((unsigned char)0xB7);
1991   emit_operand(dst, src);
1992 }
1993 
1994 void Assembler::movzwl(Register dst, Register src) { // movzxw
1995   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1996   emit_int8(0x0F);
1997   emit_int8((unsigned char)0xB7);
1998   emit_int8(0xC0 | encode);
1999 }
2000 
2001 void Assembler::mull(Address src) {
2002   InstructionMark im(this);
2003   prefix(src);
2004   emit_int8((unsigned char)0xF7);
2005   emit_operand(rsp, src);
2006 }
2007 
2008 void Assembler::mull(Register src) {
2009   int encode = prefix_and_encode(src->encoding());
2010   emit_int8((unsigned char)0xF7);
2011   emit_int8((unsigned char)(0xE0 | encode));
2012 }
2013 
2014 void Assembler::mulsd(XMMRegister dst, Address src) {
2015   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2016   emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
2017 }
2018 
2019 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
2020   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2021   emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
2022 }
2023 
2024 void Assembler::mulss(XMMRegister dst, Address src) {
2025   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2026   emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
2027 }
2028 
2029 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
2030   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2031   emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
2032 }
2033 
2034 void Assembler::negl(Register dst) {
2035   int encode = prefix_and_encode(dst->encoding());
2036   emit_int8((unsigned char)0xF7);
2037   emit_int8((unsigned char)(0xD8 | encode));
2038 }
2039 
2040 void Assembler::nop(int i) {
2041 #ifdef ASSERT
2042   assert(i > 0, " ");
2043   // The fancy nops aren't currently recognized by debuggers making it a
2044   // pain to disassemble code while debugging. If asserts are on clearly
2045   // speed is not an issue so simply use the single byte traditional nop
2046   // to do alignment.
2047 
2048   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
2049   return;
2050 
2051 #endif // ASSERT
2052 
2053   if (UseAddressNop && VM_Version::is_intel()) {
2054     //
2055     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
2056     //  1: 0x90
2057     //  2: 0x66 0x90
2058     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2059     //  4: 0x0F 0x1F 0x40 0x00
2060     //  5: 0x0F 0x1F 0x44 0x00 0x00
2061     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2062     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2063     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2064     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2065     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2066     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2067 
2068     // The rest coding is Intel specific - don't use consecutive address nops
2069 
2070     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2071     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2072     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2073     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2074 
2075     while(i >= 15) {
2076       // For Intel don't generate consecutive addess nops (mix with regular nops)
2077       i -= 15;
2078       emit_int8(0x66);   // size prefix
2079       emit_int8(0x66);   // size prefix
2080       emit_int8(0x66);   // size prefix
2081       addr_nop_8();
2082       emit_int8(0x66);   // size prefix
2083       emit_int8(0x66);   // size prefix
2084       emit_int8(0x66);   // size prefix
2085       emit_int8((unsigned char)0x90);
2086                          // nop
2087     }
2088     switch (i) {
2089       case 14:
2090         emit_int8(0x66); // size prefix
2091       case 13:
2092         emit_int8(0x66); // size prefix
2093       case 12:
2094         addr_nop_8();
2095         emit_int8(0x66); // size prefix
2096         emit_int8(0x66); // size prefix
2097         emit_int8(0x66); // size prefix
2098         emit_int8((unsigned char)0x90);
2099                          // nop
2100         break;
2101       case 11:
2102         emit_int8(0x66); // size prefix
2103       case 10:
2104         emit_int8(0x66); // size prefix
2105       case 9:
2106         emit_int8(0x66); // size prefix
2107       case 8:
2108         addr_nop_8();
2109         break;
2110       case 7:
2111         addr_nop_7();
2112         break;
2113       case 6:
2114         emit_int8(0x66); // size prefix
2115       case 5:
2116         addr_nop_5();
2117         break;
2118       case 4:
2119         addr_nop_4();
2120         break;
2121       case 3:
2122         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2123         emit_int8(0x66); // size prefix
2124       case 2:
2125         emit_int8(0x66); // size prefix
2126       case 1:
2127         emit_int8((unsigned char)0x90);
2128                          // nop
2129         break;
2130       default:
2131         assert(i == 0, " ");
2132     }
2133     return;
2134   }
2135   if (UseAddressNop && VM_Version::is_amd()) {
2136     //
2137     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2138     //  1: 0x90
2139     //  2: 0x66 0x90
2140     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2141     //  4: 0x0F 0x1F 0x40 0x00
2142     //  5: 0x0F 0x1F 0x44 0x00 0x00
2143     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2144     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2145     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2146     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2147     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2148     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2149 
2150     // The rest coding is AMD specific - use consecutive address nops
2151 
2152     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2153     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2154     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2155     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2156     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2157     //     Size prefixes (0x66) are added for larger sizes
2158 
2159     while(i >= 22) {
2160       i -= 11;
2161       emit_int8(0x66); // size prefix
2162       emit_int8(0x66); // size prefix
2163       emit_int8(0x66); // size prefix
2164       addr_nop_8();
2165     }
2166     // Generate first nop for size between 21-12
2167     switch (i) {
2168       case 21:
2169         i -= 1;
2170         emit_int8(0x66); // size prefix
2171       case 20:
2172       case 19:
2173         i -= 1;
2174         emit_int8(0x66); // size prefix
2175       case 18:
2176       case 17:
2177         i -= 1;
2178         emit_int8(0x66); // size prefix
2179       case 16:
2180       case 15:
2181         i -= 8;
2182         addr_nop_8();
2183         break;
2184       case 14:
2185       case 13:
2186         i -= 7;
2187         addr_nop_7();
2188         break;
2189       case 12:
2190         i -= 6;
2191         emit_int8(0x66); // size prefix
2192         addr_nop_5();
2193         break;
2194       default:
2195         assert(i < 12, " ");
2196     }
2197 
2198     // Generate second nop for size between 11-1
2199     switch (i) {
2200       case 11:
2201         emit_int8(0x66); // size prefix
2202       case 10:
2203         emit_int8(0x66); // size prefix
2204       case 9:
2205         emit_int8(0x66); // size prefix
2206       case 8:
2207         addr_nop_8();
2208         break;
2209       case 7:
2210         addr_nop_7();
2211         break;
2212       case 6:
2213         emit_int8(0x66); // size prefix
2214       case 5:
2215         addr_nop_5();
2216         break;
2217       case 4:
2218         addr_nop_4();
2219         break;
2220       case 3:
2221         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2222         emit_int8(0x66); // size prefix
2223       case 2:
2224         emit_int8(0x66); // size prefix
2225       case 1:
2226         emit_int8((unsigned char)0x90);
2227                          // nop
2228         break;
2229       default:
2230         assert(i == 0, " ");
2231     }
2232     return;
2233   }
2234 
2235   // Using nops with size prefixes "0x66 0x90".
2236   // From AMD Optimization Guide:
2237   //  1: 0x90
2238   //  2: 0x66 0x90
2239   //  3: 0x66 0x66 0x90
2240   //  4: 0x66 0x66 0x66 0x90
2241   //  5: 0x66 0x66 0x90 0x66 0x90
2242   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
2243   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
2244   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
2245   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2246   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2247   //
2248   while(i > 12) {
2249     i -= 4;
2250     emit_int8(0x66); // size prefix
2251     emit_int8(0x66);
2252     emit_int8(0x66);
2253     emit_int8((unsigned char)0x90);
2254                      // nop
2255   }
2256   // 1 - 12 nops
2257   if(i > 8) {
2258     if(i > 9) {
2259       i -= 1;
2260       emit_int8(0x66);
2261     }
2262     i -= 3;
2263     emit_int8(0x66);
2264     emit_int8(0x66);
2265     emit_int8((unsigned char)0x90);
2266   }
2267   // 1 - 8 nops
2268   if(i > 4) {
2269     if(i > 6) {
2270       i -= 1;
2271       emit_int8(0x66);
2272     }
2273     i -= 3;
2274     emit_int8(0x66);
2275     emit_int8(0x66);
2276     emit_int8((unsigned char)0x90);
2277   }
2278   switch (i) {
2279     case 4:
2280       emit_int8(0x66);
2281     case 3:
2282       emit_int8(0x66);
2283     case 2:
2284       emit_int8(0x66);
2285     case 1:
2286       emit_int8((unsigned char)0x90);
2287       break;
2288     default:
2289       assert(i == 0, " ");
2290   }
2291 }
2292 
2293 void Assembler::notl(Register dst) {
2294   int encode = prefix_and_encode(dst->encoding());
2295   emit_int8((unsigned char)0xF7);
2296   emit_int8((unsigned char)(0xD0 | encode));
2297 }
2298 
2299 void Assembler::orl(Address dst, int32_t imm32) {
2300   InstructionMark im(this);
2301   prefix(dst);
2302   emit_arith_operand(0x81, rcx, dst, imm32);
2303 }
2304 
2305 void Assembler::orl(Register dst, int32_t imm32) {
2306   prefix(dst);
2307   emit_arith(0x81, 0xC8, dst, imm32);
2308 }
2309 
2310 void Assembler::orl(Register dst, Address src) {
2311   InstructionMark im(this);
2312   prefix(src, dst);
2313   emit_int8(0x0B);
2314   emit_operand(dst, src);
2315 }
2316 
2317 void Assembler::orl(Register dst, Register src) {
2318   (void) prefix_and_encode(dst->encoding(), src->encoding());
2319   emit_arith(0x0B, 0xC0, dst, src);
2320 }
2321 
2322 void Assembler::packuswb(XMMRegister dst, Address src) {
2323   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2324   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2325   emit_simd_arith(0x67, dst, src, VEX_SIMD_66);
2326 }
2327 
2328 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
2329   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2330   emit_simd_arith(0x67, dst, src, VEX_SIMD_66);
2331 }
2332 
2333 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
2334   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
2335   emit_vex_arith(0x67, dst, nds, src, VEX_SIMD_66, vector256);
2336 }
2337 
2338 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, bool vector256) {
2339   assert(VM_Version::supports_avx2(), "");
2340   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, true, vector256);
2341   emit_int8(0x00);
2342   emit_int8(0xC0 | encode);
2343   emit_int8(imm8);
2344 }
2345 
2346 void Assembler::pause() {
2347   emit_int8((unsigned char)0xF3);
2348   emit_int8((unsigned char)0x90);
2349 }
2350 
2351 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
2352   assert(VM_Version::supports_sse4_2(), "");
2353   InstructionMark im(this);
2354   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
2355   emit_int8(0x61);
2356   emit_operand(dst, src);
2357   emit_int8(imm8);
2358 }
2359 
2360 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
2361   assert(VM_Version::supports_sse4_2(), "");
2362   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
2363   emit_int8(0x61);
2364   emit_int8((unsigned char)(0xC0 | encode));
2365   emit_int8(imm8);
2366 }
2367 
2368 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
2369   assert(VM_Version::supports_sse4_1(), "");
2370   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, false);
2371   emit_int8(0x16);
2372   emit_int8((unsigned char)(0xC0 | encode));
2373   emit_int8(imm8);
2374 }
2375 
2376 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
2377   assert(VM_Version::supports_sse4_1(), "");
2378   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, true);
2379   emit_int8(0x16);
2380   emit_int8((unsigned char)(0xC0 | encode));
2381   emit_int8(imm8);
2382 }
2383 
2384 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
2385   assert(VM_Version::supports_sse4_1(), "");
2386   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, false);
2387   emit_int8(0x22);
2388   emit_int8((unsigned char)(0xC0 | encode));
2389   emit_int8(imm8);
2390 }
2391 
2392 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
2393   assert(VM_Version::supports_sse4_1(), "");
2394   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, true);
2395   emit_int8(0x22);
2396   emit_int8((unsigned char)(0xC0 | encode));
2397   emit_int8(imm8);
2398 }
2399 
2400 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
2401   assert(VM_Version::supports_sse4_1(), "");
2402   InstructionMark im(this);
2403   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2404   emit_int8(0x30);
2405   emit_operand(dst, src);
2406 }
2407 
2408 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
2409   assert(VM_Version::supports_sse4_1(), "");
2410   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2411   emit_int8(0x30);
2412   emit_int8((unsigned char)(0xC0 | encode));
2413 }
2414 
2415 // generic
2416 void Assembler::pop(Register dst) {
2417   int encode = prefix_and_encode(dst->encoding());
2418   emit_int8(0x58 | encode);
2419 }
2420 
2421 void Assembler::popcntl(Register dst, Address src) {
2422   assert(VM_Version::supports_popcnt(), "must support");
2423   InstructionMark im(this);
2424   emit_int8((unsigned char)0xF3);
2425   prefix(src, dst);
2426   emit_int8(0x0F);
2427   emit_int8((unsigned char)0xB8);
2428   emit_operand(dst, src);
2429 }
2430 
2431 void Assembler::popcntl(Register dst, Register src) {
2432   assert(VM_Version::supports_popcnt(), "must support");
2433   emit_int8((unsigned char)0xF3);
2434   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2435   emit_int8(0x0F);
2436   emit_int8((unsigned char)0xB8);
2437   emit_int8((unsigned char)(0xC0 | encode));
2438 }
2439 
2440 void Assembler::popf() {
2441   emit_int8((unsigned char)0x9D);
2442 }
2443 
2444 #ifndef _LP64 // no 32bit push/pop on amd64
2445 void Assembler::popl(Address dst) {
2446   // NOTE: this will adjust stack by 8byte on 64bits
2447   InstructionMark im(this);
2448   prefix(dst);
2449   emit_int8((unsigned char)0x8F);
2450   emit_operand(rax, dst);
2451 }
2452 #endif
2453 
2454 void Assembler::prefetch_prefix(Address src) {
2455   prefix(src);
2456   emit_int8(0x0F);
2457 }
2458 
2459 void Assembler::prefetchnta(Address src) {
2460   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2461   InstructionMark im(this);
2462   prefetch_prefix(src);
2463   emit_int8(0x18);
2464   emit_operand(rax, src); // 0, src
2465 }
2466 
2467 void Assembler::prefetchr(Address src) {
2468   assert(VM_Version::supports_3dnow_prefetch(), "must support");
2469   InstructionMark im(this);
2470   prefetch_prefix(src);
2471   emit_int8(0x0D);
2472   emit_operand(rax, src); // 0, src
2473 }
2474 
2475 void Assembler::prefetcht0(Address src) {
2476   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2477   InstructionMark im(this);
2478   prefetch_prefix(src);
2479   emit_int8(0x18);
2480   emit_operand(rcx, src); // 1, src
2481 }
2482 
2483 void Assembler::prefetcht1(Address src) {
2484   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2485   InstructionMark im(this);
2486   prefetch_prefix(src);
2487   emit_int8(0x18);
2488   emit_operand(rdx, src); // 2, src
2489 }
2490 
2491 void Assembler::prefetcht2(Address src) {
2492   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2493   InstructionMark im(this);
2494   prefetch_prefix(src);
2495   emit_int8(0x18);
2496   emit_operand(rbx, src); // 3, src
2497 }
2498 
2499 void Assembler::prefetchw(Address src) {
2500   assert(VM_Version::supports_3dnow_prefetch(), "must support");
2501   InstructionMark im(this);
2502   prefetch_prefix(src);
2503   emit_int8(0x0D);
2504   emit_operand(rcx, src); // 1, src
2505 }
2506 
2507 void Assembler::prefix(Prefix p) {
2508   emit_int8(p);
2509 }
2510 
2511 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
2512   assert(VM_Version::supports_ssse3(), "");
2513   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2514   emit_int8(0x00);
2515   emit_int8((unsigned char)(0xC0 | encode));
2516 }
2517 
2518 void Assembler::pshufb(XMMRegister dst, Address src) {
2519   assert(VM_Version::supports_ssse3(), "");
2520   InstructionMark im(this);
2521   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2522   emit_int8(0x00);
2523   emit_operand(dst, src);
2524 }
2525 
2526 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
2527   assert(isByte(mode), "invalid value");
2528   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2529   emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_66);
2530   emit_int8(mode & 0xFF);
2531 
2532 }
2533 
2534 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
2535   assert(isByte(mode), "invalid value");
2536   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2537   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2538   InstructionMark im(this);
2539   simd_prefix(dst, src, VEX_SIMD_66);
2540   emit_int8(0x70);
2541   emit_operand(dst, src);
2542   emit_int8(mode & 0xFF);
2543 }
2544 
2545 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
2546   assert(isByte(mode), "invalid value");
2547   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2548   emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_F2);
2549   emit_int8(mode & 0xFF);
2550 }
2551 
2552 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
2553   assert(isByte(mode), "invalid value");
2554   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2555   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2556   InstructionMark im(this);
2557   simd_prefix(dst, src, VEX_SIMD_F2);
2558   emit_int8(0x70);
2559   emit_operand(dst, src);
2560   emit_int8(mode & 0xFF);
2561 }
2562 
2563 void Assembler::psrldq(XMMRegister dst, int shift) {
2564   // Shift 128 bit value in xmm register by number of bytes.
2565   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2566   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66);
2567   emit_int8(0x73);
2568   emit_int8((unsigned char)(0xC0 | encode));
2569   emit_int8(shift);
2570 }
2571 
2572 void Assembler::ptest(XMMRegister dst, Address src) {
2573   assert(VM_Version::supports_sse4_1(), "");
2574   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2575   InstructionMark im(this);
2576   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2577   emit_int8(0x17);
2578   emit_operand(dst, src);
2579 }
2580 
2581 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
2582   assert(VM_Version::supports_sse4_1(), "");
2583   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2584   emit_int8(0x17);
2585   emit_int8((unsigned char)(0xC0 | encode));
2586 }
2587 
2588 void Assembler::vptest(XMMRegister dst, Address src) {
2589   assert(VM_Version::supports_avx(), "");
2590   InstructionMark im(this);
2591   bool vector256 = true;
2592   assert(dst != xnoreg, "sanity");
2593   int dst_enc = dst->encoding();
2594   // swap src<->dst for encoding
2595   vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector256);
2596   emit_int8(0x17);
2597   emit_operand(dst, src);
2598 }
2599 
2600 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
2601   assert(VM_Version::supports_avx(), "");
2602   bool vector256 = true;
2603   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_38);
2604   emit_int8(0x17);
2605   emit_int8((unsigned char)(0xC0 | encode));
2606 }
2607 
2608 void Assembler::punpcklbw(XMMRegister dst, Address src) {
2609   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2610   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2611   emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
2612 }
2613 
2614 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
2615   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2616   emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
2617 }
2618 
2619 void Assembler::punpckldq(XMMRegister dst, Address src) {
2620   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2621   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2622   emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
2623 }
2624 
2625 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
2626   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2627   emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
2628 }
2629 
2630 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
2631   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2632   emit_simd_arith(0x6C, dst, src, VEX_SIMD_66);
2633 }
2634 
2635 void Assembler::push(int32_t imm32) {
2636   // in 64bits we push 64bits onto the stack but only
2637   // take a 32bit immediate
2638   emit_int8(0x68);
2639   emit_int32(imm32);
2640 }
2641 
2642 void Assembler::push(Register src) {
2643   int encode = prefix_and_encode(src->encoding());
2644 
2645   emit_int8(0x50 | encode);
2646 }
2647 
2648 void Assembler::pushf() {
2649   emit_int8((unsigned char)0x9C);
2650 }
2651 
2652 #ifndef _LP64 // no 32bit push/pop on amd64
2653 void Assembler::pushl(Address src) {
2654   // Note this will push 64bit on 64bit
2655   InstructionMark im(this);
2656   prefix(src);
2657   emit_int8((unsigned char)0xFF);
2658   emit_operand(rsi, src);
2659 }
2660 #endif
2661 
2662 void Assembler::rcll(Register dst, int imm8) {
2663   assert(isShiftCount(imm8), "illegal shift count");
2664   int encode = prefix_and_encode(dst->encoding());
2665   if (imm8 == 1) {
2666     emit_int8((unsigned char)0xD1);
2667     emit_int8((unsigned char)(0xD0 | encode));
2668   } else {
2669     emit_int8((unsigned char)0xC1);
2670     emit_int8((unsigned char)0xD0 | encode);
2671     emit_int8(imm8);
2672   }
2673 }
2674 
2675 void Assembler::rdtsc() {
2676   emit_int8((unsigned char)0x0F);
2677   emit_int8((unsigned char)0x31);
2678 }
2679 
2680 // copies data from [esi] to [edi] using rcx pointer sized words
2681 // generic
2682 void Assembler::rep_mov() {
2683   emit_int8((unsigned char)0xF3);
2684   // MOVSQ
2685   LP64_ONLY(prefix(REX_W));
2686   emit_int8((unsigned char)0xA5);
2687 }
2688 
2689 // sets rcx bytes with rax, value at [edi]
2690 void Assembler::rep_stosb() {
2691   emit_int8((unsigned char)0xF3); // REP
2692   LP64_ONLY(prefix(REX_W));
2693   emit_int8((unsigned char)0xAA); // STOSB
2694 }
2695 
2696 // sets rcx pointer sized words with rax, value at [edi]
2697 // generic
2698 void Assembler::rep_stos() {
2699   emit_int8((unsigned char)0xF3); // REP
2700   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
2701   emit_int8((unsigned char)0xAB);
2702 }
2703 
2704 // scans rcx pointer sized words at [edi] for occurance of rax,
2705 // generic
2706 void Assembler::repne_scan() { // repne_scan
2707   emit_int8((unsigned char)0xF2);
2708   // SCASQ
2709   LP64_ONLY(prefix(REX_W));
2710   emit_int8((unsigned char)0xAF);
2711 }
2712 
2713 #ifdef _LP64
2714 // scans rcx 4 byte words at [edi] for occurance of rax,
2715 // generic
2716 void Assembler::repne_scanl() { // repne_scan
2717   emit_int8((unsigned char)0xF2);
2718   // SCASL
2719   emit_int8((unsigned char)0xAF);
2720 }
2721 #endif
2722 
2723 void Assembler::ret(int imm16) {
2724   if (imm16 == 0) {
2725     emit_int8((unsigned char)0xC3);
2726   } else {
2727     emit_int8((unsigned char)0xC2);
2728     emit_int16(imm16);
2729   }
2730 }
2731 
2732 void Assembler::sahf() {
2733 #ifdef _LP64
2734   // Not supported in 64bit mode
2735   ShouldNotReachHere();
2736 #endif
2737   emit_int8((unsigned char)0x9E);
2738 }
2739 
2740 void Assembler::sarl(Register dst, int imm8) {
2741   int encode = prefix_and_encode(dst->encoding());
2742   assert(isShiftCount(imm8), "illegal shift count");
2743   if (imm8 == 1) {
2744     emit_int8((unsigned char)0xD1);
2745     emit_int8((unsigned char)(0xF8 | encode));
2746   } else {
2747     emit_int8((unsigned char)0xC1);
2748     emit_int8((unsigned char)(0xF8 | encode));
2749     emit_int8(imm8);
2750   }
2751 }
2752 
2753 void Assembler::sarl(Register dst) {
2754   int encode = prefix_and_encode(dst->encoding());
2755   emit_int8((unsigned char)0xD3);
2756   emit_int8((unsigned char)(0xF8 | encode));
2757 }
2758 
2759 void Assembler::sbbl(Address dst, int32_t imm32) {
2760   InstructionMark im(this);
2761   prefix(dst);
2762   emit_arith_operand(0x81, rbx, dst, imm32);
2763 }
2764 
2765 void Assembler::sbbl(Register dst, int32_t imm32) {
2766   prefix(dst);
2767   emit_arith(0x81, 0xD8, dst, imm32);
2768 }
2769 
2770 
2771 void Assembler::sbbl(Register dst, Address src) {
2772   InstructionMark im(this);
2773   prefix(src, dst);
2774   emit_int8(0x1B);
2775   emit_operand(dst, src);
2776 }
2777 
2778 void Assembler::sbbl(Register dst, Register src) {
2779   (void) prefix_and_encode(dst->encoding(), src->encoding());
2780   emit_arith(0x1B, 0xC0, dst, src);
2781 }
2782 
2783 void Assembler::setb(Condition cc, Register dst) {
2784   assert(0 <= cc && cc < 16, "illegal cc");
2785   int encode = prefix_and_encode(dst->encoding(), true);
2786   emit_int8(0x0F);
2787   emit_int8((unsigned char)0x90 | cc);
2788   emit_int8((unsigned char)(0xC0 | encode));
2789 }
2790 
2791 void Assembler::shll(Register dst, int imm8) {
2792   assert(isShiftCount(imm8), "illegal shift count");
2793   int encode = prefix_and_encode(dst->encoding());
2794   if (imm8 == 1 ) {
2795     emit_int8((unsigned char)0xD1);
2796     emit_int8((unsigned char)(0xE0 | encode));
2797   } else {
2798     emit_int8((unsigned char)0xC1);
2799     emit_int8((unsigned char)(0xE0 | encode));
2800     emit_int8(imm8);
2801   }
2802 }
2803 
2804 void Assembler::shll(Register dst) {
2805   int encode = prefix_and_encode(dst->encoding());
2806   emit_int8((unsigned char)0xD3);
2807   emit_int8((unsigned char)(0xE0 | encode));
2808 }
2809 
2810 void Assembler::shrl(Register dst, int imm8) {
2811   assert(isShiftCount(imm8), "illegal shift count");
2812   int encode = prefix_and_encode(dst->encoding());
2813   emit_int8((unsigned char)0xC1);
2814   emit_int8((unsigned char)(0xE8 | encode));
2815   emit_int8(imm8);
2816 }
2817 
2818 void Assembler::shrl(Register dst) {
2819   int encode = prefix_and_encode(dst->encoding());
2820   emit_int8((unsigned char)0xD3);
2821   emit_int8((unsigned char)(0xE8 | encode));
2822 }
2823 
2824 // copies a single word from [esi] to [edi]
2825 void Assembler::smovl() {
2826   emit_int8((unsigned char)0xA5);
2827 }
2828 
2829 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
2830   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2831   emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
2832 }
2833 
2834 void Assembler::sqrtsd(XMMRegister dst, Address src) {
2835   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2836   emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
2837 }
2838 
2839 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
2840   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2841   emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
2842 }
2843 
2844 void Assembler::std() {
2845   emit_int8((unsigned char)0xFD);
2846 }
2847 
2848 void Assembler::sqrtss(XMMRegister dst, Address src) {
2849   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2850   emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
2851 }
2852 
2853 void Assembler::stmxcsr( Address dst) {
2854   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2855   InstructionMark im(this);
2856   prefix(dst);
2857   emit_int8(0x0F);
2858   emit_int8((unsigned char)0xAE);
2859   emit_operand(as_Register(3), dst);
2860 }
2861 
2862 void Assembler::subl(Address dst, int32_t imm32) {
2863   InstructionMark im(this);
2864   prefix(dst);
2865   emit_arith_operand(0x81, rbp, dst, imm32);
2866 }
2867 
2868 void Assembler::subl(Address dst, Register src) {
2869   InstructionMark im(this);
2870   prefix(dst, src);
2871   emit_int8(0x29);
2872   emit_operand(src, dst);
2873 }
2874 
2875 void Assembler::subl(Register dst, int32_t imm32) {
2876   prefix(dst);
2877   emit_arith(0x81, 0xE8, dst, imm32);
2878 }
2879 
2880 // Force generation of a 4 byte immediate value even if it fits into 8bit
2881 void Assembler::subl_imm32(Register dst, int32_t imm32) {
2882   prefix(dst);
2883   emit_arith_imm32(0x81, 0xE8, dst, imm32);
2884 }
2885 
2886 void Assembler::subl(Register dst, Address src) {
2887   InstructionMark im(this);
2888   prefix(src, dst);
2889   emit_int8(0x2B);
2890   emit_operand(dst, src);
2891 }
2892 
2893 void Assembler::subl(Register dst, Register src) {
2894   (void) prefix_and_encode(dst->encoding(), src->encoding());
2895   emit_arith(0x2B, 0xC0, dst, src);
2896 }
2897 
2898 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
2899   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2900   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
2901 }
2902 
2903 void Assembler::subsd(XMMRegister dst, Address src) {
2904   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2905   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
2906 }
2907 
2908 void Assembler::subss(XMMRegister dst, XMMRegister src) {
2909   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2910   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
2911 }
2912 
2913 void Assembler::subss(XMMRegister dst, Address src) {
2914   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2915   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
2916 }
2917 
2918 void Assembler::testb(Register dst, int imm8) {
2919   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2920   (void) prefix_and_encode(dst->encoding(), true);
2921   emit_arith_b(0xF6, 0xC0, dst, imm8);
2922 }
2923 
2924 void Assembler::testl(Register dst, int32_t imm32) {
2925   // not using emit_arith because test
2926   // doesn't support sign-extension of
2927   // 8bit operands
2928   int encode = dst->encoding();
2929   if (encode == 0) {
2930     emit_int8((unsigned char)0xA9);
2931   } else {
2932     encode = prefix_and_encode(encode);
2933     emit_int8((unsigned char)0xF7);
2934     emit_int8((unsigned char)(0xC0 | encode));
2935   }
2936   emit_int32(imm32);
2937 }
2938 
2939 void Assembler::testl(Register dst, Register src) {
2940   (void) prefix_and_encode(dst->encoding(), src->encoding());
2941   emit_arith(0x85, 0xC0, dst, src);
2942 }
2943 
2944 void Assembler::testl(Register dst, Address  src) {
2945   InstructionMark im(this);
2946   prefix(src, dst);
2947   emit_int8((unsigned char)0x85);
2948   emit_operand(dst, src);
2949 }
2950 
2951 void Assembler::tzcntl(Register dst, Register src) {
2952   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
2953   emit_int8((unsigned char)0xF3);
2954   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2955   emit_int8(0x0F);
2956   emit_int8((unsigned char)0xBC);
2957   emit_int8((unsigned char)0xC0 | encode);
2958 }
2959 
2960 void Assembler::tzcntq(Register dst, Register src) {
2961   assert(VM_Version::supports_bmi1(), "tzcnt instruction not supported");
2962   emit_int8((unsigned char)0xF3);
2963   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
2964   emit_int8(0x0F);
2965   emit_int8((unsigned char)0xBC);
2966   emit_int8((unsigned char)(0xC0 | encode));
2967 }
2968 
2969 void Assembler::ucomisd(XMMRegister dst, Address src) {
2970   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2971   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
2972 }
2973 
2974 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
2975   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2976   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
2977 }
2978 
2979 void Assembler::ucomiss(XMMRegister dst, Address src) {
2980   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2981   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE);
2982 }
2983 
2984 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
2985   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2986   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE);
2987 }
2988 
2989 void Assembler::xabort(int8_t imm8) {
2990   emit_int8((unsigned char)0xC6);
2991   emit_int8((unsigned char)0xF8);
2992   emit_int8((unsigned char)(imm8 & 0xFF));
2993 }
2994 
2995 void Assembler::xaddl(Address dst, Register src) {
2996   InstructionMark im(this);
2997   prefix(dst, src);
2998   emit_int8(0x0F);
2999   emit_int8((unsigned char)0xC1);
3000   emit_operand(src, dst);
3001 }
3002 
3003 void Assembler::xbegin(Label& abort, relocInfo::relocType rtype) {
3004   InstructionMark im(this);
3005   relocate(rtype);
3006   if (abort.is_bound())
3007   {
3008     address entry = target(abort);
3009     assert(entry != NULL, "abort entry NULL");
3010     intptr_t offset = entry - pc();
3011     emit_int8((unsigned char)0xC7);
3012     emit_int8((unsigned char)0xF8);
3013     emit_int32(offset - 6); // 2 opcode + 4 address
3014   }
3015   else
3016   {
3017     abort.add_patch_at(code(), locator());
3018     emit_int8((unsigned char)0xC7);
3019     emit_int8((unsigned char)0xF8);
3020     emit_int32(0);
3021   }
3022 }
3023 
3024 void Assembler::xchgl(Register dst, Address src) { // xchg
3025   InstructionMark im(this);
3026   prefix(src, dst);
3027   emit_int8((unsigned char)0x87);
3028   emit_operand(dst, src);
3029 }
3030 
3031 void Assembler::xchgl(Register dst, Register src) {
3032   int encode = prefix_and_encode(dst->encoding(), src->encoding());
3033   emit_int8((unsigned char)0x87);
3034   emit_int8((unsigned char)(0xC0 | encode));
3035 }
3036 
3037 void Assembler::xgetbv() {
3038   emit_int8(0x0F);
3039   emit_int8(0x01);
3040   emit_int8((unsigned char)0xD0);
3041 }
3042 
3043 void Assembler::xend() {
3044   emit_int8((unsigned char)0x0F);
3045   emit_int8((unsigned char)0x01);
3046   emit_int8((unsigned char)0xD5);
3047 }
3048 
3049 void Assembler::xorl(Register dst, int32_t imm32) {
3050   prefix(dst);
3051   emit_arith(0x81, 0xF0, dst, imm32);
3052 }
3053 
3054 void Assembler::xorl(Register dst, Address src) {
3055   InstructionMark im(this);
3056   prefix(src, dst);
3057   emit_int8(0x33);
3058   emit_operand(dst, src);
3059 }
3060 
3061 void Assembler::xorl(Register dst, Register src) {
3062   (void) prefix_and_encode(dst->encoding(), src->encoding());
3063   emit_arith(0x33, 0xC0, dst, src);
3064 }
3065 
3066 
3067 // AVX 3-operands scalar float-point arithmetic instructions
3068 
3069 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
3070   assert(VM_Version::supports_avx(), "");
3071   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
3072 }
3073 
3074 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3075   assert(VM_Version::supports_avx(), "");
3076   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
3077 }
3078 
3079 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
3080   assert(VM_Version::supports_avx(), "");
3081   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3082 }
3083 
3084 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3085   assert(VM_Version::supports_avx(), "");
3086   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3087 }
3088 
3089 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
3090   assert(VM_Version::supports_avx(), "");
3091   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
3092 }
3093 
3094 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3095   assert(VM_Version::supports_avx(), "");
3096   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
3097 }
3098 
3099 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
3100   assert(VM_Version::supports_avx(), "");
3101   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3102 }
3103 
3104 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3105   assert(VM_Version::supports_avx(), "");
3106   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3107 }
3108 
3109 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
3110   assert(VM_Version::supports_avx(), "");
3111   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
3112 }
3113 
3114 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3115   assert(VM_Version::supports_avx(), "");
3116   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
3117 }
3118 
3119 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
3120   assert(VM_Version::supports_avx(), "");
3121   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3122 }
3123 
3124 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3125   assert(VM_Version::supports_avx(), "");
3126   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3127 }
3128 
3129 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
3130   assert(VM_Version::supports_avx(), "");
3131   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
3132 }
3133 
3134 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3135   assert(VM_Version::supports_avx(), "");
3136   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
3137 }
3138 
3139 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
3140   assert(VM_Version::supports_avx(), "");
3141   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3142 }
3143 
3144 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3145   assert(VM_Version::supports_avx(), "");
3146   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3147 }
3148 
3149 //====================VECTOR ARITHMETIC=====================================
3150 
3151 // Float-point vector arithmetic
3152 
3153 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
3154   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3155   emit_simd_arith(0x58, dst, src, VEX_SIMD_66);
3156 }
3157 
3158 void Assembler::addps(XMMRegister dst, XMMRegister src) {
3159   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3160   emit_simd_arith(0x58, dst, src, VEX_SIMD_NONE);
3161 }
3162 
3163 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3164   assert(VM_Version::supports_avx(), "");
3165   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector256);
3166 }
3167 
3168 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3169   assert(VM_Version::supports_avx(), "");
3170   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector256);
3171 }
3172 
3173 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3174   assert(VM_Version::supports_avx(), "");
3175   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector256);
3176 }
3177 
3178 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3179   assert(VM_Version::supports_avx(), "");
3180   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector256);
3181 }
3182 
3183 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
3184   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3185   emit_simd_arith(0x5C, dst, src, VEX_SIMD_66);
3186 }
3187 
3188 void Assembler::subps(XMMRegister dst, XMMRegister src) {
3189   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3190   emit_simd_arith(0x5C, dst, src, VEX_SIMD_NONE);
3191 }
3192 
3193 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3194   assert(VM_Version::supports_avx(), "");
3195   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector256);
3196 }
3197 
3198 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3199   assert(VM_Version::supports_avx(), "");
3200   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector256);
3201 }
3202 
3203 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3204   assert(VM_Version::supports_avx(), "");
3205   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector256);
3206 }
3207 
3208 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3209   assert(VM_Version::supports_avx(), "");
3210   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector256);
3211 }
3212 
3213 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
3214   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3215   emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
3216 }
3217 
3218 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
3219   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3220   emit_simd_arith(0x59, dst, src, VEX_SIMD_NONE);
3221 }
3222 
3223 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3224   assert(VM_Version::supports_avx(), "");
3225   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector256);
3226 }
3227 
3228 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3229   assert(VM_Version::supports_avx(), "");
3230   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector256);
3231 }
3232 
3233 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3234   assert(VM_Version::supports_avx(), "");
3235   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector256);
3236 }
3237 
3238 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3239   assert(VM_Version::supports_avx(), "");
3240   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector256);
3241 }
3242 
3243 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
3244   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3245   emit_simd_arith(0x5E, dst, src, VEX_SIMD_66);
3246 }
3247 
3248 void Assembler::divps(XMMRegister dst, XMMRegister src) {
3249   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3250   emit_simd_arith(0x5E, dst, src, VEX_SIMD_NONE);
3251 }
3252 
3253 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3254   assert(VM_Version::supports_avx(), "");
3255   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector256);
3256 }
3257 
3258 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3259   assert(VM_Version::supports_avx(), "");
3260   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector256);
3261 }
3262 
3263 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3264   assert(VM_Version::supports_avx(), "");
3265   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector256);
3266 }
3267 
3268 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3269   assert(VM_Version::supports_avx(), "");
3270   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector256);
3271 }
3272 
3273 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
3274   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3275   emit_simd_arith(0x54, dst, src, VEX_SIMD_66);
3276 }
3277 
3278 void Assembler::andps(XMMRegister dst, XMMRegister src) {
3279   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3280   emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE);
3281 }
3282 
3283 void Assembler::andps(XMMRegister dst, Address src) {
3284   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3285   emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE);
3286 }
3287 
3288 void Assembler::andpd(XMMRegister dst, Address src) {
3289   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3290   emit_simd_arith(0x54, dst, src, VEX_SIMD_66);
3291 }
3292 
3293 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3294   assert(VM_Version::supports_avx(), "");
3295   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector256);
3296 }
3297 
3298 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3299   assert(VM_Version::supports_avx(), "");
3300   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector256);
3301 }
3302 
3303 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3304   assert(VM_Version::supports_avx(), "");
3305   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector256);
3306 }
3307 
3308 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3309   assert(VM_Version::supports_avx(), "");
3310   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector256);
3311 }
3312 
3313 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
3314   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3315   emit_simd_arith(0x57, dst, src, VEX_SIMD_66);
3316 }
3317 
3318 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
3319   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3320   emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE);
3321 }
3322 
3323 void Assembler::xorpd(XMMRegister dst, Address src) {
3324   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3325   emit_simd_arith(0x57, dst, src, VEX_SIMD_66);
3326 }
3327 
3328 void Assembler::xorps(XMMRegister dst, Address src) {
3329   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3330   emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE);
3331 }
3332 
3333 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3334   assert(VM_Version::supports_avx(), "");
3335   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector256);
3336 }
3337 
3338 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3339   assert(VM_Version::supports_avx(), "");
3340   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector256);
3341 }
3342 
3343 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3344   assert(VM_Version::supports_avx(), "");
3345   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector256);
3346 }
3347 
3348 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3349   assert(VM_Version::supports_avx(), "");
3350   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector256);
3351 }
3352 
3353 
3354 // Integer vector arithmetic
3355 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
3356   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3357   emit_simd_arith(0xFC, dst, src, VEX_SIMD_66);
3358 }
3359 
3360 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
3361   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3362   emit_simd_arith(0xFD, dst, src, VEX_SIMD_66);
3363 }
3364 
3365 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
3366   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3367   emit_simd_arith(0xFE, dst, src, VEX_SIMD_66);
3368 }
3369 
3370 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
3371   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3372   emit_simd_arith(0xD4, dst, src, VEX_SIMD_66);
3373 }
3374 
3375 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3376   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3377   emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector256);
3378 }
3379 
3380 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3381   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3382   emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector256);
3383 }
3384 
3385 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3386   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3387   emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector256);
3388 }
3389 
3390 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3391   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3392   emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector256);
3393 }
3394 
3395 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3396   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3397   emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector256);
3398 }
3399 
3400 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3401   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3402   emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector256);
3403 }
3404 
3405 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3406   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3407   emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector256);
3408 }
3409 
3410 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3411   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3412   emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector256);
3413 }
3414 
3415 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
3416   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3417   emit_simd_arith(0xF8, dst, src, VEX_SIMD_66);
3418 }
3419 
3420 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
3421   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3422   emit_simd_arith(0xF9, dst, src, VEX_SIMD_66);
3423 }
3424 
3425 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
3426   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3427   emit_simd_arith(0xFA, dst, src, VEX_SIMD_66);
3428 }
3429 
3430 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
3431   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3432   emit_simd_arith(0xFB, dst, src, VEX_SIMD_66);
3433 }
3434 
3435 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3436   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3437   emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector256);
3438 }
3439 
3440 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3441   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3442   emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector256);
3443 }
3444 
3445 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3446   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3447   emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector256);
3448 }
3449 
3450 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3451   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3452   emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector256);
3453 }
3454 
3455 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3456   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3457   emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector256);
3458 }
3459 
3460 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3461   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3462   emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector256);
3463 }
3464 
3465 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3466   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3467   emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector256);
3468 }
3469 
3470 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3471   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3472   emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector256);
3473 }
3474 
3475 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
3476   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3477   emit_simd_arith(0xD5, dst, src, VEX_SIMD_66);
3478 }
3479 
3480 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
3481   assert(VM_Version::supports_sse4_1(), "");
3482   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
3483   emit_int8(0x40);
3484   emit_int8((unsigned char)(0xC0 | encode));
3485 }
3486 
3487 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3488   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3489   emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector256);
3490 }
3491 
3492 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3493   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3494   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_38);
3495   emit_int8(0x40);
3496   emit_int8((unsigned char)(0xC0 | encode));
3497 }
3498 
3499 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3500   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3501   emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector256);
3502 }
3503 
3504 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3505   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3506   InstructionMark im(this);
3507   int dst_enc = dst->encoding();
3508   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3509   vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector256);
3510   emit_int8(0x40);
3511   emit_operand(dst, src);
3512 }
3513 
3514 // Shift packed integers left by specified number of bits.
3515 void Assembler::psllw(XMMRegister dst, int shift) {
3516   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3517   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
3518   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
3519   emit_int8(0x71);
3520   emit_int8((unsigned char)(0xC0 | encode));
3521   emit_int8(shift & 0xFF);
3522 }
3523 
3524 void Assembler::pslld(XMMRegister dst, int shift) {
3525   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3526   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
3527   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
3528   emit_int8(0x72);
3529   emit_int8((unsigned char)(0xC0 | encode));
3530   emit_int8(shift & 0xFF);
3531 }
3532 
3533 void Assembler::psllq(XMMRegister dst, int shift) {
3534   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3535   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
3536   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
3537   emit_int8(0x73);
3538   emit_int8((unsigned char)(0xC0 | encode));
3539   emit_int8(shift & 0xFF);
3540 }
3541 
3542 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
3543   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3544   emit_simd_arith(0xF1, dst, shift, VEX_SIMD_66);
3545 }
3546 
3547 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
3548   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3549   emit_simd_arith(0xF2, dst, shift, VEX_SIMD_66);
3550 }
3551 
3552 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
3553   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3554   emit_simd_arith(0xF3, dst, shift, VEX_SIMD_66);
3555 }
3556 
3557 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3558   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3559   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
3560   emit_vex_arith(0x71, xmm6, dst, src, VEX_SIMD_66, vector256);
3561   emit_int8(shift & 0xFF);
3562 }
3563 
3564 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3565   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3566   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
3567   emit_vex_arith(0x72, xmm6, dst, src, VEX_SIMD_66, vector256);
3568   emit_int8(shift & 0xFF);
3569 }
3570 
3571 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3572   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3573   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
3574   emit_vex_arith(0x73, xmm6, dst, src, VEX_SIMD_66, vector256);
3575   emit_int8(shift & 0xFF);
3576 }
3577 
3578 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3579   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3580   emit_vex_arith(0xF1, dst, src, shift, VEX_SIMD_66, vector256);
3581 }
3582 
3583 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3584   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3585   emit_vex_arith(0xF2, dst, src, shift, VEX_SIMD_66, vector256);
3586 }
3587 
3588 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3589   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3590   emit_vex_arith(0xF3, dst, src, shift, VEX_SIMD_66, vector256);
3591 }
3592 
3593 // Shift packed integers logically right by specified number of bits.
3594 void Assembler::psrlw(XMMRegister dst, int shift) {
3595   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3596   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
3597   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
3598   emit_int8(0x71);
3599   emit_int8((unsigned char)(0xC0 | encode));
3600   emit_int8(shift & 0xFF);
3601 }
3602 
3603 void Assembler::psrld(XMMRegister dst, int shift) {
3604   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3605   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
3606   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
3607   emit_int8(0x72);
3608   emit_int8((unsigned char)(0xC0 | encode));
3609   emit_int8(shift & 0xFF);
3610 }
3611 
3612 void Assembler::psrlq(XMMRegister dst, int shift) {
3613   // Do not confuse it with psrldq SSE2 instruction which
3614   // shifts 128 bit value in xmm register by number of bytes.
3615   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3616   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3617   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
3618   emit_int8(0x73);
3619   emit_int8((unsigned char)(0xC0 | encode));
3620   emit_int8(shift & 0xFF);
3621 }
3622 
3623 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
3624   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3625   emit_simd_arith(0xD1, dst, shift, VEX_SIMD_66);
3626 }
3627 
3628 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
3629   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3630   emit_simd_arith(0xD2, dst, shift, VEX_SIMD_66);
3631 }
3632 
3633 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
3634   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3635   emit_simd_arith(0xD3, dst, shift, VEX_SIMD_66);
3636 }
3637 
3638 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3639   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3640   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3641   emit_vex_arith(0x71, xmm2, dst, src, VEX_SIMD_66, vector256);
3642   emit_int8(shift & 0xFF);
3643 }
3644 
3645 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3646   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3647   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3648   emit_vex_arith(0x72, xmm2, dst, src, VEX_SIMD_66, vector256);
3649   emit_int8(shift & 0xFF);
3650 }
3651 
3652 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3653   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3654   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3655   emit_vex_arith(0x73, xmm2, dst, src, VEX_SIMD_66, vector256);
3656   emit_int8(shift & 0xFF);
3657 }
3658 
3659 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3660   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3661   emit_vex_arith(0xD1, dst, src, shift, VEX_SIMD_66, vector256);
3662 }
3663 
3664 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3665   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3666   emit_vex_arith(0xD2, dst, src, shift, VEX_SIMD_66, vector256);
3667 }
3668 
3669 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3670   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3671   emit_vex_arith(0xD3, dst, src, shift, VEX_SIMD_66, vector256);
3672 }
3673 
3674 // Shift packed integers arithmetically right by specified number of bits.
3675 void Assembler::psraw(XMMRegister dst, int shift) {
3676   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3677   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
3678   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66);
3679   emit_int8(0x71);
3680   emit_int8((unsigned char)(0xC0 | encode));
3681   emit_int8(shift & 0xFF);
3682 }
3683 
3684 void Assembler::psrad(XMMRegister dst, int shift) {
3685   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3686   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
3687   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66);
3688   emit_int8(0x72);
3689   emit_int8((unsigned char)(0xC0 | encode));
3690   emit_int8(shift & 0xFF);
3691 }
3692 
3693 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
3694   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3695   emit_simd_arith(0xE1, dst, shift, VEX_SIMD_66);
3696 }
3697 
3698 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
3699   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3700   emit_simd_arith(0xE2, dst, shift, VEX_SIMD_66);
3701 }
3702 
3703 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3704   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3705   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
3706   emit_vex_arith(0x71, xmm4, dst, src, VEX_SIMD_66, vector256);
3707   emit_int8(shift & 0xFF);
3708 }
3709 
3710 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3711   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3712   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
3713   emit_vex_arith(0x72, xmm4, dst, src, VEX_SIMD_66, vector256);
3714   emit_int8(shift & 0xFF);
3715 }
3716 
3717 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3718   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3719   emit_vex_arith(0xE1, dst, src, shift, VEX_SIMD_66, vector256);
3720 }
3721 
3722 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3723   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3724   emit_vex_arith(0xE2, dst, src, shift, VEX_SIMD_66, vector256);
3725 }
3726 
3727 
3728 // AND packed integers
3729 void Assembler::pand(XMMRegister dst, XMMRegister src) {
3730   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3731   emit_simd_arith(0xDB, dst, src, VEX_SIMD_66);
3732 }
3733 
3734 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3735   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3736   emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector256);
3737 }
3738 
3739 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3740   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3741   emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector256);
3742 }
3743 
3744 void Assembler::por(XMMRegister dst, XMMRegister src) {
3745   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3746   emit_simd_arith(0xEB, dst, src, VEX_SIMD_66);
3747 }
3748 
3749 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3750   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3751   emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector256);
3752 }
3753 
3754 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3755   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3756   emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector256);
3757 }
3758 
3759 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
3760   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3761   emit_simd_arith(0xEF, dst, src, VEX_SIMD_66);
3762 }
3763 
3764 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3765   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3766   emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector256);
3767 }
3768 
3769 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3770   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3771   emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector256);
3772 }
3773 
3774 
3775 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3776   assert(VM_Version::supports_avx(), "");
3777   bool vector256 = true;
3778   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
3779   emit_int8(0x18);
3780   emit_int8((unsigned char)(0xC0 | encode));
3781   // 0x00 - insert into lower 128 bits
3782   // 0x01 - insert into upper 128 bits
3783   emit_int8(0x01);
3784 }
3785 
3786 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
3787   assert(VM_Version::supports_avx(), "");
3788   InstructionMark im(this);
3789   bool vector256 = true;
3790   assert(dst != xnoreg, "sanity");
3791   int dst_enc = dst->encoding();
3792   // swap src<->dst for encoding
3793   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3794   emit_int8(0x18);
3795   emit_operand(dst, src);
3796   // 0x01 - insert into upper 128 bits
3797   emit_int8(0x01);
3798 }
3799 
3800 void Assembler::vextractf128h(Address dst, XMMRegister src) {
3801   assert(VM_Version::supports_avx(), "");
3802   InstructionMark im(this);
3803   bool vector256 = true;
3804   assert(src != xnoreg, "sanity");
3805   int src_enc = src->encoding();
3806   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3807   emit_int8(0x19);
3808   emit_operand(src, dst);
3809   // 0x01 - extract from upper 128 bits
3810   emit_int8(0x01);
3811 }
3812 
3813 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3814   assert(VM_Version::supports_avx2(), "");
3815   bool vector256 = true;
3816   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
3817   emit_int8(0x38);
3818   emit_int8((unsigned char)(0xC0 | encode));
3819   // 0x00 - insert into lower 128 bits
3820   // 0x01 - insert into upper 128 bits
3821   emit_int8(0x01);
3822 }
3823 
3824 void Assembler::vinserti128h(XMMRegister dst, Address src) {
3825   assert(VM_Version::supports_avx2(), "");
3826   InstructionMark im(this);
3827   bool vector256 = true;
3828   assert(dst != xnoreg, "sanity");
3829   int dst_enc = dst->encoding();
3830   // swap src<->dst for encoding
3831   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3832   emit_int8(0x38);
3833   emit_operand(dst, src);
3834   // 0x01 - insert into upper 128 bits
3835   emit_int8(0x01);
3836 }
3837 
3838 void Assembler::vextracti128h(Address dst, XMMRegister src) {
3839   assert(VM_Version::supports_avx2(), "");
3840   InstructionMark im(this);
3841   bool vector256 = true;
3842   assert(src != xnoreg, "sanity");
3843   int src_enc = src->encoding();
3844   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3845   emit_int8(0x39);
3846   emit_operand(src, dst);
3847   // 0x01 - extract from upper 128 bits
3848   emit_int8(0x01);
3849 }
3850 
3851 // duplicate 4-bytes integer data from src into 8 locations in dest
3852 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
3853   assert(VM_Version::supports_avx2(), "");
3854   bool vector256 = true;
3855   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_38);
3856   emit_int8(0x58);
3857   emit_int8((unsigned char)(0xC0 | encode));
3858 }
3859 
3860 // Carry-Less Multiplication Quadword
3861 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
3862   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
3863   bool vector256 = false;
3864   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
3865   emit_int8(0x44);
3866   emit_int8((unsigned char)(0xC0 | encode));
3867   emit_int8((unsigned char)mask);
3868 }
3869 
3870 void Assembler::vzeroupper() {
3871   assert(VM_Version::supports_avx(), "");
3872   (void)vex_prefix_and_encode(xmm0, xmm0, xmm0, VEX_SIMD_NONE);
3873   emit_int8(0x77);
3874 }
3875 
3876 
3877 #ifndef _LP64
3878 // 32bit only pieces of the assembler
3879 
3880 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
3881   // NO PREFIX AS NEVER 64BIT
3882   InstructionMark im(this);
3883   emit_int8((unsigned char)0x81);
3884   emit_int8((unsigned char)(0xF8 | src1->encoding()));
3885   emit_data(imm32, rspec, 0);
3886 }
3887 
3888 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
3889   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
3890   InstructionMark im(this);
3891   emit_int8((unsigned char)0x81);
3892   emit_operand(rdi, src1);
3893   emit_data(imm32, rspec, 0);
3894 }
3895 
3896 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
3897 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
3898 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
3899 void Assembler::cmpxchg8(Address adr) {
3900   InstructionMark im(this);
3901   emit_int8(0x0F);
3902   emit_int8((unsigned char)0xC7);
3903   emit_operand(rcx, adr);
3904 }
3905 
3906 void Assembler::decl(Register dst) {
3907   // Don't use it directly. Use MacroAssembler::decrementl() instead.
3908  emit_int8(0x48 | dst->encoding());
3909 }
3910 
3911 #endif // _LP64
3912 
3913 // 64bit typically doesn't use the x87 but needs to for the trig funcs
3914 
3915 void Assembler::fabs() {
3916   emit_int8((unsigned char)0xD9);
3917   emit_int8((unsigned char)0xE1);
3918 }
3919 
3920 void Assembler::fadd(int i) {
3921   emit_farith(0xD8, 0xC0, i);
3922 }
3923 
3924 void Assembler::fadd_d(Address src) {
3925   InstructionMark im(this);
3926   emit_int8((unsigned char)0xDC);
3927   emit_operand32(rax, src);
3928 }
3929 
3930 void Assembler::fadd_s(Address src) {
3931   InstructionMark im(this);
3932   emit_int8((unsigned char)0xD8);
3933   emit_operand32(rax, src);
3934 }
3935 
3936 void Assembler::fadda(int i) {
3937   emit_farith(0xDC, 0xC0, i);
3938 }
3939 
3940 void Assembler::faddp(int i) {
3941   emit_farith(0xDE, 0xC0, i);
3942 }
3943 
3944 void Assembler::fchs() {
3945   emit_int8((unsigned char)0xD9);
3946   emit_int8((unsigned char)0xE0);
3947 }
3948 
3949 void Assembler::fcom(int i) {
3950   emit_farith(0xD8, 0xD0, i);
3951 }
3952 
3953 void Assembler::fcomp(int i) {
3954   emit_farith(0xD8, 0xD8, i);
3955 }
3956 
3957 void Assembler::fcomp_d(Address src) {
3958   InstructionMark im(this);
3959   emit_int8((unsigned char)0xDC);
3960   emit_operand32(rbx, src);
3961 }
3962 
3963 void Assembler::fcomp_s(Address src) {
3964   InstructionMark im(this);
3965   emit_int8((unsigned char)0xD8);
3966   emit_operand32(rbx, src);
3967 }
3968 
3969 void Assembler::fcompp() {
3970   emit_int8((unsigned char)0xDE);
3971   emit_int8((unsigned char)0xD9);
3972 }
3973 
3974 void Assembler::fcos() {
3975   emit_int8((unsigned char)0xD9);
3976   emit_int8((unsigned char)0xFF);
3977 }
3978 
3979 void Assembler::fdecstp() {
3980   emit_int8((unsigned char)0xD9);
3981   emit_int8((unsigned char)0xF6);
3982 }
3983 
3984 void Assembler::fdiv(int i) {
3985   emit_farith(0xD8, 0xF0, i);
3986 }
3987 
3988 void Assembler::fdiv_d(Address src) {
3989   InstructionMark im(this);
3990   emit_int8((unsigned char)0xDC);
3991   emit_operand32(rsi, src);
3992 }
3993 
3994 void Assembler::fdiv_s(Address src) {
3995   InstructionMark im(this);
3996   emit_int8((unsigned char)0xD8);
3997   emit_operand32(rsi, src);
3998 }
3999 
4000 void Assembler::fdiva(int i) {
4001   emit_farith(0xDC, 0xF8, i);
4002 }
4003 
4004 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
4005 //       is erroneous for some of the floating-point instructions below.
4006 
4007 void Assembler::fdivp(int i) {
4008   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
4009 }
4010 
4011 void Assembler::fdivr(int i) {
4012   emit_farith(0xD8, 0xF8, i);
4013 }
4014 
4015 void Assembler::fdivr_d(Address src) {
4016   InstructionMark im(this);
4017   emit_int8((unsigned char)0xDC);
4018   emit_operand32(rdi, src);
4019 }
4020 
4021 void Assembler::fdivr_s(Address src) {
4022   InstructionMark im(this);
4023   emit_int8((unsigned char)0xD8);
4024   emit_operand32(rdi, src);
4025 }
4026 
4027 void Assembler::fdivra(int i) {
4028   emit_farith(0xDC, 0xF0, i);
4029 }
4030 
4031 void Assembler::fdivrp(int i) {
4032   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
4033 }
4034 
4035 void Assembler::ffree(int i) {
4036   emit_farith(0xDD, 0xC0, i);
4037 }
4038 
4039 void Assembler::fild_d(Address adr) {
4040   InstructionMark im(this);
4041   emit_int8((unsigned char)0xDF);
4042   emit_operand32(rbp, adr);
4043 }
4044 
4045 void Assembler::fild_s(Address adr) {
4046   InstructionMark im(this);
4047   emit_int8((unsigned char)0xDB);
4048   emit_operand32(rax, adr);
4049 }
4050 
4051 void Assembler::fincstp() {
4052   emit_int8((unsigned char)0xD9);
4053   emit_int8((unsigned char)0xF7);
4054 }
4055 
4056 void Assembler::finit() {
4057   emit_int8((unsigned char)0x9B);
4058   emit_int8((unsigned char)0xDB);
4059   emit_int8((unsigned char)0xE3);
4060 }
4061 
4062 void Assembler::fist_s(Address adr) {
4063   InstructionMark im(this);
4064   emit_int8((unsigned char)0xDB);
4065   emit_operand32(rdx, adr);
4066 }
4067 
4068 void Assembler::fistp_d(Address adr) {
4069   InstructionMark im(this);
4070   emit_int8((unsigned char)0xDF);
4071   emit_operand32(rdi, adr);
4072 }
4073 
4074 void Assembler::fistp_s(Address adr) {
4075   InstructionMark im(this);
4076   emit_int8((unsigned char)0xDB);
4077   emit_operand32(rbx, adr);
4078 }
4079 
4080 void Assembler::fld1() {
4081   emit_int8((unsigned char)0xD9);
4082   emit_int8((unsigned char)0xE8);
4083 }
4084 
4085 void Assembler::fld_d(Address adr) {
4086   InstructionMark im(this);
4087   emit_int8((unsigned char)0xDD);
4088   emit_operand32(rax, adr);
4089 }
4090 
4091 void Assembler::fld_s(Address adr) {
4092   InstructionMark im(this);
4093   emit_int8((unsigned char)0xD9);
4094   emit_operand32(rax, adr);
4095 }
4096 
4097 
4098 void Assembler::fld_s(int index) {
4099   emit_farith(0xD9, 0xC0, index);
4100 }
4101 
4102 void Assembler::fld_x(Address adr) {
4103   InstructionMark im(this);
4104   emit_int8((unsigned char)0xDB);
4105   emit_operand32(rbp, adr);
4106 }
4107 
4108 void Assembler::fldcw(Address src) {
4109   InstructionMark im(this);
4110   emit_int8((unsigned char)0xD9);
4111   emit_operand32(rbp, src);
4112 }
4113 
4114 void Assembler::fldenv(Address src) {
4115   InstructionMark im(this);
4116   emit_int8((unsigned char)0xD9);
4117   emit_operand32(rsp, src);
4118 }
4119 
4120 void Assembler::fldlg2() {
4121   emit_int8((unsigned char)0xD9);
4122   emit_int8((unsigned char)0xEC);
4123 }
4124 
4125 void Assembler::fldln2() {
4126   emit_int8((unsigned char)0xD9);
4127   emit_int8((unsigned char)0xED);
4128 }
4129 
4130 void Assembler::fldz() {
4131   emit_int8((unsigned char)0xD9);
4132   emit_int8((unsigned char)0xEE);
4133 }
4134 
4135 void Assembler::flog() {
4136   fldln2();
4137   fxch();
4138   fyl2x();
4139 }
4140 
4141 void Assembler::flog10() {
4142   fldlg2();
4143   fxch();
4144   fyl2x();
4145 }
4146 
4147 void Assembler::fmul(int i) {
4148   emit_farith(0xD8, 0xC8, i);
4149 }
4150 
4151 void Assembler::fmul_d(Address src) {
4152   InstructionMark im(this);
4153   emit_int8((unsigned char)0xDC);
4154   emit_operand32(rcx, src);
4155 }
4156 
4157 void Assembler::fmul_s(Address src) {
4158   InstructionMark im(this);
4159   emit_int8((unsigned char)0xD8);
4160   emit_operand32(rcx, src);
4161 }
4162 
4163 void Assembler::fmula(int i) {
4164   emit_farith(0xDC, 0xC8, i);
4165 }
4166 
4167 void Assembler::fmulp(int i) {
4168   emit_farith(0xDE, 0xC8, i);
4169 }
4170 
4171 void Assembler::fnsave(Address dst) {
4172   InstructionMark im(this);
4173   emit_int8((unsigned char)0xDD);
4174   emit_operand32(rsi, dst);
4175 }
4176 
4177 void Assembler::fnstcw(Address src) {
4178   InstructionMark im(this);
4179   emit_int8((unsigned char)0x9B);
4180   emit_int8((unsigned char)0xD9);
4181   emit_operand32(rdi, src);
4182 }
4183 
4184 void Assembler::fnstsw_ax() {
4185   emit_int8((unsigned char)0xDF);
4186   emit_int8((unsigned char)0xE0);
4187 }
4188 
4189 void Assembler::fprem() {
4190   emit_int8((unsigned char)0xD9);
4191   emit_int8((unsigned char)0xF8);
4192 }
4193 
4194 void Assembler::fprem1() {
4195   emit_int8((unsigned char)0xD9);
4196   emit_int8((unsigned char)0xF5);
4197 }
4198 
4199 void Assembler::frstor(Address src) {
4200   InstructionMark im(this);
4201   emit_int8((unsigned char)0xDD);
4202   emit_operand32(rsp, src);
4203 }
4204 
4205 void Assembler::fsin() {
4206   emit_int8((unsigned char)0xD9);
4207   emit_int8((unsigned char)0xFE);
4208 }
4209 
4210 void Assembler::fsqrt() {
4211   emit_int8((unsigned char)0xD9);
4212   emit_int8((unsigned char)0xFA);
4213 }
4214 
4215 void Assembler::fst_d(Address adr) {
4216   InstructionMark im(this);
4217   emit_int8((unsigned char)0xDD);
4218   emit_operand32(rdx, adr);
4219 }
4220 
4221 void Assembler::fst_s(Address adr) {
4222   InstructionMark im(this);
4223   emit_int8((unsigned char)0xD9);
4224   emit_operand32(rdx, adr);
4225 }
4226 
4227 void Assembler::fstp_d(Address adr) {
4228   InstructionMark im(this);
4229   emit_int8((unsigned char)0xDD);
4230   emit_operand32(rbx, adr);
4231 }
4232 
4233 void Assembler::fstp_d(int index) {
4234   emit_farith(0xDD, 0xD8, index);
4235 }
4236 
4237 void Assembler::fstp_s(Address adr) {
4238   InstructionMark im(this);
4239   emit_int8((unsigned char)0xD9);
4240   emit_operand32(rbx, adr);
4241 }
4242 
4243 void Assembler::fstp_x(Address adr) {
4244   InstructionMark im(this);
4245   emit_int8((unsigned char)0xDB);
4246   emit_operand32(rdi, adr);
4247 }
4248 
4249 void Assembler::fsub(int i) {
4250   emit_farith(0xD8, 0xE0, i);
4251 }
4252 
4253 void Assembler::fsub_d(Address src) {
4254   InstructionMark im(this);
4255   emit_int8((unsigned char)0xDC);
4256   emit_operand32(rsp, src);
4257 }
4258 
4259 void Assembler::fsub_s(Address src) {
4260   InstructionMark im(this);
4261   emit_int8((unsigned char)0xD8);
4262   emit_operand32(rsp, src);
4263 }
4264 
4265 void Assembler::fsuba(int i) {
4266   emit_farith(0xDC, 0xE8, i);
4267 }
4268 
4269 void Assembler::fsubp(int i) {
4270   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
4271 }
4272 
4273 void Assembler::fsubr(int i) {
4274   emit_farith(0xD8, 0xE8, i);
4275 }
4276 
4277 void Assembler::fsubr_d(Address src) {
4278   InstructionMark im(this);
4279   emit_int8((unsigned char)0xDC);
4280   emit_operand32(rbp, src);
4281 }
4282 
4283 void Assembler::fsubr_s(Address src) {
4284   InstructionMark im(this);
4285   emit_int8((unsigned char)0xD8);
4286   emit_operand32(rbp, src);
4287 }
4288 
4289 void Assembler::fsubra(int i) {
4290   emit_farith(0xDC, 0xE0, i);
4291 }
4292 
4293 void Assembler::fsubrp(int i) {
4294   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
4295 }
4296 
4297 void Assembler::ftan() {
4298   emit_int8((unsigned char)0xD9);
4299   emit_int8((unsigned char)0xF2);
4300   emit_int8((unsigned char)0xDD);
4301   emit_int8((unsigned char)0xD8);
4302 }
4303 
4304 void Assembler::ftst() {
4305   emit_int8((unsigned char)0xD9);
4306   emit_int8((unsigned char)0xE4);
4307 }
4308 
4309 void Assembler::fucomi(int i) {
4310   // make sure the instruction is supported (introduced for P6, together with cmov)
4311   guarantee(VM_Version::supports_cmov(), "illegal instruction");
4312   emit_farith(0xDB, 0xE8, i);
4313 }
4314 
4315 void Assembler::fucomip(int i) {
4316   // make sure the instruction is supported (introduced for P6, together with cmov)
4317   guarantee(VM_Version::supports_cmov(), "illegal instruction");
4318   emit_farith(0xDF, 0xE8, i);
4319 }
4320 
4321 void Assembler::fwait() {
4322   emit_int8((unsigned char)0x9B);
4323 }
4324 
4325 void Assembler::fxch(int i) {
4326   emit_farith(0xD9, 0xC8, i);
4327 }
4328 
4329 void Assembler::fyl2x() {
4330   emit_int8((unsigned char)0xD9);
4331   emit_int8((unsigned char)0xF1);
4332 }
4333 
4334 void Assembler::frndint() {
4335   emit_int8((unsigned char)0xD9);
4336   emit_int8((unsigned char)0xFC);
4337 }
4338 
4339 void Assembler::f2xm1() {
4340   emit_int8((unsigned char)0xD9);
4341   emit_int8((unsigned char)0xF0);
4342 }
4343 
4344 void Assembler::fldl2e() {
4345   emit_int8((unsigned char)0xD9);
4346   emit_int8((unsigned char)0xEA);
4347 }
4348 
4349 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
4350 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
4351 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
4352 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
4353 
4354 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
4355 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
4356   if (pre > 0) {
4357     emit_int8(simd_pre[pre]);
4358   }
4359   if (rex_w) {
4360     prefixq(adr, xreg);
4361   } else {
4362     prefix(adr, xreg);
4363   }
4364   if (opc > 0) {
4365     emit_int8(0x0F);
4366     int opc2 = simd_opc[opc];
4367     if (opc2 > 0) {
4368       emit_int8(opc2);
4369     }
4370   }
4371 }
4372 
4373 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
4374   if (pre > 0) {
4375     emit_int8(simd_pre[pre]);
4376   }
4377   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) :
4378                           prefix_and_encode(dst_enc, src_enc);
4379   if (opc > 0) {
4380     emit_int8(0x0F);
4381     int opc2 = simd_opc[opc];
4382     if (opc2 > 0) {
4383       emit_int8(opc2);
4384     }
4385   }
4386   return encode;
4387 }
4388 
4389 
4390 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, int nds_enc, VexSimdPrefix pre, VexOpcode opc, bool vector256) {
4391   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
4392     prefix(VEX_3bytes);
4393 
4394     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
4395     byte1 = (~byte1) & 0xE0;
4396     byte1 |= opc;
4397     emit_int8(byte1);
4398 
4399     int byte2 = ((~nds_enc) & 0xf) << 3;
4400     byte2 |= (vex_w ? VEX_W : 0) | (vector256 ? 4 : 0) | pre;
4401     emit_int8(byte2);
4402   } else {
4403     prefix(VEX_2bytes);
4404 
4405     int byte1 = vex_r ? VEX_R : 0;
4406     byte1 = (~byte1) & 0x80;
4407     byte1 |= ((~nds_enc) & 0xf) << 3;
4408     byte1 |= (vector256 ? 4 : 0) | pre;
4409     emit_int8(byte1);
4410   }
4411 }
4412 
4413 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, bool vex_w, bool vector256){
4414   bool vex_r = (xreg_enc >= 8);
4415   bool vex_b = adr.base_needs_rex();
4416   bool vex_x = adr.index_needs_rex();
4417   vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector256);
4418 }
4419 
4420 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool vex_w, bool vector256) {
4421   bool vex_r = (dst_enc >= 8);
4422   bool vex_b = (src_enc >= 8);
4423   bool vex_x = false;
4424   vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector256);
4425   return (((dst_enc & 7) << 3) | (src_enc & 7));
4426 }
4427 
4428 
4429 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre, VexOpcode opc, bool rex_w, bool vector256) {
4430   if (UseAVX > 0) {
4431     int xreg_enc = xreg->encoding();
4432     int  nds_enc = nds->is_valid() ? nds->encoding() : 0;
4433     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, rex_w, vector256);
4434   } else {
4435     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
4436     rex_prefix(adr, xreg, pre, opc, rex_w);
4437   }
4438 }
4439 
4440 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre, VexOpcode opc, bool rex_w, bool vector256) {
4441   int dst_enc = dst->encoding();
4442   int src_enc = src->encoding();
4443   if (UseAVX > 0) {
4444     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4445     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector256);
4446   } else {
4447     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
4448     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, rex_w);
4449   }
4450 }
4451 
4452 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre) {
4453   InstructionMark im(this);
4454   simd_prefix(dst, dst, src, pre);
4455   emit_int8(opcode);
4456   emit_operand(dst, src);
4457 }
4458 
4459 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre) {
4460   int encode = simd_prefix_and_encode(dst, dst, src, pre);
4461   emit_int8(opcode);
4462   emit_int8((unsigned char)(0xC0 | encode));
4463 }
4464 
4465 // Versions with no second source register (non-destructive source).
4466 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre) {
4467   InstructionMark im(this);
4468   simd_prefix(dst, xnoreg, src, pre);
4469   emit_int8(opcode);
4470   emit_operand(dst, src);
4471 }
4472 
4473 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre) {
4474   int encode = simd_prefix_and_encode(dst, xnoreg, src, pre);
4475   emit_int8(opcode);
4476   emit_int8((unsigned char)(0xC0 | encode));
4477 }
4478 
4479 // 3-operands AVX instructions
4480 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds,
4481                                Address src, VexSimdPrefix pre, bool vector256) {
4482   InstructionMark im(this);
4483   vex_prefix(dst, nds, src, pre, vector256);
4484   emit_int8(opcode);
4485   emit_operand(dst, src);
4486 }
4487 
4488 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds,
4489                                XMMRegister src, VexSimdPrefix pre, bool vector256) {
4490   int encode = vex_prefix_and_encode(dst, nds, src, pre, vector256);
4491   emit_int8(opcode);
4492   emit_int8((unsigned char)(0xC0 | encode));
4493 }
4494 
4495 #ifndef _LP64
4496 
4497 void Assembler::incl(Register dst) {
4498   // Don't use it directly. Use MacroAssembler::incrementl() instead.
4499   emit_int8(0x40 | dst->encoding());
4500 }
4501 
4502 void Assembler::lea(Register dst, Address src) {
4503   leal(dst, src);
4504 }
4505 
4506 void Assembler::mov_literal32(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
4507   InstructionMark im(this);
4508   emit_int8((unsigned char)0xC7);
4509   emit_operand(rax, dst);
4510   emit_data((int)imm32, rspec, 0);
4511 }
4512 
4513 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
4514   InstructionMark im(this);
4515   int encode = prefix_and_encode(dst->encoding());
4516   emit_int8((unsigned char)(0xB8 | encode));
4517   emit_data((int)imm32, rspec, 0);
4518 }
4519 
4520 void Assembler::popa() { // 32bit
4521   emit_int8(0x61);
4522 }
4523 
4524 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
4525   InstructionMark im(this);
4526   emit_int8(0x68);
4527   emit_data(imm32, rspec, 0);
4528 }
4529 
4530 void Assembler::pusha() { // 32bit
4531   emit_int8(0x60);
4532 }
4533 
4534 void Assembler::set_byte_if_not_zero(Register dst) {
4535   emit_int8(0x0F);
4536   emit_int8((unsigned char)0x95);
4537   emit_int8((unsigned char)(0xE0 | dst->encoding()));
4538 }
4539 
4540 void Assembler::shldl(Register dst, Register src) {
4541   emit_int8(0x0F);
4542   emit_int8((unsigned char)0xA5);
4543   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
4544 }
4545 
4546 void Assembler::shrdl(Register dst, Register src) {
4547   emit_int8(0x0F);
4548   emit_int8((unsigned char)0xAD);
4549   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
4550 }
4551 
4552 #else // LP64
4553 
4554 void Assembler::set_byte_if_not_zero(Register dst) {
4555   int enc = prefix_and_encode(dst->encoding(), true);
4556   emit_int8(0x0F);
4557   emit_int8((unsigned char)0x95);
4558   emit_int8((unsigned char)(0xE0 | enc));
4559 }
4560 
4561 // 64bit only pieces of the assembler
4562 // This should only be used by 64bit instructions that can use rip-relative
4563 // it cannot be used by instructions that want an immediate value.
4564 
4565 bool Assembler::reachable(AddressLiteral adr) {
4566   int64_t disp;
4567   // None will force a 64bit literal to the code stream. Likely a placeholder
4568   // for something that will be patched later and we need to certain it will
4569   // always be reachable.
4570   if (adr.reloc() == relocInfo::none) {
4571     return false;
4572   }
4573   if (adr.reloc() == relocInfo::internal_word_type) {
4574     // This should be rip relative and easily reachable.
4575     return true;
4576   }
4577   if (adr.reloc() == relocInfo::virtual_call_type ||
4578       adr.reloc() == relocInfo::opt_virtual_call_type ||
4579       adr.reloc() == relocInfo::static_call_type ||
4580       adr.reloc() == relocInfo::static_stub_type ) {
4581     // This should be rip relative within the code cache and easily
4582     // reachable until we get huge code caches. (At which point
4583     // ic code is going to have issues).
4584     return true;
4585   }
4586   if (adr.reloc() != relocInfo::external_word_type &&
4587       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
4588       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
4589       adr.reloc() != relocInfo::runtime_call_type ) {
4590     return false;
4591   }
4592 
4593   // Stress the correction code
4594   if (ForceUnreachable) {
4595     // Must be runtimecall reloc, see if it is in the codecache
4596     // Flipping stuff in the codecache to be unreachable causes issues
4597     // with things like inline caches where the additional instructions
4598     // are not handled.
4599     if (CodeCache::find_blob(adr._target) == NULL) {
4600       return false;
4601     }
4602   }
4603   // For external_word_type/runtime_call_type if it is reachable from where we
4604   // are now (possibly a temp buffer) and where we might end up
4605   // anywhere in the codeCache then we are always reachable.
4606   // This would have to change if we ever save/restore shared code
4607   // to be more pessimistic.
4608   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
4609   if (!is_simm32(disp)) return false;
4610   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
4611   if (!is_simm32(disp)) return false;
4612 
4613   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
4614 
4615   // Because rip relative is a disp + address_of_next_instruction and we
4616   // don't know the value of address_of_next_instruction we apply a fudge factor
4617   // to make sure we will be ok no matter the size of the instruction we get placed into.
4618   // We don't have to fudge the checks above here because they are already worst case.
4619 
4620   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
4621   // + 4 because better safe than sorry.
4622   const int fudge = 12 + 4;
4623   if (disp < 0) {
4624     disp -= fudge;
4625   } else {
4626     disp += fudge;
4627   }
4628   return is_simm32(disp);
4629 }
4630 
4631 // Check if the polling page is not reachable from the code cache using rip-relative
4632 // addressing.
4633 bool Assembler::is_polling_page_far() {
4634   intptr_t addr = (intptr_t)os::get_polling_page();
4635   return ForceUnreachable ||
4636          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
4637          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
4638 }
4639 
4640 void Assembler::emit_data64(jlong data,
4641                             relocInfo::relocType rtype,
4642                             int format) {
4643   if (rtype == relocInfo::none) {
4644     emit_int64(data);
4645   } else {
4646     emit_data64(data, Relocation::spec_simple(rtype), format);
4647   }
4648 }
4649 
4650 void Assembler::emit_data64(jlong data,
4651                             RelocationHolder const& rspec,
4652                             int format) {
4653   assert(imm_operand == 0, "default format must be immediate in this file");
4654   assert(imm_operand == format, "must be immediate");
4655   assert(inst_mark() != NULL, "must be inside InstructionMark");
4656   // Do not use AbstractAssembler::relocate, which is not intended for
4657   // embedded words.  Instead, relocate to the enclosing instruction.
4658   code_section()->relocate(inst_mark(), rspec, format);
4659 #ifdef ASSERT
4660   check_relocation(rspec, format);
4661 #endif
4662   emit_int64(data);
4663 }
4664 
4665 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
4666   if (reg_enc >= 8) {
4667     prefix(REX_B);
4668     reg_enc -= 8;
4669   } else if (byteinst && reg_enc >= 4) {
4670     prefix(REX);
4671   }
4672   return reg_enc;
4673 }
4674 
4675 int Assembler::prefixq_and_encode(int reg_enc) {
4676   if (reg_enc < 8) {
4677     prefix(REX_W);
4678   } else {
4679     prefix(REX_WB);
4680     reg_enc -= 8;
4681   }
4682   return reg_enc;
4683 }
4684 
4685 int Assembler::prefix_and_encode(int dst_enc, int src_enc, bool byteinst) {
4686   if (dst_enc < 8) {
4687     if (src_enc >= 8) {
4688       prefix(REX_B);
4689       src_enc -= 8;
4690     } else if (byteinst && src_enc >= 4) {
4691       prefix(REX);
4692     }
4693   } else {
4694     if (src_enc < 8) {
4695       prefix(REX_R);
4696     } else {
4697       prefix(REX_RB);
4698       src_enc -= 8;
4699     }
4700     dst_enc -= 8;
4701   }
4702   return dst_enc << 3 | src_enc;
4703 }
4704 
4705 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
4706   if (dst_enc < 8) {
4707     if (src_enc < 8) {
4708       prefix(REX_W);
4709     } else {
4710       prefix(REX_WB);
4711       src_enc -= 8;
4712     }
4713   } else {
4714     if (src_enc < 8) {
4715       prefix(REX_WR);
4716     } else {
4717       prefix(REX_WRB);
4718       src_enc -= 8;
4719     }
4720     dst_enc -= 8;
4721   }
4722   return dst_enc << 3 | src_enc;
4723 }
4724 
4725 void Assembler::prefix(Register reg) {
4726   if (reg->encoding() >= 8) {
4727     prefix(REX_B);
4728   }
4729 }
4730 
4731 void Assembler::prefix(Address adr) {
4732   if (adr.base_needs_rex()) {
4733     if (adr.index_needs_rex()) {
4734       prefix(REX_XB);
4735     } else {
4736       prefix(REX_B);
4737     }
4738   } else {
4739     if (adr.index_needs_rex()) {
4740       prefix(REX_X);
4741     }
4742   }
4743 }
4744 
4745 void Assembler::prefixq(Address adr) {
4746   if (adr.base_needs_rex()) {
4747     if (adr.index_needs_rex()) {
4748       prefix(REX_WXB);
4749     } else {
4750       prefix(REX_WB);
4751     }
4752   } else {
4753     if (adr.index_needs_rex()) {
4754       prefix(REX_WX);
4755     } else {
4756       prefix(REX_W);
4757     }
4758   }
4759 }
4760 
4761 
4762 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
4763   if (reg->encoding() < 8) {
4764     if (adr.base_needs_rex()) {
4765       if (adr.index_needs_rex()) {
4766         prefix(REX_XB);
4767       } else {
4768         prefix(REX_B);
4769       }
4770     } else {
4771       if (adr.index_needs_rex()) {
4772         prefix(REX_X);
4773       } else if (byteinst && reg->encoding() >= 4 ) {
4774         prefix(REX);
4775       }
4776     }
4777   } else {
4778     if (adr.base_needs_rex()) {
4779       if (adr.index_needs_rex()) {
4780         prefix(REX_RXB);
4781       } else {
4782         prefix(REX_RB);
4783       }
4784     } else {
4785       if (adr.index_needs_rex()) {
4786         prefix(REX_RX);
4787       } else {
4788         prefix(REX_R);
4789       }
4790     }
4791   }
4792 }
4793 
4794 void Assembler::prefixq(Address adr, Register src) {
4795   if (src->encoding() < 8) {
4796     if (adr.base_needs_rex()) {
4797       if (adr.index_needs_rex()) {
4798         prefix(REX_WXB);
4799       } else {
4800         prefix(REX_WB);
4801       }
4802     } else {
4803       if (adr.index_needs_rex()) {
4804         prefix(REX_WX);
4805       } else {
4806         prefix(REX_W);
4807       }
4808     }
4809   } else {
4810     if (adr.base_needs_rex()) {
4811       if (adr.index_needs_rex()) {
4812         prefix(REX_WRXB);
4813       } else {
4814         prefix(REX_WRB);
4815       }
4816     } else {
4817       if (adr.index_needs_rex()) {
4818         prefix(REX_WRX);
4819       } else {
4820         prefix(REX_WR);
4821       }
4822     }
4823   }
4824 }
4825 
4826 void Assembler::prefix(Address adr, XMMRegister reg) {
4827   if (reg->encoding() < 8) {
4828     if (adr.base_needs_rex()) {
4829       if (adr.index_needs_rex()) {
4830         prefix(REX_XB);
4831       } else {
4832         prefix(REX_B);
4833       }
4834     } else {
4835       if (adr.index_needs_rex()) {
4836         prefix(REX_X);
4837       }
4838     }
4839   } else {
4840     if (adr.base_needs_rex()) {
4841       if (adr.index_needs_rex()) {
4842         prefix(REX_RXB);
4843       } else {
4844         prefix(REX_RB);
4845       }
4846     } else {
4847       if (adr.index_needs_rex()) {
4848         prefix(REX_RX);
4849       } else {
4850         prefix(REX_R);
4851       }
4852     }
4853   }
4854 }
4855 
4856 void Assembler::prefixq(Address adr, XMMRegister src) {
4857   if (src->encoding() < 8) {
4858     if (adr.base_needs_rex()) {
4859       if (adr.index_needs_rex()) {
4860         prefix(REX_WXB);
4861       } else {
4862         prefix(REX_WB);
4863       }
4864     } else {
4865       if (adr.index_needs_rex()) {
4866         prefix(REX_WX);
4867       } else {
4868         prefix(REX_W);
4869       }
4870     }
4871   } else {
4872     if (adr.base_needs_rex()) {
4873       if (adr.index_needs_rex()) {
4874         prefix(REX_WRXB);
4875       } else {
4876         prefix(REX_WRB);
4877       }
4878     } else {
4879       if (adr.index_needs_rex()) {
4880         prefix(REX_WRX);
4881       } else {
4882         prefix(REX_WR);
4883       }
4884     }
4885   }
4886 }
4887 
4888 void Assembler::adcq(Register dst, int32_t imm32) {
4889   (void) prefixq_and_encode(dst->encoding());
4890   emit_arith(0x81, 0xD0, dst, imm32);
4891 }
4892 
4893 void Assembler::adcq(Register dst, Address src) {
4894   InstructionMark im(this);
4895   prefixq(src, dst);
4896   emit_int8(0x13);
4897   emit_operand(dst, src);
4898 }
4899 
4900 void Assembler::adcq(Register dst, Register src) {
4901   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4902   emit_arith(0x13, 0xC0, dst, src);
4903 }
4904 
4905 void Assembler::addq(Address dst, int32_t imm32) {
4906   InstructionMark im(this);
4907   prefixq(dst);
4908   emit_arith_operand(0x81, rax, dst,imm32);
4909 }
4910 
4911 void Assembler::addq(Address dst, Register src) {
4912   InstructionMark im(this);
4913   prefixq(dst, src);
4914   emit_int8(0x01);
4915   emit_operand(src, dst);
4916 }
4917 
4918 void Assembler::addq(Register dst, int32_t imm32) {
4919   (void) prefixq_and_encode(dst->encoding());
4920   emit_arith(0x81, 0xC0, dst, imm32);
4921 }
4922 
4923 void Assembler::addq(Register dst, Address src) {
4924   InstructionMark im(this);
4925   prefixq(src, dst);
4926   emit_int8(0x03);
4927   emit_operand(dst, src);
4928 }
4929 
4930 void Assembler::addq(Register dst, Register src) {
4931   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4932   emit_arith(0x03, 0xC0, dst, src);
4933 }
4934 
4935 void Assembler::andq(Address dst, int32_t imm32) {
4936   InstructionMark im(this);
4937   prefixq(dst);
4938   emit_int8((unsigned char)0x81);
4939   emit_operand(rsp, dst, 4);
4940   emit_int32(imm32);
4941 }
4942 
4943 void Assembler::andq(Register dst, int32_t imm32) {
4944   (void) prefixq_and_encode(dst->encoding());
4945   emit_arith(0x81, 0xE0, dst, imm32);
4946 }
4947 
4948 void Assembler::andq(Register dst, Address src) {
4949   InstructionMark im(this);
4950   prefixq(src, dst);
4951   emit_int8(0x23);
4952   emit_operand(dst, src);
4953 }
4954 
4955 void Assembler::andq(Register dst, Register src) {
4956   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4957   emit_arith(0x23, 0xC0, dst, src);
4958 }
4959 
4960 void Assembler::andnq(Register dst, Register src1, Register src2) {
4961   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
4962   int encode = vex_prefix_0F38_and_encode_q(dst, src1, src2);
4963   emit_int8((unsigned char)0xF2);
4964   emit_int8((unsigned char)(0xC0 | encode));
4965 }
4966 
4967 void Assembler::andnq(Register dst, Register src1, Address src2) {
4968   InstructionMark im(this);
4969   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
4970   vex_prefix_0F38_q(dst, src1, src2);
4971   emit_int8((unsigned char)0xF2);
4972   emit_operand(dst, src2);
4973 }
4974 
4975 void Assembler::bsfq(Register dst, Register src) {
4976   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4977   emit_int8(0x0F);
4978   emit_int8((unsigned char)0xBC);
4979   emit_int8((unsigned char)(0xC0 | encode));
4980 }
4981 
4982 void Assembler::bsrq(Register dst, Register src) {
4983   assert(!VM_Version::supports_lzcnt(), "encoding is treated as LZCNT");
4984   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4985   emit_int8(0x0F);
4986   emit_int8((unsigned char)0xBD);
4987   emit_int8((unsigned char)(0xC0 | encode));
4988 }
4989 
4990 void Assembler::bswapq(Register reg) {
4991   int encode = prefixq_and_encode(reg->encoding());
4992   emit_int8(0x0F);
4993   emit_int8((unsigned char)(0xC8 | encode));
4994 }
4995 
4996 void Assembler::blsiq(Register dst, Register src) {
4997   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
4998   int encode = vex_prefix_0F38_and_encode_q(rbx, dst, src);
4999   emit_int8((unsigned char)0xF3);
5000   emit_int8((unsigned char)(0xC0 | encode));
5001 }
5002 
5003 void Assembler::blsiq(Register dst, Address src) {
5004   InstructionMark im(this);
5005   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
5006   vex_prefix_0F38_q(rbx, dst, src);
5007   emit_int8((unsigned char)0xF3);
5008   emit_operand(rbx, src);
5009 }
5010 
5011 void Assembler::blsmskq(Register dst, Register src) {
5012   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
5013   int encode = vex_prefix_0F38_and_encode_q(rdx, dst, src);
5014   emit_int8((unsigned char)0xF3);
5015   emit_int8((unsigned char)(0xC0 | encode));
5016 }
5017 
5018 void Assembler::blsmskq(Register dst, Address src) {
5019   InstructionMark im(this);
5020   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
5021   vex_prefix_0F38_q(rdx, dst, src);
5022   emit_int8((unsigned char)0xF3);
5023   emit_operand(rdx, src);
5024 }
5025 
5026 void Assembler::blsrq(Register dst, Register src) {
5027   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
5028   int encode = vex_prefix_0F38_and_encode_q(rcx, dst, src);
5029   emit_int8((unsigned char)0xF3);
5030   emit_int8((unsigned char)(0xC0 | encode));
5031 }
5032 
5033 void Assembler::blsrq(Register dst, Address src) {
5034   InstructionMark im(this);
5035   assert(VM_Version::supports_bmi1(), "bit manipulation instructions not supported");
5036   vex_prefix_0F38_q(rcx, dst, src);
5037   emit_int8((unsigned char)0xF3);
5038   emit_operand(rcx, src);
5039 }
5040 
5041 void Assembler::cdqq() {
5042   prefix(REX_W);
5043   emit_int8((unsigned char)0x99);
5044 }
5045 
5046 void Assembler::clflush(Address adr) {
5047   prefix(adr);
5048   emit_int8(0x0F);
5049   emit_int8((unsigned char)0xAE);
5050   emit_operand(rdi, adr);
5051 }
5052 
5053 void Assembler::cmovq(Condition cc, Register dst, Register src) {
5054   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5055   emit_int8(0x0F);
5056   emit_int8(0x40 | cc);
5057   emit_int8((unsigned char)(0xC0 | encode));
5058 }
5059 
5060 void Assembler::cmovq(Condition cc, Register dst, Address src) {
5061   InstructionMark im(this);
5062   prefixq(src, dst);
5063   emit_int8(0x0F);
5064   emit_int8(0x40 | cc);
5065   emit_operand(dst, src);
5066 }
5067 
5068 void Assembler::cmpq(Address dst, int32_t imm32) {
5069   InstructionMark im(this);
5070   prefixq(dst);
5071   emit_int8((unsigned char)0x81);
5072   emit_operand(rdi, dst, 4);
5073   emit_int32(imm32);
5074 }
5075 
5076 void Assembler::cmpq(Register dst, int32_t imm32) {
5077   (void) prefixq_and_encode(dst->encoding());
5078   emit_arith(0x81, 0xF8, dst, imm32);
5079 }
5080 
5081 void Assembler::cmpq(Address dst, Register src) {
5082   InstructionMark im(this);
5083   prefixq(dst, src);
5084   emit_int8(0x3B);
5085   emit_operand(src, dst);
5086 }
5087 
5088 void Assembler::cmpq(Register dst, Register src) {
5089   (void) prefixq_and_encode(dst->encoding(), src->encoding());
5090   emit_arith(0x3B, 0xC0, dst, src);
5091 }
5092 
5093 void Assembler::cmpq(Register dst, Address  src) {
5094   InstructionMark im(this);
5095   prefixq(src, dst);
5096   emit_int8(0x3B);
5097   emit_operand(dst, src);
5098 }
5099 
5100 void Assembler::cmpxchgq(Register reg, Address adr) {
5101   InstructionMark im(this);
5102   prefixq(adr, reg);
5103   emit_int8(0x0F);
5104   emit_int8((unsigned char)0xB1);
5105   emit_operand(reg, adr);
5106 }
5107 
5108 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
5109   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5110   int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F2);
5111   emit_int8(0x2A);
5112   emit_int8((unsigned char)(0xC0 | encode));
5113 }
5114 
5115 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
5116   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5117   InstructionMark im(this);
5118   simd_prefix_q(dst, dst, src, VEX_SIMD_F2);
5119   emit_int8(0x2A);
5120   emit_operand(dst, src);
5121 }
5122 
5123 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
5124   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5125   int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F3);
5126   emit_int8(0x2A);
5127   emit_int8((unsigned char)(0xC0 | encode));
5128 }
5129 
5130 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
5131   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5132   InstructionMark im(this);
5133   simd_prefix_q(dst, dst, src, VEX_SIMD_F3);
5134   emit_int8(0x2A);
5135   emit_operand(dst, src);
5136 }
5137 
5138 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
5139   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5140   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F2);
5141   emit_int8(0x2C);
5142   emit_int8((unsigned char)(0xC0 | encode));
5143 }
5144 
5145 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
5146   NOT_LP64(assert(VM_Version::supports_sse(), ""));
5147   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F3);
5148   emit_int8(0x2C);
5149   emit_int8((unsigned char)(0xC0 | encode));
5150 }
5151 
5152 void Assembler::decl(Register dst) {
5153   // Don't use it directly. Use MacroAssembler::decrementl() instead.
5154   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
5155   int encode = prefix_and_encode(dst->encoding());
5156   emit_int8((unsigned char)0xFF);
5157   emit_int8((unsigned char)(0xC8 | encode));
5158 }
5159 
5160 void Assembler::decq(Register dst) {
5161   // Don't use it directly. Use MacroAssembler::decrementq() instead.
5162   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
5163   int encode = prefixq_and_encode(dst->encoding());
5164   emit_int8((unsigned char)0xFF);
5165   emit_int8(0xC8 | encode);
5166 }
5167 
5168 void Assembler::decq(Address dst) {
5169   // Don't use it directly. Use MacroAssembler::decrementq() instead.
5170   InstructionMark im(this);
5171   prefixq(dst);
5172   emit_int8((unsigned char)0xFF);
5173   emit_operand(rcx, dst);
5174 }
5175 
5176 void Assembler::fxrstor(Address src) {
5177   prefixq(src);
5178   emit_int8(0x0F);
5179   emit_int8((unsigned char)0xAE);
5180   emit_operand(as_Register(1), src);
5181 }
5182 
5183 void Assembler::fxsave(Address dst) {
5184   prefixq(dst);
5185   emit_int8(0x0F);
5186   emit_int8((unsigned char)0xAE);
5187   emit_operand(as_Register(0), dst);
5188 }
5189 
5190 void Assembler::idivq(Register src) {
5191   int encode = prefixq_and_encode(src->encoding());
5192   emit_int8((unsigned char)0xF7);
5193   emit_int8((unsigned char)(0xF8 | encode));
5194 }
5195 
5196 void Assembler::imulq(Register dst, Register src) {
5197   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5198   emit_int8(0x0F);
5199   emit_int8((unsigned char)0xAF);
5200   emit_int8((unsigned char)(0xC0 | encode));
5201 }
5202 
5203 void Assembler::imulq(Register dst, Register src, int value) {
5204   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5205   if (is8bit(value)) {
5206     emit_int8(0x6B);
5207     emit_int8((unsigned char)(0xC0 | encode));
5208     emit_int8(value & 0xFF);
5209   } else {
5210     emit_int8(0x69);
5211     emit_int8((unsigned char)(0xC0 | encode));
5212     emit_int32(value);
5213   }
5214 }
5215 
5216 void Assembler::imulq(Register dst, Address src) {
5217   InstructionMark im(this);
5218   prefixq(src, dst);
5219   emit_int8(0x0F);
5220   emit_int8((unsigned char) 0xAF);
5221   emit_operand(dst, src);
5222 }
5223 
5224 void Assembler::incl(Register dst) {
5225   // Don't use it directly. Use MacroAssembler::incrementl() instead.
5226   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
5227   int encode = prefix_and_encode(dst->encoding());
5228   emit_int8((unsigned char)0xFF);
5229   emit_int8((unsigned char)(0xC0 | encode));
5230 }
5231 
5232 void Assembler::incq(Register dst) {
5233   // Don't use it directly. Use MacroAssembler::incrementq() instead.
5234   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
5235   int encode = prefixq_and_encode(dst->encoding());
5236   emit_int8((unsigned char)0xFF);
5237   emit_int8((unsigned char)(0xC0 | encode));
5238 }
5239 
5240 void Assembler::incq(Address dst) {
5241   // Don't use it directly. Use MacroAssembler::incrementq() instead.
5242   InstructionMark im(this);
5243   prefixq(dst);
5244   emit_int8((unsigned char)0xFF);
5245   emit_operand(rax, dst);
5246 }
5247 
5248 void Assembler::lea(Register dst, Address src) {
5249   leaq(dst, src);
5250 }
5251 
5252 void Assembler::leaq(Register dst, Address src) {
5253   InstructionMark im(this);
5254   prefixq(src, dst);
5255   emit_int8((unsigned char)0x8D);
5256   emit_operand(dst, src);
5257 }
5258 
5259 void Assembler::mov64(Register dst, int64_t imm64) {
5260   InstructionMark im(this);
5261   int encode = prefixq_and_encode(dst->encoding());
5262   emit_int8((unsigned char)(0xB8 | encode));
5263   emit_int64(imm64);
5264 }
5265 
5266 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
5267   InstructionMark im(this);
5268   int encode = prefixq_and_encode(dst->encoding());
5269   emit_int8(0xB8 | encode);
5270   emit_data64(imm64, rspec);
5271 }
5272 
5273 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
5274   InstructionMark im(this);
5275   int encode = prefix_and_encode(dst->encoding());
5276   emit_int8((unsigned char)(0xB8 | encode));
5277   emit_data((int)imm32, rspec, narrow_oop_operand);
5278 }
5279 
5280 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
5281   InstructionMark im(this);
5282   prefix(dst);
5283   emit_int8((unsigned char)0xC7);
5284   emit_operand(rax, dst, 4);
5285   emit_data((int)imm32, rspec, narrow_oop_operand);
5286 }
5287 
5288 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
5289   InstructionMark im(this);
5290   int encode = prefix_and_encode(src1->encoding());
5291   emit_int8((unsigned char)0x81);
5292   emit_int8((unsigned char)(0xF8 | encode));
5293   emit_data((int)imm32, rspec, narrow_oop_operand);
5294 }
5295 
5296 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
5297   InstructionMark im(this);
5298   prefix(src1);
5299   emit_int8((unsigned char)0x81);
5300   emit_operand(rax, src1, 4);
5301   emit_data((int)imm32, rspec, narrow_oop_operand);
5302 }
5303 
5304 void Assembler::lzcntq(Register dst, Register src) {
5305   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
5306   emit_int8((unsigned char)0xF3);
5307   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5308   emit_int8(0x0F);
5309   emit_int8((unsigned char)0xBD);
5310   emit_int8((unsigned char)(0xC0 | encode));
5311 }
5312 
5313 void Assembler::movdq(XMMRegister dst, Register src) {
5314   // table D-1 says MMX/SSE2
5315   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5316   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_66);
5317   emit_int8(0x6E);
5318   emit_int8((unsigned char)(0xC0 | encode));
5319 }
5320 
5321 void Assembler::movdq(Register dst, XMMRegister src) {
5322   // table D-1 says MMX/SSE2
5323   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5324   // swap src/dst to get correct prefix
5325   int encode = simd_prefix_and_encode_q(src, dst, VEX_SIMD_66);
5326   emit_int8(0x7E);
5327   emit_int8((unsigned char)(0xC0 | encode));
5328 }
5329 
5330 void Assembler::movq(Register dst, Register src) {
5331   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5332   emit_int8((unsigned char)0x8B);
5333   emit_int8((unsigned char)(0xC0 | encode));
5334 }
5335 
5336 void Assembler::movq(Register dst, Address src) {
5337   InstructionMark im(this);
5338   prefixq(src, dst);
5339   emit_int8((unsigned char)0x8B);
5340   emit_operand(dst, src);
5341 }
5342 
5343 void Assembler::movq(Address dst, Register src) {
5344   InstructionMark im(this);
5345   prefixq(dst, src);
5346   emit_int8((unsigned char)0x89);
5347   emit_operand(src, dst);
5348 }
5349 
5350 void Assembler::movsbq(Register dst, Address src) {
5351   InstructionMark im(this);
5352   prefixq(src, dst);
5353   emit_int8(0x0F);
5354   emit_int8((unsigned char)0xBE);
5355   emit_operand(dst, src);
5356 }
5357 
5358 void Assembler::movsbq(Register dst, Register src) {
5359   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5360   emit_int8(0x0F);
5361   emit_int8((unsigned char)0xBE);
5362   emit_int8((unsigned char)(0xC0 | encode));
5363 }
5364 
5365 void Assembler::movslq(Register dst, int32_t imm32) {
5366   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
5367   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
5368   // as a result we shouldn't use until tested at runtime...
5369   ShouldNotReachHere();
5370   InstructionMark im(this);
5371   int encode = prefixq_and_encode(dst->encoding());
5372   emit_int8((unsigned char)(0xC7 | encode));
5373   emit_int32(imm32);
5374 }
5375 
5376 void Assembler::movslq(Address dst, int32_t imm32) {
5377   assert(is_simm32(imm32), "lost bits");
5378   InstructionMark im(this);
5379   prefixq(dst);
5380   emit_int8((unsigned char)0xC7);
5381   emit_operand(rax, dst, 4);
5382   emit_int32(imm32);
5383 }
5384 
5385 void Assembler::movslq(Register dst, Address src) {
5386   InstructionMark im(this);
5387   prefixq(src, dst);
5388   emit_int8(0x63);
5389   emit_operand(dst, src);
5390 }
5391 
5392 void Assembler::movslq(Register dst, Register src) {
5393   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5394   emit_int8(0x63);
5395   emit_int8((unsigned char)(0xC0 | encode));
5396 }
5397 
5398 void Assembler::movswq(Register dst, Address src) {
5399   InstructionMark im(this);
5400   prefixq(src, dst);
5401   emit_int8(0x0F);
5402   emit_int8((unsigned char)0xBF);
5403   emit_operand(dst, src);
5404 }
5405 
5406 void Assembler::movswq(Register dst, Register src) {
5407   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5408   emit_int8((unsigned char)0x0F);
5409   emit_int8((unsigned char)0xBF);
5410   emit_int8((unsigned char)(0xC0 | encode));
5411 }
5412 
5413 void Assembler::movzbq(Register dst, Address src) {
5414   InstructionMark im(this);
5415   prefixq(src, dst);
5416   emit_int8((unsigned char)0x0F);
5417   emit_int8((unsigned char)0xB6);
5418   emit_operand(dst, src);
5419 }
5420 
5421 void Assembler::movzbq(Register dst, Register src) {
5422   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5423   emit_int8(0x0F);
5424   emit_int8((unsigned char)0xB6);
5425   emit_int8(0xC0 | encode);
5426 }
5427 
5428 void Assembler::movzwq(Register dst, Address src) {
5429   InstructionMark im(this);
5430   prefixq(src, dst);
5431   emit_int8((unsigned char)0x0F);
5432   emit_int8((unsigned char)0xB7);
5433   emit_operand(dst, src);
5434 }
5435 
5436 void Assembler::movzwq(Register dst, Register src) {
5437   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5438   emit_int8((unsigned char)0x0F);
5439   emit_int8((unsigned char)0xB7);
5440   emit_int8((unsigned char)(0xC0 | encode));
5441 }
5442 
5443 void Assembler::negq(Register dst) {
5444   int encode = prefixq_and_encode(dst->encoding());
5445   emit_int8((unsigned char)0xF7);
5446   emit_int8((unsigned char)(0xD8 | encode));
5447 }
5448 
5449 void Assembler::notq(Register dst) {
5450   int encode = prefixq_and_encode(dst->encoding());
5451   emit_int8((unsigned char)0xF7);
5452   emit_int8((unsigned char)(0xD0 | encode));
5453 }
5454 
5455 void Assembler::orq(Address dst, int32_t imm32) {
5456   InstructionMark im(this);
5457   prefixq(dst);
5458   emit_int8((unsigned char)0x81);
5459   emit_operand(rcx, dst, 4);
5460   emit_int32(imm32);
5461 }
5462 
5463 void Assembler::orq(Register dst, int32_t imm32) {
5464   (void) prefixq_and_encode(dst->encoding());
5465   emit_arith(0x81, 0xC8, dst, imm32);
5466 }
5467 
5468 void Assembler::orq(Register dst, Address src) {
5469   InstructionMark im(this);
5470   prefixq(src, dst);
5471   emit_int8(0x0B);
5472   emit_operand(dst, src);
5473 }
5474 
5475 void Assembler::orq(Register dst, Register src) {
5476   (void) prefixq_and_encode(dst->encoding(), src->encoding());
5477   emit_arith(0x0B, 0xC0, dst, src);
5478 }
5479 
5480 void Assembler::popa() { // 64bit
5481   movq(r15, Address(rsp, 0));
5482   movq(r14, Address(rsp, wordSize));
5483   movq(r13, Address(rsp, 2 * wordSize));
5484   movq(r12, Address(rsp, 3 * wordSize));
5485   movq(r11, Address(rsp, 4 * wordSize));
5486   movq(r10, Address(rsp, 5 * wordSize));
5487   movq(r9,  Address(rsp, 6 * wordSize));
5488   movq(r8,  Address(rsp, 7 * wordSize));
5489   movq(rdi, Address(rsp, 8 * wordSize));
5490   movq(rsi, Address(rsp, 9 * wordSize));
5491   movq(rbp, Address(rsp, 10 * wordSize));
5492   // skip rsp
5493   movq(rbx, Address(rsp, 12 * wordSize));
5494   movq(rdx, Address(rsp, 13 * wordSize));
5495   movq(rcx, Address(rsp, 14 * wordSize));
5496   movq(rax, Address(rsp, 15 * wordSize));
5497 
5498   addq(rsp, 16 * wordSize);
5499 }
5500 
5501 void Assembler::popcntq(Register dst, Address src) {
5502   assert(VM_Version::supports_popcnt(), "must support");
5503   InstructionMark im(this);
5504   emit_int8((unsigned char)0xF3);
5505   prefixq(src, dst);
5506   emit_int8((unsigned char)0x0F);
5507   emit_int8((unsigned char)0xB8);
5508   emit_operand(dst, src);
5509 }
5510 
5511 void Assembler::popcntq(Register dst, Register src) {
5512   assert(VM_Version::supports_popcnt(), "must support");
5513   emit_int8((unsigned char)0xF3);
5514   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5515   emit_int8((unsigned char)0x0F);
5516   emit_int8((unsigned char)0xB8);
5517   emit_int8((unsigned char)(0xC0 | encode));
5518 }
5519 
5520 void Assembler::popq(Address dst) {
5521   InstructionMark im(this);
5522   prefixq(dst);
5523   emit_int8((unsigned char)0x8F);
5524   emit_operand(rax, dst);
5525 }
5526 
5527 void Assembler::pusha() { // 64bit
5528   // we have to store original rsp.  ABI says that 128 bytes
5529   // below rsp are local scratch.
5530   movq(Address(rsp, -5 * wordSize), rsp);
5531 
5532   subq(rsp, 16 * wordSize);
5533 
5534   movq(Address(rsp, 15 * wordSize), rax);
5535   movq(Address(rsp, 14 * wordSize), rcx);
5536   movq(Address(rsp, 13 * wordSize), rdx);
5537   movq(Address(rsp, 12 * wordSize), rbx);
5538   // skip rsp
5539   movq(Address(rsp, 10 * wordSize), rbp);
5540   movq(Address(rsp, 9 * wordSize), rsi);
5541   movq(Address(rsp, 8 * wordSize), rdi);
5542   movq(Address(rsp, 7 * wordSize), r8);
5543   movq(Address(rsp, 6 * wordSize), r9);
5544   movq(Address(rsp, 5 * wordSize), r10);
5545   movq(Address(rsp, 4 * wordSize), r11);
5546   movq(Address(rsp, 3 * wordSize), r12);
5547   movq(Address(rsp, 2 * wordSize), r13);
5548   movq(Address(rsp, wordSize), r14);
5549   movq(Address(rsp, 0), r15);
5550 }
5551 
5552 void Assembler::pushq(Address src) {
5553   InstructionMark im(this);
5554   prefixq(src);
5555   emit_int8((unsigned char)0xFF);
5556   emit_operand(rsi, src);
5557 }
5558 
5559 void Assembler::rclq(Register dst, int imm8) {
5560   assert(isShiftCount(imm8 >> 1), "illegal shift count");
5561   int encode = prefixq_and_encode(dst->encoding());
5562   if (imm8 == 1) {
5563     emit_int8((unsigned char)0xD1);
5564     emit_int8((unsigned char)(0xD0 | encode));
5565   } else {
5566     emit_int8((unsigned char)0xC1);
5567     emit_int8((unsigned char)(0xD0 | encode));
5568     emit_int8(imm8);
5569   }
5570 }
5571 void Assembler::sarq(Register dst, int imm8) {
5572   assert(isShiftCount(imm8 >> 1), "illegal shift count");
5573   int encode = prefixq_and_encode(dst->encoding());
5574   if (imm8 == 1) {
5575     emit_int8((unsigned char)0xD1);
5576     emit_int8((unsigned char)(0xF8 | encode));
5577   } else {
5578     emit_int8((unsigned char)0xC1);
5579     emit_int8((unsigned char)(0xF8 | encode));
5580     emit_int8(imm8);
5581   }
5582 }
5583 
5584 void Assembler::sarq(Register dst) {
5585   int encode = prefixq_and_encode(dst->encoding());
5586   emit_int8((unsigned char)0xD3);
5587   emit_int8((unsigned char)(0xF8 | encode));
5588 }
5589 
5590 void Assembler::sbbq(Address dst, int32_t imm32) {
5591   InstructionMark im(this);
5592   prefixq(dst);
5593   emit_arith_operand(0x81, rbx, dst, imm32);
5594 }
5595 
5596 void Assembler::sbbq(Register dst, int32_t imm32) {
5597   (void) prefixq_and_encode(dst->encoding());
5598   emit_arith(0x81, 0xD8, dst, imm32);
5599 }
5600 
5601 void Assembler::sbbq(Register dst, Address src) {
5602   InstructionMark im(this);
5603   prefixq(src, dst);
5604   emit_int8(0x1B);
5605   emit_operand(dst, src);
5606 }
5607 
5608 void Assembler::sbbq(Register dst, Register src) {
5609   (void) prefixq_and_encode(dst->encoding(), src->encoding());
5610   emit_arith(0x1B, 0xC0, dst, src);
5611 }
5612 
5613 void Assembler::shlq(Register dst, int imm8) {
5614   assert(isShiftCount(imm8 >> 1), "illegal shift count");
5615   int encode = prefixq_and_encode(dst->encoding());
5616   if (imm8 == 1) {
5617     emit_int8((unsigned char)0xD1);
5618     emit_int8((unsigned char)(0xE0 | encode));
5619   } else {
5620     emit_int8((unsigned char)0xC1);
5621     emit_int8((unsigned char)(0xE0 | encode));
5622     emit_int8(imm8);
5623   }
5624 }
5625 
5626 void Assembler::shlq(Register dst) {
5627   int encode = prefixq_and_encode(dst->encoding());
5628   emit_int8((unsigned char)0xD3);
5629   emit_int8((unsigned char)(0xE0 | encode));
5630 }
5631 
5632 void Assembler::shrq(Register dst, int imm8) {
5633   assert(isShiftCount(imm8 >> 1), "illegal shift count");
5634   int encode = prefixq_and_encode(dst->encoding());
5635   emit_int8((unsigned char)0xC1);
5636   emit_int8((unsigned char)(0xE8 | encode));
5637   emit_int8(imm8);
5638 }
5639 
5640 void Assembler::shrq(Register dst) {
5641   int encode = prefixq_and_encode(dst->encoding());
5642   emit_int8((unsigned char)0xD3);
5643   emit_int8(0xE8 | encode);
5644 }
5645 
5646 void Assembler::subq(Address dst, int32_t imm32) {
5647   InstructionMark im(this);
5648   prefixq(dst);
5649   emit_arith_operand(0x81, rbp, dst, imm32);
5650 }
5651 
5652 void Assembler::subq(Address dst, Register src) {
5653   InstructionMark im(this);
5654   prefixq(dst, src);
5655   emit_int8(0x29);
5656   emit_operand(src, dst);
5657 }
5658 
5659 void Assembler::subq(Register dst, int32_t imm32) {
5660   (void) prefixq_and_encode(dst->encoding());
5661   emit_arith(0x81, 0xE8, dst, imm32);
5662 }
5663 
5664 // Force generation of a 4 byte immediate value even if it fits into 8bit
5665 void Assembler::subq_imm32(Register dst, int32_t imm32) {
5666   (void) prefixq_and_encode(dst->encoding());
5667   emit_arith_imm32(0x81, 0xE8, dst, imm32);
5668 }
5669 
5670 void Assembler::subq(Register dst, Address src) {
5671   InstructionMark im(this);
5672   prefixq(src, dst);
5673   emit_int8(0x2B);
5674   emit_operand(dst, src);
5675 }
5676 
5677 void Assembler::subq(Register dst, Register src) {
5678   (void) prefixq_and_encode(dst->encoding(), src->encoding());
5679   emit_arith(0x2B, 0xC0, dst, src);
5680 }
5681 
5682 void Assembler::testq(Register dst, int32_t imm32) {
5683   // not using emit_arith because test
5684   // doesn't support sign-extension of
5685   // 8bit operands
5686   int encode = dst->encoding();
5687   if (encode == 0) {
5688     prefix(REX_W);
5689     emit_int8((unsigned char)0xA9);
5690   } else {
5691     encode = prefixq_and_encode(encode);
5692     emit_int8((unsigned char)0xF7);
5693     emit_int8((unsigned char)(0xC0 | encode));
5694   }
5695   emit_int32(imm32);
5696 }
5697 
5698 void Assembler::testq(Register dst, Register src) {
5699   (void) prefixq_and_encode(dst->encoding(), src->encoding());
5700   emit_arith(0x85, 0xC0, dst, src);
5701 }
5702 
5703 void Assembler::xaddq(Address dst, Register src) {
5704   InstructionMark im(this);
5705   prefixq(dst, src);
5706   emit_int8(0x0F);
5707   emit_int8((unsigned char)0xC1);
5708   emit_operand(src, dst);
5709 }
5710 
5711 void Assembler::xchgq(Register dst, Address src) {
5712   InstructionMark im(this);
5713   prefixq(src, dst);
5714   emit_int8((unsigned char)0x87);
5715   emit_operand(dst, src);
5716 }
5717 
5718 void Assembler::xchgq(Register dst, Register src) {
5719   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5720   emit_int8((unsigned char)0x87);
5721   emit_int8((unsigned char)(0xc0 | encode));
5722 }
5723 
5724 void Assembler::xorq(Register dst, Register src) {
5725   (void) prefixq_and_encode(dst->encoding(), src->encoding());
5726   emit_arith(0x33, 0xC0, dst, src);
5727 }
5728 
5729 void Assembler::xorq(Register dst, Address src) {
5730   InstructionMark im(this);
5731   prefixq(src, dst);
5732   emit_int8(0x33);
5733   emit_operand(dst, src);
5734 }
5735 
5736 #endif // !LP64