1 /*
   2  * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "gc_interface/collectedHeap.inline.hpp"
  29 #include "interpreter/interpreter.hpp"
  30 #include "memory/cardTableModRefBS.hpp"
  31 #include "memory/resourceArea.hpp"
  32 #include "prims/methodHandles.hpp"
  33 #include "runtime/biasedLocking.hpp"
  34 #include "runtime/interfaceSupport.hpp"
  35 #include "runtime/objectMonitor.hpp"
  36 #include "runtime/os.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubRoutines.hpp"
  39 #include "utilities/macros.hpp"
  40 #if INCLUDE_ALL_GCS
  41 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
  42 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
  43 #include "gc_implementation/g1/heapRegion.hpp"
  44 #endif // INCLUDE_ALL_GCS
  45 
  46 #ifdef PRODUCT
  47 #define BLOCK_COMMENT(str) /* nothing */
  48 #define STOP(error) stop(error)
  49 #else
  50 #define BLOCK_COMMENT(str) block_comment(str)
  51 #define STOP(error) block_comment(error); stop(error)
  52 #endif
  53 
  54 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  55 // Implementation of AddressLiteral
  56 
  57 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  58   _is_lval = false;
  59   _target = target;
  60   switch (rtype) {
  61   case relocInfo::oop_type:
  62   case relocInfo::metadata_type:
  63     // Oops are a special case. Normally they would be their own section
  64     // but in cases like icBuffer they are literals in the code stream that
  65     // we don't have a section for. We use none so that we get a literal address
  66     // which is always patchable.
  67     break;
  68   case relocInfo::external_word_type:
  69     _rspec = external_word_Relocation::spec(target);
  70     break;
  71   case relocInfo::internal_word_type:
  72     _rspec = internal_word_Relocation::spec(target);
  73     break;
  74   case relocInfo::opt_virtual_call_type:
  75     _rspec = opt_virtual_call_Relocation::spec();
  76     break;
  77   case relocInfo::static_call_type:
  78     _rspec = static_call_Relocation::spec();
  79     break;
  80   case relocInfo::runtime_call_type:
  81     _rspec = runtime_call_Relocation::spec();
  82     break;
  83   case relocInfo::poll_type:
  84   case relocInfo::poll_return_type:
  85     _rspec = Relocation::spec_simple(rtype);
  86     break;
  87   case relocInfo::none:
  88     break;
  89   default:
  90     ShouldNotReachHere();
  91     break;
  92   }
  93 }
  94 
  95 // Implementation of Address
  96 
  97 #ifdef _LP64
  98 
  99 Address Address::make_array(ArrayAddress adr) {
 100   // Not implementable on 64bit machines
 101   // Should have been handled higher up the call chain.
 102   ShouldNotReachHere();
 103   return Address();
 104 }
 105 
 106 // exceedingly dangerous constructor
 107 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
 108   _base  = noreg;
 109   _index = noreg;
 110   _scale = no_scale;
 111   _disp  = disp;
 112   switch (rtype) {
 113     case relocInfo::external_word_type:
 114       _rspec = external_word_Relocation::spec(loc);
 115       break;
 116     case relocInfo::internal_word_type:
 117       _rspec = internal_word_Relocation::spec(loc);
 118       break;
 119     case relocInfo::runtime_call_type:
 120       // HMM
 121       _rspec = runtime_call_Relocation::spec();
 122       break;
 123     case relocInfo::poll_type:
 124     case relocInfo::poll_return_type:
 125       _rspec = Relocation::spec_simple(rtype);
 126       break;
 127     case relocInfo::none:
 128       break;
 129     default:
 130       ShouldNotReachHere();
 131   }
 132 }
 133 #else // LP64
 134 
 135 Address Address::make_array(ArrayAddress adr) {
 136   AddressLiteral base = adr.base();
 137   Address index = adr.index();
 138   assert(index._disp == 0, "must not have disp"); // maybe it can?
 139   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 140   array._rspec = base._rspec;
 141   return array;
 142 }
 143 
 144 // exceedingly dangerous constructor
 145 Address::Address(address loc, RelocationHolder spec) {
 146   _base  = noreg;
 147   _index = noreg;
 148   _scale = no_scale;
 149   _disp  = (intptr_t) loc;
 150   _rspec = spec;
 151 }
 152 
 153 #endif // _LP64
 154 
 155 
 156 
 157 // Convert the raw encoding form into the form expected by the constructor for
 158 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 159 // that to noreg for the Address constructor.
 160 Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 161   RelocationHolder rspec;
 162   if (disp_reloc != relocInfo::none) {
 163     rspec = Relocation::spec_simple(disp_reloc);
 164   }
 165   bool valid_index = index != rsp->encoding();
 166   if (valid_index) {
 167     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 168     madr._rspec = rspec;
 169     return madr;
 170   } else {
 171     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 172     madr._rspec = rspec;
 173     return madr;
 174   }
 175 }
 176 
 177 // Implementation of Assembler
 178 
 179 int AbstractAssembler::code_fill_byte() {
 180   return (u_char)'\xF4'; // hlt
 181 }
 182 
 183 // make this go away someday
 184 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 185   if (rtype == relocInfo::none)
 186         emit_int32(data);
 187   else  emit_data(data, Relocation::spec_simple(rtype), format);
 188 }
 189 
 190 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 191   assert(imm_operand == 0, "default format must be immediate in this file");
 192   assert(inst_mark() != NULL, "must be inside InstructionMark");
 193   if (rspec.type() !=  relocInfo::none) {
 194     #ifdef ASSERT
 195       check_relocation(rspec, format);
 196     #endif
 197     // Do not use AbstractAssembler::relocate, which is not intended for
 198     // embedded words.  Instead, relocate to the enclosing instruction.
 199 
 200     // hack. call32 is too wide for mask so use disp32
 201     if (format == call32_operand)
 202       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 203     else
 204       code_section()->relocate(inst_mark(), rspec, format);
 205   }
 206   emit_int32(data);
 207 }
 208 
 209 static int encode(Register r) {
 210   int enc = r->encoding();
 211   if (enc >= 8) {
 212     enc -= 8;
 213   }
 214   return enc;
 215 }
 216 
 217 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 218   assert(dst->has_byte_register(), "must have byte register");
 219   assert(isByte(op1) && isByte(op2), "wrong opcode");
 220   assert(isByte(imm8), "not a byte");
 221   assert((op1 & 0x01) == 0, "should be 8bit operation");
 222   emit_int8(op1);
 223   emit_int8(op2 | encode(dst));
 224   emit_int8(imm8);
 225 }
 226 
 227 
 228 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 229   assert(isByte(op1) && isByte(op2), "wrong opcode");
 230   assert((op1 & 0x01) == 1, "should be 32bit operation");
 231   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 232   if (is8bit(imm32)) {
 233     emit_int8(op1 | 0x02); // set sign bit
 234     emit_int8(op2 | encode(dst));
 235     emit_int8(imm32 & 0xFF);
 236   } else {
 237     emit_int8(op1);
 238     emit_int8(op2 | encode(dst));
 239     emit_int32(imm32);
 240   }
 241 }
 242 
 243 // Force generation of a 4 byte immediate value even if it fits into 8bit
 244 void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
 245   assert(isByte(op1) && isByte(op2), "wrong opcode");
 246   assert((op1 & 0x01) == 1, "should be 32bit operation");
 247   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 248   emit_int8(op1);
 249   emit_int8(op2 | encode(dst));
 250   emit_int32(imm32);
 251 }
 252 
 253 // immediate-to-memory forms
 254 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 255   assert((op1 & 0x01) == 1, "should be 32bit operation");
 256   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 257   if (is8bit(imm32)) {
 258     emit_int8(op1 | 0x02); // set sign bit
 259     emit_operand(rm, adr, 1);
 260     emit_int8(imm32 & 0xFF);
 261   } else {
 262     emit_int8(op1);
 263     emit_operand(rm, adr, 4);
 264     emit_int32(imm32);
 265   }
 266 }
 267 
 268 
 269 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 270   assert(isByte(op1) && isByte(op2), "wrong opcode");
 271   emit_int8(op1);
 272   emit_int8(op2 | encode(dst) << 3 | encode(src));
 273 }
 274 
 275 
 276 void Assembler::emit_operand(Register reg, Register base, Register index,
 277                              Address::ScaleFactor scale, int disp,
 278                              RelocationHolder const& rspec,
 279                              int rip_relative_correction) {
 280   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 281 
 282   // Encode the registers as needed in the fields they are used in
 283 
 284   int regenc = encode(reg) << 3;
 285   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 286   int baseenc = base->is_valid() ? encode(base) : 0;
 287 
 288   if (base->is_valid()) {
 289     if (index->is_valid()) {
 290       assert(scale != Address::no_scale, "inconsistent address");
 291       // [base + index*scale + disp]
 292       if (disp == 0 && rtype == relocInfo::none  &&
 293           base != rbp LP64_ONLY(&& base != r13)) {
 294         // [base + index*scale]
 295         // [00 reg 100][ss index base]
 296         assert(index != rsp, "illegal addressing mode");
 297         emit_int8(0x04 | regenc);
 298         emit_int8(scale << 6 | indexenc | baseenc);
 299       } else if (is8bit(disp) && rtype == relocInfo::none) {
 300         // [base + index*scale + imm8]
 301         // [01 reg 100][ss index base] imm8
 302         assert(index != rsp, "illegal addressing mode");
 303         emit_int8(0x44 | regenc);
 304         emit_int8(scale << 6 | indexenc | baseenc);
 305         emit_int8(disp & 0xFF);
 306       } else {
 307         // [base + index*scale + disp32]
 308         // [10 reg 100][ss index base] disp32
 309         assert(index != rsp, "illegal addressing mode");
 310         emit_int8(0x84 | regenc);
 311         emit_int8(scale << 6 | indexenc | baseenc);
 312         emit_data(disp, rspec, disp32_operand);
 313       }
 314     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 315       // [rsp + disp]
 316       if (disp == 0 && rtype == relocInfo::none) {
 317         // [rsp]
 318         // [00 reg 100][00 100 100]
 319         emit_int8(0x04 | regenc);
 320         emit_int8(0x24);
 321       } else if (is8bit(disp) && rtype == relocInfo::none) {
 322         // [rsp + imm8]
 323         // [01 reg 100][00 100 100] disp8
 324         emit_int8(0x44 | regenc);
 325         emit_int8(0x24);
 326         emit_int8(disp & 0xFF);
 327       } else {
 328         // [rsp + imm32]
 329         // [10 reg 100][00 100 100] disp32
 330         emit_int8(0x84 | regenc);
 331         emit_int8(0x24);
 332         emit_data(disp, rspec, disp32_operand);
 333       }
 334     } else {
 335       // [base + disp]
 336       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 337       if (disp == 0 && rtype == relocInfo::none &&
 338           base != rbp LP64_ONLY(&& base != r13)) {
 339         // [base]
 340         // [00 reg base]
 341         emit_int8(0x00 | regenc | baseenc);
 342       } else if (is8bit(disp) && rtype == relocInfo::none) {
 343         // [base + disp8]
 344         // [01 reg base] disp8
 345         emit_int8(0x40 | regenc | baseenc);
 346         emit_int8(disp & 0xFF);
 347       } else {
 348         // [base + disp32]
 349         // [10 reg base] disp32
 350         emit_int8(0x80 | regenc | baseenc);
 351         emit_data(disp, rspec, disp32_operand);
 352       }
 353     }
 354   } else {
 355     if (index->is_valid()) {
 356       assert(scale != Address::no_scale, "inconsistent address");
 357       // [index*scale + disp]
 358       // [00 reg 100][ss index 101] disp32
 359       assert(index != rsp, "illegal addressing mode");
 360       emit_int8(0x04 | regenc);
 361       emit_int8(scale << 6 | indexenc | 0x05);
 362       emit_data(disp, rspec, disp32_operand);
 363     } else if (rtype != relocInfo::none ) {
 364       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 365       // [00 000 101] disp32
 366 
 367       emit_int8(0x05 | regenc);
 368       // Note that the RIP-rel. correction applies to the generated
 369       // disp field, but _not_ to the target address in the rspec.
 370 
 371       // disp was created by converting the target address minus the pc
 372       // at the start of the instruction. That needs more correction here.
 373       // intptr_t disp = target - next_ip;
 374       assert(inst_mark() != NULL, "must be inside InstructionMark");
 375       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 376       int64_t adjusted = disp;
 377       // Do rip-rel adjustment for 64bit
 378       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 379       assert(is_simm32(adjusted),
 380              "must be 32bit offset (RIP relative address)");
 381       emit_data((int32_t) adjusted, rspec, disp32_operand);
 382 
 383     } else {
 384       // 32bit never did this, did everything as the rip-rel/disp code above
 385       // [disp] ABSOLUTE
 386       // [00 reg 100][00 100 101] disp32
 387       emit_int8(0x04 | regenc);
 388       emit_int8(0x25);
 389       emit_data(disp, rspec, disp32_operand);
 390     }
 391   }
 392 }
 393 
 394 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 395                              Address::ScaleFactor scale, int disp,
 396                              RelocationHolder const& rspec) {
 397   emit_operand((Register)reg, base, index, scale, disp, rspec);
 398 }
 399 
 400 // Secret local extension to Assembler::WhichOperand:
 401 #define end_pc_operand (_WhichOperand_limit)
 402 
 403 address Assembler::locate_operand(address inst, WhichOperand which) {
 404   // Decode the given instruction, and return the address of
 405   // an embedded 32-bit operand word.
 406 
 407   // If "which" is disp32_operand, selects the displacement portion
 408   // of an effective address specifier.
 409   // If "which" is imm64_operand, selects the trailing immediate constant.
 410   // If "which" is call32_operand, selects the displacement of a call or jump.
 411   // Caller is responsible for ensuring that there is such an operand,
 412   // and that it is 32/64 bits wide.
 413 
 414   // If "which" is end_pc_operand, find the end of the instruction.
 415 
 416   address ip = inst;
 417   bool is_64bit = false;
 418 
 419   debug_only(bool has_disp32 = false);
 420   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 421 
 422   again_after_prefix:
 423   switch (0xFF & *ip++) {
 424 
 425   // These convenience macros generate groups of "case" labels for the switch.
 426 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 427 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 428              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 429 #define REP16(x) REP8((x)+0): \
 430               case REP8((x)+8)
 431 
 432   case CS_segment:
 433   case SS_segment:
 434   case DS_segment:
 435   case ES_segment:
 436   case FS_segment:
 437   case GS_segment:
 438     // Seems dubious
 439     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 440     assert(ip == inst+1, "only one prefix allowed");
 441     goto again_after_prefix;
 442 
 443   case 0x67:
 444   case REX:
 445   case REX_B:
 446   case REX_X:
 447   case REX_XB:
 448   case REX_R:
 449   case REX_RB:
 450   case REX_RX:
 451   case REX_RXB:
 452     NOT_LP64(assert(false, "64bit prefixes"));
 453     goto again_after_prefix;
 454 
 455   case REX_W:
 456   case REX_WB:
 457   case REX_WX:
 458   case REX_WXB:
 459   case REX_WR:
 460   case REX_WRB:
 461   case REX_WRX:
 462   case REX_WRXB:
 463     NOT_LP64(assert(false, "64bit prefixes"));
 464     is_64bit = true;
 465     goto again_after_prefix;
 466 
 467   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 468   case 0x88: // movb a, r
 469   case 0x89: // movl a, r
 470   case 0x8A: // movb r, a
 471   case 0x8B: // movl r, a
 472   case 0x8F: // popl a
 473     debug_only(has_disp32 = true);
 474     break;
 475 
 476   case 0x68: // pushq #32
 477     if (which == end_pc_operand) {
 478       return ip + 4;
 479     }
 480     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 481     return ip;                  // not produced by emit_operand
 482 
 483   case 0x66: // movw ... (size prefix)
 484     again_after_size_prefix2:
 485     switch (0xFF & *ip++) {
 486     case REX:
 487     case REX_B:
 488     case REX_X:
 489     case REX_XB:
 490     case REX_R:
 491     case REX_RB:
 492     case REX_RX:
 493     case REX_RXB:
 494     case REX_W:
 495     case REX_WB:
 496     case REX_WX:
 497     case REX_WXB:
 498     case REX_WR:
 499     case REX_WRB:
 500     case REX_WRX:
 501     case REX_WRXB:
 502       NOT_LP64(assert(false, "64bit prefix found"));
 503       goto again_after_size_prefix2;
 504     case 0x8B: // movw r, a
 505     case 0x89: // movw a, r
 506       debug_only(has_disp32 = true);
 507       break;
 508     case 0xC7: // movw a, #16
 509       debug_only(has_disp32 = true);
 510       tail_size = 2;  // the imm16
 511       break;
 512     case 0x0F: // several SSE/SSE2 variants
 513       ip--;    // reparse the 0x0F
 514       goto again_after_prefix;
 515     default:
 516       ShouldNotReachHere();
 517     }
 518     break;
 519 
 520   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 521     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 522     // these asserts are somewhat nonsensical
 523 #ifndef _LP64
 524     assert(which == imm_operand || which == disp32_operand,
 525            err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, ip));
 526 #else
 527     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 528            which == narrow_oop_operand && !is_64bit,
 529            err_msg("which %d is_64_bit %d ip " INTPTR_FORMAT, which, is_64bit, ip));
 530 #endif // _LP64
 531     return ip;
 532 
 533   case 0x69: // imul r, a, #32
 534   case 0xC7: // movl a, #32(oop?)
 535     tail_size = 4;
 536     debug_only(has_disp32 = true); // has both kinds of operands!
 537     break;
 538 
 539   case 0x0F: // movx..., etc.
 540     switch (0xFF & *ip++) {
 541     case 0x3A: // pcmpestri
 542       tail_size = 1;
 543     case 0x38: // ptest, pmovzxbw
 544       ip++; // skip opcode
 545       debug_only(has_disp32 = true); // has both kinds of operands!
 546       break;
 547 
 548     case 0x70: // pshufd r, r/a, #8
 549       debug_only(has_disp32 = true); // has both kinds of operands!
 550     case 0x73: // psrldq r, #8
 551       tail_size = 1;
 552       break;
 553 
 554     case 0x12: // movlps
 555     case 0x28: // movaps
 556     case 0x2E: // ucomiss
 557     case 0x2F: // comiss
 558     case 0x54: // andps
 559     case 0x55: // andnps
 560     case 0x56: // orps
 561     case 0x57: // xorps
 562     case 0x6E: // movd
 563     case 0x7E: // movd
 564     case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
 565       debug_only(has_disp32 = true);
 566       break;
 567 
 568     case 0xAD: // shrd r, a, %cl
 569     case 0xAF: // imul r, a
 570     case 0xBE: // movsbl r, a (movsxb)
 571     case 0xBF: // movswl r, a (movsxw)
 572     case 0xB6: // movzbl r, a (movzxb)
 573     case 0xB7: // movzwl r, a (movzxw)
 574     case REP16(0x40): // cmovl cc, r, a
 575     case 0xB0: // cmpxchgb
 576     case 0xB1: // cmpxchg
 577     case 0xC1: // xaddl
 578     case 0xC7: // cmpxchg8
 579     case REP16(0x90): // setcc a
 580       debug_only(has_disp32 = true);
 581       // fall out of the switch to decode the address
 582       break;
 583 
 584     case 0xC4: // pinsrw r, a, #8
 585       debug_only(has_disp32 = true);
 586     case 0xC5: // pextrw r, r, #8
 587       tail_size = 1;  // the imm8
 588       break;
 589 
 590     case 0xAC: // shrd r, a, #8
 591       debug_only(has_disp32 = true);
 592       tail_size = 1;  // the imm8
 593       break;
 594 
 595     case REP16(0x80): // jcc rdisp32
 596       if (which == end_pc_operand)  return ip + 4;
 597       assert(which == call32_operand, "jcc has no disp32 or imm");
 598       return ip;
 599     default:
 600       ShouldNotReachHere();
 601     }
 602     break;
 603 
 604   case 0x81: // addl a, #32; addl r, #32
 605     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 606     // on 32bit in the case of cmpl, the imm might be an oop
 607     tail_size = 4;
 608     debug_only(has_disp32 = true); // has both kinds of operands!
 609     break;
 610 
 611   case 0x83: // addl a, #8; addl r, #8
 612     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 613     debug_only(has_disp32 = true); // has both kinds of operands!
 614     tail_size = 1;
 615     break;
 616 
 617   case 0x9B:
 618     switch (0xFF & *ip++) {
 619     case 0xD9: // fnstcw a
 620       debug_only(has_disp32 = true);
 621       break;
 622     default:
 623       ShouldNotReachHere();
 624     }
 625     break;
 626 
 627   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 628   case REP4(0x10): // adc...
 629   case REP4(0x20): // and...
 630   case REP4(0x30): // xor...
 631   case REP4(0x08): // or...
 632   case REP4(0x18): // sbb...
 633   case REP4(0x28): // sub...
 634   case 0xF7: // mull a
 635   case 0x8D: // lea r, a
 636   case 0x87: // xchg r, a
 637   case REP4(0x38): // cmp...
 638   case 0x85: // test r, a
 639     debug_only(has_disp32 = true); // has both kinds of operands!
 640     break;
 641 
 642   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 643   case 0xC6: // movb a, #8
 644   case 0x80: // cmpb a, #8
 645   case 0x6B: // imul r, a, #8
 646     debug_only(has_disp32 = true); // has both kinds of operands!
 647     tail_size = 1; // the imm8
 648     break;
 649 
 650   case 0xC4: // VEX_3bytes
 651   case 0xC5: // VEX_2bytes
 652     assert((UseAVX > 0), "shouldn't have VEX prefix");
 653     assert(ip == inst+1, "no prefixes allowed");
 654     // C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
 655     // but they have prefix 0x0F and processed when 0x0F processed above.
 656     //
 657     // In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
 658     // instructions (these instructions are not supported in 64-bit mode).
 659     // To distinguish them bits [7:6] are set in the VEX second byte since
 660     // ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
 661     // those VEX bits REX and vvvv bits are inverted.
 662     //
 663     // Fortunately C2 doesn't generate these instructions so we don't need
 664     // to check for them in product version.
 665 
 666     // Check second byte
 667     NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
 668 
 669     // First byte
 670     if ((0xFF & *inst) == VEX_3bytes) {
 671       ip++; // third byte
 672       is_64bit = ((VEX_W & *ip) == VEX_W);
 673     }
 674     ip++; // opcode
 675     // To find the end of instruction (which == end_pc_operand).
 676     switch (0xFF & *ip) {
 677     case 0x61: // pcmpestri r, r/a, #8
 678     case 0x70: // pshufd r, r/a, #8
 679     case 0x73: // psrldq r, #8
 680       tail_size = 1;  // the imm8
 681       break;
 682     default:
 683       break;
 684     }
 685     ip++; // skip opcode
 686     debug_only(has_disp32 = true); // has both kinds of operands!
 687     break;
 688 
 689   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 690   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 691   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 692   case 0xDD: // fld_d a; fst_d a; fstp_d a
 693   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 694   case 0xDF: // fild_d a; fistp_d a
 695   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 696   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 697   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 698     debug_only(has_disp32 = true);
 699     break;
 700 
 701   case 0xE8: // call rdisp32
 702   case 0xE9: // jmp  rdisp32
 703     if (which == end_pc_operand)  return ip + 4;
 704     assert(which == call32_operand, "call has no disp32 or imm");
 705     return ip;
 706 
 707   case 0xF0:                    // Lock
 708     assert(os::is_MP(), "only on MP");
 709     goto again_after_prefix;
 710 
 711   case 0xF3:                    // For SSE
 712   case 0xF2:                    // For SSE2
 713     switch (0xFF & *ip++) {
 714     case REX:
 715     case REX_B:
 716     case REX_X:
 717     case REX_XB:
 718     case REX_R:
 719     case REX_RB:
 720     case REX_RX:
 721     case REX_RXB:
 722     case REX_W:
 723     case REX_WB:
 724     case REX_WX:
 725     case REX_WXB:
 726     case REX_WR:
 727     case REX_WRB:
 728     case REX_WRX:
 729     case REX_WRXB:
 730       NOT_LP64(assert(false, "found 64bit prefix"));
 731       ip++;
 732     default:
 733       ip++;
 734     }
 735     debug_only(has_disp32 = true); // has both kinds of operands!
 736     break;
 737 
 738   default:
 739     ShouldNotReachHere();
 740 
 741 #undef REP8
 742 #undef REP16
 743   }
 744 
 745   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 746 #ifdef _LP64
 747   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 748 #else
 749   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 750   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 751 #endif // LP64
 752   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 753 
 754   // parse the output of emit_operand
 755   int op2 = 0xFF & *ip++;
 756   int base = op2 & 0x07;
 757   int op3 = -1;
 758   const int b100 = 4;
 759   const int b101 = 5;
 760   if (base == b100 && (op2 >> 6) != 3) {
 761     op3 = 0xFF & *ip++;
 762     base = op3 & 0x07;   // refetch the base
 763   }
 764   // now ip points at the disp (if any)
 765 
 766   switch (op2 >> 6) {
 767   case 0:
 768     // [00 reg  100][ss index base]
 769     // [00 reg  100][00   100  esp]
 770     // [00 reg base]
 771     // [00 reg  100][ss index  101][disp32]
 772     // [00 reg  101]               [disp32]
 773 
 774     if (base == b101) {
 775       if (which == disp32_operand)
 776         return ip;              // caller wants the disp32
 777       ip += 4;                  // skip the disp32
 778     }
 779     break;
 780 
 781   case 1:
 782     // [01 reg  100][ss index base][disp8]
 783     // [01 reg  100][00   100  esp][disp8]
 784     // [01 reg base]               [disp8]
 785     ip += 1;                    // skip the disp8
 786     break;
 787 
 788   case 2:
 789     // [10 reg  100][ss index base][disp32]
 790     // [10 reg  100][00   100  esp][disp32]
 791     // [10 reg base]               [disp32]
 792     if (which == disp32_operand)
 793       return ip;                // caller wants the disp32
 794     ip += 4;                    // skip the disp32
 795     break;
 796 
 797   case 3:
 798     // [11 reg base]  (not a memory addressing mode)
 799     break;
 800   }
 801 
 802   if (which == end_pc_operand) {
 803     return ip + tail_size;
 804   }
 805 
 806 #ifdef _LP64
 807   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 808 #else
 809   assert(which == imm_operand, "instruction has only an imm field");
 810 #endif // LP64
 811   return ip;
 812 }
 813 
 814 address Assembler::locate_next_instruction(address inst) {
 815   // Secretly share code with locate_operand:
 816   return locate_operand(inst, end_pc_operand);
 817 }
 818 
 819 
 820 #ifdef ASSERT
 821 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 822   address inst = inst_mark();
 823   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 824   address opnd;
 825 
 826   Relocation* r = rspec.reloc();
 827   if (r->type() == relocInfo::none) {
 828     return;
 829   } else if (r->is_call() || format == call32_operand) {
 830     // assert(format == imm32_operand, "cannot specify a nonzero format");
 831     opnd = locate_operand(inst, call32_operand);
 832   } else if (r->is_data()) {
 833     assert(format == imm_operand || format == disp32_operand
 834            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 835     opnd = locate_operand(inst, (WhichOperand)format);
 836   } else {
 837     assert(format == imm_operand, "cannot specify a format");
 838     return;
 839   }
 840   assert(opnd == pc(), "must put operand where relocs can find it");
 841 }
 842 #endif // ASSERT
 843 
 844 void Assembler::emit_operand32(Register reg, Address adr) {
 845   assert(reg->encoding() < 8, "no extended registers");
 846   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
 847   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 848                adr._rspec);
 849 }
 850 
 851 void Assembler::emit_operand(Register reg, Address adr,
 852                              int rip_relative_correction) {
 853   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 854                adr._rspec,
 855                rip_relative_correction);
 856 }
 857 
 858 void Assembler::emit_operand(XMMRegister reg, Address adr) {
 859   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 860                adr._rspec);
 861 }
 862 
 863 // MMX operations
 864 void Assembler::emit_operand(MMXRegister reg, Address adr) {
 865   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
 866   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
 867 }
 868 
 869 // work around gcc (3.2.1-7a) bug
 870 void Assembler::emit_operand(Address adr, MMXRegister reg) {
 871   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
 872   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
 873 }
 874 
 875 
 876 void Assembler::emit_farith(int b1, int b2, int i) {
 877   assert(isByte(b1) && isByte(b2), "wrong opcode");
 878   assert(0 <= i &&  i < 8, "illegal stack offset");
 879   emit_int8(b1);
 880   emit_int8(b2 + i);
 881 }
 882 
 883 
 884 // Now the Assembler instructions (identical for 32/64 bits)
 885 
 886 void Assembler::adcl(Address dst, int32_t imm32) {
 887   InstructionMark im(this);
 888   prefix(dst);
 889   emit_arith_operand(0x81, rdx, dst, imm32);
 890 }
 891 
 892 void Assembler::adcl(Address dst, Register src) {
 893   InstructionMark im(this);
 894   prefix(dst, src);
 895   emit_int8(0x11);
 896   emit_operand(src, dst);
 897 }
 898 
 899 void Assembler::adcl(Register dst, int32_t imm32) {
 900   prefix(dst);
 901   emit_arith(0x81, 0xD0, dst, imm32);
 902 }
 903 
 904 void Assembler::adcl(Register dst, Address src) {
 905   InstructionMark im(this);
 906   prefix(src, dst);
 907   emit_int8(0x13);
 908   emit_operand(dst, src);
 909 }
 910 
 911 void Assembler::adcl(Register dst, Register src) {
 912   (void) prefix_and_encode(dst->encoding(), src->encoding());
 913   emit_arith(0x13, 0xC0, dst, src);
 914 }
 915 
 916 void Assembler::addl(Address dst, int32_t imm32) {
 917   InstructionMark im(this);
 918   prefix(dst);
 919   emit_arith_operand(0x81, rax, dst, imm32);
 920 }
 921 
 922 void Assembler::addl(Address dst, Register src) {
 923   InstructionMark im(this);
 924   prefix(dst, src);
 925   emit_int8(0x01);
 926   emit_operand(src, dst);
 927 }
 928 
 929 void Assembler::addl(Register dst, int32_t imm32) {
 930   prefix(dst);
 931   emit_arith(0x81, 0xC0, dst, imm32);
 932 }
 933 
 934 void Assembler::addl(Register dst, Address src) {
 935   InstructionMark im(this);
 936   prefix(src, dst);
 937   emit_int8(0x03);
 938   emit_operand(dst, src);
 939 }
 940 
 941 void Assembler::addl(Register dst, Register src) {
 942   (void) prefix_and_encode(dst->encoding(), src->encoding());
 943   emit_arith(0x03, 0xC0, dst, src);
 944 }
 945 
 946 void Assembler::addr_nop_4() {
 947   assert(UseAddressNop, "no CPU support");
 948   // 4 bytes: NOP DWORD PTR [EAX+0]
 949   emit_int8(0x0F);
 950   emit_int8(0x1F);
 951   emit_int8(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 952   emit_int8(0);    // 8-bits offset (1 byte)
 953 }
 954 
 955 void Assembler::addr_nop_5() {
 956   assert(UseAddressNop, "no CPU support");
 957   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 958   emit_int8(0x0F);
 959   emit_int8(0x1F);
 960   emit_int8(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 961   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 962   emit_int8(0);    // 8-bits offset (1 byte)
 963 }
 964 
 965 void Assembler::addr_nop_7() {
 966   assert(UseAddressNop, "no CPU support");
 967   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 968   emit_int8(0x0F);
 969   emit_int8(0x1F);
 970   emit_int8((unsigned char)0x80);
 971                    // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 972   emit_int32(0);   // 32-bits offset (4 bytes)
 973 }
 974 
 975 void Assembler::addr_nop_8() {
 976   assert(UseAddressNop, "no CPU support");
 977   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 978   emit_int8(0x0F);
 979   emit_int8(0x1F);
 980   emit_int8((unsigned char)0x84);
 981                    // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 982   emit_int8(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 983   emit_int32(0);   // 32-bits offset (4 bytes)
 984 }
 985 
 986 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 987   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 988   emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
 989 }
 990 
 991 void Assembler::addsd(XMMRegister dst, Address src) {
 992   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 993   emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
 994 }
 995 
 996 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 997   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 998   emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
 999 }
1000 
1001 void Assembler::addss(XMMRegister dst, Address src) {
1002   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1003   emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
1004 }
1005 
1006 void Assembler::aesdec(XMMRegister dst, Address src) {
1007   assert(VM_Version::supports_aes(), "");
1008   InstructionMark im(this);
1009   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1010   emit_int8((unsigned char)0xDE);
1011   emit_operand(dst, src);
1012 }
1013 
1014 void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
1015   assert(VM_Version::supports_aes(), "");
1016   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1017   emit_int8((unsigned char)0xDE);
1018   emit_int8(0xC0 | encode);
1019 }
1020 
1021 void Assembler::aesdeclast(XMMRegister dst, Address src) {
1022   assert(VM_Version::supports_aes(), "");
1023   InstructionMark im(this);
1024   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1025   emit_int8((unsigned char)0xDF);
1026   emit_operand(dst, src);
1027 }
1028 
1029 void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
1030   assert(VM_Version::supports_aes(), "");
1031   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1032   emit_int8((unsigned char)0xDF);
1033   emit_int8((unsigned char)(0xC0 | encode));
1034 }
1035 
1036 void Assembler::aesenc(XMMRegister dst, Address src) {
1037   assert(VM_Version::supports_aes(), "");
1038   InstructionMark im(this);
1039   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1040   emit_int8((unsigned char)0xDC);
1041   emit_operand(dst, src);
1042 }
1043 
1044 void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
1045   assert(VM_Version::supports_aes(), "");
1046   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1047   emit_int8((unsigned char)0xDC);
1048   emit_int8(0xC0 | encode);
1049 }
1050 
1051 void Assembler::aesenclast(XMMRegister dst, Address src) {
1052   assert(VM_Version::supports_aes(), "");
1053   InstructionMark im(this);
1054   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1055   emit_int8((unsigned char)0xDD);
1056   emit_operand(dst, src);
1057 }
1058 
1059 void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
1060   assert(VM_Version::supports_aes(), "");
1061   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
1062   emit_int8((unsigned char)0xDD);
1063   emit_int8((unsigned char)(0xC0 | encode));
1064 }
1065 
1066 
1067 void Assembler::andl(Address dst, int32_t imm32) {
1068   InstructionMark im(this);
1069   prefix(dst);
1070   emit_int8((unsigned char)0x81);
1071   emit_operand(rsp, dst, 4);
1072   emit_int32(imm32);
1073 }
1074 
1075 void Assembler::andl(Register dst, int32_t imm32) {
1076   prefix(dst);
1077   emit_arith(0x81, 0xE0, dst, imm32);
1078 }
1079 
1080 void Assembler::andl(Register dst, Address src) {
1081   InstructionMark im(this);
1082   prefix(src, dst);
1083   emit_int8(0x23);
1084   emit_operand(dst, src);
1085 }
1086 
1087 void Assembler::andl(Register dst, Register src) {
1088   (void) prefix_and_encode(dst->encoding(), src->encoding());
1089   emit_arith(0x23, 0xC0, dst, src);
1090 }
1091 
1092 void Assembler::bsfl(Register dst, Register src) {
1093   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1094   emit_int8(0x0F);
1095   emit_int8((unsigned char)0xBC);
1096   emit_int8((unsigned char)(0xC0 | encode));
1097 }
1098 
1099 void Assembler::bsrl(Register dst, Register src) {
1100   assert(!VM_Version::supports_lzcnt(), "encoding is treated as LZCNT");
1101   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1102   emit_int8(0x0F);
1103   emit_int8((unsigned char)0xBD);
1104   emit_int8((unsigned char)(0xC0 | encode));
1105 }
1106 
1107 void Assembler::bswapl(Register reg) { // bswap
1108   int encode = prefix_and_encode(reg->encoding());
1109   emit_int8(0x0F);
1110   emit_int8((unsigned char)(0xC8 | encode));
1111 }
1112 
1113 void Assembler::call(Label& L, relocInfo::relocType rtype) {
1114   // suspect disp32 is always good
1115   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
1116 
1117   if (L.is_bound()) {
1118     const int long_size = 5;
1119     int offs = (int)( target(L) - pc() );
1120     assert(offs <= 0, "assembler error");
1121     InstructionMark im(this);
1122     // 1110 1000 #32-bit disp
1123     emit_int8((unsigned char)0xE8);
1124     emit_data(offs - long_size, rtype, operand);
1125   } else {
1126     InstructionMark im(this);
1127     // 1110 1000 #32-bit disp
1128     L.add_patch_at(code(), locator());
1129 
1130     emit_int8((unsigned char)0xE8);
1131     emit_data(int(0), rtype, operand);
1132   }
1133 }
1134 
1135 void Assembler::call(Register dst) {
1136   int encode = prefix_and_encode(dst->encoding());
1137   emit_int8((unsigned char)0xFF);
1138   emit_int8((unsigned char)(0xD0 | encode));
1139 }
1140 
1141 
1142 void Assembler::call(Address adr) {
1143   InstructionMark im(this);
1144   prefix(adr);
1145   emit_int8((unsigned char)0xFF);
1146   emit_operand(rdx, adr);
1147 }
1148 
1149 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1150   assert(entry != NULL, "call most probably wrong");
1151   InstructionMark im(this);
1152   emit_int8((unsigned char)0xE8);
1153   intptr_t disp = entry - (pc() + sizeof(int32_t));
1154   assert(is_simm32(disp), "must be 32bit offset (call2)");
1155   // Technically, should use call32_operand, but this format is
1156   // implied by the fact that we're emitting a call instruction.
1157 
1158   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1159   emit_data((int) disp, rspec, operand);
1160 }
1161 
1162 void Assembler::cdql() {
1163   emit_int8((unsigned char)0x99);
1164 }
1165 
1166 void Assembler::cld() {
1167   emit_int8((unsigned char)0xFC);
1168 }
1169 
1170 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1171   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1172   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1173   emit_int8(0x0F);
1174   emit_int8(0x40 | cc);
1175   emit_int8((unsigned char)(0xC0 | encode));
1176 }
1177 
1178 
1179 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1180   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1181   prefix(src, dst);
1182   emit_int8(0x0F);
1183   emit_int8(0x40 | cc);
1184   emit_operand(dst, src);
1185 }
1186 
1187 void Assembler::cmpb(Address dst, int imm8) {
1188   InstructionMark im(this);
1189   prefix(dst);
1190   emit_int8((unsigned char)0x80);
1191   emit_operand(rdi, dst, 1);
1192   emit_int8(imm8);
1193 }
1194 
1195 void Assembler::cmpl(Address dst, int32_t imm32) {
1196   InstructionMark im(this);
1197   prefix(dst);
1198   emit_int8((unsigned char)0x81);
1199   emit_operand(rdi, dst, 4);
1200   emit_int32(imm32);
1201 }
1202 
1203 void Assembler::cmpl(Register dst, int32_t imm32) {
1204   prefix(dst);
1205   emit_arith(0x81, 0xF8, dst, imm32);
1206 }
1207 
1208 void Assembler::cmpl(Register dst, Register src) {
1209   (void) prefix_and_encode(dst->encoding(), src->encoding());
1210   emit_arith(0x3B, 0xC0, dst, src);
1211 }
1212 
1213 
1214 void Assembler::cmpl(Register dst, Address  src) {
1215   InstructionMark im(this);
1216   prefix(src, dst);
1217   emit_int8((unsigned char)0x3B);
1218   emit_operand(dst, src);
1219 }
1220 
1221 void Assembler::cmpw(Address dst, int imm16) {
1222   InstructionMark im(this);
1223   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1224   emit_int8(0x66);
1225   emit_int8((unsigned char)0x81);
1226   emit_operand(rdi, dst, 2);
1227   emit_int16(imm16);
1228 }
1229 
1230 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1231 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1232 // The ZF is set if the compared values were equal, and cleared otherwise.
1233 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1234   InstructionMark im(this);
1235   prefix(adr, reg);
1236   emit_int8(0x0F);
1237   emit_int8((unsigned char)0xB1);
1238   emit_operand(reg, adr);
1239 }
1240 
1241 void Assembler::comisd(XMMRegister dst, Address src) {
1242   // NOTE: dbx seems to decode this as comiss even though the
1243   // 0x66 is there. Strangly ucomisd comes out correct
1244   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1245   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1246 }
1247 
1248 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
1249   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1250   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
1251 }
1252 
1253 void Assembler::comiss(XMMRegister dst, Address src) {
1254   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1255   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE);
1256 }
1257 
1258 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
1259   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1260   emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE);
1261 }
1262 
1263 void Assembler::cpuid() {
1264   emit_int8(0x0F);
1265   emit_int8((unsigned char)0xA2);
1266 }
1267 
1268 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1269   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1270   emit_simd_arith_nonds(0xE6, dst, src, VEX_SIMD_F3);
1271 }
1272 
1273 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1274   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1275   emit_simd_arith_nonds(0x5B, dst, src, VEX_SIMD_NONE);
1276 }
1277 
1278 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1279   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1280   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1281 }
1282 
1283 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
1284   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1285   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
1286 }
1287 
1288 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1289   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1290   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
1291   emit_int8(0x2A);
1292   emit_int8((unsigned char)(0xC0 | encode));
1293 }
1294 
1295 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
1296   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1297   emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2);
1298 }
1299 
1300 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1301   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1302   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
1303   emit_int8(0x2A);
1304   emit_int8((unsigned char)(0xC0 | encode));
1305 }
1306 
1307 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
1308   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1309   emit_simd_arith(0x2A, dst, src, VEX_SIMD_F3);
1310 }
1311 
1312 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1313   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1314   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
1315 }
1316 
1317 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
1318   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1319   emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
1320 }
1321 
1322 
1323 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1324   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1325   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2);
1326   emit_int8(0x2C);
1327   emit_int8((unsigned char)(0xC0 | encode));
1328 }
1329 
1330 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1331   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1332   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);
1333   emit_int8(0x2C);
1334   emit_int8((unsigned char)(0xC0 | encode));
1335 }
1336 
1337 void Assembler::decl(Address dst) {
1338   // Don't use it directly. Use MacroAssembler::decrement() instead.
1339   InstructionMark im(this);
1340   prefix(dst);
1341   emit_int8((unsigned char)0xFF);
1342   emit_operand(rcx, dst);
1343 }
1344 
1345 void Assembler::divsd(XMMRegister dst, Address src) {
1346   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1347   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1348 }
1349 
1350 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1351   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1352   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
1353 }
1354 
1355 void Assembler::divss(XMMRegister dst, Address src) {
1356   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1357   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
1358 }
1359 
1360 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1361   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1362   emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
1363 }
1364 
1365 void Assembler::emms() {
1366   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1367   emit_int8(0x0F);
1368   emit_int8(0x77);
1369 }
1370 
1371 void Assembler::hlt() {
1372   emit_int8((unsigned char)0xF4);
1373 }
1374 
1375 void Assembler::idivl(Register src) {
1376   int encode = prefix_and_encode(src->encoding());
1377   emit_int8((unsigned char)0xF7);
1378   emit_int8((unsigned char)(0xF8 | encode));
1379 }
1380 
1381 void Assembler::divl(Register src) { // Unsigned
1382   int encode = prefix_and_encode(src->encoding());
1383   emit_int8((unsigned char)0xF7);
1384   emit_int8((unsigned char)(0xF0 | encode));
1385 }
1386 
1387 void Assembler::imull(Register dst, Register src) {
1388   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1389   emit_int8(0x0F);
1390   emit_int8((unsigned char)0xAF);
1391   emit_int8((unsigned char)(0xC0 | encode));
1392 }
1393 
1394 
1395 void Assembler::imull(Register dst, Register src, int value) {
1396   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1397   if (is8bit(value)) {
1398     emit_int8(0x6B);
1399     emit_int8((unsigned char)(0xC0 | encode));
1400     emit_int8(value & 0xFF);
1401   } else {
1402     emit_int8(0x69);
1403     emit_int8((unsigned char)(0xC0 | encode));
1404     emit_int32(value);
1405   }
1406 }
1407 
1408 void Assembler::imull(Register dst, Address src) {
1409   InstructionMark im(this);
1410   prefix(src, dst);
1411   emit_int8(0x0F);
1412   emit_int8((unsigned char) 0xAF);
1413   emit_operand(dst, src);
1414 }
1415 
1416 
1417 void Assembler::incl(Address dst) {
1418   // Don't use it directly. Use MacroAssembler::increment() instead.
1419   InstructionMark im(this);
1420   prefix(dst);
1421   emit_int8((unsigned char)0xFF);
1422   emit_operand(rax, dst);
1423 }
1424 
1425 void Assembler::jcc(Condition cc, Label& L, bool maybe_short) {
1426   InstructionMark im(this);
1427   assert((0 <= cc) && (cc < 16), "illegal cc");
1428   if (L.is_bound()) {
1429     address dst = target(L);
1430     assert(dst != NULL, "jcc most probably wrong");
1431 
1432     const int short_size = 2;
1433     const int long_size = 6;
1434     intptr_t offs = (intptr_t)dst - (intptr_t)pc();
1435     if (maybe_short && is8bit(offs - short_size)) {
1436       // 0111 tttn #8-bit disp
1437       emit_int8(0x70 | cc);
1438       emit_int8((offs - short_size) & 0xFF);
1439     } else {
1440       // 0000 1111 1000 tttn #32-bit disp
1441       assert(is_simm32(offs - long_size),
1442              "must be 32bit offset (call4)");
1443       emit_int8(0x0F);
1444       emit_int8((unsigned char)(0x80 | cc));
1445       emit_int32(offs - long_size);
1446     }
1447   } else {
1448     // Note: could eliminate cond. jumps to this jump if condition
1449     //       is the same however, seems to be rather unlikely case.
1450     // Note: use jccb() if label to be bound is very close to get
1451     //       an 8-bit displacement
1452     L.add_patch_at(code(), locator());
1453     emit_int8(0x0F);
1454     emit_int8((unsigned char)(0x80 | cc));
1455     emit_int32(0);
1456   }
1457 }
1458 
1459 void Assembler::jccb(Condition cc, Label& L) {
1460   if (L.is_bound()) {
1461     const int short_size = 2;
1462     address entry = target(L);
1463 #ifdef ASSERT
1464     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1465     intptr_t delta = short_branch_delta();
1466     if (delta != 0) {
1467       dist += (dist < 0 ? (-delta) :delta);
1468     }
1469     assert(is8bit(dist), "Dispacement too large for a short jmp");
1470 #endif
1471     intptr_t offs = (intptr_t)entry - (intptr_t)pc();
1472     // 0111 tttn #8-bit disp
1473     emit_int8(0x70 | cc);
1474     emit_int8((offs - short_size) & 0xFF);
1475   } else {
1476     InstructionMark im(this);
1477     L.add_patch_at(code(), locator());
1478     emit_int8(0x70 | cc);
1479     emit_int8(0);
1480   }
1481 }
1482 
1483 void Assembler::jmp(Address adr) {
1484   InstructionMark im(this);
1485   prefix(adr);
1486   emit_int8((unsigned char)0xFF);
1487   emit_operand(rsp, adr);
1488 }
1489 
1490 void Assembler::jmp(Label& L, bool maybe_short) {
1491   if (L.is_bound()) {
1492     address entry = target(L);
1493     assert(entry != NULL, "jmp most probably wrong");
1494     InstructionMark im(this);
1495     const int short_size = 2;
1496     const int long_size = 5;
1497     intptr_t offs = entry - pc();
1498     if (maybe_short && is8bit(offs - short_size)) {
1499       emit_int8((unsigned char)0xEB);
1500       emit_int8((offs - short_size) & 0xFF);
1501     } else {
1502       emit_int8((unsigned char)0xE9);
1503       emit_int32(offs - long_size);
1504     }
1505   } else {
1506     // By default, forward jumps are always 32-bit displacements, since
1507     // we can't yet know where the label will be bound.  If you're sure that
1508     // the forward jump will not run beyond 256 bytes, use jmpb to
1509     // force an 8-bit displacement.
1510     InstructionMark im(this);
1511     L.add_patch_at(code(), locator());
1512     emit_int8((unsigned char)0xE9);
1513     emit_int32(0);
1514   }
1515 }
1516 
1517 void Assembler::jmp(Register entry) {
1518   int encode = prefix_and_encode(entry->encoding());
1519   emit_int8((unsigned char)0xFF);
1520   emit_int8((unsigned char)(0xE0 | encode));
1521 }
1522 
1523 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
1524   InstructionMark im(this);
1525   emit_int8((unsigned char)0xE9);
1526   assert(dest != NULL, "must have a target");
1527   intptr_t disp = dest - (pc() + sizeof(int32_t));
1528   assert(is_simm32(disp), "must be 32bit offset (jmp)");
1529   emit_data(disp, rspec.reloc(), call32_operand);
1530 }
1531 
1532 void Assembler::jmpb(Label& L) {
1533   if (L.is_bound()) {
1534     const int short_size = 2;
1535     address entry = target(L);
1536     assert(entry != NULL, "jmp most probably wrong");
1537 #ifdef ASSERT
1538     intptr_t dist = (intptr_t)entry - ((intptr_t)pc() + short_size);
1539     intptr_t delta = short_branch_delta();
1540     if (delta != 0) {
1541       dist += (dist < 0 ? (-delta) :delta);
1542     }
1543     assert(is8bit(dist), "Dispacement too large for a short jmp");
1544 #endif
1545     intptr_t offs = entry - pc();
1546     emit_int8((unsigned char)0xEB);
1547     emit_int8((offs - short_size) & 0xFF);
1548   } else {
1549     InstructionMark im(this);
1550     L.add_patch_at(code(), locator());
1551     emit_int8((unsigned char)0xEB);
1552     emit_int8(0);
1553   }
1554 }
1555 
1556 void Assembler::ldmxcsr( Address src) {
1557   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1558   InstructionMark im(this);
1559   prefix(src);
1560   emit_int8(0x0F);
1561   emit_int8((unsigned char)0xAE);
1562   emit_operand(as_Register(2), src);
1563 }
1564 
1565 void Assembler::leal(Register dst, Address src) {
1566   InstructionMark im(this);
1567 #ifdef _LP64
1568   emit_int8(0x67); // addr32
1569   prefix(src, dst);
1570 #endif // LP64
1571   emit_int8((unsigned char)0x8D);
1572   emit_operand(dst, src);
1573 }
1574 
1575 void Assembler::lfence() {
1576   emit_int8(0x0F);
1577   emit_int8((unsigned char)0xAE);
1578   emit_int8((unsigned char)0xE8);
1579 }
1580 
1581 void Assembler::lock() {
1582   emit_int8((unsigned char)0xF0);
1583 }
1584 
1585 void Assembler::lzcntl(Register dst, Register src) {
1586   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
1587   emit_int8((unsigned char)0xF3);
1588   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1589   emit_int8(0x0F);
1590   emit_int8((unsigned char)0xBD);
1591   emit_int8((unsigned char)(0xC0 | encode));
1592 }
1593 
1594 // Emit mfence instruction
1595 void Assembler::mfence() {
1596   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
1597   emit_int8(0x0F);
1598   emit_int8((unsigned char)0xAE);
1599   emit_int8((unsigned char)0xF0);
1600 }
1601 
1602 void Assembler::mov(Register dst, Register src) {
1603   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
1604 }
1605 
1606 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
1607   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1608   emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_66);
1609 }
1610 
1611 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
1612   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1613   emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_NONE);
1614 }
1615 
1616 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
1617   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1618   int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE);
1619   emit_int8(0x16);
1620   emit_int8((unsigned char)(0xC0 | encode));
1621 }
1622 
1623 void Assembler::movb(Register dst, Address src) {
1624   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
1625   InstructionMark im(this);
1626   prefix(src, dst, true);
1627   emit_int8((unsigned char)0x8A);
1628   emit_operand(dst, src);
1629 }
1630 
1631 
1632 void Assembler::movb(Address dst, int imm8) {
1633   InstructionMark im(this);
1634    prefix(dst);
1635   emit_int8((unsigned char)0xC6);
1636   emit_operand(rax, dst, 1);
1637   emit_int8(imm8);
1638 }
1639 
1640 
1641 void Assembler::movb(Address dst, Register src) {
1642   assert(src->has_byte_register(), "must have byte register");
1643   InstructionMark im(this);
1644   prefix(dst, src, true);
1645   emit_int8((unsigned char)0x88);
1646   emit_operand(src, dst);
1647 }
1648 
1649 void Assembler::movdl(XMMRegister dst, Register src) {
1650   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1651   int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
1652   emit_int8(0x6E);
1653   emit_int8((unsigned char)(0xC0 | encode));
1654 }
1655 
1656 void Assembler::movdl(Register dst, XMMRegister src) {
1657   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1658   // swap src/dst to get correct prefix
1659   int encode = simd_prefix_and_encode(src, dst, VEX_SIMD_66);
1660   emit_int8(0x7E);
1661   emit_int8((unsigned char)(0xC0 | encode));
1662 }
1663 
1664 void Assembler::movdl(XMMRegister dst, Address src) {
1665   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1666   InstructionMark im(this);
1667   simd_prefix(dst, src, VEX_SIMD_66);
1668   emit_int8(0x6E);
1669   emit_operand(dst, src);
1670 }
1671 
1672 void Assembler::movdl(Address dst, XMMRegister src) {
1673   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1674   InstructionMark im(this);
1675   simd_prefix(dst, src, VEX_SIMD_66);
1676   emit_int8(0x7E);
1677   emit_operand(src, dst);
1678 }
1679 
1680 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
1681   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1682   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
1683 }
1684 
1685 void Assembler::movdqa(XMMRegister dst, Address src) {
1686   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1687   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
1688 }
1689 
1690 void Assembler::movdqu(XMMRegister dst, Address src) {
1691   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1692   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
1693 }
1694 
1695 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
1696   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1697   emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
1698 }
1699 
1700 void Assembler::movdqu(Address dst, XMMRegister src) {
1701   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1702   InstructionMark im(this);
1703   simd_prefix(dst, src, VEX_SIMD_F3);
1704   emit_int8(0x7F);
1705   emit_operand(src, dst);
1706 }
1707 
1708 // Move Unaligned 256bit Vector
1709 void Assembler::vmovdqu(XMMRegister dst, XMMRegister src) {
1710   assert(UseAVX, "");
1711   bool vector256 = true;
1712   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_F3, vector256);
1713   emit_int8(0x6F);
1714   emit_int8((unsigned char)(0xC0 | encode));
1715 }
1716 
1717 void Assembler::vmovdqu(XMMRegister dst, Address src) {
1718   assert(UseAVX, "");
1719   InstructionMark im(this);
1720   bool vector256 = true;
1721   vex_prefix(dst, xnoreg, src, VEX_SIMD_F3, vector256);
1722   emit_int8(0x6F);
1723   emit_operand(dst, src);
1724 }
1725 
1726 void Assembler::vmovdqu(Address dst, XMMRegister src) {
1727   assert(UseAVX, "");
1728   InstructionMark im(this);
1729   bool vector256 = true;
1730   // swap src<->dst for encoding
1731   assert(src != xnoreg, "sanity");
1732   vex_prefix(src, xnoreg, dst, VEX_SIMD_F3, vector256);
1733   emit_int8(0x7F);
1734   emit_operand(src, dst);
1735 }
1736 
1737 // Uses zero extension on 64bit
1738 
1739 void Assembler::movl(Register dst, int32_t imm32) {
1740   int encode = prefix_and_encode(dst->encoding());
1741   emit_int8((unsigned char)(0xB8 | encode));
1742   emit_int32(imm32);
1743 }
1744 
1745 void Assembler::movl(Register dst, Register src) {
1746   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1747   emit_int8((unsigned char)0x8B);
1748   emit_int8((unsigned char)(0xC0 | encode));
1749 }
1750 
1751 void Assembler::movl(Register dst, Address src) {
1752   InstructionMark im(this);
1753   prefix(src, dst);
1754   emit_int8((unsigned char)0x8B);
1755   emit_operand(dst, src);
1756 }
1757 
1758 void Assembler::movl(Address dst, int32_t imm32) {
1759   InstructionMark im(this);
1760   prefix(dst);
1761   emit_int8((unsigned char)0xC7);
1762   emit_operand(rax, dst, 4);
1763   emit_int32(imm32);
1764 }
1765 
1766 void Assembler::movl(Address dst, Register src) {
1767   InstructionMark im(this);
1768   prefix(dst, src);
1769   emit_int8((unsigned char)0x89);
1770   emit_operand(src, dst);
1771 }
1772 
1773 // New cpus require to use movsd and movss to avoid partial register stall
1774 // when loading from memory. But for old Opteron use movlpd instead of movsd.
1775 // The selection is done in MacroAssembler::movdbl() and movflt().
1776 void Assembler::movlpd(XMMRegister dst, Address src) {
1777   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1778   emit_simd_arith(0x12, dst, src, VEX_SIMD_66);
1779 }
1780 
1781 void Assembler::movq( MMXRegister dst, Address src ) {
1782   assert( VM_Version::supports_mmx(), "" );
1783   emit_int8(0x0F);
1784   emit_int8(0x6F);
1785   emit_operand(dst, src);
1786 }
1787 
1788 void Assembler::movq( Address dst, MMXRegister src ) {
1789   assert( VM_Version::supports_mmx(), "" );
1790   emit_int8(0x0F);
1791   emit_int8(0x7F);
1792   // workaround gcc (3.2.1-7a) bug
1793   // In that version of gcc with only an emit_operand(MMX, Address)
1794   // gcc will tail jump and try and reverse the parameters completely
1795   // obliterating dst in the process. By having a version available
1796   // that doesn't need to swap the args at the tail jump the bug is
1797   // avoided.
1798   emit_operand(dst, src);
1799 }
1800 
1801 void Assembler::movq(XMMRegister dst, Address src) {
1802   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1803   InstructionMark im(this);
1804   simd_prefix(dst, src, VEX_SIMD_F3);
1805   emit_int8(0x7E);
1806   emit_operand(dst, src);
1807 }
1808 
1809 void Assembler::movq(Address dst, XMMRegister src) {
1810   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1811   InstructionMark im(this);
1812   simd_prefix(dst, src, VEX_SIMD_66);
1813   emit_int8((unsigned char)0xD6);
1814   emit_operand(src, dst);
1815 }
1816 
1817 void Assembler::movsbl(Register dst, Address src) { // movsxb
1818   InstructionMark im(this);
1819   prefix(src, dst);
1820   emit_int8(0x0F);
1821   emit_int8((unsigned char)0xBE);
1822   emit_operand(dst, src);
1823 }
1824 
1825 void Assembler::movsbl(Register dst, Register src) { // movsxb
1826   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
1827   int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
1828   emit_int8(0x0F);
1829   emit_int8((unsigned char)0xBE);
1830   emit_int8((unsigned char)(0xC0 | encode));
1831 }
1832 
1833 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
1834   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1835   emit_simd_arith(0x10, dst, src, VEX_SIMD_F2);
1836 }
1837 
1838 void Assembler::movsd(XMMRegister dst, Address src) {
1839   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1840   emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F2);
1841 }
1842 
1843 void Assembler::movsd(Address dst, XMMRegister src) {
1844   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1845   InstructionMark im(this);
1846   simd_prefix(dst, src, VEX_SIMD_F2);
1847   emit_int8(0x11);
1848   emit_operand(src, dst);
1849 }
1850 
1851 void Assembler::movss(XMMRegister dst, XMMRegister src) {
1852   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1853   emit_simd_arith(0x10, dst, src, VEX_SIMD_F3);
1854 }
1855 
1856 void Assembler::movss(XMMRegister dst, Address src) {
1857   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1858   emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F3);
1859 }
1860 
1861 void Assembler::movss(Address dst, XMMRegister src) {
1862   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1863   InstructionMark im(this);
1864   simd_prefix(dst, src, VEX_SIMD_F3);
1865   emit_int8(0x11);
1866   emit_operand(src, dst);
1867 }
1868 
1869 void Assembler::movswl(Register dst, Address src) { // movsxw
1870   InstructionMark im(this);
1871   prefix(src, dst);
1872   emit_int8(0x0F);
1873   emit_int8((unsigned char)0xBF);
1874   emit_operand(dst, src);
1875 }
1876 
1877 void Assembler::movswl(Register dst, Register src) { // movsxw
1878   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1879   emit_int8(0x0F);
1880   emit_int8((unsigned char)0xBF);
1881   emit_int8((unsigned char)(0xC0 | encode));
1882 }
1883 
1884 void Assembler::movw(Address dst, int imm16) {
1885   InstructionMark im(this);
1886 
1887   emit_int8(0x66); // switch to 16-bit mode
1888   prefix(dst);
1889   emit_int8((unsigned char)0xC7);
1890   emit_operand(rax, dst, 2);
1891   emit_int16(imm16);
1892 }
1893 
1894 void Assembler::movw(Register dst, Address src) {
1895   InstructionMark im(this);
1896   emit_int8(0x66);
1897   prefix(src, dst);
1898   emit_int8((unsigned char)0x8B);
1899   emit_operand(dst, src);
1900 }
1901 
1902 void Assembler::movw(Address dst, Register src) {
1903   InstructionMark im(this);
1904   emit_int8(0x66);
1905   prefix(dst, src);
1906   emit_int8((unsigned char)0x89);
1907   emit_operand(src, dst);
1908 }
1909 
1910 void Assembler::movzbl(Register dst, Address src) { // movzxb
1911   InstructionMark im(this);
1912   prefix(src, dst);
1913   emit_int8(0x0F);
1914   emit_int8((unsigned char)0xB6);
1915   emit_operand(dst, src);
1916 }
1917 
1918 void Assembler::movzbl(Register dst, Register src) { // movzxb
1919   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
1920   int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
1921   emit_int8(0x0F);
1922   emit_int8((unsigned char)0xB6);
1923   emit_int8(0xC0 | encode);
1924 }
1925 
1926 void Assembler::movzwl(Register dst, Address src) { // movzxw
1927   InstructionMark im(this);
1928   prefix(src, dst);
1929   emit_int8(0x0F);
1930   emit_int8((unsigned char)0xB7);
1931   emit_operand(dst, src);
1932 }
1933 
1934 void Assembler::movzwl(Register dst, Register src) { // movzxw
1935   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1936   emit_int8(0x0F);
1937   emit_int8((unsigned char)0xB7);
1938   emit_int8(0xC0 | encode);
1939 }
1940 
1941 void Assembler::mull(Address src) {
1942   InstructionMark im(this);
1943   prefix(src);
1944   emit_int8((unsigned char)0xF7);
1945   emit_operand(rsp, src);
1946 }
1947 
1948 void Assembler::mull(Register src) {
1949   int encode = prefix_and_encode(src->encoding());
1950   emit_int8((unsigned char)0xF7);
1951   emit_int8((unsigned char)(0xE0 | encode));
1952 }
1953 
1954 void Assembler::mulsd(XMMRegister dst, Address src) {
1955   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1956   emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
1957 }
1958 
1959 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
1960   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1961   emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
1962 }
1963 
1964 void Assembler::mulss(XMMRegister dst, Address src) {
1965   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1966   emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
1967 }
1968 
1969 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
1970   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1971   emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
1972 }
1973 
1974 void Assembler::negl(Register dst) {
1975   int encode = prefix_and_encode(dst->encoding());
1976   emit_int8((unsigned char)0xF7);
1977   emit_int8((unsigned char)(0xD8 | encode));
1978 }
1979 
1980 void Assembler::nop(int i) {
1981 #ifdef ASSERT
1982   assert(i > 0, " ");
1983   // The fancy nops aren't currently recognized by debuggers making it a
1984   // pain to disassemble code while debugging. If asserts are on clearly
1985   // speed is not an issue so simply use the single byte traditional nop
1986   // to do alignment.
1987 
1988   for (; i > 0 ; i--) emit_int8((unsigned char)0x90);
1989   return;
1990 
1991 #endif // ASSERT
1992 
1993   if (UseAddressNop && VM_Version::is_intel()) {
1994     //
1995     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
1996     //  1: 0x90
1997     //  2: 0x66 0x90
1998     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
1999     //  4: 0x0F 0x1F 0x40 0x00
2000     //  5: 0x0F 0x1F 0x44 0x00 0x00
2001     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2002     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2003     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2004     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2005     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2006     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2007 
2008     // The rest coding is Intel specific - don't use consecutive address nops
2009 
2010     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2011     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2012     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2013     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
2014 
2015     while(i >= 15) {
2016       // For Intel don't generate consecutive addess nops (mix with regular nops)
2017       i -= 15;
2018       emit_int8(0x66);   // size prefix
2019       emit_int8(0x66);   // size prefix
2020       emit_int8(0x66);   // size prefix
2021       addr_nop_8();
2022       emit_int8(0x66);   // size prefix
2023       emit_int8(0x66);   // size prefix
2024       emit_int8(0x66);   // size prefix
2025       emit_int8((unsigned char)0x90);
2026                          // nop
2027     }
2028     switch (i) {
2029       case 14:
2030         emit_int8(0x66); // size prefix
2031       case 13:
2032         emit_int8(0x66); // size prefix
2033       case 12:
2034         addr_nop_8();
2035         emit_int8(0x66); // size prefix
2036         emit_int8(0x66); // size prefix
2037         emit_int8(0x66); // size prefix
2038         emit_int8((unsigned char)0x90);
2039                          // nop
2040         break;
2041       case 11:
2042         emit_int8(0x66); // size prefix
2043       case 10:
2044         emit_int8(0x66); // size prefix
2045       case 9:
2046         emit_int8(0x66); // size prefix
2047       case 8:
2048         addr_nop_8();
2049         break;
2050       case 7:
2051         addr_nop_7();
2052         break;
2053       case 6:
2054         emit_int8(0x66); // size prefix
2055       case 5:
2056         addr_nop_5();
2057         break;
2058       case 4:
2059         addr_nop_4();
2060         break;
2061       case 3:
2062         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2063         emit_int8(0x66); // size prefix
2064       case 2:
2065         emit_int8(0x66); // size prefix
2066       case 1:
2067         emit_int8((unsigned char)0x90);
2068                          // nop
2069         break;
2070       default:
2071         assert(i == 0, " ");
2072     }
2073     return;
2074   }
2075   if (UseAddressNop && VM_Version::is_amd()) {
2076     //
2077     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2078     //  1: 0x90
2079     //  2: 0x66 0x90
2080     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2081     //  4: 0x0F 0x1F 0x40 0x00
2082     //  5: 0x0F 0x1F 0x44 0x00 0x00
2083     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2084     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2085     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2086     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2087     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2088     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2089 
2090     // The rest coding is AMD specific - use consecutive address nops
2091 
2092     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2093     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2094     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2095     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2096     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2097     //     Size prefixes (0x66) are added for larger sizes
2098 
2099     while(i >= 22) {
2100       i -= 11;
2101       emit_int8(0x66); // size prefix
2102       emit_int8(0x66); // size prefix
2103       emit_int8(0x66); // size prefix
2104       addr_nop_8();
2105     }
2106     // Generate first nop for size between 21-12
2107     switch (i) {
2108       case 21:
2109         i -= 1;
2110         emit_int8(0x66); // size prefix
2111       case 20:
2112       case 19:
2113         i -= 1;
2114         emit_int8(0x66); // size prefix
2115       case 18:
2116       case 17:
2117         i -= 1;
2118         emit_int8(0x66); // size prefix
2119       case 16:
2120       case 15:
2121         i -= 8;
2122         addr_nop_8();
2123         break;
2124       case 14:
2125       case 13:
2126         i -= 7;
2127         addr_nop_7();
2128         break;
2129       case 12:
2130         i -= 6;
2131         emit_int8(0x66); // size prefix
2132         addr_nop_5();
2133         break;
2134       default:
2135         assert(i < 12, " ");
2136     }
2137 
2138     // Generate second nop for size between 11-1
2139     switch (i) {
2140       case 11:
2141         emit_int8(0x66); // size prefix
2142       case 10:
2143         emit_int8(0x66); // size prefix
2144       case 9:
2145         emit_int8(0x66); // size prefix
2146       case 8:
2147         addr_nop_8();
2148         break;
2149       case 7:
2150         addr_nop_7();
2151         break;
2152       case 6:
2153         emit_int8(0x66); // size prefix
2154       case 5:
2155         addr_nop_5();
2156         break;
2157       case 4:
2158         addr_nop_4();
2159         break;
2160       case 3:
2161         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2162         emit_int8(0x66); // size prefix
2163       case 2:
2164         emit_int8(0x66); // size prefix
2165       case 1:
2166         emit_int8((unsigned char)0x90);
2167                          // nop
2168         break;
2169       default:
2170         assert(i == 0, " ");
2171     }
2172     return;
2173   }
2174 
2175   // Using nops with size prefixes "0x66 0x90".
2176   // From AMD Optimization Guide:
2177   //  1: 0x90
2178   //  2: 0x66 0x90
2179   //  3: 0x66 0x66 0x90
2180   //  4: 0x66 0x66 0x66 0x90
2181   //  5: 0x66 0x66 0x90 0x66 0x90
2182   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
2183   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
2184   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
2185   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2186   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2187   //
2188   while(i > 12) {
2189     i -= 4;
2190     emit_int8(0x66); // size prefix
2191     emit_int8(0x66);
2192     emit_int8(0x66);
2193     emit_int8((unsigned char)0x90);
2194                      // nop
2195   }
2196   // 1 - 12 nops
2197   if(i > 8) {
2198     if(i > 9) {
2199       i -= 1;
2200       emit_int8(0x66);
2201     }
2202     i -= 3;
2203     emit_int8(0x66);
2204     emit_int8(0x66);
2205     emit_int8((unsigned char)0x90);
2206   }
2207   // 1 - 8 nops
2208   if(i > 4) {
2209     if(i > 6) {
2210       i -= 1;
2211       emit_int8(0x66);
2212     }
2213     i -= 3;
2214     emit_int8(0x66);
2215     emit_int8(0x66);
2216     emit_int8((unsigned char)0x90);
2217   }
2218   switch (i) {
2219     case 4:
2220       emit_int8(0x66);
2221     case 3:
2222       emit_int8(0x66);
2223     case 2:
2224       emit_int8(0x66);
2225     case 1:
2226       emit_int8((unsigned char)0x90);
2227       break;
2228     default:
2229       assert(i == 0, " ");
2230   }
2231 }
2232 
2233 void Assembler::notl(Register dst) {
2234   int encode = prefix_and_encode(dst->encoding());
2235   emit_int8((unsigned char)0xF7);
2236   emit_int8((unsigned char)(0xD0 | encode));
2237 }
2238 
2239 void Assembler::orl(Address dst, int32_t imm32) {
2240   InstructionMark im(this);
2241   prefix(dst);
2242   emit_arith_operand(0x81, rcx, dst, imm32);
2243 }
2244 
2245 void Assembler::orl(Register dst, int32_t imm32) {
2246   prefix(dst);
2247   emit_arith(0x81, 0xC8, dst, imm32);
2248 }
2249 
2250 void Assembler::orl(Register dst, Address src) {
2251   InstructionMark im(this);
2252   prefix(src, dst);
2253   emit_int8(0x0B);
2254   emit_operand(dst, src);
2255 }
2256 
2257 void Assembler::orl(Register dst, Register src) {
2258   (void) prefix_and_encode(dst->encoding(), src->encoding());
2259   emit_arith(0x0B, 0xC0, dst, src);
2260 }
2261 
2262 void Assembler::packuswb(XMMRegister dst, Address src) {
2263   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2264   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2265   emit_simd_arith(0x67, dst, src, VEX_SIMD_66);
2266 }
2267 
2268 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
2269   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2270   emit_simd_arith(0x67, dst, src, VEX_SIMD_66);
2271 }
2272 
2273 void Assembler::vpackuswb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
2274   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
2275   emit_vex_arith(0x67, dst, nds, src, VEX_SIMD_66, vector256);
2276 }
2277 
2278 void Assembler::vpermq(XMMRegister dst, XMMRegister src, int imm8, bool vector256) {
2279   assert(VM_Version::supports_avx2(), "");
2280   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, true, vector256);
2281   emit_int8(0x00);
2282   emit_int8(0xC0 | encode);
2283   emit_int8(imm8);
2284 }
2285 
2286 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
2287   assert(VM_Version::supports_sse4_2(), "");
2288   InstructionMark im(this);
2289   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
2290   emit_int8(0x61);
2291   emit_operand(dst, src);
2292   emit_int8(imm8);
2293 }
2294 
2295 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
2296   assert(VM_Version::supports_sse4_2(), "");
2297   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
2298   emit_int8(0x61);
2299   emit_int8((unsigned char)(0xC0 | encode));
2300   emit_int8(imm8);
2301 }
2302 
2303 void Assembler::pextrd(Register dst, XMMRegister src, int imm8) {
2304   assert(VM_Version::supports_sse4_1(), "");
2305   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, false);
2306   emit_int8(0x16);
2307   emit_int8((unsigned char)(0xC0 | encode));
2308   emit_int8(imm8);
2309 }
2310 
2311 void Assembler::pextrq(Register dst, XMMRegister src, int imm8) {
2312   assert(VM_Version::supports_sse4_1(), "");
2313   int encode = simd_prefix_and_encode(as_XMMRegister(dst->encoding()), xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A, true);
2314   emit_int8(0x16);
2315   emit_int8((unsigned char)(0xC0 | encode));
2316   emit_int8(imm8);
2317 }
2318 
2319 void Assembler::pinsrd(XMMRegister dst, Register src, int imm8) {
2320   assert(VM_Version::supports_sse4_1(), "");
2321   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, false);
2322   emit_int8(0x22);
2323   emit_int8((unsigned char)(0xC0 | encode));
2324   emit_int8(imm8);
2325 }
2326 
2327 void Assembler::pinsrq(XMMRegister dst, Register src, int imm8) {
2328   assert(VM_Version::supports_sse4_1(), "");
2329   int encode = simd_prefix_and_encode(dst, dst, as_XMMRegister(src->encoding()), VEX_SIMD_66, VEX_OPCODE_0F_3A, true);
2330   emit_int8(0x22);
2331   emit_int8((unsigned char)(0xC0 | encode));
2332   emit_int8(imm8);
2333 }
2334 
2335 void Assembler::pmovzxbw(XMMRegister dst, Address src) {
2336   assert(VM_Version::supports_sse4_1(), "");
2337   InstructionMark im(this);
2338   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2339   emit_int8(0x30);
2340   emit_operand(dst, src);
2341 }
2342 
2343 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
2344   assert(VM_Version::supports_sse4_1(), "");
2345   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2346   emit_int8(0x30);
2347   emit_int8((unsigned char)(0xC0 | encode));
2348 }
2349 
2350 // generic
2351 void Assembler::pop(Register dst) {
2352   int encode = prefix_and_encode(dst->encoding());
2353   emit_int8(0x58 | encode);
2354 }
2355 
2356 void Assembler::popcntl(Register dst, Address src) {
2357   assert(VM_Version::supports_popcnt(), "must support");
2358   InstructionMark im(this);
2359   emit_int8((unsigned char)0xF3);
2360   prefix(src, dst);
2361   emit_int8(0x0F);
2362   emit_int8((unsigned char)0xB8);
2363   emit_operand(dst, src);
2364 }
2365 
2366 void Assembler::popcntl(Register dst, Register src) {
2367   assert(VM_Version::supports_popcnt(), "must support");
2368   emit_int8((unsigned char)0xF3);
2369   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2370   emit_int8(0x0F);
2371   emit_int8((unsigned char)0xB8);
2372   emit_int8((unsigned char)(0xC0 | encode));
2373 }
2374 
2375 void Assembler::popf() {
2376   emit_int8((unsigned char)0x9D);
2377 }
2378 
2379 #ifndef _LP64 // no 32bit push/pop on amd64
2380 void Assembler::popl(Address dst) {
2381   // NOTE: this will adjust stack by 8byte on 64bits
2382   InstructionMark im(this);
2383   prefix(dst);
2384   emit_int8((unsigned char)0x8F);
2385   emit_operand(rax, dst);
2386 }
2387 #endif
2388 
2389 void Assembler::prefetch_prefix(Address src) {
2390   prefix(src);
2391   emit_int8(0x0F);
2392 }
2393 
2394 void Assembler::prefetchnta(Address src) {
2395   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2396   InstructionMark im(this);
2397   prefetch_prefix(src);
2398   emit_int8(0x18);
2399   emit_operand(rax, src); // 0, src
2400 }
2401 
2402 void Assembler::prefetchr(Address src) {
2403   assert(VM_Version::supports_3dnow_prefetch(), "must support");
2404   InstructionMark im(this);
2405   prefetch_prefix(src);
2406   emit_int8(0x0D);
2407   emit_operand(rax, src); // 0, src
2408 }
2409 
2410 void Assembler::prefetcht0(Address src) {
2411   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2412   InstructionMark im(this);
2413   prefetch_prefix(src);
2414   emit_int8(0x18);
2415   emit_operand(rcx, src); // 1, src
2416 }
2417 
2418 void Assembler::prefetcht1(Address src) {
2419   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2420   InstructionMark im(this);
2421   prefetch_prefix(src);
2422   emit_int8(0x18);
2423   emit_operand(rdx, src); // 2, src
2424 }
2425 
2426 void Assembler::prefetcht2(Address src) {
2427   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2428   InstructionMark im(this);
2429   prefetch_prefix(src);
2430   emit_int8(0x18);
2431   emit_operand(rbx, src); // 3, src
2432 }
2433 
2434 void Assembler::prefetchw(Address src) {
2435   assert(VM_Version::supports_3dnow_prefetch(), "must support");
2436   InstructionMark im(this);
2437   prefetch_prefix(src);
2438   emit_int8(0x0D);
2439   emit_operand(rcx, src); // 1, src
2440 }
2441 
2442 void Assembler::prefix(Prefix p) {
2443   emit_int8(p);
2444 }
2445 
2446 void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
2447   assert(VM_Version::supports_ssse3(), "");
2448   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2449   emit_int8(0x00);
2450   emit_int8((unsigned char)(0xC0 | encode));
2451 }
2452 
2453 void Assembler::pshufb(XMMRegister dst, Address src) {
2454   assert(VM_Version::supports_ssse3(), "");
2455   InstructionMark im(this);
2456   simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2457   emit_int8(0x00);
2458   emit_operand(dst, src);
2459 }
2460 
2461 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
2462   assert(isByte(mode), "invalid value");
2463   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2464   emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_66);
2465   emit_int8(mode & 0xFF);
2466 
2467 }
2468 
2469 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
2470   assert(isByte(mode), "invalid value");
2471   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2472   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2473   InstructionMark im(this);
2474   simd_prefix(dst, src, VEX_SIMD_66);
2475   emit_int8(0x70);
2476   emit_operand(dst, src);
2477   emit_int8(mode & 0xFF);
2478 }
2479 
2480 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
2481   assert(isByte(mode), "invalid value");
2482   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2483   emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_F2);
2484   emit_int8(mode & 0xFF);
2485 }
2486 
2487 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
2488   assert(isByte(mode), "invalid value");
2489   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2490   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2491   InstructionMark im(this);
2492   simd_prefix(dst, src, VEX_SIMD_F2);
2493   emit_int8(0x70);
2494   emit_operand(dst, src);
2495   emit_int8(mode & 0xFF);
2496 }
2497 
2498 void Assembler::psrldq(XMMRegister dst, int shift) {
2499   // Shift 128 bit value in xmm register by number of bytes.
2500   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2501   int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66);
2502   emit_int8(0x73);
2503   emit_int8((unsigned char)(0xC0 | encode));
2504   emit_int8(shift);
2505 }
2506 
2507 void Assembler::ptest(XMMRegister dst, Address src) {
2508   assert(VM_Version::supports_sse4_1(), "");
2509   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2510   InstructionMark im(this);
2511   simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2512   emit_int8(0x17);
2513   emit_operand(dst, src);
2514 }
2515 
2516 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
2517   assert(VM_Version::supports_sse4_1(), "");
2518   int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
2519   emit_int8(0x17);
2520   emit_int8((unsigned char)(0xC0 | encode));
2521 }
2522 
2523 void Assembler::vptest(XMMRegister dst, Address src) {
2524   assert(VM_Version::supports_avx(), "");
2525   InstructionMark im(this);
2526   bool vector256 = true;
2527   assert(dst != xnoreg, "sanity");
2528   int dst_enc = dst->encoding();
2529   // swap src<->dst for encoding
2530   vex_prefix(src, 0, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector256);
2531   emit_int8(0x17);
2532   emit_operand(dst, src);
2533 }
2534 
2535 void Assembler::vptest(XMMRegister dst, XMMRegister src) {
2536   assert(VM_Version::supports_avx(), "");
2537   bool vector256 = true;
2538   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_38);
2539   emit_int8(0x17);
2540   emit_int8((unsigned char)(0xC0 | encode));
2541 }
2542 
2543 void Assembler::punpcklbw(XMMRegister dst, Address src) {
2544   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2545   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2546   emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
2547 }
2548 
2549 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
2550   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2551   emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
2552 }
2553 
2554 void Assembler::punpckldq(XMMRegister dst, Address src) {
2555   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2556   assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
2557   emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
2558 }
2559 
2560 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
2561   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2562   emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
2563 }
2564 
2565 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
2566   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2567   emit_simd_arith(0x6C, dst, src, VEX_SIMD_66);
2568 }
2569 
2570 void Assembler::push(int32_t imm32) {
2571   // in 64bits we push 64bits onto the stack but only
2572   // take a 32bit immediate
2573   emit_int8(0x68);
2574   emit_int32(imm32);
2575 }
2576 
2577 void Assembler::push(Register src) {
2578   int encode = prefix_and_encode(src->encoding());
2579 
2580   emit_int8(0x50 | encode);
2581 }
2582 
2583 void Assembler::pushf() {
2584   emit_int8((unsigned char)0x9C);
2585 }
2586 
2587 #ifndef _LP64 // no 32bit push/pop on amd64
2588 void Assembler::pushl(Address src) {
2589   // Note this will push 64bit on 64bit
2590   InstructionMark im(this);
2591   prefix(src);
2592   emit_int8((unsigned char)0xFF);
2593   emit_operand(rsi, src);
2594 }
2595 #endif
2596 
2597 void Assembler::rcll(Register dst, int imm8) {
2598   assert(isShiftCount(imm8), "illegal shift count");
2599   int encode = prefix_and_encode(dst->encoding());
2600   if (imm8 == 1) {
2601     emit_int8((unsigned char)0xD1);
2602     emit_int8((unsigned char)(0xD0 | encode));
2603   } else {
2604     emit_int8((unsigned char)0xC1);
2605     emit_int8((unsigned char)0xD0 | encode);
2606     emit_int8(imm8);
2607   }
2608 }
2609 
2610 // copies data from [esi] to [edi] using rcx pointer sized words
2611 // generic
2612 void Assembler::rep_mov() {
2613   emit_int8((unsigned char)0xF3);
2614   // MOVSQ
2615   LP64_ONLY(prefix(REX_W));
2616   emit_int8((unsigned char)0xA5);
2617 }
2618 
2619 // sets rcx bytes with rax, value at [edi]
2620 void Assembler::rep_stosb() {
2621   emit_int8((unsigned char)0xF3); // REP
2622   LP64_ONLY(prefix(REX_W));
2623   emit_int8((unsigned char)0xAA); // STOSB
2624 }
2625 
2626 // sets rcx pointer sized words with rax, value at [edi]
2627 // generic
2628 void Assembler::rep_stos() {
2629   emit_int8((unsigned char)0xF3); // REP
2630   LP64_ONLY(prefix(REX_W));       // LP64:STOSQ, LP32:STOSD
2631   emit_int8((unsigned char)0xAB);
2632 }
2633 
2634 // scans rcx pointer sized words at [edi] for occurance of rax,
2635 // generic
2636 void Assembler::repne_scan() { // repne_scan
2637   emit_int8((unsigned char)0xF2);
2638   // SCASQ
2639   LP64_ONLY(prefix(REX_W));
2640   emit_int8((unsigned char)0xAF);
2641 }
2642 
2643 #ifdef _LP64
2644 // scans rcx 4 byte words at [edi] for occurance of rax,
2645 // generic
2646 void Assembler::repne_scanl() { // repne_scan
2647   emit_int8((unsigned char)0xF2);
2648   // SCASL
2649   emit_int8((unsigned char)0xAF);
2650 }
2651 #endif
2652 
2653 void Assembler::ret(int imm16) {
2654   if (imm16 == 0) {
2655     emit_int8((unsigned char)0xC3);
2656   } else {
2657     emit_int8((unsigned char)0xC2);
2658     emit_int16(imm16);
2659   }
2660 }
2661 
2662 void Assembler::sahf() {
2663 #ifdef _LP64
2664   // Not supported in 64bit mode
2665   ShouldNotReachHere();
2666 #endif
2667   emit_int8((unsigned char)0x9E);
2668 }
2669 
2670 void Assembler::sarl(Register dst, int imm8) {
2671   int encode = prefix_and_encode(dst->encoding());
2672   assert(isShiftCount(imm8), "illegal shift count");
2673   if (imm8 == 1) {
2674     emit_int8((unsigned char)0xD1);
2675     emit_int8((unsigned char)(0xF8 | encode));
2676   } else {
2677     emit_int8((unsigned char)0xC1);
2678     emit_int8((unsigned char)(0xF8 | encode));
2679     emit_int8(imm8);
2680   }
2681 }
2682 
2683 void Assembler::sarl(Register dst) {
2684   int encode = prefix_and_encode(dst->encoding());
2685   emit_int8((unsigned char)0xD3);
2686   emit_int8((unsigned char)(0xF8 | encode));
2687 }
2688 
2689 void Assembler::sbbl(Address dst, int32_t imm32) {
2690   InstructionMark im(this);
2691   prefix(dst);
2692   emit_arith_operand(0x81, rbx, dst, imm32);
2693 }
2694 
2695 void Assembler::sbbl(Register dst, int32_t imm32) {
2696   prefix(dst);
2697   emit_arith(0x81, 0xD8, dst, imm32);
2698 }
2699 
2700 
2701 void Assembler::sbbl(Register dst, Address src) {
2702   InstructionMark im(this);
2703   prefix(src, dst);
2704   emit_int8(0x1B);
2705   emit_operand(dst, src);
2706 }
2707 
2708 void Assembler::sbbl(Register dst, Register src) {
2709   (void) prefix_and_encode(dst->encoding(), src->encoding());
2710   emit_arith(0x1B, 0xC0, dst, src);
2711 }
2712 
2713 void Assembler::setb(Condition cc, Register dst) {
2714   assert(0 <= cc && cc < 16, "illegal cc");
2715   int encode = prefix_and_encode(dst->encoding(), true);
2716   emit_int8(0x0F);
2717   emit_int8((unsigned char)0x90 | cc);
2718   emit_int8((unsigned char)(0xC0 | encode));
2719 }
2720 
2721 void Assembler::shll(Register dst, int imm8) {
2722   assert(isShiftCount(imm8), "illegal shift count");
2723   int encode = prefix_and_encode(dst->encoding());
2724   if (imm8 == 1 ) {
2725     emit_int8((unsigned char)0xD1);
2726     emit_int8((unsigned char)(0xE0 | encode));
2727   } else {
2728     emit_int8((unsigned char)0xC1);
2729     emit_int8((unsigned char)(0xE0 | encode));
2730     emit_int8(imm8);
2731   }
2732 }
2733 
2734 void Assembler::shll(Register dst) {
2735   int encode = prefix_and_encode(dst->encoding());
2736   emit_int8((unsigned char)0xD3);
2737   emit_int8((unsigned char)(0xE0 | encode));
2738 }
2739 
2740 void Assembler::shrl(Register dst, int imm8) {
2741   assert(isShiftCount(imm8), "illegal shift count");
2742   int encode = prefix_and_encode(dst->encoding());
2743   emit_int8((unsigned char)0xC1);
2744   emit_int8((unsigned char)(0xE8 | encode));
2745   emit_int8(imm8);
2746 }
2747 
2748 void Assembler::shrl(Register dst) {
2749   int encode = prefix_and_encode(dst->encoding());
2750   emit_int8((unsigned char)0xD3);
2751   emit_int8((unsigned char)(0xE8 | encode));
2752 }
2753 
2754 // copies a single word from [esi] to [edi]
2755 void Assembler::smovl() {
2756   emit_int8((unsigned char)0xA5);
2757 }
2758 
2759 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
2760   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2761   emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
2762 }
2763 
2764 void Assembler::sqrtsd(XMMRegister dst, Address src) {
2765   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2766   emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
2767 }
2768 
2769 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
2770   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2771   emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
2772 }
2773 
2774 void Assembler::std() {
2775   emit_int8((unsigned char)0xFD);
2776 }
2777 
2778 void Assembler::sqrtss(XMMRegister dst, Address src) {
2779   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2780   emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
2781 }
2782 
2783 void Assembler::stmxcsr( Address dst) {
2784   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2785   InstructionMark im(this);
2786   prefix(dst);
2787   emit_int8(0x0F);
2788   emit_int8((unsigned char)0xAE);
2789   emit_operand(as_Register(3), dst);
2790 }
2791 
2792 void Assembler::subl(Address dst, int32_t imm32) {
2793   InstructionMark im(this);
2794   prefix(dst);
2795   emit_arith_operand(0x81, rbp, dst, imm32);
2796 }
2797 
2798 void Assembler::subl(Address dst, Register src) {
2799   InstructionMark im(this);
2800   prefix(dst, src);
2801   emit_int8(0x29);
2802   emit_operand(src, dst);
2803 }
2804 
2805 void Assembler::subl(Register dst, int32_t imm32) {
2806   prefix(dst);
2807   emit_arith(0x81, 0xE8, dst, imm32);
2808 }
2809 
2810 // Force generation of a 4 byte immediate value even if it fits into 8bit
2811 void Assembler::subl_imm32(Register dst, int32_t imm32) {
2812   prefix(dst);
2813   emit_arith_imm32(0x81, 0xE8, dst, imm32);
2814 }
2815 
2816 void Assembler::subl(Register dst, Address src) {
2817   InstructionMark im(this);
2818   prefix(src, dst);
2819   emit_int8(0x2B);
2820   emit_operand(dst, src);
2821 }
2822 
2823 void Assembler::subl(Register dst, Register src) {
2824   (void) prefix_and_encode(dst->encoding(), src->encoding());
2825   emit_arith(0x2B, 0xC0, dst, src);
2826 }
2827 
2828 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
2829   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2830   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
2831 }
2832 
2833 void Assembler::subsd(XMMRegister dst, Address src) {
2834   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2835   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
2836 }
2837 
2838 void Assembler::subss(XMMRegister dst, XMMRegister src) {
2839   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2840   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
2841 }
2842 
2843 void Assembler::subss(XMMRegister dst, Address src) {
2844   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2845   emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
2846 }
2847 
2848 void Assembler::testb(Register dst, int imm8) {
2849   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2850   (void) prefix_and_encode(dst->encoding(), true);
2851   emit_arith_b(0xF6, 0xC0, dst, imm8);
2852 }
2853 
2854 void Assembler::testl(Register dst, int32_t imm32) {
2855   // not using emit_arith because test
2856   // doesn't support sign-extension of
2857   // 8bit operands
2858   int encode = dst->encoding();
2859   if (encode == 0) {
2860     emit_int8((unsigned char)0xA9);
2861   } else {
2862     encode = prefix_and_encode(encode);
2863     emit_int8((unsigned char)0xF7);
2864     emit_int8((unsigned char)(0xC0 | encode));
2865   }
2866   emit_int32(imm32);
2867 }
2868 
2869 void Assembler::testl(Register dst, Register src) {
2870   (void) prefix_and_encode(dst->encoding(), src->encoding());
2871   emit_arith(0x85, 0xC0, dst, src);
2872 }
2873 
2874 void Assembler::testl(Register dst, Address  src) {
2875   InstructionMark im(this);
2876   prefix(src, dst);
2877   emit_int8((unsigned char)0x85);
2878   emit_operand(dst, src);
2879 }
2880 
2881 void Assembler::ucomisd(XMMRegister dst, Address src) {
2882   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2883   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
2884 }
2885 
2886 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
2887   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2888   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
2889 }
2890 
2891 void Assembler::ucomiss(XMMRegister dst, Address src) {
2892   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2893   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE);
2894 }
2895 
2896 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
2897   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2898   emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE);
2899 }
2900 
2901 
2902 void Assembler::xaddl(Address dst, Register src) {
2903   InstructionMark im(this);
2904   prefix(dst, src);
2905   emit_int8(0x0F);
2906   emit_int8((unsigned char)0xC1);
2907   emit_operand(src, dst);
2908 }
2909 
2910 void Assembler::xchgl(Register dst, Address src) { // xchg
2911   InstructionMark im(this);
2912   prefix(src, dst);
2913   emit_int8((unsigned char)0x87);
2914   emit_operand(dst, src);
2915 }
2916 
2917 void Assembler::xchgl(Register dst, Register src) {
2918   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2919   emit_int8((unsigned char)0x87);
2920   emit_int8((unsigned char)(0xC0 | encode));
2921 }
2922 
2923 void Assembler::xgetbv() {
2924   emit_int8(0x0F);
2925   emit_int8(0x01);
2926   emit_int8((unsigned char)0xD0);
2927 }
2928 
2929 void Assembler::xorl(Register dst, int32_t imm32) {
2930   prefix(dst);
2931   emit_arith(0x81, 0xF0, dst, imm32);
2932 }
2933 
2934 void Assembler::xorl(Register dst, Address src) {
2935   InstructionMark im(this);
2936   prefix(src, dst);
2937   emit_int8(0x33);
2938   emit_operand(dst, src);
2939 }
2940 
2941 void Assembler::xorl(Register dst, Register src) {
2942   (void) prefix_and_encode(dst->encoding(), src->encoding());
2943   emit_arith(0x33, 0xC0, dst, src);
2944 }
2945 
2946 
2947 // AVX 3-operands scalar float-point arithmetic instructions
2948 
2949 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
2950   assert(VM_Version::supports_avx(), "");
2951   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2952 }
2953 
2954 void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2955   assert(VM_Version::supports_avx(), "");
2956   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2957 }
2958 
2959 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
2960   assert(VM_Version::supports_avx(), "");
2961   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2962 }
2963 
2964 void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2965   assert(VM_Version::supports_avx(), "");
2966   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2967 }
2968 
2969 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
2970   assert(VM_Version::supports_avx(), "");
2971   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2972 }
2973 
2974 void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2975   assert(VM_Version::supports_avx(), "");
2976   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2977 }
2978 
2979 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
2980   assert(VM_Version::supports_avx(), "");
2981   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2982 }
2983 
2984 void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2985   assert(VM_Version::supports_avx(), "");
2986   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
2987 }
2988 
2989 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
2990   assert(VM_Version::supports_avx(), "");
2991   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2992 }
2993 
2994 void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
2995   assert(VM_Version::supports_avx(), "");
2996   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
2997 }
2998 
2999 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
3000   assert(VM_Version::supports_avx(), "");
3001   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3002 }
3003 
3004 void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3005   assert(VM_Version::supports_avx(), "");
3006   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3007 }
3008 
3009 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
3010   assert(VM_Version::supports_avx(), "");
3011   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
3012 }
3013 
3014 void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3015   assert(VM_Version::supports_avx(), "");
3016   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
3017 }
3018 
3019 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
3020   assert(VM_Version::supports_avx(), "");
3021   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3022 }
3023 
3024 void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3025   assert(VM_Version::supports_avx(), "");
3026   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
3027 }
3028 
3029 //====================VECTOR ARITHMETIC=====================================
3030 
3031 // Float-point vector arithmetic
3032 
3033 void Assembler::addpd(XMMRegister dst, XMMRegister src) {
3034   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3035   emit_simd_arith(0x58, dst, src, VEX_SIMD_66);
3036 }
3037 
3038 void Assembler::addps(XMMRegister dst, XMMRegister src) {
3039   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3040   emit_simd_arith(0x58, dst, src, VEX_SIMD_NONE);
3041 }
3042 
3043 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3044   assert(VM_Version::supports_avx(), "");
3045   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector256);
3046 }
3047 
3048 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3049   assert(VM_Version::supports_avx(), "");
3050   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector256);
3051 }
3052 
3053 void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3054   assert(VM_Version::supports_avx(), "");
3055   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector256);
3056 }
3057 
3058 void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3059   assert(VM_Version::supports_avx(), "");
3060   emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector256);
3061 }
3062 
3063 void Assembler::subpd(XMMRegister dst, XMMRegister src) {
3064   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3065   emit_simd_arith(0x5C, dst, src, VEX_SIMD_66);
3066 }
3067 
3068 void Assembler::subps(XMMRegister dst, XMMRegister src) {
3069   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3070   emit_simd_arith(0x5C, dst, src, VEX_SIMD_NONE);
3071 }
3072 
3073 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3074   assert(VM_Version::supports_avx(), "");
3075   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector256);
3076 }
3077 
3078 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3079   assert(VM_Version::supports_avx(), "");
3080   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector256);
3081 }
3082 
3083 void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3084   assert(VM_Version::supports_avx(), "");
3085   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector256);
3086 }
3087 
3088 void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3089   assert(VM_Version::supports_avx(), "");
3090   emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector256);
3091 }
3092 
3093 void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
3094   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3095   emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
3096 }
3097 
3098 void Assembler::mulps(XMMRegister dst, XMMRegister src) {
3099   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3100   emit_simd_arith(0x59, dst, src, VEX_SIMD_NONE);
3101 }
3102 
3103 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3104   assert(VM_Version::supports_avx(), "");
3105   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector256);
3106 }
3107 
3108 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3109   assert(VM_Version::supports_avx(), "");
3110   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector256);
3111 }
3112 
3113 void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3114   assert(VM_Version::supports_avx(), "");
3115   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector256);
3116 }
3117 
3118 void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3119   assert(VM_Version::supports_avx(), "");
3120   emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector256);
3121 }
3122 
3123 void Assembler::divpd(XMMRegister dst, XMMRegister src) {
3124   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3125   emit_simd_arith(0x5E, dst, src, VEX_SIMD_66);
3126 }
3127 
3128 void Assembler::divps(XMMRegister dst, XMMRegister src) {
3129   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3130   emit_simd_arith(0x5E, dst, src, VEX_SIMD_NONE);
3131 }
3132 
3133 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3134   assert(VM_Version::supports_avx(), "");
3135   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector256);
3136 }
3137 
3138 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3139   assert(VM_Version::supports_avx(), "");
3140   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector256);
3141 }
3142 
3143 void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3144   assert(VM_Version::supports_avx(), "");
3145   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector256);
3146 }
3147 
3148 void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3149   assert(VM_Version::supports_avx(), "");
3150   emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector256);
3151 }
3152 
3153 void Assembler::andpd(XMMRegister dst, XMMRegister src) {
3154   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3155   emit_simd_arith(0x54, dst, src, VEX_SIMD_66);
3156 }
3157 
3158 void Assembler::andps(XMMRegister dst, XMMRegister src) {
3159   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3160   emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE);
3161 }
3162 
3163 void Assembler::andps(XMMRegister dst, Address src) {
3164   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3165   emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE);
3166 }
3167 
3168 void Assembler::andpd(XMMRegister dst, Address src) {
3169   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3170   emit_simd_arith(0x54, dst, src, VEX_SIMD_66);
3171 }
3172 
3173 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3174   assert(VM_Version::supports_avx(), "");
3175   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector256);
3176 }
3177 
3178 void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3179   assert(VM_Version::supports_avx(), "");
3180   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector256);
3181 }
3182 
3183 void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3184   assert(VM_Version::supports_avx(), "");
3185   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector256);
3186 }
3187 
3188 void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3189   assert(VM_Version::supports_avx(), "");
3190   emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector256);
3191 }
3192 
3193 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
3194   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3195   emit_simd_arith(0x57, dst, src, VEX_SIMD_66);
3196 }
3197 
3198 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
3199   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3200   emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE);
3201 }
3202 
3203 void Assembler::xorpd(XMMRegister dst, Address src) {
3204   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3205   emit_simd_arith(0x57, dst, src, VEX_SIMD_66);
3206 }
3207 
3208 void Assembler::xorps(XMMRegister dst, Address src) {
3209   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3210   emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE);
3211 }
3212 
3213 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3214   assert(VM_Version::supports_avx(), "");
3215   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector256);
3216 }
3217 
3218 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3219   assert(VM_Version::supports_avx(), "");
3220   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector256);
3221 }
3222 
3223 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3224   assert(VM_Version::supports_avx(), "");
3225   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector256);
3226 }
3227 
3228 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3229   assert(VM_Version::supports_avx(), "");
3230   emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector256);
3231 }
3232 
3233 
3234 // Integer vector arithmetic
3235 void Assembler::paddb(XMMRegister dst, XMMRegister src) {
3236   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3237   emit_simd_arith(0xFC, dst, src, VEX_SIMD_66);
3238 }
3239 
3240 void Assembler::paddw(XMMRegister dst, XMMRegister src) {
3241   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3242   emit_simd_arith(0xFD, dst, src, VEX_SIMD_66);
3243 }
3244 
3245 void Assembler::paddd(XMMRegister dst, XMMRegister src) {
3246   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3247   emit_simd_arith(0xFE, dst, src, VEX_SIMD_66);
3248 }
3249 
3250 void Assembler::paddq(XMMRegister dst, XMMRegister src) {
3251   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3252   emit_simd_arith(0xD4, dst, src, VEX_SIMD_66);
3253 }
3254 
3255 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3256   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3257   emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector256);
3258 }
3259 
3260 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3261   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3262   emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector256);
3263 }
3264 
3265 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3266   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3267   emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector256);
3268 }
3269 
3270 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3271   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3272   emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector256);
3273 }
3274 
3275 void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3276   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3277   emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector256);
3278 }
3279 
3280 void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3281   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3282   emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector256);
3283 }
3284 
3285 void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3286   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3287   emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector256);
3288 }
3289 
3290 void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3291   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3292   emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector256);
3293 }
3294 
3295 void Assembler::psubb(XMMRegister dst, XMMRegister src) {
3296   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3297   emit_simd_arith(0xF8, dst, src, VEX_SIMD_66);
3298 }
3299 
3300 void Assembler::psubw(XMMRegister dst, XMMRegister src) {
3301   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3302   emit_simd_arith(0xF9, dst, src, VEX_SIMD_66);
3303 }
3304 
3305 void Assembler::psubd(XMMRegister dst, XMMRegister src) {
3306   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3307   emit_simd_arith(0xFA, dst, src, VEX_SIMD_66);
3308 }
3309 
3310 void Assembler::psubq(XMMRegister dst, XMMRegister src) {
3311   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3312   emit_simd_arith(0xFB, dst, src, VEX_SIMD_66);
3313 }
3314 
3315 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3316   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3317   emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector256);
3318 }
3319 
3320 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3321   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3322   emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector256);
3323 }
3324 
3325 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3326   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3327   emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector256);
3328 }
3329 
3330 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3331   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3332   emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector256);
3333 }
3334 
3335 void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3336   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3337   emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector256);
3338 }
3339 
3340 void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3341   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3342   emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector256);
3343 }
3344 
3345 void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3346   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3347   emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector256);
3348 }
3349 
3350 void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3351   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3352   emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector256);
3353 }
3354 
3355 void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
3356   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3357   emit_simd_arith(0xD5, dst, src, VEX_SIMD_66);
3358 }
3359 
3360 void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
3361   assert(VM_Version::supports_sse4_1(), "");
3362   int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
3363   emit_int8(0x40);
3364   emit_int8((unsigned char)(0xC0 | encode));
3365 }
3366 
3367 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3368   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3369   emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector256);
3370 }
3371 
3372 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3373   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3374   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_38);
3375   emit_int8(0x40);
3376   emit_int8((unsigned char)(0xC0 | encode));
3377 }
3378 
3379 void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3380   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3381   emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector256);
3382 }
3383 
3384 void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3385   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3386   InstructionMark im(this);
3387   int dst_enc = dst->encoding();
3388   int nds_enc = nds->is_valid() ? nds->encoding() : 0;
3389   vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector256);
3390   emit_int8(0x40);
3391   emit_operand(dst, src);
3392 }
3393 
3394 // Shift packed integers left by specified number of bits.
3395 void Assembler::psllw(XMMRegister dst, int shift) {
3396   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3397   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
3398   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
3399   emit_int8(0x71);
3400   emit_int8((unsigned char)(0xC0 | encode));
3401   emit_int8(shift & 0xFF);
3402 }
3403 
3404 void Assembler::pslld(XMMRegister dst, int shift) {
3405   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3406   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
3407   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
3408   emit_int8(0x72);
3409   emit_int8((unsigned char)(0xC0 | encode));
3410   emit_int8(shift & 0xFF);
3411 }
3412 
3413 void Assembler::psllq(XMMRegister dst, int shift) {
3414   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3415   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
3416   int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
3417   emit_int8(0x73);
3418   emit_int8((unsigned char)(0xC0 | encode));
3419   emit_int8(shift & 0xFF);
3420 }
3421 
3422 void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
3423   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3424   emit_simd_arith(0xF1, dst, shift, VEX_SIMD_66);
3425 }
3426 
3427 void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
3428   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3429   emit_simd_arith(0xF2, dst, shift, VEX_SIMD_66);
3430 }
3431 
3432 void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
3433   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3434   emit_simd_arith(0xF3, dst, shift, VEX_SIMD_66);
3435 }
3436 
3437 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3438   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3439   // XMM6 is for /6 encoding: 66 0F 71 /6 ib
3440   emit_vex_arith(0x71, xmm6, dst, src, VEX_SIMD_66, vector256);
3441   emit_int8(shift & 0xFF);
3442 }
3443 
3444 void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3445   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3446   // XMM6 is for /6 encoding: 66 0F 72 /6 ib
3447   emit_vex_arith(0x72, xmm6, dst, src, VEX_SIMD_66, vector256);
3448   emit_int8(shift & 0xFF);
3449 }
3450 
3451 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3452   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3453   // XMM6 is for /6 encoding: 66 0F 73 /6 ib
3454   emit_vex_arith(0x73, xmm6, dst, src, VEX_SIMD_66, vector256);
3455   emit_int8(shift & 0xFF);
3456 }
3457 
3458 void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3459   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3460   emit_vex_arith(0xF1, dst, src, shift, VEX_SIMD_66, vector256);
3461 }
3462 
3463 void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3464   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3465   emit_vex_arith(0xF2, dst, src, shift, VEX_SIMD_66, vector256);
3466 }
3467 
3468 void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3469   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3470   emit_vex_arith(0xF3, dst, src, shift, VEX_SIMD_66, vector256);
3471 }
3472 
3473 // Shift packed integers logically right by specified number of bits.
3474 void Assembler::psrlw(XMMRegister dst, int shift) {
3475   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3476   // XMM2 is for /2 encoding: 66 0F 71 /2 ib
3477   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
3478   emit_int8(0x71);
3479   emit_int8((unsigned char)(0xC0 | encode));
3480   emit_int8(shift & 0xFF);
3481 }
3482 
3483 void Assembler::psrld(XMMRegister dst, int shift) {
3484   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3485   // XMM2 is for /2 encoding: 66 0F 72 /2 ib
3486   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
3487   emit_int8(0x72);
3488   emit_int8((unsigned char)(0xC0 | encode));
3489   emit_int8(shift & 0xFF);
3490 }
3491 
3492 void Assembler::psrlq(XMMRegister dst, int shift) {
3493   // Do not confuse it with psrldq SSE2 instruction which
3494   // shifts 128 bit value in xmm register by number of bytes.
3495   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3496   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3497   int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
3498   emit_int8(0x73);
3499   emit_int8((unsigned char)(0xC0 | encode));
3500   emit_int8(shift & 0xFF);
3501 }
3502 
3503 void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
3504   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3505   emit_simd_arith(0xD1, dst, shift, VEX_SIMD_66);
3506 }
3507 
3508 void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
3509   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3510   emit_simd_arith(0xD2, dst, shift, VEX_SIMD_66);
3511 }
3512 
3513 void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
3514   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3515   emit_simd_arith(0xD3, dst, shift, VEX_SIMD_66);
3516 }
3517 
3518 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3519   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3520   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3521   emit_vex_arith(0x71, xmm2, dst, src, VEX_SIMD_66, vector256);
3522   emit_int8(shift & 0xFF);
3523 }
3524 
3525 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3526   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3527   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3528   emit_vex_arith(0x72, xmm2, dst, src, VEX_SIMD_66, vector256);
3529   emit_int8(shift & 0xFF);
3530 }
3531 
3532 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3533   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3534   // XMM2 is for /2 encoding: 66 0F 73 /2 ib
3535   emit_vex_arith(0x73, xmm2, dst, src, VEX_SIMD_66, vector256);
3536   emit_int8(shift & 0xFF);
3537 }
3538 
3539 void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3540   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3541   emit_vex_arith(0xD1, dst, src, shift, VEX_SIMD_66, vector256);
3542 }
3543 
3544 void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3545   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3546   emit_vex_arith(0xD2, dst, src, shift, VEX_SIMD_66, vector256);
3547 }
3548 
3549 void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3550   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3551   emit_vex_arith(0xD3, dst, src, shift, VEX_SIMD_66, vector256);
3552 }
3553 
3554 // Shift packed integers arithmetically right by specified number of bits.
3555 void Assembler::psraw(XMMRegister dst, int shift) {
3556   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3557   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
3558   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66);
3559   emit_int8(0x71);
3560   emit_int8((unsigned char)(0xC0 | encode));
3561   emit_int8(shift & 0xFF);
3562 }
3563 
3564 void Assembler::psrad(XMMRegister dst, int shift) {
3565   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3566   // XMM4 is for /4 encoding: 66 0F 72 /4 ib
3567   int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66);
3568   emit_int8(0x72);
3569   emit_int8((unsigned char)(0xC0 | encode));
3570   emit_int8(shift & 0xFF);
3571 }
3572 
3573 void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
3574   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3575   emit_simd_arith(0xE1, dst, shift, VEX_SIMD_66);
3576 }
3577 
3578 void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
3579   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3580   emit_simd_arith(0xE2, dst, shift, VEX_SIMD_66);
3581 }
3582 
3583 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3584   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3585   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
3586   emit_vex_arith(0x71, xmm4, dst, src, VEX_SIMD_66, vector256);
3587   emit_int8(shift & 0xFF);
3588 }
3589 
3590 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
3591   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3592   // XMM4 is for /4 encoding: 66 0F 71 /4 ib
3593   emit_vex_arith(0x72, xmm4, dst, src, VEX_SIMD_66, vector256);
3594   emit_int8(shift & 0xFF);
3595 }
3596 
3597 void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3598   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3599   emit_vex_arith(0xE1, dst, src, shift, VEX_SIMD_66, vector256);
3600 }
3601 
3602 void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
3603   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3604   emit_vex_arith(0xE2, dst, src, shift, VEX_SIMD_66, vector256);
3605 }
3606 
3607 
3608 // AND packed integers
3609 void Assembler::pand(XMMRegister dst, XMMRegister src) {
3610   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3611   emit_simd_arith(0xDB, dst, src, VEX_SIMD_66);
3612 }
3613 
3614 void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3615   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3616   emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector256);
3617 }
3618 
3619 void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3620   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3621   emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector256);
3622 }
3623 
3624 void Assembler::por(XMMRegister dst, XMMRegister src) {
3625   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3626   emit_simd_arith(0xEB, dst, src, VEX_SIMD_66);
3627 }
3628 
3629 void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3630   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3631   emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector256);
3632 }
3633 
3634 void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3635   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3636   emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector256);
3637 }
3638 
3639 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
3640   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3641   emit_simd_arith(0xEF, dst, src, VEX_SIMD_66);
3642 }
3643 
3644 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
3645   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3646   emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector256);
3647 }
3648 
3649 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
3650   assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
3651   emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector256);
3652 }
3653 
3654 
3655 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3656   assert(VM_Version::supports_avx(), "");
3657   bool vector256 = true;
3658   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
3659   emit_int8(0x18);
3660   emit_int8((unsigned char)(0xC0 | encode));
3661   // 0x00 - insert into lower 128 bits
3662   // 0x01 - insert into upper 128 bits
3663   emit_int8(0x01);
3664 }
3665 
3666 void Assembler::vinsertf128h(XMMRegister dst, Address src) {
3667   assert(VM_Version::supports_avx(), "");
3668   InstructionMark im(this);
3669   bool vector256 = true;
3670   assert(dst != xnoreg, "sanity");
3671   int dst_enc = dst->encoding();
3672   // swap src<->dst for encoding
3673   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3674   emit_int8(0x18);
3675   emit_operand(dst, src);
3676   // 0x01 - insert into upper 128 bits
3677   emit_int8(0x01);
3678 }
3679 
3680 void Assembler::vextractf128h(Address dst, XMMRegister src) {
3681   assert(VM_Version::supports_avx(), "");
3682   InstructionMark im(this);
3683   bool vector256 = true;
3684   assert(src != xnoreg, "sanity");
3685   int src_enc = src->encoding();
3686   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3687   emit_int8(0x19);
3688   emit_operand(src, dst);
3689   // 0x01 - extract from upper 128 bits
3690   emit_int8(0x01);
3691 }
3692 
3693 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
3694   assert(VM_Version::supports_avx2(), "");
3695   bool vector256 = true;
3696   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
3697   emit_int8(0x38);
3698   emit_int8((unsigned char)(0xC0 | encode));
3699   // 0x00 - insert into lower 128 bits
3700   // 0x01 - insert into upper 128 bits
3701   emit_int8(0x01);
3702 }
3703 
3704 void Assembler::vinserti128h(XMMRegister dst, Address src) {
3705   assert(VM_Version::supports_avx2(), "");
3706   InstructionMark im(this);
3707   bool vector256 = true;
3708   assert(dst != xnoreg, "sanity");
3709   int dst_enc = dst->encoding();
3710   // swap src<->dst for encoding
3711   vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3712   emit_int8(0x38);
3713   emit_operand(dst, src);
3714   // 0x01 - insert into upper 128 bits
3715   emit_int8(0x01);
3716 }
3717 
3718 void Assembler::vextracti128h(Address dst, XMMRegister src) {
3719   assert(VM_Version::supports_avx2(), "");
3720   InstructionMark im(this);
3721   bool vector256 = true;
3722   assert(src != xnoreg, "sanity");
3723   int src_enc = src->encoding();
3724   vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
3725   emit_int8(0x39);
3726   emit_operand(src, dst);
3727   // 0x01 - extract from upper 128 bits
3728   emit_int8(0x01);
3729 }
3730 
3731 // duplicate 4-bytes integer data from src into 8 locations in dest
3732 void Assembler::vpbroadcastd(XMMRegister dst, XMMRegister src) {
3733   assert(VM_Version::supports_avx2(), "");
3734   bool vector256 = true;
3735   int encode = vex_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_38);
3736   emit_int8(0x58);
3737   emit_int8((unsigned char)(0xC0 | encode));
3738 }
3739 
3740 // Carry-Less Multiplication Quadword
3741 void Assembler::vpclmulqdq(XMMRegister dst, XMMRegister nds, XMMRegister src, int mask) {
3742   assert(VM_Version::supports_avx() && VM_Version::supports_clmul(), "");
3743   bool vector256 = false;
3744   int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
3745   emit_int8(0x44);
3746   emit_int8((unsigned char)(0xC0 | encode));
3747   emit_int8((unsigned char)mask);
3748 }
3749 
3750 void Assembler::vzeroupper() {
3751   assert(VM_Version::supports_avx(), "");
3752   (void)vex_prefix_and_encode(xmm0, xmm0, xmm0, VEX_SIMD_NONE);
3753   emit_int8(0x77);
3754 }
3755 
3756 
3757 #ifndef _LP64
3758 // 32bit only pieces of the assembler
3759 
3760 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
3761   // NO PREFIX AS NEVER 64BIT
3762   InstructionMark im(this);
3763   emit_int8((unsigned char)0x81);
3764   emit_int8((unsigned char)(0xF8 | src1->encoding()));
3765   emit_data(imm32, rspec, 0);
3766 }
3767 
3768 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
3769   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
3770   InstructionMark im(this);
3771   emit_int8((unsigned char)0x81);
3772   emit_operand(rdi, src1);
3773   emit_data(imm32, rspec, 0);
3774 }
3775 
3776 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
3777 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
3778 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
3779 void Assembler::cmpxchg8(Address adr) {
3780   InstructionMark im(this);
3781   emit_int8(0x0F);
3782   emit_int8((unsigned char)0xC7);
3783   emit_operand(rcx, adr);
3784 }
3785 
3786 void Assembler::decl(Register dst) {
3787   // Don't use it directly. Use MacroAssembler::decrementl() instead.
3788  emit_int8(0x48 | dst->encoding());
3789 }
3790 
3791 #endif // _LP64
3792 
3793 // 64bit typically doesn't use the x87 but needs to for the trig funcs
3794 
3795 void Assembler::fabs() {
3796   emit_int8((unsigned char)0xD9);
3797   emit_int8((unsigned char)0xE1);
3798 }
3799 
3800 void Assembler::fadd(int i) {
3801   emit_farith(0xD8, 0xC0, i);
3802 }
3803 
3804 void Assembler::fadd_d(Address src) {
3805   InstructionMark im(this);
3806   emit_int8((unsigned char)0xDC);
3807   emit_operand32(rax, src);
3808 }
3809 
3810 void Assembler::fadd_s(Address src) {
3811   InstructionMark im(this);
3812   emit_int8((unsigned char)0xD8);
3813   emit_operand32(rax, src);
3814 }
3815 
3816 void Assembler::fadda(int i) {
3817   emit_farith(0xDC, 0xC0, i);
3818 }
3819 
3820 void Assembler::faddp(int i) {
3821   emit_farith(0xDE, 0xC0, i);
3822 }
3823 
3824 void Assembler::fchs() {
3825   emit_int8((unsigned char)0xD9);
3826   emit_int8((unsigned char)0xE0);
3827 }
3828 
3829 void Assembler::fcom(int i) {
3830   emit_farith(0xD8, 0xD0, i);
3831 }
3832 
3833 void Assembler::fcomp(int i) {
3834   emit_farith(0xD8, 0xD8, i);
3835 }
3836 
3837 void Assembler::fcomp_d(Address src) {
3838   InstructionMark im(this);
3839   emit_int8((unsigned char)0xDC);
3840   emit_operand32(rbx, src);
3841 }
3842 
3843 void Assembler::fcomp_s(Address src) {
3844   InstructionMark im(this);
3845   emit_int8((unsigned char)0xD8);
3846   emit_operand32(rbx, src);
3847 }
3848 
3849 void Assembler::fcompp() {
3850   emit_int8((unsigned char)0xDE);
3851   emit_int8((unsigned char)0xD9);
3852 }
3853 
3854 void Assembler::fcos() {
3855   emit_int8((unsigned char)0xD9);
3856   emit_int8((unsigned char)0xFF);
3857 }
3858 
3859 void Assembler::fdecstp() {
3860   emit_int8((unsigned char)0xD9);
3861   emit_int8((unsigned char)0xF6);
3862 }
3863 
3864 void Assembler::fdiv(int i) {
3865   emit_farith(0xD8, 0xF0, i);
3866 }
3867 
3868 void Assembler::fdiv_d(Address src) {
3869   InstructionMark im(this);
3870   emit_int8((unsigned char)0xDC);
3871   emit_operand32(rsi, src);
3872 }
3873 
3874 void Assembler::fdiv_s(Address src) {
3875   InstructionMark im(this);
3876   emit_int8((unsigned char)0xD8);
3877   emit_operand32(rsi, src);
3878 }
3879 
3880 void Assembler::fdiva(int i) {
3881   emit_farith(0xDC, 0xF8, i);
3882 }
3883 
3884 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
3885 //       is erroneous for some of the floating-point instructions below.
3886 
3887 void Assembler::fdivp(int i) {
3888   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
3889 }
3890 
3891 void Assembler::fdivr(int i) {
3892   emit_farith(0xD8, 0xF8, i);
3893 }
3894 
3895 void Assembler::fdivr_d(Address src) {
3896   InstructionMark im(this);
3897   emit_int8((unsigned char)0xDC);
3898   emit_operand32(rdi, src);
3899 }
3900 
3901 void Assembler::fdivr_s(Address src) {
3902   InstructionMark im(this);
3903   emit_int8((unsigned char)0xD8);
3904   emit_operand32(rdi, src);
3905 }
3906 
3907 void Assembler::fdivra(int i) {
3908   emit_farith(0xDC, 0xF0, i);
3909 }
3910 
3911 void Assembler::fdivrp(int i) {
3912   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
3913 }
3914 
3915 void Assembler::ffree(int i) {
3916   emit_farith(0xDD, 0xC0, i);
3917 }
3918 
3919 void Assembler::fild_d(Address adr) {
3920   InstructionMark im(this);
3921   emit_int8((unsigned char)0xDF);
3922   emit_operand32(rbp, adr);
3923 }
3924 
3925 void Assembler::fild_s(Address adr) {
3926   InstructionMark im(this);
3927   emit_int8((unsigned char)0xDB);
3928   emit_operand32(rax, adr);
3929 }
3930 
3931 void Assembler::fincstp() {
3932   emit_int8((unsigned char)0xD9);
3933   emit_int8((unsigned char)0xF7);
3934 }
3935 
3936 void Assembler::finit() {
3937   emit_int8((unsigned char)0x9B);
3938   emit_int8((unsigned char)0xDB);
3939   emit_int8((unsigned char)0xE3);
3940 }
3941 
3942 void Assembler::fist_s(Address adr) {
3943   InstructionMark im(this);
3944   emit_int8((unsigned char)0xDB);
3945   emit_operand32(rdx, adr);
3946 }
3947 
3948 void Assembler::fistp_d(Address adr) {
3949   InstructionMark im(this);
3950   emit_int8((unsigned char)0xDF);
3951   emit_operand32(rdi, adr);
3952 }
3953 
3954 void Assembler::fistp_s(Address adr) {
3955   InstructionMark im(this);
3956   emit_int8((unsigned char)0xDB);
3957   emit_operand32(rbx, adr);
3958 }
3959 
3960 void Assembler::fld1() {
3961   emit_int8((unsigned char)0xD9);
3962   emit_int8((unsigned char)0xE8);
3963 }
3964 
3965 void Assembler::fld_d(Address adr) {
3966   InstructionMark im(this);
3967   emit_int8((unsigned char)0xDD);
3968   emit_operand32(rax, adr);
3969 }
3970 
3971 void Assembler::fld_s(Address adr) {
3972   InstructionMark im(this);
3973   emit_int8((unsigned char)0xD9);
3974   emit_operand32(rax, adr);
3975 }
3976 
3977 
3978 void Assembler::fld_s(int index) {
3979   emit_farith(0xD9, 0xC0, index);
3980 }
3981 
3982 void Assembler::fld_x(Address adr) {
3983   InstructionMark im(this);
3984   emit_int8((unsigned char)0xDB);
3985   emit_operand32(rbp, adr);
3986 }
3987 
3988 void Assembler::fldcw(Address src) {
3989   InstructionMark im(this);
3990   emit_int8((unsigned char)0xD9);
3991   emit_operand32(rbp, src);
3992 }
3993 
3994 void Assembler::fldenv(Address src) {
3995   InstructionMark im(this);
3996   emit_int8((unsigned char)0xD9);
3997   emit_operand32(rsp, src);
3998 }
3999 
4000 void Assembler::fldlg2() {
4001   emit_int8((unsigned char)0xD9);
4002   emit_int8((unsigned char)0xEC);
4003 }
4004 
4005 void Assembler::fldln2() {
4006   emit_int8((unsigned char)0xD9);
4007   emit_int8((unsigned char)0xED);
4008 }
4009 
4010 void Assembler::fldz() {
4011   emit_int8((unsigned char)0xD9);
4012   emit_int8((unsigned char)0xEE);
4013 }
4014 
4015 void Assembler::flog() {
4016   fldln2();
4017   fxch();
4018   fyl2x();
4019 }
4020 
4021 void Assembler::flog10() {
4022   fldlg2();
4023   fxch();
4024   fyl2x();
4025 }
4026 
4027 void Assembler::fmul(int i) {
4028   emit_farith(0xD8, 0xC8, i);
4029 }
4030 
4031 void Assembler::fmul_d(Address src) {
4032   InstructionMark im(this);
4033   emit_int8((unsigned char)0xDC);
4034   emit_operand32(rcx, src);
4035 }
4036 
4037 void Assembler::fmul_s(Address src) {
4038   InstructionMark im(this);
4039   emit_int8((unsigned char)0xD8);
4040   emit_operand32(rcx, src);
4041 }
4042 
4043 void Assembler::fmula(int i) {
4044   emit_farith(0xDC, 0xC8, i);
4045 }
4046 
4047 void Assembler::fmulp(int i) {
4048   emit_farith(0xDE, 0xC8, i);
4049 }
4050 
4051 void Assembler::fnsave(Address dst) {
4052   InstructionMark im(this);
4053   emit_int8((unsigned char)0xDD);
4054   emit_operand32(rsi, dst);
4055 }
4056 
4057 void Assembler::fnstcw(Address src) {
4058   InstructionMark im(this);
4059   emit_int8((unsigned char)0x9B);
4060   emit_int8((unsigned char)0xD9);
4061   emit_operand32(rdi, src);
4062 }
4063 
4064 void Assembler::fnstsw_ax() {
4065   emit_int8((unsigned char)0xDF);
4066   emit_int8((unsigned char)0xE0);
4067 }
4068 
4069 void Assembler::fprem() {
4070   emit_int8((unsigned char)0xD9);
4071   emit_int8((unsigned char)0xF8);
4072 }
4073 
4074 void Assembler::fprem1() {
4075   emit_int8((unsigned char)0xD9);
4076   emit_int8((unsigned char)0xF5);
4077 }
4078 
4079 void Assembler::frstor(Address src) {
4080   InstructionMark im(this);
4081   emit_int8((unsigned char)0xDD);
4082   emit_operand32(rsp, src);
4083 }
4084 
4085 void Assembler::fsin() {
4086   emit_int8((unsigned char)0xD9);
4087   emit_int8((unsigned char)0xFE);
4088 }
4089 
4090 void Assembler::fsqrt() {
4091   emit_int8((unsigned char)0xD9);
4092   emit_int8((unsigned char)0xFA);
4093 }
4094 
4095 void Assembler::fst_d(Address adr) {
4096   InstructionMark im(this);
4097   emit_int8((unsigned char)0xDD);
4098   emit_operand32(rdx, adr);
4099 }
4100 
4101 void Assembler::fst_s(Address adr) {
4102   InstructionMark im(this);
4103   emit_int8((unsigned char)0xD9);
4104   emit_operand32(rdx, adr);
4105 }
4106 
4107 void Assembler::fstp_d(Address adr) {
4108   InstructionMark im(this);
4109   emit_int8((unsigned char)0xDD);
4110   emit_operand32(rbx, adr);
4111 }
4112 
4113 void Assembler::fstp_d(int index) {
4114   emit_farith(0xDD, 0xD8, index);
4115 }
4116 
4117 void Assembler::fstp_s(Address adr) {
4118   InstructionMark im(this);
4119   emit_int8((unsigned char)0xD9);
4120   emit_operand32(rbx, adr);
4121 }
4122 
4123 void Assembler::fstp_x(Address adr) {
4124   InstructionMark im(this);
4125   emit_int8((unsigned char)0xDB);
4126   emit_operand32(rdi, adr);
4127 }
4128 
4129 void Assembler::fsub(int i) {
4130   emit_farith(0xD8, 0xE0, i);
4131 }
4132 
4133 void Assembler::fsub_d(Address src) {
4134   InstructionMark im(this);
4135   emit_int8((unsigned char)0xDC);
4136   emit_operand32(rsp, src);
4137 }
4138 
4139 void Assembler::fsub_s(Address src) {
4140   InstructionMark im(this);
4141   emit_int8((unsigned char)0xD8);
4142   emit_operand32(rsp, src);
4143 }
4144 
4145 void Assembler::fsuba(int i) {
4146   emit_farith(0xDC, 0xE8, i);
4147 }
4148 
4149 void Assembler::fsubp(int i) {
4150   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
4151 }
4152 
4153 void Assembler::fsubr(int i) {
4154   emit_farith(0xD8, 0xE8, i);
4155 }
4156 
4157 void Assembler::fsubr_d(Address src) {
4158   InstructionMark im(this);
4159   emit_int8((unsigned char)0xDC);
4160   emit_operand32(rbp, src);
4161 }
4162 
4163 void Assembler::fsubr_s(Address src) {
4164   InstructionMark im(this);
4165   emit_int8((unsigned char)0xD8);
4166   emit_operand32(rbp, src);
4167 }
4168 
4169 void Assembler::fsubra(int i) {
4170   emit_farith(0xDC, 0xE0, i);
4171 }
4172 
4173 void Assembler::fsubrp(int i) {
4174   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
4175 }
4176 
4177 void Assembler::ftan() {
4178   emit_int8((unsigned char)0xD9);
4179   emit_int8((unsigned char)0xF2);
4180   emit_int8((unsigned char)0xDD);
4181   emit_int8((unsigned char)0xD8);
4182 }
4183 
4184 void Assembler::ftst() {
4185   emit_int8((unsigned char)0xD9);
4186   emit_int8((unsigned char)0xE4);
4187 }
4188 
4189 void Assembler::fucomi(int i) {
4190   // make sure the instruction is supported (introduced for P6, together with cmov)
4191   guarantee(VM_Version::supports_cmov(), "illegal instruction");
4192   emit_farith(0xDB, 0xE8, i);
4193 }
4194 
4195 void Assembler::fucomip(int i) {
4196   // make sure the instruction is supported (introduced for P6, together with cmov)
4197   guarantee(VM_Version::supports_cmov(), "illegal instruction");
4198   emit_farith(0xDF, 0xE8, i);
4199 }
4200 
4201 void Assembler::fwait() {
4202   emit_int8((unsigned char)0x9B);
4203 }
4204 
4205 void Assembler::fxch(int i) {
4206   emit_farith(0xD9, 0xC8, i);
4207 }
4208 
4209 void Assembler::fyl2x() {
4210   emit_int8((unsigned char)0xD9);
4211   emit_int8((unsigned char)0xF1);
4212 }
4213 
4214 void Assembler::frndint() {
4215   emit_int8((unsigned char)0xD9);
4216   emit_int8((unsigned char)0xFC);
4217 }
4218 
4219 void Assembler::f2xm1() {
4220   emit_int8((unsigned char)0xD9);
4221   emit_int8((unsigned char)0xF0);
4222 }
4223 
4224 void Assembler::fldl2e() {
4225   emit_int8((unsigned char)0xD9);
4226   emit_int8((unsigned char)0xEA);
4227 }
4228 
4229 // SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
4230 static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
4231 // SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
4232 static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
4233 
4234 // Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
4235 void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
4236   if (pre > 0) {
4237     emit_int8(simd_pre[pre]);
4238   }
4239   if (rex_w) {
4240     prefixq(adr, xreg);
4241   } else {
4242     prefix(adr, xreg);
4243   }
4244   if (opc > 0) {
4245     emit_int8(0x0F);
4246     int opc2 = simd_opc[opc];
4247     if (opc2 > 0) {
4248       emit_int8(opc2);
4249     }
4250   }
4251 }
4252 
4253 int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
4254   if (pre > 0) {
4255     emit_int8(simd_pre[pre]);
4256   }
4257   int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) :
4258                           prefix_and_encode(dst_enc, src_enc);
4259   if (opc > 0) {
4260     emit_int8(0x0F);
4261     int opc2 = simd_opc[opc];
4262     if (opc2 > 0) {
4263       emit_int8(opc2);
4264     }
4265   }
4266   return encode;
4267 }
4268 
4269 
4270 void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, int nds_enc, VexSimdPrefix pre, VexOpcode opc, bool vector256) {
4271   if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
4272     prefix(VEX_3bytes);
4273 
4274     int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
4275     byte1 = (~byte1) & 0xE0;
4276     byte1 |= opc;
4277     emit_int8(byte1);
4278 
4279     int byte2 = ((~nds_enc) & 0xf) << 3;
4280     byte2 |= (vex_w ? VEX_W : 0) | (vector256 ? 4 : 0) | pre;
4281     emit_int8(byte2);
4282   } else {
4283     prefix(VEX_2bytes);
4284 
4285     int byte1 = vex_r ? VEX_R : 0;
4286     byte1 = (~byte1) & 0x80;
4287     byte1 |= ((~nds_enc) & 0xf) << 3;
4288     byte1 |= (vector256 ? 4 : 0) | pre;
4289     emit_int8(byte1);
4290   }
4291 }
4292 
4293 void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, bool vex_w, bool vector256){
4294   bool vex_r = (xreg_enc >= 8);
4295   bool vex_b = adr.base_needs_rex();
4296   bool vex_x = adr.index_needs_rex();
4297   vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector256);
4298 }
4299 
4300 int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool vex_w, bool vector256) {
4301   bool vex_r = (dst_enc >= 8);
4302   bool vex_b = (src_enc >= 8);
4303   bool vex_x = false;
4304   vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector256);
4305   return (((dst_enc & 7) << 3) | (src_enc & 7));
4306 }
4307 
4308 
4309 void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre, VexOpcode opc, bool rex_w, bool vector256) {
4310   if (UseAVX > 0) {
4311     int xreg_enc = xreg->encoding();
4312     int  nds_enc = nds->is_valid() ? nds->encoding() : 0;
4313     vex_prefix(adr, nds_enc, xreg_enc, pre, opc, rex_w, vector256);
4314   } else {
4315     assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
4316     rex_prefix(adr, xreg, pre, opc, rex_w);
4317   }
4318 }
4319 
4320 int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre, VexOpcode opc, bool rex_w, bool vector256) {
4321   int dst_enc = dst->encoding();
4322   int src_enc = src->encoding();
4323   if (UseAVX > 0) {
4324     int nds_enc = nds->is_valid() ? nds->encoding() : 0;
4325     return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector256);
4326   } else {
4327     assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
4328     return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, rex_w);
4329   }
4330 }
4331 
4332 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre) {
4333   InstructionMark im(this);
4334   simd_prefix(dst, dst, src, pre);
4335   emit_int8(opcode);
4336   emit_operand(dst, src);
4337 }
4338 
4339 void Assembler::emit_simd_arith(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre) {
4340   int encode = simd_prefix_and_encode(dst, dst, src, pre);
4341   emit_int8(opcode);
4342   emit_int8((unsigned char)(0xC0 | encode));
4343 }
4344 
4345 // Versions with no second source register (non-destructive source).
4346 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre) {
4347   InstructionMark im(this);
4348   simd_prefix(dst, xnoreg, src, pre);
4349   emit_int8(opcode);
4350   emit_operand(dst, src);
4351 }
4352 
4353 void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre) {
4354   int encode = simd_prefix_and_encode(dst, xnoreg, src, pre);
4355   emit_int8(opcode);
4356   emit_int8((unsigned char)(0xC0 | encode));
4357 }
4358 
4359 // 3-operands AVX instructions
4360 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds,
4361                                Address src, VexSimdPrefix pre, bool vector256) {
4362   InstructionMark im(this);
4363   vex_prefix(dst, nds, src, pre, vector256);
4364   emit_int8(opcode);
4365   emit_operand(dst, src);
4366 }
4367 
4368 void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds,
4369                                XMMRegister src, VexSimdPrefix pre, bool vector256) {
4370   int encode = vex_prefix_and_encode(dst, nds, src, pre, vector256);
4371   emit_int8(opcode);
4372   emit_int8((unsigned char)(0xC0 | encode));
4373 }
4374 
4375 #ifndef _LP64
4376 
4377 void Assembler::incl(Register dst) {
4378   // Don't use it directly. Use MacroAssembler::incrementl() instead.
4379   emit_int8(0x40 | dst->encoding());
4380 }
4381 
4382 void Assembler::lea(Register dst, Address src) {
4383   leal(dst, src);
4384 }
4385 
4386 void Assembler::mov_literal32(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
4387   InstructionMark im(this);
4388   emit_int8((unsigned char)0xC7);
4389   emit_operand(rax, dst);
4390   emit_data((int)imm32, rspec, 0);
4391 }
4392 
4393 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
4394   InstructionMark im(this);
4395   int encode = prefix_and_encode(dst->encoding());
4396   emit_int8((unsigned char)(0xB8 | encode));
4397   emit_data((int)imm32, rspec, 0);
4398 }
4399 
4400 void Assembler::popa() { // 32bit
4401   emit_int8(0x61);
4402 }
4403 
4404 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
4405   InstructionMark im(this);
4406   emit_int8(0x68);
4407   emit_data(imm32, rspec, 0);
4408 }
4409 
4410 void Assembler::pusha() { // 32bit
4411   emit_int8(0x60);
4412 }
4413 
4414 void Assembler::set_byte_if_not_zero(Register dst) {
4415   emit_int8(0x0F);
4416   emit_int8((unsigned char)0x95);
4417   emit_int8((unsigned char)(0xE0 | dst->encoding()));
4418 }
4419 
4420 void Assembler::shldl(Register dst, Register src) {
4421   emit_int8(0x0F);
4422   emit_int8((unsigned char)0xA5);
4423   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
4424 }
4425 
4426 void Assembler::shrdl(Register dst, Register src) {
4427   emit_int8(0x0F);
4428   emit_int8((unsigned char)0xAD);
4429   emit_int8((unsigned char)(0xC0 | src->encoding() << 3 | dst->encoding()));
4430 }
4431 
4432 #else // LP64
4433 
4434 void Assembler::set_byte_if_not_zero(Register dst) {
4435   int enc = prefix_and_encode(dst->encoding(), true);
4436   emit_int8(0x0F);
4437   emit_int8((unsigned char)0x95);
4438   emit_int8((unsigned char)(0xE0 | enc));
4439 }
4440 
4441 // 64bit only pieces of the assembler
4442 // This should only be used by 64bit instructions that can use rip-relative
4443 // it cannot be used by instructions that want an immediate value.
4444 
4445 bool Assembler::reachable(AddressLiteral adr) {
4446   int64_t disp;
4447   // None will force a 64bit literal to the code stream. Likely a placeholder
4448   // for something that will be patched later and we need to certain it will
4449   // always be reachable.
4450   if (adr.reloc() == relocInfo::none) {
4451     return false;
4452   }
4453   if (adr.reloc() == relocInfo::internal_word_type) {
4454     // This should be rip relative and easily reachable.
4455     return true;
4456   }
4457   if (adr.reloc() == relocInfo::virtual_call_type ||
4458       adr.reloc() == relocInfo::opt_virtual_call_type ||
4459       adr.reloc() == relocInfo::static_call_type ||
4460       adr.reloc() == relocInfo::static_stub_type ) {
4461     // This should be rip relative within the code cache and easily
4462     // reachable until we get huge code caches. (At which point
4463     // ic code is going to have issues).
4464     return true;
4465   }
4466   if (adr.reloc() != relocInfo::external_word_type &&
4467       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
4468       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
4469       adr.reloc() != relocInfo::runtime_call_type ) {
4470     return false;
4471   }
4472 
4473   // Stress the correction code
4474   if (ForceUnreachable) {
4475     // Must be runtimecall reloc, see if it is in the codecache
4476     // Flipping stuff in the codecache to be unreachable causes issues
4477     // with things like inline caches where the additional instructions
4478     // are not handled.
4479     if (CodeCache::find_blob(adr._target) == NULL) {
4480       return false;
4481     }
4482   }
4483   // For external_word_type/runtime_call_type if it is reachable from where we
4484   // are now (possibly a temp buffer) and where we might end up
4485   // anywhere in the codeCache then we are always reachable.
4486   // This would have to change if we ever save/restore shared code
4487   // to be more pessimistic.
4488   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
4489   if (!is_simm32(disp)) return false;
4490   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
4491   if (!is_simm32(disp)) return false;
4492 
4493   disp = (int64_t)adr._target - ((int64_t)pc() + sizeof(int));
4494 
4495   // Because rip relative is a disp + address_of_next_instruction and we
4496   // don't know the value of address_of_next_instruction we apply a fudge factor
4497   // to make sure we will be ok no matter the size of the instruction we get placed into.
4498   // We don't have to fudge the checks above here because they are already worst case.
4499 
4500   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
4501   // + 4 because better safe than sorry.
4502   const int fudge = 12 + 4;
4503   if (disp < 0) {
4504     disp -= fudge;
4505   } else {
4506     disp += fudge;
4507   }
4508   return is_simm32(disp);
4509 }
4510 
4511 // Check if the polling page is not reachable from the code cache using rip-relative
4512 // addressing.
4513 bool Assembler::is_polling_page_far() {
4514   intptr_t addr = (intptr_t)os::get_polling_page();
4515   return ForceUnreachable ||
4516          !is_simm32(addr - (intptr_t)CodeCache::low_bound()) ||
4517          !is_simm32(addr - (intptr_t)CodeCache::high_bound());
4518 }
4519 
4520 void Assembler::emit_data64(jlong data,
4521                             relocInfo::relocType rtype,
4522                             int format) {
4523   if (rtype == relocInfo::none) {
4524     emit_int64(data);
4525   } else {
4526     emit_data64(data, Relocation::spec_simple(rtype), format);
4527   }
4528 }
4529 
4530 void Assembler::emit_data64(jlong data,
4531                             RelocationHolder const& rspec,
4532                             int format) {
4533   assert(imm_operand == 0, "default format must be immediate in this file");
4534   assert(imm_operand == format, "must be immediate");
4535   assert(inst_mark() != NULL, "must be inside InstructionMark");
4536   // Do not use AbstractAssembler::relocate, which is not intended for
4537   // embedded words.  Instead, relocate to the enclosing instruction.
4538   code_section()->relocate(inst_mark(), rspec, format);
4539 #ifdef ASSERT
4540   check_relocation(rspec, format);
4541 #endif
4542   emit_int64(data);
4543 }
4544 
4545 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
4546   if (reg_enc >= 8) {
4547     prefix(REX_B);
4548     reg_enc -= 8;
4549   } else if (byteinst && reg_enc >= 4) {
4550     prefix(REX);
4551   }
4552   return reg_enc;
4553 }
4554 
4555 int Assembler::prefixq_and_encode(int reg_enc) {
4556   if (reg_enc < 8) {
4557     prefix(REX_W);
4558   } else {
4559     prefix(REX_WB);
4560     reg_enc -= 8;
4561   }
4562   return reg_enc;
4563 }
4564 
4565 int Assembler::prefix_and_encode(int dst_enc, int src_enc, bool byteinst) {
4566   if (dst_enc < 8) {
4567     if (src_enc >= 8) {
4568       prefix(REX_B);
4569       src_enc -= 8;
4570     } else if (byteinst && src_enc >= 4) {
4571       prefix(REX);
4572     }
4573   } else {
4574     if (src_enc < 8) {
4575       prefix(REX_R);
4576     } else {
4577       prefix(REX_RB);
4578       src_enc -= 8;
4579     }
4580     dst_enc -= 8;
4581   }
4582   return dst_enc << 3 | src_enc;
4583 }
4584 
4585 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
4586   if (dst_enc < 8) {
4587     if (src_enc < 8) {
4588       prefix(REX_W);
4589     } else {
4590       prefix(REX_WB);
4591       src_enc -= 8;
4592     }
4593   } else {
4594     if (src_enc < 8) {
4595       prefix(REX_WR);
4596     } else {
4597       prefix(REX_WRB);
4598       src_enc -= 8;
4599     }
4600     dst_enc -= 8;
4601   }
4602   return dst_enc << 3 | src_enc;
4603 }
4604 
4605 void Assembler::prefix(Register reg) {
4606   if (reg->encoding() >= 8) {
4607     prefix(REX_B);
4608   }
4609 }
4610 
4611 void Assembler::prefix(Address adr) {
4612   if (adr.base_needs_rex()) {
4613     if (adr.index_needs_rex()) {
4614       prefix(REX_XB);
4615     } else {
4616       prefix(REX_B);
4617     }
4618   } else {
4619     if (adr.index_needs_rex()) {
4620       prefix(REX_X);
4621     }
4622   }
4623 }
4624 
4625 void Assembler::prefixq(Address adr) {
4626   if (adr.base_needs_rex()) {
4627     if (adr.index_needs_rex()) {
4628       prefix(REX_WXB);
4629     } else {
4630       prefix(REX_WB);
4631     }
4632   } else {
4633     if (adr.index_needs_rex()) {
4634       prefix(REX_WX);
4635     } else {
4636       prefix(REX_W);
4637     }
4638   }
4639 }
4640 
4641 
4642 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
4643   if (reg->encoding() < 8) {
4644     if (adr.base_needs_rex()) {
4645       if (adr.index_needs_rex()) {
4646         prefix(REX_XB);
4647       } else {
4648         prefix(REX_B);
4649       }
4650     } else {
4651       if (adr.index_needs_rex()) {
4652         prefix(REX_X);
4653       } else if (byteinst && reg->encoding() >= 4 ) {
4654         prefix(REX);
4655       }
4656     }
4657   } else {
4658     if (adr.base_needs_rex()) {
4659       if (adr.index_needs_rex()) {
4660         prefix(REX_RXB);
4661       } else {
4662         prefix(REX_RB);
4663       }
4664     } else {
4665       if (adr.index_needs_rex()) {
4666         prefix(REX_RX);
4667       } else {
4668         prefix(REX_R);
4669       }
4670     }
4671   }
4672 }
4673 
4674 void Assembler::prefixq(Address adr, Register src) {
4675   if (src->encoding() < 8) {
4676     if (adr.base_needs_rex()) {
4677       if (adr.index_needs_rex()) {
4678         prefix(REX_WXB);
4679       } else {
4680         prefix(REX_WB);
4681       }
4682     } else {
4683       if (adr.index_needs_rex()) {
4684         prefix(REX_WX);
4685       } else {
4686         prefix(REX_W);
4687       }
4688     }
4689   } else {
4690     if (adr.base_needs_rex()) {
4691       if (adr.index_needs_rex()) {
4692         prefix(REX_WRXB);
4693       } else {
4694         prefix(REX_WRB);
4695       }
4696     } else {
4697       if (adr.index_needs_rex()) {
4698         prefix(REX_WRX);
4699       } else {
4700         prefix(REX_WR);
4701       }
4702     }
4703   }
4704 }
4705 
4706 void Assembler::prefix(Address adr, XMMRegister reg) {
4707   if (reg->encoding() < 8) {
4708     if (adr.base_needs_rex()) {
4709       if (adr.index_needs_rex()) {
4710         prefix(REX_XB);
4711       } else {
4712         prefix(REX_B);
4713       }
4714     } else {
4715       if (adr.index_needs_rex()) {
4716         prefix(REX_X);
4717       }
4718     }
4719   } else {
4720     if (adr.base_needs_rex()) {
4721       if (adr.index_needs_rex()) {
4722         prefix(REX_RXB);
4723       } else {
4724         prefix(REX_RB);
4725       }
4726     } else {
4727       if (adr.index_needs_rex()) {
4728         prefix(REX_RX);
4729       } else {
4730         prefix(REX_R);
4731       }
4732     }
4733   }
4734 }
4735 
4736 void Assembler::prefixq(Address adr, XMMRegister src) {
4737   if (src->encoding() < 8) {
4738     if (adr.base_needs_rex()) {
4739       if (adr.index_needs_rex()) {
4740         prefix(REX_WXB);
4741       } else {
4742         prefix(REX_WB);
4743       }
4744     } else {
4745       if (adr.index_needs_rex()) {
4746         prefix(REX_WX);
4747       } else {
4748         prefix(REX_W);
4749       }
4750     }
4751   } else {
4752     if (adr.base_needs_rex()) {
4753       if (adr.index_needs_rex()) {
4754         prefix(REX_WRXB);
4755       } else {
4756         prefix(REX_WRB);
4757       }
4758     } else {
4759       if (adr.index_needs_rex()) {
4760         prefix(REX_WRX);
4761       } else {
4762         prefix(REX_WR);
4763       }
4764     }
4765   }
4766 }
4767 
4768 void Assembler::adcq(Register dst, int32_t imm32) {
4769   (void) prefixq_and_encode(dst->encoding());
4770   emit_arith(0x81, 0xD0, dst, imm32);
4771 }
4772 
4773 void Assembler::adcq(Register dst, Address src) {
4774   InstructionMark im(this);
4775   prefixq(src, dst);
4776   emit_int8(0x13);
4777   emit_operand(dst, src);
4778 }
4779 
4780 void Assembler::adcq(Register dst, Register src) {
4781   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4782   emit_arith(0x13, 0xC0, dst, src);
4783 }
4784 
4785 void Assembler::addq(Address dst, int32_t imm32) {
4786   InstructionMark im(this);
4787   prefixq(dst);
4788   emit_arith_operand(0x81, rax, dst,imm32);
4789 }
4790 
4791 void Assembler::addq(Address dst, Register src) {
4792   InstructionMark im(this);
4793   prefixq(dst, src);
4794   emit_int8(0x01);
4795   emit_operand(src, dst);
4796 }
4797 
4798 void Assembler::addq(Register dst, int32_t imm32) {
4799   (void) prefixq_and_encode(dst->encoding());
4800   emit_arith(0x81, 0xC0, dst, imm32);
4801 }
4802 
4803 void Assembler::addq(Register dst, Address src) {
4804   InstructionMark im(this);
4805   prefixq(src, dst);
4806   emit_int8(0x03);
4807   emit_operand(dst, src);
4808 }
4809 
4810 void Assembler::addq(Register dst, Register src) {
4811   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4812   emit_arith(0x03, 0xC0, dst, src);
4813 }
4814 
4815 void Assembler::andq(Address dst, int32_t imm32) {
4816   InstructionMark im(this);
4817   prefixq(dst);
4818   emit_int8((unsigned char)0x81);
4819   emit_operand(rsp, dst, 4);
4820   emit_int32(imm32);
4821 }
4822 
4823 void Assembler::andq(Register dst, int32_t imm32) {
4824   (void) prefixq_and_encode(dst->encoding());
4825   emit_arith(0x81, 0xE0, dst, imm32);
4826 }
4827 
4828 void Assembler::andq(Register dst, Address src) {
4829   InstructionMark im(this);
4830   prefixq(src, dst);
4831   emit_int8(0x23);
4832   emit_operand(dst, src);
4833 }
4834 
4835 void Assembler::andq(Register dst, Register src) {
4836   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4837   emit_arith(0x23, 0xC0, dst, src);
4838 }
4839 
4840 void Assembler::bsfq(Register dst, Register src) {
4841   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4842   emit_int8(0x0F);
4843   emit_int8((unsigned char)0xBC);
4844   emit_int8((unsigned char)(0xC0 | encode));
4845 }
4846 
4847 void Assembler::bsrq(Register dst, Register src) {
4848   assert(!VM_Version::supports_lzcnt(), "encoding is treated as LZCNT");
4849   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4850   emit_int8(0x0F);
4851   emit_int8((unsigned char)0xBD);
4852   emit_int8((unsigned char)(0xC0 | encode));
4853 }
4854 
4855 void Assembler::bswapq(Register reg) {
4856   int encode = prefixq_and_encode(reg->encoding());
4857   emit_int8(0x0F);
4858   emit_int8((unsigned char)(0xC8 | encode));
4859 }
4860 
4861 void Assembler::cdqq() {
4862   prefix(REX_W);
4863   emit_int8((unsigned char)0x99);
4864 }
4865 
4866 void Assembler::clflush(Address adr) {
4867   prefix(adr);
4868   emit_int8(0x0F);
4869   emit_int8((unsigned char)0xAE);
4870   emit_operand(rdi, adr);
4871 }
4872 
4873 void Assembler::cmovq(Condition cc, Register dst, Register src) {
4874   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4875   emit_int8(0x0F);
4876   emit_int8(0x40 | cc);
4877   emit_int8((unsigned char)(0xC0 | encode));
4878 }
4879 
4880 void Assembler::cmovq(Condition cc, Register dst, Address src) {
4881   InstructionMark im(this);
4882   prefixq(src, dst);
4883   emit_int8(0x0F);
4884   emit_int8(0x40 | cc);
4885   emit_operand(dst, src);
4886 }
4887 
4888 void Assembler::cmpq(Address dst, int32_t imm32) {
4889   InstructionMark im(this);
4890   prefixq(dst);
4891   emit_int8((unsigned char)0x81);
4892   emit_operand(rdi, dst, 4);
4893   emit_int32(imm32);
4894 }
4895 
4896 void Assembler::cmpq(Register dst, int32_t imm32) {
4897   (void) prefixq_and_encode(dst->encoding());
4898   emit_arith(0x81, 0xF8, dst, imm32);
4899 }
4900 
4901 void Assembler::cmpq(Address dst, Register src) {
4902   InstructionMark im(this);
4903   prefixq(dst, src);
4904   emit_int8(0x3B);
4905   emit_operand(src, dst);
4906 }
4907 
4908 void Assembler::cmpq(Register dst, Register src) {
4909   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4910   emit_arith(0x3B, 0xC0, dst, src);
4911 }
4912 
4913 void Assembler::cmpq(Register dst, Address  src) {
4914   InstructionMark im(this);
4915   prefixq(src, dst);
4916   emit_int8(0x3B);
4917   emit_operand(dst, src);
4918 }
4919 
4920 void Assembler::cmpxchgq(Register reg, Address adr) {
4921   InstructionMark im(this);
4922   prefixq(adr, reg);
4923   emit_int8(0x0F);
4924   emit_int8((unsigned char)0xB1);
4925   emit_operand(reg, adr);
4926 }
4927 
4928 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
4929   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4930   int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F2);
4931   emit_int8(0x2A);
4932   emit_int8((unsigned char)(0xC0 | encode));
4933 }
4934 
4935 void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
4936   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4937   InstructionMark im(this);
4938   simd_prefix_q(dst, dst, src, VEX_SIMD_F2);
4939   emit_int8(0x2A);
4940   emit_operand(dst, src);
4941 }
4942 
4943 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
4944   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4945   int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F3);
4946   emit_int8(0x2A);
4947   emit_int8((unsigned char)(0xC0 | encode));
4948 }
4949 
4950 void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
4951   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4952   InstructionMark im(this);
4953   simd_prefix_q(dst, dst, src, VEX_SIMD_F3);
4954   emit_int8(0x2A);
4955   emit_operand(dst, src);
4956 }
4957 
4958 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
4959   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4960   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F2);
4961   emit_int8(0x2C);
4962   emit_int8((unsigned char)(0xC0 | encode));
4963 }
4964 
4965 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
4966   NOT_LP64(assert(VM_Version::supports_sse(), ""));
4967   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F3);
4968   emit_int8(0x2C);
4969   emit_int8((unsigned char)(0xC0 | encode));
4970 }
4971 
4972 void Assembler::decl(Register dst) {
4973   // Don't use it directly. Use MacroAssembler::decrementl() instead.
4974   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
4975   int encode = prefix_and_encode(dst->encoding());
4976   emit_int8((unsigned char)0xFF);
4977   emit_int8((unsigned char)(0xC8 | encode));
4978 }
4979 
4980 void Assembler::decq(Register dst) {
4981   // Don't use it directly. Use MacroAssembler::decrementq() instead.
4982   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
4983   int encode = prefixq_and_encode(dst->encoding());
4984   emit_int8((unsigned char)0xFF);
4985   emit_int8(0xC8 | encode);
4986 }
4987 
4988 void Assembler::decq(Address dst) {
4989   // Don't use it directly. Use MacroAssembler::decrementq() instead.
4990   InstructionMark im(this);
4991   prefixq(dst);
4992   emit_int8((unsigned char)0xFF);
4993   emit_operand(rcx, dst);
4994 }
4995 
4996 void Assembler::fxrstor(Address src) {
4997   prefixq(src);
4998   emit_int8(0x0F);
4999   emit_int8((unsigned char)0xAE);
5000   emit_operand(as_Register(1), src);
5001 }
5002 
5003 void Assembler::fxsave(Address dst) {
5004   prefixq(dst);
5005   emit_int8(0x0F);
5006   emit_int8((unsigned char)0xAE);
5007   emit_operand(as_Register(0), dst);
5008 }
5009 
5010 void Assembler::idivq(Register src) {
5011   int encode = prefixq_and_encode(src->encoding());
5012   emit_int8((unsigned char)0xF7);
5013   emit_int8((unsigned char)(0xF8 | encode));
5014 }
5015 
5016 void Assembler::imulq(Register dst, Register src) {
5017   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5018   emit_int8(0x0F);
5019   emit_int8((unsigned char)0xAF);
5020   emit_int8((unsigned char)(0xC0 | encode));
5021 }
5022 
5023 void Assembler::imulq(Register dst, Register src, int value) {
5024   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5025   if (is8bit(value)) {
5026     emit_int8(0x6B);
5027     emit_int8((unsigned char)(0xC0 | encode));
5028     emit_int8(value & 0xFF);
5029   } else {
5030     emit_int8(0x69);
5031     emit_int8((unsigned char)(0xC0 | encode));
5032     emit_int32(value);
5033   }
5034 }
5035 
5036 void Assembler::imulq(Register dst, Address src) {
5037   InstructionMark im(this);
5038   prefixq(src, dst);
5039   emit_int8(0x0F);
5040   emit_int8((unsigned char) 0xAF);
5041   emit_operand(dst, src);
5042 }
5043 
5044 void Assembler::incl(Register dst) {
5045   // Don't use it directly. Use MacroAssembler::incrementl() instead.
5046   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
5047   int encode = prefix_and_encode(dst->encoding());
5048   emit_int8((unsigned char)0xFF);
5049   emit_int8((unsigned char)(0xC0 | encode));
5050 }
5051 
5052 void Assembler::incq(Register dst) {
5053   // Don't use it directly. Use MacroAssembler::incrementq() instead.
5054   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
5055   int encode = prefixq_and_encode(dst->encoding());
5056   emit_int8((unsigned char)0xFF);
5057   emit_int8((unsigned char)(0xC0 | encode));
5058 }
5059 
5060 void Assembler::incq(Address dst) {
5061   // Don't use it directly. Use MacroAssembler::incrementq() instead.
5062   InstructionMark im(this);
5063   prefixq(dst);
5064   emit_int8((unsigned char)0xFF);
5065   emit_operand(rax, dst);
5066 }
5067 
5068 void Assembler::lea(Register dst, Address src) {
5069   leaq(dst, src);
5070 }
5071 
5072 void Assembler::leaq(Register dst, Address src) {
5073   InstructionMark im(this);
5074   prefixq(src, dst);
5075   emit_int8((unsigned char)0x8D);
5076   emit_operand(dst, src);
5077 }
5078 
5079 void Assembler::mov64(Register dst, int64_t imm64) {
5080   InstructionMark im(this);
5081   int encode = prefixq_and_encode(dst->encoding());
5082   emit_int8((unsigned char)(0xB8 | encode));
5083   emit_int64(imm64);
5084 }
5085 
5086 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
5087   InstructionMark im(this);
5088   int encode = prefixq_and_encode(dst->encoding());
5089   emit_int8(0xB8 | encode);
5090   emit_data64(imm64, rspec);
5091 }
5092 
5093 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
5094   InstructionMark im(this);
5095   int encode = prefix_and_encode(dst->encoding());
5096   emit_int8((unsigned char)(0xB8 | encode));
5097   emit_data((int)imm32, rspec, narrow_oop_operand);
5098 }
5099 
5100 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
5101   InstructionMark im(this);
5102   prefix(dst);
5103   emit_int8((unsigned char)0xC7);
5104   emit_operand(rax, dst, 4);
5105   emit_data((int)imm32, rspec, narrow_oop_operand);
5106 }
5107 
5108 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
5109   InstructionMark im(this);
5110   int encode = prefix_and_encode(src1->encoding());
5111   emit_int8((unsigned char)0x81);
5112   emit_int8((unsigned char)(0xF8 | encode));
5113   emit_data((int)imm32, rspec, narrow_oop_operand);
5114 }
5115 
5116 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
5117   InstructionMark im(this);
5118   prefix(src1);
5119   emit_int8((unsigned char)0x81);
5120   emit_operand(rax, src1, 4);
5121   emit_data((int)imm32, rspec, narrow_oop_operand);
5122 }
5123 
5124 void Assembler::lzcntq(Register dst, Register src) {
5125   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
5126   emit_int8((unsigned char)0xF3);
5127   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5128   emit_int8(0x0F);
5129   emit_int8((unsigned char)0xBD);
5130   emit_int8((unsigned char)(0xC0 | encode));
5131 }
5132 
5133 void Assembler::movdq(XMMRegister dst, Register src) {
5134   // table D-1 says MMX/SSE2
5135   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5136   int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_66);
5137   emit_int8(0x6E);
5138   emit_int8((unsigned char)(0xC0 | encode));
5139 }
5140 
5141 void Assembler::movdq(Register dst, XMMRegister src) {
5142   // table D-1 says MMX/SSE2
5143   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
5144   // swap src/dst to get correct prefix
5145   int encode = simd_prefix_and_encode_q(src, dst, VEX_SIMD_66);
5146   emit_int8(0x7E);
5147   emit_int8((unsigned char)(0xC0 | encode));
5148 }
5149 
5150 void Assembler::movq(Register dst, Register src) {
5151   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5152   emit_int8((unsigned char)0x8B);
5153   emit_int8((unsigned char)(0xC0 | encode));
5154 }
5155 
5156 void Assembler::movq(Register dst, Address src) {
5157   InstructionMark im(this);
5158   prefixq(src, dst);
5159   emit_int8((unsigned char)0x8B);
5160   emit_operand(dst, src);
5161 }
5162 
5163 void Assembler::movq(Address dst, Register src) {
5164   InstructionMark im(this);
5165   prefixq(dst, src);
5166   emit_int8((unsigned char)0x89);
5167   emit_operand(src, dst);
5168 }
5169 
5170 void Assembler::movsbq(Register dst, Address src) {
5171   InstructionMark im(this);
5172   prefixq(src, dst);
5173   emit_int8(0x0F);
5174   emit_int8((unsigned char)0xBE);
5175   emit_operand(dst, src);
5176 }
5177 
5178 void Assembler::movsbq(Register dst, Register src) {
5179   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5180   emit_int8(0x0F);
5181   emit_int8((unsigned char)0xBE);
5182   emit_int8((unsigned char)(0xC0 | encode));
5183 }
5184 
5185 void Assembler::movslq(Register dst, int32_t imm32) {
5186   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
5187   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
5188   // as a result we shouldn't use until tested at runtime...
5189   ShouldNotReachHere();
5190   InstructionMark im(this);
5191   int encode = prefixq_and_encode(dst->encoding());
5192   emit_int8((unsigned char)(0xC7 | encode));
5193   emit_int32(imm32);
5194 }
5195 
5196 void Assembler::movslq(Address dst, int32_t imm32) {
5197   assert(is_simm32(imm32), "lost bits");
5198   InstructionMark im(this);
5199   prefixq(dst);
5200   emit_int8((unsigned char)0xC7);
5201   emit_operand(rax, dst, 4);
5202   emit_int32(imm32);
5203 }
5204 
5205 void Assembler::movslq(Register dst, Address src) {
5206   InstructionMark im(this);
5207   prefixq(src, dst);
5208   emit_int8(0x63);
5209   emit_operand(dst, src);
5210 }
5211 
5212 void Assembler::movslq(Register dst, Register src) {
5213   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5214   emit_int8(0x63);
5215   emit_int8((unsigned char)(0xC0 | encode));
5216 }
5217 
5218 void Assembler::movswq(Register dst, Address src) {
5219   InstructionMark im(this);
5220   prefixq(src, dst);
5221   emit_int8(0x0F);
5222   emit_int8((unsigned char)0xBF);
5223   emit_operand(dst, src);
5224 }
5225 
5226 void Assembler::movswq(Register dst, Register src) {
5227   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5228   emit_int8((unsigned char)0x0F);
5229   emit_int8((unsigned char)0xBF);
5230   emit_int8((unsigned char)(0xC0 | encode));
5231 }
5232 
5233 void Assembler::movzbq(Register dst, Address src) {
5234   InstructionMark im(this);
5235   prefixq(src, dst);
5236   emit_int8((unsigned char)0x0F);
5237   emit_int8((unsigned char)0xB6);
5238   emit_operand(dst, src);
5239 }
5240 
5241 void Assembler::movzbq(Register dst, Register src) {
5242   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5243   emit_int8(0x0F);
5244   emit_int8((unsigned char)0xB6);
5245   emit_int8(0xC0 | encode);
5246 }
5247 
5248 void Assembler::movzwq(Register dst, Address src) {
5249   InstructionMark im(this);
5250   prefixq(src, dst);
5251   emit_int8((unsigned char)0x0F);
5252   emit_int8((unsigned char)0xB7);
5253   emit_operand(dst, src);
5254 }
5255 
5256 void Assembler::movzwq(Register dst, Register src) {
5257   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5258   emit_int8((unsigned char)0x0F);
5259   emit_int8((unsigned char)0xB7);
5260   emit_int8((unsigned char)(0xC0 | encode));
5261 }
5262 
5263 void Assembler::negq(Register dst) {
5264   int encode = prefixq_and_encode(dst->encoding());
5265   emit_int8((unsigned char)0xF7);
5266   emit_int8((unsigned char)(0xD8 | encode));
5267 }
5268 
5269 void Assembler::notq(Register dst) {
5270   int encode = prefixq_and_encode(dst->encoding());
5271   emit_int8((unsigned char)0xF7);
5272   emit_int8((unsigned char)(0xD0 | encode));
5273 }
5274 
5275 void Assembler::orq(Address dst, int32_t imm32) {
5276   InstructionMark im(this);
5277   prefixq(dst);
5278   emit_int8((unsigned char)0x81);
5279   emit_operand(rcx, dst, 4);
5280   emit_int32(imm32);
5281 }
5282 
5283 void Assembler::orq(Register dst, int32_t imm32) {
5284   (void) prefixq_and_encode(dst->encoding());
5285   emit_arith(0x81, 0xC8, dst, imm32);
5286 }
5287 
5288 void Assembler::orq(Register dst, Address src) {
5289   InstructionMark im(this);
5290   prefixq(src, dst);
5291   emit_int8(0x0B);
5292   emit_operand(dst, src);
5293 }
5294 
5295 void Assembler::orq(Register dst, Register src) {
5296   (void) prefixq_and_encode(dst->encoding(), src->encoding());
5297   emit_arith(0x0B, 0xC0, dst, src);
5298 }
5299 
5300 void Assembler::popa() { // 64bit
5301   movq(r15, Address(rsp, 0));
5302   movq(r14, Address(rsp, wordSize));
5303   movq(r13, Address(rsp, 2 * wordSize));
5304   movq(r12, Address(rsp, 3 * wordSize));
5305   movq(r11, Address(rsp, 4 * wordSize));
5306   movq(r10, Address(rsp, 5 * wordSize));
5307   movq(r9,  Address(rsp, 6 * wordSize));
5308   movq(r8,  Address(rsp, 7 * wordSize));
5309   movq(rdi, Address(rsp, 8 * wordSize));
5310   movq(rsi, Address(rsp, 9 * wordSize));
5311   movq(rbp, Address(rsp, 10 * wordSize));
5312   // skip rsp
5313   movq(rbx, Address(rsp, 12 * wordSize));
5314   movq(rdx, Address(rsp, 13 * wordSize));
5315   movq(rcx, Address(rsp, 14 * wordSize));
5316   movq(rax, Address(rsp, 15 * wordSize));
5317 
5318   addq(rsp, 16 * wordSize);
5319 }
5320 
5321 void Assembler::popcntq(Register dst, Address src) {
5322   assert(VM_Version::supports_popcnt(), "must support");
5323   InstructionMark im(this);
5324   emit_int8((unsigned char)0xF3);
5325   prefixq(src, dst);
5326   emit_int8((unsigned char)0x0F);
5327   emit_int8((unsigned char)0xB8);
5328   emit_operand(dst, src);
5329 }
5330 
5331 void Assembler::popcntq(Register dst, Register src) {
5332   assert(VM_Version::supports_popcnt(), "must support");
5333   emit_int8((unsigned char)0xF3);
5334   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5335   emit_int8((unsigned char)0x0F);
5336   emit_int8((unsigned char)0xB8);
5337   emit_int8((unsigned char)(0xC0 | encode));
5338 }
5339 
5340 void Assembler::popq(Address dst) {
5341   InstructionMark im(this);
5342   prefixq(dst);
5343   emit_int8((unsigned char)0x8F);
5344   emit_operand(rax, dst);
5345 }
5346 
5347 void Assembler::pusha() { // 64bit
5348   // we have to store original rsp.  ABI says that 128 bytes
5349   // below rsp are local scratch.
5350   movq(Address(rsp, -5 * wordSize), rsp);
5351 
5352   subq(rsp, 16 * wordSize);
5353 
5354   movq(Address(rsp, 15 * wordSize), rax);
5355   movq(Address(rsp, 14 * wordSize), rcx);
5356   movq(Address(rsp, 13 * wordSize), rdx);
5357   movq(Address(rsp, 12 * wordSize), rbx);
5358   // skip rsp
5359   movq(Address(rsp, 10 * wordSize), rbp);
5360   movq(Address(rsp, 9 * wordSize), rsi);
5361   movq(Address(rsp, 8 * wordSize), rdi);
5362   movq(Address(rsp, 7 * wordSize), r8);
5363   movq(Address(rsp, 6 * wordSize), r9);
5364   movq(Address(rsp, 5 * wordSize), r10);
5365   movq(Address(rsp, 4 * wordSize), r11);
5366   movq(Address(rsp, 3 * wordSize), r12);
5367   movq(Address(rsp, 2 * wordSize), r13);
5368   movq(Address(rsp, wordSize), r14);
5369   movq(Address(rsp, 0), r15);
5370 }
5371 
5372 void Assembler::pushq(Address src) {
5373   InstructionMark im(this);
5374   prefixq(src);
5375   emit_int8((unsigned char)0xFF);
5376   emit_operand(rsi, src);
5377 }
5378 
5379 void Assembler::rclq(Register dst, int imm8) {
5380   assert(isShiftCount(imm8 >> 1), "illegal shift count");
5381   int encode = prefixq_and_encode(dst->encoding());
5382   if (imm8 == 1) {
5383     emit_int8((unsigned char)0xD1);
5384     emit_int8((unsigned char)(0xD0 | encode));
5385   } else {
5386     emit_int8((unsigned char)0xC1);
5387     emit_int8((unsigned char)(0xD0 | encode));
5388     emit_int8(imm8);
5389   }
5390 }
5391 void Assembler::sarq(Register dst, int imm8) {
5392   assert(isShiftCount(imm8 >> 1), "illegal shift count");
5393   int encode = prefixq_and_encode(dst->encoding());
5394   if (imm8 == 1) {
5395     emit_int8((unsigned char)0xD1);
5396     emit_int8((unsigned char)(0xF8 | encode));
5397   } else {
5398     emit_int8((unsigned char)0xC1);
5399     emit_int8((unsigned char)(0xF8 | encode));
5400     emit_int8(imm8);
5401   }
5402 }
5403 
5404 void Assembler::sarq(Register dst) {
5405   int encode = prefixq_and_encode(dst->encoding());
5406   emit_int8((unsigned char)0xD3);
5407   emit_int8((unsigned char)(0xF8 | encode));
5408 }
5409 
5410 void Assembler::sbbq(Address dst, int32_t imm32) {
5411   InstructionMark im(this);
5412   prefixq(dst);
5413   emit_arith_operand(0x81, rbx, dst, imm32);
5414 }
5415 
5416 void Assembler::sbbq(Register dst, int32_t imm32) {
5417   (void) prefixq_and_encode(dst->encoding());
5418   emit_arith(0x81, 0xD8, dst, imm32);
5419 }
5420 
5421 void Assembler::sbbq(Register dst, Address src) {
5422   InstructionMark im(this);
5423   prefixq(src, dst);
5424   emit_int8(0x1B);
5425   emit_operand(dst, src);
5426 }
5427 
5428 void Assembler::sbbq(Register dst, Register src) {
5429   (void) prefixq_and_encode(dst->encoding(), src->encoding());
5430   emit_arith(0x1B, 0xC0, dst, src);
5431 }
5432 
5433 void Assembler::shlq(Register dst, int imm8) {
5434   assert(isShiftCount(imm8 >> 1), "illegal shift count");
5435   int encode = prefixq_and_encode(dst->encoding());
5436   if (imm8 == 1) {
5437     emit_int8((unsigned char)0xD1);
5438     emit_int8((unsigned char)(0xE0 | encode));
5439   } else {
5440     emit_int8((unsigned char)0xC1);
5441     emit_int8((unsigned char)(0xE0 | encode));
5442     emit_int8(imm8);
5443   }
5444 }
5445 
5446 void Assembler::shlq(Register dst) {
5447   int encode = prefixq_and_encode(dst->encoding());
5448   emit_int8((unsigned char)0xD3);
5449   emit_int8((unsigned char)(0xE0 | encode));
5450 }
5451 
5452 void Assembler::shrq(Register dst, int imm8) {
5453   assert(isShiftCount(imm8 >> 1), "illegal shift count");
5454   int encode = prefixq_and_encode(dst->encoding());
5455   emit_int8((unsigned char)0xC1);
5456   emit_int8((unsigned char)(0xE8 | encode));
5457   emit_int8(imm8);
5458 }
5459 
5460 void Assembler::shrq(Register dst) {
5461   int encode = prefixq_and_encode(dst->encoding());
5462   emit_int8((unsigned char)0xD3);
5463   emit_int8(0xE8 | encode);
5464 }
5465 
5466 void Assembler::subq(Address dst, int32_t imm32) {
5467   InstructionMark im(this);
5468   prefixq(dst);
5469   emit_arith_operand(0x81, rbp, dst, imm32);
5470 }
5471 
5472 void Assembler::subq(Address dst, Register src) {
5473   InstructionMark im(this);
5474   prefixq(dst, src);
5475   emit_int8(0x29);
5476   emit_operand(src, dst);
5477 }
5478 
5479 void Assembler::subq(Register dst, int32_t imm32) {
5480   (void) prefixq_and_encode(dst->encoding());
5481   emit_arith(0x81, 0xE8, dst, imm32);
5482 }
5483 
5484 // Force generation of a 4 byte immediate value even if it fits into 8bit
5485 void Assembler::subq_imm32(Register dst, int32_t imm32) {
5486   (void) prefixq_and_encode(dst->encoding());
5487   emit_arith_imm32(0x81, 0xE8, dst, imm32);
5488 }
5489 
5490 void Assembler::subq(Register dst, Address src) {
5491   InstructionMark im(this);
5492   prefixq(src, dst);
5493   emit_int8(0x2B);
5494   emit_operand(dst, src);
5495 }
5496 
5497 void Assembler::subq(Register dst, Register src) {
5498   (void) prefixq_and_encode(dst->encoding(), src->encoding());
5499   emit_arith(0x2B, 0xC0, dst, src);
5500 }
5501 
5502 void Assembler::testq(Register dst, int32_t imm32) {
5503   // not using emit_arith because test
5504   // doesn't support sign-extension of
5505   // 8bit operands
5506   int encode = dst->encoding();
5507   if (encode == 0) {
5508     prefix(REX_W);
5509     emit_int8((unsigned char)0xA9);
5510   } else {
5511     encode = prefixq_and_encode(encode);
5512     emit_int8((unsigned char)0xF7);
5513     emit_int8((unsigned char)(0xC0 | encode));
5514   }
5515   emit_int32(imm32);
5516 }
5517 
5518 void Assembler::testq(Register dst, Register src) {
5519   (void) prefixq_and_encode(dst->encoding(), src->encoding());
5520   emit_arith(0x85, 0xC0, dst, src);
5521 }
5522 
5523 void Assembler::xaddq(Address dst, Register src) {
5524   InstructionMark im(this);
5525   prefixq(dst, src);
5526   emit_int8(0x0F);
5527   emit_int8((unsigned char)0xC1);
5528   emit_operand(src, dst);
5529 }
5530 
5531 void Assembler::xchgq(Register dst, Address src) {
5532   InstructionMark im(this);
5533   prefixq(src, dst);
5534   emit_int8((unsigned char)0x87);
5535   emit_operand(dst, src);
5536 }
5537 
5538 void Assembler::xchgq(Register dst, Register src) {
5539   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
5540   emit_int8((unsigned char)0x87);
5541   emit_int8((unsigned char)(0xc0 | encode));
5542 }
5543 
5544 void Assembler::xorq(Register dst, Register src) {
5545   (void) prefixq_and_encode(dst->encoding(), src->encoding());
5546   emit_arith(0x33, 0xC0, dst, src);
5547 }
5548 
5549 void Assembler::xorq(Register dst, Address src) {
5550   InstructionMark im(this);
5551   prefixq(src, dst);
5552   emit_int8(0x33);
5553   emit_operand(dst, src);
5554 }
5555 
5556 #endif // !LP64