1 /*
   2  * Copyright 1997-2009 Sun Microsystems, Inc.  All Rights Reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  20  * CA 95054 USA or visit www.sun.com if you need additional information or
  21  * have any questions.
  22  *
  23  */
  24 
  25 #include "incls/_precompiled.incl"
  26 #include "incls/_assembler_x86.cpp.incl"
  27 
  28 // Implementation of AddressLiteral
  29 
  30 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
  31   _is_lval = false;
  32   _target = target;
  33   switch (rtype) {
  34   case relocInfo::oop_type:
  35     // Oops are a special case. Normally they would be their own section
  36     // but in cases like icBuffer they are literals in the code stream that
  37     // we don't have a section for. We use none so that we get a literal address
  38     // which is always patchable.
  39     break;
  40   case relocInfo::external_word_type:
  41     _rspec = external_word_Relocation::spec(target);
  42     break;
  43   case relocInfo::internal_word_type:
  44     _rspec = internal_word_Relocation::spec(target);
  45     break;
  46   case relocInfo::opt_virtual_call_type:
  47     _rspec = opt_virtual_call_Relocation::spec();
  48     break;
  49   case relocInfo::static_call_type:
  50     _rspec = static_call_Relocation::spec();
  51     break;
  52   case relocInfo::runtime_call_type:
  53     _rspec = runtime_call_Relocation::spec();
  54     break;
  55   case relocInfo::poll_type:
  56   case relocInfo::poll_return_type:
  57     _rspec = Relocation::spec_simple(rtype);
  58     break;
  59   case relocInfo::none:
  60     break;
  61   default:
  62     ShouldNotReachHere();
  63     break;
  64   }
  65 }
  66 
  67 // Implementation of Address
  68 
  69 #ifdef _LP64
  70 
  71 Address Address::make_array(ArrayAddress adr) {
  72   // Not implementable on 64bit machines
  73   // Should have been handled higher up the call chain.
  74   ShouldNotReachHere();
  75   return Address();
  76 }
  77 
  78 // exceedingly dangerous constructor
  79 Address::Address(int disp, address loc, relocInfo::relocType rtype) {
  80   _base  = noreg;
  81   _index = noreg;
  82   _scale = no_scale;
  83   _disp  = disp;
  84   switch (rtype) {
  85     case relocInfo::external_word_type:
  86       _rspec = external_word_Relocation::spec(loc);
  87       break;
  88     case relocInfo::internal_word_type:
  89       _rspec = internal_word_Relocation::spec(loc);
  90       break;
  91     case relocInfo::runtime_call_type:
  92       // HMM
  93       _rspec = runtime_call_Relocation::spec();
  94       break;
  95     case relocInfo::poll_type:
  96     case relocInfo::poll_return_type:
  97       _rspec = Relocation::spec_simple(rtype);
  98       break;
  99     case relocInfo::none:
 100       break;
 101     default:
 102       ShouldNotReachHere();
 103   }
 104 }
 105 #else // LP64
 106 
 107 Address Address::make_array(ArrayAddress adr) {
 108   AddressLiteral base = adr.base();
 109   Address index = adr.index();
 110   assert(index._disp == 0, "must not have disp"); // maybe it can?
 111   Address array(index._base, index._index, index._scale, (intptr_t) base.target());
 112   array._rspec = base._rspec;
 113   return array;
 114 }
 115 
 116 // exceedingly dangerous constructor
 117 Address::Address(address loc, RelocationHolder spec) {
 118   _base  = noreg;
 119   _index = noreg;
 120   _scale = no_scale;
 121   _disp  = (intptr_t) loc;
 122   _rspec = spec;
 123 }
 124 
 125 #endif // _LP64
 126 
 127 
 128 
 129 // Convert the raw encoding form into the form expected by the constructor for
 130 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 131 // that to noreg for the Address constructor.
 132 Address Address::make_raw(int base, int index, int scale, int disp, bool disp_is_oop) {
 133   RelocationHolder rspec;
 134   if (disp_is_oop) {
 135     rspec = Relocation::spec_simple(relocInfo::oop_type);
 136   }
 137   bool valid_index = index != rsp->encoding();
 138   if (valid_index) {
 139     Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 140     madr._rspec = rspec;
 141     return madr;
 142   } else {
 143     Address madr(as_Register(base), noreg, Address::no_scale, in_ByteSize(disp));
 144     madr._rspec = rspec;
 145     return madr;
 146   }
 147 }
 148 
 149 // Implementation of Assembler
 150 
 151 int AbstractAssembler::code_fill_byte() {
 152   return (u_char)'\xF4'; // hlt
 153 }
 154 
 155 // make this go away someday
 156 void Assembler::emit_data(jint data, relocInfo::relocType rtype, int format) {
 157   if (rtype == relocInfo::none)
 158         emit_long(data);
 159   else  emit_data(data, Relocation::spec_simple(rtype), format);
 160 }
 161 
 162 void Assembler::emit_data(jint data, RelocationHolder const& rspec, int format) {
 163   assert(imm_operand == 0, "default format must be immediate in this file");
 164   assert(inst_mark() != NULL, "must be inside InstructionMark");
 165   if (rspec.type() !=  relocInfo::none) {
 166     #ifdef ASSERT
 167       check_relocation(rspec, format);
 168     #endif
 169     // Do not use AbstractAssembler::relocate, which is not intended for
 170     // embedded words.  Instead, relocate to the enclosing instruction.
 171 
 172     // hack. call32 is too wide for mask so use disp32
 173     if (format == call32_operand)
 174       code_section()->relocate(inst_mark(), rspec, disp32_operand);
 175     else
 176       code_section()->relocate(inst_mark(), rspec, format);
 177   }
 178   emit_long(data);
 179 }
 180 
 181 static int encode(Register r) {
 182   int enc = r->encoding();
 183   if (enc >= 8) {
 184     enc -= 8;
 185   }
 186   return enc;
 187 }
 188 
 189 static int encode(XMMRegister r) {
 190   int enc = r->encoding();
 191   if (enc >= 8) {
 192     enc -= 8;
 193   }
 194   return enc;
 195 }
 196 
 197 void Assembler::emit_arith_b(int op1, int op2, Register dst, int imm8) {
 198   assert(dst->has_byte_register(), "must have byte register");
 199   assert(isByte(op1) && isByte(op2), "wrong opcode");
 200   assert(isByte(imm8), "not a byte");
 201   assert((op1 & 0x01) == 0, "should be 8bit operation");
 202   emit_byte(op1);
 203   emit_byte(op2 | encode(dst));
 204   emit_byte(imm8);
 205 }
 206 
 207 
 208 void Assembler::emit_arith(int op1, int op2, Register dst, int32_t imm32) {
 209   assert(isByte(op1) && isByte(op2), "wrong opcode");
 210   assert((op1 & 0x01) == 1, "should be 32bit operation");
 211   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 212   if (is8bit(imm32)) {
 213     emit_byte(op1 | 0x02); // set sign bit
 214     emit_byte(op2 | encode(dst));
 215     emit_byte(imm32 & 0xFF);
 216   } else {
 217     emit_byte(op1);
 218     emit_byte(op2 | encode(dst));
 219     emit_long(imm32);
 220   }
 221 }
 222 
 223 // immediate-to-memory forms
 224 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 225   assert((op1 & 0x01) == 1, "should be 32bit operation");
 226   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 227   if (is8bit(imm32)) {
 228     emit_byte(op1 | 0x02); // set sign bit
 229     emit_operand(rm, adr, 1);
 230     emit_byte(imm32 & 0xFF);
 231   } else {
 232     emit_byte(op1);
 233     emit_operand(rm, adr, 4);
 234     emit_long(imm32);
 235   }
 236 }
 237 
 238 void Assembler::emit_arith(int op1, int op2, Register dst, jobject obj) {
 239   LP64_ONLY(ShouldNotReachHere());
 240   assert(isByte(op1) && isByte(op2), "wrong opcode");
 241   assert((op1 & 0x01) == 1, "should be 32bit operation");
 242   assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
 243   InstructionMark im(this);
 244   emit_byte(op1);
 245   emit_byte(op2 | encode(dst));
 246   emit_data((intptr_t)obj, relocInfo::oop_type, 0);
 247 }
 248 
 249 
 250 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 251   assert(isByte(op1) && isByte(op2), "wrong opcode");
 252   emit_byte(op1);
 253   emit_byte(op2 | encode(dst) << 3 | encode(src));
 254 }
 255 
 256 
 257 void Assembler::emit_operand(Register reg, Register base, Register index,
 258                              Address::ScaleFactor scale, int disp,
 259                              RelocationHolder const& rspec,
 260                              int rip_relative_correction) {
 261   relocInfo::relocType rtype = (relocInfo::relocType) rspec.type();
 262 
 263   // Encode the registers as needed in the fields they are used in
 264 
 265   int regenc = encode(reg) << 3;
 266   int indexenc = index->is_valid() ? encode(index) << 3 : 0;
 267   int baseenc = base->is_valid() ? encode(base) : 0;
 268 
 269   if (base->is_valid()) {
 270     if (index->is_valid()) {
 271       assert(scale != Address::no_scale, "inconsistent address");
 272       // [base + index*scale + disp]
 273       if (disp == 0 && rtype == relocInfo::none  &&
 274           base != rbp LP64_ONLY(&& base != r13)) {
 275         // [base + index*scale]
 276         // [00 reg 100][ss index base]
 277         assert(index != rsp, "illegal addressing mode");
 278         emit_byte(0x04 | regenc);
 279         emit_byte(scale << 6 | indexenc | baseenc);
 280       } else if (is8bit(disp) && rtype == relocInfo::none) {
 281         // [base + index*scale + imm8]
 282         // [01 reg 100][ss index base] imm8
 283         assert(index != rsp, "illegal addressing mode");
 284         emit_byte(0x44 | regenc);
 285         emit_byte(scale << 6 | indexenc | baseenc);
 286         emit_byte(disp & 0xFF);
 287       } else {
 288         // [base + index*scale + disp32]
 289         // [10 reg 100][ss index base] disp32
 290         assert(index != rsp, "illegal addressing mode");
 291         emit_byte(0x84 | regenc);
 292         emit_byte(scale << 6 | indexenc | baseenc);
 293         emit_data(disp, rspec, disp32_operand);
 294       }
 295     } else if (base == rsp LP64_ONLY(|| base == r12)) {
 296       // [rsp + disp]
 297       if (disp == 0 && rtype == relocInfo::none) {
 298         // [rsp]
 299         // [00 reg 100][00 100 100]
 300         emit_byte(0x04 | regenc);
 301         emit_byte(0x24);
 302       } else if (is8bit(disp) && rtype == relocInfo::none) {
 303         // [rsp + imm8]
 304         // [01 reg 100][00 100 100] disp8
 305         emit_byte(0x44 | regenc);
 306         emit_byte(0x24);
 307         emit_byte(disp & 0xFF);
 308       } else {
 309         // [rsp + imm32]
 310         // [10 reg 100][00 100 100] disp32
 311         emit_byte(0x84 | regenc);
 312         emit_byte(0x24);
 313         emit_data(disp, rspec, disp32_operand);
 314       }
 315     } else {
 316       // [base + disp]
 317       assert(base != rsp LP64_ONLY(&& base != r12), "illegal addressing mode");
 318       if (disp == 0 && rtype == relocInfo::none &&
 319           base != rbp LP64_ONLY(&& base != r13)) {
 320         // [base]
 321         // [00 reg base]
 322         emit_byte(0x00 | regenc | baseenc);
 323       } else if (is8bit(disp) && rtype == relocInfo::none) {
 324         // [base + disp8]
 325         // [01 reg base] disp8
 326         emit_byte(0x40 | regenc | baseenc);
 327         emit_byte(disp & 0xFF);
 328       } else {
 329         // [base + disp32]
 330         // [10 reg base] disp32
 331         emit_byte(0x80 | regenc | baseenc);
 332         emit_data(disp, rspec, disp32_operand);
 333       }
 334     }
 335   } else {
 336     if (index->is_valid()) {
 337       assert(scale != Address::no_scale, "inconsistent address");
 338       // [index*scale + disp]
 339       // [00 reg 100][ss index 101] disp32
 340       assert(index != rsp, "illegal addressing mode");
 341       emit_byte(0x04 | regenc);
 342       emit_byte(scale << 6 | indexenc | 0x05);
 343       emit_data(disp, rspec, disp32_operand);
 344     } else if (rtype != relocInfo::none ) {
 345       // [disp] (64bit) RIP-RELATIVE (32bit) abs
 346       // [00 000 101] disp32
 347 
 348       emit_byte(0x05 | regenc);
 349       // Note that the RIP-rel. correction applies to the generated
 350       // disp field, but _not_ to the target address in the rspec.
 351 
 352       // disp was created by converting the target address minus the pc
 353       // at the start of the instruction. That needs more correction here.
 354       // intptr_t disp = target - next_ip;
 355       assert(inst_mark() != NULL, "must be inside InstructionMark");
 356       address next_ip = pc() + sizeof(int32_t) + rip_relative_correction;
 357       int64_t adjusted = disp;
 358       // Do rip-rel adjustment for 64bit
 359       LP64_ONLY(adjusted -=  (next_ip - inst_mark()));
 360       assert(is_simm32(adjusted),
 361              "must be 32bit offset (RIP relative address)");
 362       emit_data((int32_t) adjusted, rspec, disp32_operand);
 363 
 364     } else {
 365       // 32bit never did this, did everything as the rip-rel/disp code above
 366       // [disp] ABSOLUTE
 367       // [00 reg 100][00 100 101] disp32
 368       emit_byte(0x04 | regenc);
 369       emit_byte(0x25);
 370       emit_data(disp, rspec, disp32_operand);
 371     }
 372   }
 373 }
 374 
 375 void Assembler::emit_operand(XMMRegister reg, Register base, Register index,
 376                              Address::ScaleFactor scale, int disp,
 377                              RelocationHolder const& rspec) {
 378   emit_operand((Register)reg, base, index, scale, disp, rspec);
 379 }
 380 
 381 // Secret local extension to Assembler::WhichOperand:
 382 #define end_pc_operand (_WhichOperand_limit)
 383 
 384 address Assembler::locate_operand(address inst, WhichOperand which) {
 385   // Decode the given instruction, and return the address of
 386   // an embedded 32-bit operand word.
 387 
 388   // If "which" is disp32_operand, selects the displacement portion
 389   // of an effective address specifier.
 390   // If "which" is imm64_operand, selects the trailing immediate constant.
 391   // If "which" is call32_operand, selects the displacement of a call or jump.
 392   // Caller is responsible for ensuring that there is such an operand,
 393   // and that it is 32/64 bits wide.
 394 
 395   // If "which" is end_pc_operand, find the end of the instruction.
 396 
 397   address ip = inst;
 398   bool is_64bit = false;
 399 
 400   debug_only(bool has_disp32 = false);
 401   int tail_size = 0; // other random bytes (#32, #16, etc.) at end of insn
 402 
 403   again_after_prefix:
 404   switch (0xFF & *ip++) {
 405 
 406   // These convenience macros generate groups of "case" labels for the switch.
 407 #define REP4(x) (x)+0: case (x)+1: case (x)+2: case (x)+3
 408 #define REP8(x) (x)+0: case (x)+1: case (x)+2: case (x)+3: \
 409              case (x)+4: case (x)+5: case (x)+6: case (x)+7
 410 #define REP16(x) REP8((x)+0): \
 411               case REP8((x)+8)
 412 
 413   case CS_segment:
 414   case SS_segment:
 415   case DS_segment:
 416   case ES_segment:
 417   case FS_segment:
 418   case GS_segment:
 419     // Seems dubious
 420     LP64_ONLY(assert(false, "shouldn't have that prefix"));
 421     assert(ip == inst+1, "only one prefix allowed");
 422     goto again_after_prefix;
 423 
 424   case 0x67:
 425   case REX:
 426   case REX_B:
 427   case REX_X:
 428   case REX_XB:
 429   case REX_R:
 430   case REX_RB:
 431   case REX_RX:
 432   case REX_RXB:
 433     NOT_LP64(assert(false, "64bit prefixes"));
 434     goto again_after_prefix;
 435 
 436   case REX_W:
 437   case REX_WB:
 438   case REX_WX:
 439   case REX_WXB:
 440   case REX_WR:
 441   case REX_WRB:
 442   case REX_WRX:
 443   case REX_WRXB:
 444     NOT_LP64(assert(false, "64bit prefixes"));
 445     is_64bit = true;
 446     goto again_after_prefix;
 447 
 448   case 0xFF: // pushq a; decl a; incl a; call a; jmp a
 449   case 0x88: // movb a, r
 450   case 0x89: // movl a, r
 451   case 0x8A: // movb r, a
 452   case 0x8B: // movl r, a
 453   case 0x8F: // popl a
 454     debug_only(has_disp32 = true);
 455     break;
 456 
 457   case 0x68: // pushq #32
 458     if (which == end_pc_operand) {
 459       return ip + 4;
 460     }
 461     assert(which == imm_operand && !is_64bit, "pushl has no disp32 or 64bit immediate");
 462     return ip;                  // not produced by emit_operand
 463 
 464   case 0x66: // movw ... (size prefix)
 465     again_after_size_prefix2:
 466     switch (0xFF & *ip++) {
 467     case REX:
 468     case REX_B:
 469     case REX_X:
 470     case REX_XB:
 471     case REX_R:
 472     case REX_RB:
 473     case REX_RX:
 474     case REX_RXB:
 475     case REX_W:
 476     case REX_WB:
 477     case REX_WX:
 478     case REX_WXB:
 479     case REX_WR:
 480     case REX_WRB:
 481     case REX_WRX:
 482     case REX_WRXB:
 483       NOT_LP64(assert(false, "64bit prefix found"));
 484       goto again_after_size_prefix2;
 485     case 0x8B: // movw r, a
 486     case 0x89: // movw a, r
 487       debug_only(has_disp32 = true);
 488       break;
 489     case 0xC7: // movw a, #16
 490       debug_only(has_disp32 = true);
 491       tail_size = 2;  // the imm16
 492       break;
 493     case 0x0F: // several SSE/SSE2 variants
 494       ip--;    // reparse the 0x0F
 495       goto again_after_prefix;
 496     default:
 497       ShouldNotReachHere();
 498     }
 499     break;
 500 
 501   case REP8(0xB8): // movl/q r, #32/#64(oop?)
 502     if (which == end_pc_operand)  return ip + (is_64bit ? 8 : 4);
 503     // these asserts are somewhat nonsensical
 504 #ifndef _LP64
 505     assert(which == imm_operand || which == disp32_operand, "");
 506 #else
 507     assert((which == call32_operand || which == imm_operand) && is_64bit ||
 508            which == narrow_oop_operand && !is_64bit, "");
 509 #endif // _LP64
 510     return ip;
 511 
 512   case 0x69: // imul r, a, #32
 513   case 0xC7: // movl a, #32(oop?)
 514     tail_size = 4;
 515     debug_only(has_disp32 = true); // has both kinds of operands!
 516     break;
 517 
 518   case 0x0F: // movx..., etc.
 519     switch (0xFF & *ip++) {
 520     case 0x12: // movlps
 521     case 0x28: // movaps
 522     case 0x2E: // ucomiss
 523     case 0x2F: // comiss
 524     case 0x54: // andps
 525     case 0x55: // andnps
 526     case 0x56: // orps
 527     case 0x57: // xorps
 528     case 0x6E: // movd
 529     case 0x7E: // movd
 530     case 0xAE: // ldmxcsr   a
 531       // 64bit side says it these have both operands but that doesn't
 532       // appear to be true
 533       debug_only(has_disp32 = true);
 534       break;
 535 
 536     case 0xAD: // shrd r, a, %cl
 537     case 0xAF: // imul r, a
 538     case 0xBE: // movsbl r, a (movsxb)
 539     case 0xBF: // movswl r, a (movsxw)
 540     case 0xB6: // movzbl r, a (movzxb)
 541     case 0xB7: // movzwl r, a (movzxw)
 542     case REP16(0x40): // cmovl cc, r, a
 543     case 0xB0: // cmpxchgb
 544     case 0xB1: // cmpxchg
 545     case 0xC1: // xaddl
 546     case 0xC7: // cmpxchg8
 547     case REP16(0x90): // setcc a
 548       debug_only(has_disp32 = true);
 549       // fall out of the switch to decode the address
 550       break;
 551 
 552     case 0xAC: // shrd r, a, #8
 553       debug_only(has_disp32 = true);
 554       tail_size = 1;  // the imm8
 555       break;
 556 
 557     case REP16(0x80): // jcc rdisp32
 558       if (which == end_pc_operand)  return ip + 4;
 559       assert(which == call32_operand, "jcc has no disp32 or imm");
 560       return ip;
 561     default:
 562       ShouldNotReachHere();
 563     }
 564     break;
 565 
 566   case 0x81: // addl a, #32; addl r, #32
 567     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 568     // on 32bit in the case of cmpl, the imm might be an oop
 569     tail_size = 4;
 570     debug_only(has_disp32 = true); // has both kinds of operands!
 571     break;
 572 
 573   case 0x83: // addl a, #8; addl r, #8
 574     // also: orl, adcl, sbbl, andl, subl, xorl, cmpl
 575     debug_only(has_disp32 = true); // has both kinds of operands!
 576     tail_size = 1;
 577     break;
 578 
 579   case 0x9B:
 580     switch (0xFF & *ip++) {
 581     case 0xD9: // fnstcw a
 582       debug_only(has_disp32 = true);
 583       break;
 584     default:
 585       ShouldNotReachHere();
 586     }
 587     break;
 588 
 589   case REP4(0x00): // addb a, r; addl a, r; addb r, a; addl r, a
 590   case REP4(0x10): // adc...
 591   case REP4(0x20): // and...
 592   case REP4(0x30): // xor...
 593   case REP4(0x08): // or...
 594   case REP4(0x18): // sbb...
 595   case REP4(0x28): // sub...
 596   case 0xF7: // mull a
 597   case 0x8D: // lea r, a
 598   case 0x87: // xchg r, a
 599   case REP4(0x38): // cmp...
 600   case 0x85: // test r, a
 601     debug_only(has_disp32 = true); // has both kinds of operands!
 602     break;
 603 
 604   case 0xC1: // sal a, #8; sar a, #8; shl a, #8; shr a, #8
 605   case 0xC6: // movb a, #8
 606   case 0x80: // cmpb a, #8
 607   case 0x6B: // imul r, a, #8
 608     debug_only(has_disp32 = true); // has both kinds of operands!
 609     tail_size = 1; // the imm8
 610     break;
 611 
 612   case 0xE8: // call rdisp32
 613   case 0xE9: // jmp  rdisp32
 614     if (which == end_pc_operand)  return ip + 4;
 615     assert(which == call32_operand, "call has no disp32 or imm");
 616     return ip;
 617 
 618   case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 619   case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 620   case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 621   case 0xDD: // fld_d a; fst_d a; fstp_d a
 622   case 0xDB: // fild_s a; fistp_s a; fld_x a; fstp_x a
 623   case 0xDF: // fild_d a; fistp_d a
 624   case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 625   case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 626   case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 627     debug_only(has_disp32 = true);
 628     break;
 629 
 630   case 0xF0:                    // Lock
 631     assert(os::is_MP(), "only on MP");
 632     goto again_after_prefix;
 633 
 634   case 0xF3:                    // For SSE
 635   case 0xF2:                    // For SSE2
 636     switch (0xFF & *ip++) {
 637     case REX:
 638     case REX_B:
 639     case REX_X:
 640     case REX_XB:
 641     case REX_R:
 642     case REX_RB:
 643     case REX_RX:
 644     case REX_RXB:
 645     case REX_W:
 646     case REX_WB:
 647     case REX_WX:
 648     case REX_WXB:
 649     case REX_WR:
 650     case REX_WRB:
 651     case REX_WRX:
 652     case REX_WRXB:
 653       NOT_LP64(assert(false, "found 64bit prefix"));
 654       ip++;
 655     default:
 656       ip++;
 657     }
 658     debug_only(has_disp32 = true); // has both kinds of operands!
 659     break;
 660 
 661   default:
 662     ShouldNotReachHere();
 663 
 664 #undef REP8
 665 #undef REP16
 666   }
 667 
 668   assert(which != call32_operand, "instruction is not a call, jmp, or jcc");
 669 #ifdef _LP64
 670   assert(which != imm_operand, "instruction is not a movq reg, imm64");
 671 #else
 672   // assert(which != imm_operand || has_imm32, "instruction has no imm32 field");
 673   assert(which != imm_operand || has_disp32, "instruction has no imm32 field");
 674 #endif // LP64
 675   assert(which != disp32_operand || has_disp32, "instruction has no disp32 field");
 676 
 677   // parse the output of emit_operand
 678   int op2 = 0xFF & *ip++;
 679   int base = op2 & 0x07;
 680   int op3 = -1;
 681   const int b100 = 4;
 682   const int b101 = 5;
 683   if (base == b100 && (op2 >> 6) != 3) {
 684     op3 = 0xFF & *ip++;
 685     base = op3 & 0x07;   // refetch the base
 686   }
 687   // now ip points at the disp (if any)
 688 
 689   switch (op2 >> 6) {
 690   case 0:
 691     // [00 reg  100][ss index base]
 692     // [00 reg  100][00   100  esp]
 693     // [00 reg base]
 694     // [00 reg  100][ss index  101][disp32]
 695     // [00 reg  101]               [disp32]
 696 
 697     if (base == b101) {
 698       if (which == disp32_operand)
 699         return ip;              // caller wants the disp32
 700       ip += 4;                  // skip the disp32
 701     }
 702     break;
 703 
 704   case 1:
 705     // [01 reg  100][ss index base][disp8]
 706     // [01 reg  100][00   100  esp][disp8]
 707     // [01 reg base]               [disp8]
 708     ip += 1;                    // skip the disp8
 709     break;
 710 
 711   case 2:
 712     // [10 reg  100][ss index base][disp32]
 713     // [10 reg  100][00   100  esp][disp32]
 714     // [10 reg base]               [disp32]
 715     if (which == disp32_operand)
 716       return ip;                // caller wants the disp32
 717     ip += 4;                    // skip the disp32
 718     break;
 719 
 720   case 3:
 721     // [11 reg base]  (not a memory addressing mode)
 722     break;
 723   }
 724 
 725   if (which == end_pc_operand) {
 726     return ip + tail_size;
 727   }
 728 
 729 #ifdef _LP64
 730   assert(which == narrow_oop_operand && !is_64bit, "instruction is not a movl adr, imm32");
 731 #else
 732   assert(which == imm_operand, "instruction has only an imm field");
 733 #endif // LP64
 734   return ip;
 735 }
 736 
 737 address Assembler::locate_next_instruction(address inst) {
 738   // Secretly share code with locate_operand:
 739   return locate_operand(inst, end_pc_operand);
 740 }
 741 
 742 
 743 #ifdef ASSERT
 744 void Assembler::check_relocation(RelocationHolder const& rspec, int format) {
 745   address inst = inst_mark();
 746   assert(inst != NULL && inst < pc(), "must point to beginning of instruction");
 747   address opnd;
 748 
 749   Relocation* r = rspec.reloc();
 750   if (r->type() == relocInfo::none) {
 751     return;
 752   } else if (r->is_call() || format == call32_operand) {
 753     // assert(format == imm32_operand, "cannot specify a nonzero format");
 754     opnd = locate_operand(inst, call32_operand);
 755   } else if (r->is_data()) {
 756     assert(format == imm_operand || format == disp32_operand
 757            LP64_ONLY(|| format == narrow_oop_operand), "format ok");
 758     opnd = locate_operand(inst, (WhichOperand)format);
 759   } else {
 760     assert(format == imm_operand, "cannot specify a format");
 761     return;
 762   }
 763   assert(opnd == pc(), "must put operand where relocs can find it");
 764 }
 765 #endif // ASSERT
 766 
 767 void Assembler::emit_operand32(Register reg, Address adr) {
 768   assert(reg->encoding() < 8, "no extended registers");
 769   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
 770   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 771                adr._rspec);
 772 }
 773 
 774 void Assembler::emit_operand(Register reg, Address adr,
 775                              int rip_relative_correction) {
 776   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 777                adr._rspec,
 778                rip_relative_correction);
 779 }
 780 
 781 void Assembler::emit_operand(XMMRegister reg, Address adr) {
 782   emit_operand(reg, adr._base, adr._index, adr._scale, adr._disp,
 783                adr._rspec);
 784 }
 785 
 786 // MMX operations
 787 void Assembler::emit_operand(MMXRegister reg, Address adr) {
 788   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
 789   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
 790 }
 791 
 792 // work around gcc (3.2.1-7a) bug
 793 void Assembler::emit_operand(Address adr, MMXRegister reg) {
 794   assert(!adr.base_needs_rex() && !adr.index_needs_rex(), "no extended registers");
 795   emit_operand((Register)reg, adr._base, adr._index, adr._scale, adr._disp, adr._rspec);
 796 }
 797 
 798 
 799 void Assembler::emit_farith(int b1, int b2, int i) {
 800   assert(isByte(b1) && isByte(b2), "wrong opcode");
 801   assert(0 <= i &&  i < 8, "illegal stack offset");
 802   emit_byte(b1);
 803   emit_byte(b2 + i);
 804 }
 805 
 806 
 807 // Now the Assembler instruction (identical for 32/64 bits)
 808 
 809 void Assembler::adcl(Register dst, int32_t imm32) {
 810   prefix(dst);
 811   emit_arith(0x81, 0xD0, dst, imm32);
 812 }
 813 
 814 void Assembler::adcl(Register dst, Address src) {
 815   InstructionMark im(this);
 816   prefix(src, dst);
 817   emit_byte(0x13);
 818   emit_operand(dst, src);
 819 }
 820 
 821 void Assembler::adcl(Register dst, Register src) {
 822   (void) prefix_and_encode(dst->encoding(), src->encoding());
 823   emit_arith(0x13, 0xC0, dst, src);
 824 }
 825 
 826 void Assembler::addl(Address dst, int32_t imm32) {
 827   InstructionMark im(this);
 828   prefix(dst);
 829   emit_arith_operand(0x81, rax, dst, imm32);
 830 }
 831 
 832 void Assembler::addl(Address dst, Register src) {
 833   InstructionMark im(this);
 834   prefix(dst, src);
 835   emit_byte(0x01);
 836   emit_operand(src, dst);
 837 }
 838 
 839 void Assembler::addl(Register dst, int32_t imm32) {
 840   prefix(dst);
 841   emit_arith(0x81, 0xC0, dst, imm32);
 842 }
 843 
 844 void Assembler::addl(Register dst, Address src) {
 845   InstructionMark im(this);
 846   prefix(src, dst);
 847   emit_byte(0x03);
 848   emit_operand(dst, src);
 849 }
 850 
 851 void Assembler::addl(Register dst, Register src) {
 852   (void) prefix_and_encode(dst->encoding(), src->encoding());
 853   emit_arith(0x03, 0xC0, dst, src);
 854 }
 855 
 856 void Assembler::addr_nop_4() {
 857   // 4 bytes: NOP DWORD PTR [EAX+0]
 858   emit_byte(0x0F);
 859   emit_byte(0x1F);
 860   emit_byte(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 861   emit_byte(0);    // 8-bits offset (1 byte)
 862 }
 863 
 864 void Assembler::addr_nop_5() {
 865   // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 866   emit_byte(0x0F);
 867   emit_byte(0x1F);
 868   emit_byte(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 869   emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 870   emit_byte(0);    // 8-bits offset (1 byte)
 871 }
 872 
 873 void Assembler::addr_nop_7() {
 874   // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 875   emit_byte(0x0F);
 876   emit_byte(0x1F);
 877   emit_byte(0x80); // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 878   emit_long(0);    // 32-bits offset (4 bytes)
 879 }
 880 
 881 void Assembler::addr_nop_8() {
 882   // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 883   emit_byte(0x0F);
 884   emit_byte(0x1F);
 885   emit_byte(0x84); // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 886   emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 887   emit_long(0);    // 32-bits offset (4 bytes)
 888 }
 889 
 890 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 891   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 892   emit_byte(0xF2);
 893   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 894   emit_byte(0x0F);
 895   emit_byte(0x58);
 896   emit_byte(0xC0 | encode);
 897 }
 898 
 899 void Assembler::addsd(XMMRegister dst, Address src) {
 900   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 901   InstructionMark im(this);
 902   emit_byte(0xF2);
 903   prefix(src, dst);
 904   emit_byte(0x0F);
 905   emit_byte(0x58);
 906   emit_operand(dst, src);
 907 }
 908 
 909 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 910   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 911   emit_byte(0xF3);
 912   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 913   emit_byte(0x0F);
 914   emit_byte(0x58);
 915   emit_byte(0xC0 | encode);
 916 }
 917 
 918 void Assembler::addss(XMMRegister dst, Address src) {
 919   NOT_LP64(assert(VM_Version::supports_sse(), ""));
 920   InstructionMark im(this);
 921   emit_byte(0xF3);
 922   prefix(src, dst);
 923   emit_byte(0x0F);
 924   emit_byte(0x58);
 925   emit_operand(dst, src);
 926 }
 927 
 928 void Assembler::andl(Register dst, int32_t imm32) {
 929   prefix(dst);
 930   emit_arith(0x81, 0xE0, dst, imm32);
 931 }
 932 
 933 void Assembler::andl(Register dst, Address src) {
 934   InstructionMark im(this);
 935   prefix(src, dst);
 936   emit_byte(0x23);
 937   emit_operand(dst, src);
 938 }
 939 
 940 void Assembler::andl(Register dst, Register src) {
 941   (void) prefix_and_encode(dst->encoding(), src->encoding());
 942   emit_arith(0x23, 0xC0, dst, src);
 943 }
 944 
 945 void Assembler::andpd(XMMRegister dst, Address src) {
 946   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 947   InstructionMark im(this);
 948   emit_byte(0x66);
 949   prefix(src, dst);
 950   emit_byte(0x0F);
 951   emit_byte(0x54);
 952   emit_operand(dst, src);
 953 }
 954 
 955 void Assembler::bsfl(Register dst, Register src) {
 956   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 957   emit_byte(0x0F);
 958   emit_byte(0xBC);
 959   emit_byte(0xC0 | encode);
 960 }
 961 
 962 void Assembler::bsrl(Register dst, Register src) {
 963   assert(!VM_Version::supports_lzcnt(), "encoding is treated as LZCNT");
 964   int encode = prefix_and_encode(dst->encoding(), src->encoding());
 965   emit_byte(0x0F);
 966   emit_byte(0xBD);
 967   emit_byte(0xC0 | encode);
 968 }
 969 
 970 void Assembler::bswapl(Register reg) { // bswap
 971   int encode = prefix_and_encode(reg->encoding());
 972   emit_byte(0x0F);
 973   emit_byte(0xC8 | encode);
 974 }
 975 
 976 void Assembler::call(Label& L, relocInfo::relocType rtype) {
 977   // suspect disp32 is always good
 978   int operand = LP64_ONLY(disp32_operand) NOT_LP64(imm_operand);
 979 
 980   if (L.is_bound()) {
 981     const int long_size = 5;
 982     int offs = (int)( target(L) - pc() );
 983     assert(offs <= 0, "assembler error");
 984     InstructionMark im(this);
 985     // 1110 1000 #32-bit disp
 986     emit_byte(0xE8);
 987     emit_data(offs - long_size, rtype, operand);
 988   } else {
 989     InstructionMark im(this);
 990     // 1110 1000 #32-bit disp
 991     L.add_patch_at(code(), locator());
 992 
 993     emit_byte(0xE8);
 994     emit_data(int(0), rtype, operand);
 995   }
 996 }
 997 
 998 void Assembler::call(Register dst) {
 999   // This was originally using a 32bit register encoding
1000   // and surely we want 64bit!
1001   // this is a 32bit encoding but in 64bit mode the default
1002   // operand size is 64bit so there is no need for the
1003   // wide prefix. So prefix only happens if we use the
1004   // new registers. Much like push/pop.
1005   int x = offset();
1006   // this may be true but dbx disassembles it as if it
1007   // were 32bits...
1008   // int encode = prefix_and_encode(dst->encoding());
1009   // if (offset() != x) assert(dst->encoding() >= 8, "what?");
1010   int encode = prefixq_and_encode(dst->encoding());
1011 
1012   emit_byte(0xFF);
1013   emit_byte(0xD0 | encode);
1014 }
1015 
1016 
1017 void Assembler::call(Address adr) {
1018   InstructionMark im(this);
1019   prefix(adr);
1020   emit_byte(0xFF);
1021   emit_operand(rdx, adr);
1022 }
1023 
1024 void Assembler::call_literal(address entry, RelocationHolder const& rspec) {
1025   assert(entry != NULL, "call most probably wrong");
1026   InstructionMark im(this);
1027   emit_byte(0xE8);
1028   intptr_t disp = entry - (_code_pos + sizeof(int32_t));
1029   assert(is_simm32(disp), "must be 32bit offset (call2)");
1030   // Technically, should use call32_operand, but this format is
1031   // implied by the fact that we're emitting a call instruction.
1032 
1033   int operand = LP64_ONLY(disp32_operand) NOT_LP64(call32_operand);
1034   emit_data((int) disp, rspec, operand);
1035 }
1036 
1037 void Assembler::cdql() {
1038   emit_byte(0x99);
1039 }
1040 
1041 void Assembler::cmovl(Condition cc, Register dst, Register src) {
1042   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1043   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1044   emit_byte(0x0F);
1045   emit_byte(0x40 | cc);
1046   emit_byte(0xC0 | encode);
1047 }
1048 
1049 
1050 void Assembler::cmovl(Condition cc, Register dst, Address src) {
1051   NOT_LP64(guarantee(VM_Version::supports_cmov(), "illegal instruction"));
1052   prefix(src, dst);
1053   emit_byte(0x0F);
1054   emit_byte(0x40 | cc);
1055   emit_operand(dst, src);
1056 }
1057 
1058 void Assembler::cmpb(Address dst, int imm8) {
1059   InstructionMark im(this);
1060   prefix(dst);
1061   emit_byte(0x80);
1062   emit_operand(rdi, dst, 1);
1063   emit_byte(imm8);
1064 }
1065 
1066 void Assembler::cmpl(Address dst, int32_t imm32) {
1067   InstructionMark im(this);
1068   prefix(dst);
1069   emit_byte(0x81);
1070   emit_operand(rdi, dst, 4);
1071   emit_long(imm32);
1072 }
1073 
1074 void Assembler::cmpl(Register dst, int32_t imm32) {
1075   prefix(dst);
1076   emit_arith(0x81, 0xF8, dst, imm32);
1077 }
1078 
1079 void Assembler::cmpl(Register dst, Register src) {
1080   (void) prefix_and_encode(dst->encoding(), src->encoding());
1081   emit_arith(0x3B, 0xC0, dst, src);
1082 }
1083 
1084 
1085 void Assembler::cmpl(Register dst, Address  src) {
1086   InstructionMark im(this);
1087   prefix(src, dst);
1088   emit_byte(0x3B);
1089   emit_operand(dst, src);
1090 }
1091 
1092 void Assembler::cmpw(Address dst, int imm16) {
1093   InstructionMark im(this);
1094   assert(!dst.base_needs_rex() && !dst.index_needs_rex(), "no extended registers");
1095   emit_byte(0x66);
1096   emit_byte(0x81);
1097   emit_operand(rdi, dst, 2);
1098   emit_word(imm16);
1099 }
1100 
1101 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
1102 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
1103 // The ZF is set if the compared values were equal, and cleared otherwise.
1104 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
1105   if (Atomics & 2) {
1106      // caveat: no instructionmark, so this isn't relocatable.
1107      // Emit a synthetic, non-atomic, CAS equivalent.
1108      // Beware.  The synthetic form sets all ICCs, not just ZF.
1109      // cmpxchg r,[m] is equivalent to rax, = CAS (m, rax, r)
1110      cmpl(rax, adr);
1111      movl(rax, adr);
1112      if (reg != rax) {
1113         Label L ;
1114         jcc(Assembler::notEqual, L);
1115         movl(adr, reg);
1116         bind(L);
1117      }
1118   } else {
1119      InstructionMark im(this);
1120      prefix(adr, reg);
1121      emit_byte(0x0F);
1122      emit_byte(0xB1);
1123      emit_operand(reg, adr);
1124   }
1125 }
1126 
1127 void Assembler::comisd(XMMRegister dst, Address src) {
1128   // NOTE: dbx seems to decode this as comiss even though the
1129   // 0x66 is there. Strangly ucomisd comes out correct
1130   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1131   emit_byte(0x66);
1132   comiss(dst, src);
1133 }
1134 
1135 void Assembler::comiss(XMMRegister dst, Address src) {
1136   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1137 
1138   InstructionMark im(this);
1139   prefix(src, dst);
1140   emit_byte(0x0F);
1141   emit_byte(0x2F);
1142   emit_operand(dst, src);
1143 }
1144 
1145 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
1146   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1147   emit_byte(0xF3);
1148   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1149   emit_byte(0x0F);
1150   emit_byte(0xE6);
1151   emit_byte(0xC0 | encode);
1152 }
1153 
1154 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
1155   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1156   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1157   emit_byte(0x0F);
1158   emit_byte(0x5B);
1159   emit_byte(0xC0 | encode);
1160 }
1161 
1162 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
1163   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1164   emit_byte(0xF2);
1165   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1166   emit_byte(0x0F);
1167   emit_byte(0x5A);
1168   emit_byte(0xC0 | encode);
1169 }
1170 
1171 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
1172   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1173   emit_byte(0xF2);
1174   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1175   emit_byte(0x0F);
1176   emit_byte(0x2A);
1177   emit_byte(0xC0 | encode);
1178 }
1179 
1180 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
1181   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1182   emit_byte(0xF3);
1183   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1184   emit_byte(0x0F);
1185   emit_byte(0x2A);
1186   emit_byte(0xC0 | encode);
1187 }
1188 
1189 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
1190   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1191   emit_byte(0xF3);
1192   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1193   emit_byte(0x0F);
1194   emit_byte(0x5A);
1195   emit_byte(0xC0 | encode);
1196 }
1197 
1198 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
1199   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1200   emit_byte(0xF2);
1201   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1202   emit_byte(0x0F);
1203   emit_byte(0x2C);
1204   emit_byte(0xC0 | encode);
1205 }
1206 
1207 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
1208   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1209   emit_byte(0xF3);
1210   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1211   emit_byte(0x0F);
1212   emit_byte(0x2C);
1213   emit_byte(0xC0 | encode);
1214 }
1215 
1216 void Assembler::decl(Address dst) {
1217   // Don't use it directly. Use MacroAssembler::decrement() instead.
1218   InstructionMark im(this);
1219   prefix(dst);
1220   emit_byte(0xFF);
1221   emit_operand(rcx, dst);
1222 }
1223 
1224 void Assembler::divsd(XMMRegister dst, Address src) {
1225   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1226   InstructionMark im(this);
1227   emit_byte(0xF2);
1228   prefix(src, dst);
1229   emit_byte(0x0F);
1230   emit_byte(0x5E);
1231   emit_operand(dst, src);
1232 }
1233 
1234 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
1235   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1236   emit_byte(0xF2);
1237   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1238   emit_byte(0x0F);
1239   emit_byte(0x5E);
1240   emit_byte(0xC0 | encode);
1241 }
1242 
1243 void Assembler::divss(XMMRegister dst, Address src) {
1244   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1245   InstructionMark im(this);
1246   emit_byte(0xF3);
1247   prefix(src, dst);
1248   emit_byte(0x0F);
1249   emit_byte(0x5E);
1250   emit_operand(dst, src);
1251 }
1252 
1253 void Assembler::divss(XMMRegister dst, XMMRegister src) {
1254   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1255   emit_byte(0xF3);
1256   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1257   emit_byte(0x0F);
1258   emit_byte(0x5E);
1259   emit_byte(0xC0 | encode);
1260 }
1261 
1262 void Assembler::emms() {
1263   NOT_LP64(assert(VM_Version::supports_mmx(), ""));
1264   emit_byte(0x0F);
1265   emit_byte(0x77);
1266 }
1267 
1268 void Assembler::hlt() {
1269   emit_byte(0xF4);
1270 }
1271 
1272 void Assembler::idivl(Register src) {
1273   int encode = prefix_and_encode(src->encoding());
1274   emit_byte(0xF7);
1275   emit_byte(0xF8 | encode);
1276 }
1277 
1278 void Assembler::imull(Register dst, Register src) {
1279   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1280   emit_byte(0x0F);
1281   emit_byte(0xAF);
1282   emit_byte(0xC0 | encode);
1283 }
1284 
1285 
1286 void Assembler::imull(Register dst, Register src, int value) {
1287   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1288   if (is8bit(value)) {
1289     emit_byte(0x6B);
1290     emit_byte(0xC0 | encode);
1291     emit_byte(value);
1292   } else {
1293     emit_byte(0x69);
1294     emit_byte(0xC0 | encode);
1295     emit_long(value);
1296   }
1297 }
1298 
1299 void Assembler::incl(Address dst) {
1300   // Don't use it directly. Use MacroAssembler::increment() instead.
1301   InstructionMark im(this);
1302   prefix(dst);
1303   emit_byte(0xFF);
1304   emit_operand(rax, dst);
1305 }
1306 
1307 void Assembler::jcc(Condition cc, Label& L, relocInfo::relocType rtype) {
1308   InstructionMark im(this);
1309   relocate(rtype);
1310   assert((0 <= cc) && (cc < 16), "illegal cc");
1311   if (L.is_bound()) {
1312     address dst = target(L);
1313     assert(dst != NULL, "jcc most probably wrong");
1314 
1315     const int short_size = 2;
1316     const int long_size = 6;
1317     intptr_t offs = (intptr_t)dst - (intptr_t)_code_pos;
1318     if (rtype == relocInfo::none && is8bit(offs - short_size)) {
1319       // 0111 tttn #8-bit disp
1320       emit_byte(0x70 | cc);
1321       emit_byte((offs - short_size) & 0xFF);
1322     } else {
1323       // 0000 1111 1000 tttn #32-bit disp
1324       assert(is_simm32(offs - long_size),
1325              "must be 32bit offset (call4)");
1326       emit_byte(0x0F);
1327       emit_byte(0x80 | cc);
1328       emit_long(offs - long_size);
1329     }
1330   } else {
1331     // Note: could eliminate cond. jumps to this jump if condition
1332     //       is the same however, seems to be rather unlikely case.
1333     // Note: use jccb() if label to be bound is very close to get
1334     //       an 8-bit displacement
1335     L.add_patch_at(code(), locator());
1336     emit_byte(0x0F);
1337     emit_byte(0x80 | cc);
1338     emit_long(0);
1339   }
1340 }
1341 
1342 void Assembler::jccb(Condition cc, Label& L) {
1343   if (L.is_bound()) {
1344     const int short_size = 2;
1345     address entry = target(L);
1346     assert(is8bit((intptr_t)entry - ((intptr_t)_code_pos + short_size)),
1347            "Dispacement too large for a short jmp");
1348     intptr_t offs = (intptr_t)entry - (intptr_t)_code_pos;
1349     // 0111 tttn #8-bit disp
1350     emit_byte(0x70 | cc);
1351     emit_byte((offs - short_size) & 0xFF);
1352   } else {
1353     InstructionMark im(this);
1354     L.add_patch_at(code(), locator());
1355     emit_byte(0x70 | cc);
1356     emit_byte(0);
1357   }
1358 }
1359 
1360 void Assembler::jmp(Address adr) {
1361   InstructionMark im(this);
1362   prefix(adr);
1363   emit_byte(0xFF);
1364   emit_operand(rsp, adr);
1365 }
1366 
1367 void Assembler::jmp(Label& L, relocInfo::relocType rtype) {
1368   if (L.is_bound()) {
1369     address entry = target(L);
1370     assert(entry != NULL, "jmp most probably wrong");
1371     InstructionMark im(this);
1372     const int short_size = 2;
1373     const int long_size = 5;
1374     intptr_t offs = entry - _code_pos;
1375     if (rtype == relocInfo::none && is8bit(offs - short_size)) {
1376       emit_byte(0xEB);
1377       emit_byte((offs - short_size) & 0xFF);
1378     } else {
1379       emit_byte(0xE9);
1380       emit_long(offs - long_size);
1381     }
1382   } else {
1383     // By default, forward jumps are always 32-bit displacements, since
1384     // we can't yet know where the label will be bound.  If you're sure that
1385     // the forward jump will not run beyond 256 bytes, use jmpb to
1386     // force an 8-bit displacement.
1387     InstructionMark im(this);
1388     relocate(rtype);
1389     L.add_patch_at(code(), locator());
1390     emit_byte(0xE9);
1391     emit_long(0);
1392   }
1393 }
1394 
1395 void Assembler::jmp(Register entry) {
1396   int encode = prefix_and_encode(entry->encoding());
1397   emit_byte(0xFF);
1398   emit_byte(0xE0 | encode);
1399 }
1400 
1401 void Assembler::jmp_literal(address dest, RelocationHolder const& rspec) {
1402   InstructionMark im(this);
1403   emit_byte(0xE9);
1404   assert(dest != NULL, "must have a target");
1405   intptr_t disp = dest - (_code_pos + sizeof(int32_t));
1406   assert(is_simm32(disp), "must be 32bit offset (jmp)");
1407   emit_data(disp, rspec.reloc(), call32_operand);
1408 }
1409 
1410 void Assembler::jmpb(Label& L) {
1411   if (L.is_bound()) {
1412     const int short_size = 2;
1413     address entry = target(L);
1414     assert(is8bit((entry - _code_pos) + short_size),
1415            "Dispacement too large for a short jmp");
1416     assert(entry != NULL, "jmp most probably wrong");
1417     intptr_t offs = entry - _code_pos;
1418     emit_byte(0xEB);
1419     emit_byte((offs - short_size) & 0xFF);
1420   } else {
1421     InstructionMark im(this);
1422     L.add_patch_at(code(), locator());
1423     emit_byte(0xEB);
1424     emit_byte(0);
1425   }
1426 }
1427 
1428 void Assembler::ldmxcsr( Address src) {
1429   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1430   InstructionMark im(this);
1431   prefix(src);
1432   emit_byte(0x0F);
1433   emit_byte(0xAE);
1434   emit_operand(as_Register(2), src);
1435 }
1436 
1437 void Assembler::leal(Register dst, Address src) {
1438   InstructionMark im(this);
1439 #ifdef _LP64
1440   emit_byte(0x67); // addr32
1441   prefix(src, dst);
1442 #endif // LP64
1443   emit_byte(0x8D);
1444   emit_operand(dst, src);
1445 }
1446 
1447 void Assembler::lock() {
1448   if (Atomics & 1) {
1449      // Emit either nothing, a NOP, or a NOP: prefix
1450      emit_byte(0x90) ;
1451   } else {
1452      emit_byte(0xF0);
1453   }
1454 }
1455 
1456 void Assembler::lzcntl(Register dst, Register src) {
1457   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
1458   emit_byte(0xF3);
1459   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1460   emit_byte(0x0F);
1461   emit_byte(0xBD);
1462   emit_byte(0xC0 | encode);
1463 }
1464 
1465 // Emit mfence instruction
1466 void Assembler::mfence() {
1467   NOT_LP64(assert(VM_Version::supports_sse2(), "unsupported");)
1468   emit_byte( 0x0F );
1469   emit_byte( 0xAE );
1470   emit_byte( 0xF0 );
1471 }
1472 
1473 void Assembler::mov(Register dst, Register src) {
1474   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
1475 }
1476 
1477 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
1478   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1479   int dstenc = dst->encoding();
1480   int srcenc = src->encoding();
1481   emit_byte(0x66);
1482   if (dstenc < 8) {
1483     if (srcenc >= 8) {
1484       prefix(REX_B);
1485       srcenc -= 8;
1486     }
1487   } else {
1488     if (srcenc < 8) {
1489       prefix(REX_R);
1490     } else {
1491       prefix(REX_RB);
1492       srcenc -= 8;
1493     }
1494     dstenc -= 8;
1495   }
1496   emit_byte(0x0F);
1497   emit_byte(0x28);
1498   emit_byte(0xC0 | dstenc << 3 | srcenc);
1499 }
1500 
1501 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
1502   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1503   int dstenc = dst->encoding();
1504   int srcenc = src->encoding();
1505   if (dstenc < 8) {
1506     if (srcenc >= 8) {
1507       prefix(REX_B);
1508       srcenc -= 8;
1509     }
1510   } else {
1511     if (srcenc < 8) {
1512       prefix(REX_R);
1513     } else {
1514       prefix(REX_RB);
1515       srcenc -= 8;
1516     }
1517     dstenc -= 8;
1518   }
1519   emit_byte(0x0F);
1520   emit_byte(0x28);
1521   emit_byte(0xC0 | dstenc << 3 | srcenc);
1522 }
1523 
1524 void Assembler::movb(Register dst, Address src) {
1525   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
1526   InstructionMark im(this);
1527   prefix(src, dst, true);
1528   emit_byte(0x8A);
1529   emit_operand(dst, src);
1530 }
1531 
1532 
1533 void Assembler::movb(Address dst, int imm8) {
1534   InstructionMark im(this);
1535    prefix(dst);
1536   emit_byte(0xC6);
1537   emit_operand(rax, dst, 1);
1538   emit_byte(imm8);
1539 }
1540 
1541 
1542 void Assembler::movb(Address dst, Register src) {
1543   assert(src->has_byte_register(), "must have byte register");
1544   InstructionMark im(this);
1545   prefix(dst, src, true);
1546   emit_byte(0x88);
1547   emit_operand(src, dst);
1548 }
1549 
1550 void Assembler::movdl(XMMRegister dst, Register src) {
1551   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1552   emit_byte(0x66);
1553   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1554   emit_byte(0x0F);
1555   emit_byte(0x6E);
1556   emit_byte(0xC0 | encode);
1557 }
1558 
1559 void Assembler::movdl(Register dst, XMMRegister src) {
1560   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1561   emit_byte(0x66);
1562   // swap src/dst to get correct prefix
1563   int encode = prefix_and_encode(src->encoding(), dst->encoding());
1564   emit_byte(0x0F);
1565   emit_byte(0x7E);
1566   emit_byte(0xC0 | encode);
1567 }
1568 
1569 void Assembler::movdqa(XMMRegister dst, Address src) {
1570   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1571   InstructionMark im(this);
1572   emit_byte(0x66);
1573   prefix(src, dst);
1574   emit_byte(0x0F);
1575   emit_byte(0x6F);
1576   emit_operand(dst, src);
1577 }
1578 
1579 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
1580   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1581   emit_byte(0x66);
1582   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
1583   emit_byte(0x0F);
1584   emit_byte(0x6F);
1585   emit_byte(0xC0 | encode);
1586 }
1587 
1588 void Assembler::movdqa(Address dst, XMMRegister src) {
1589   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1590   InstructionMark im(this);
1591   emit_byte(0x66);
1592   prefix(dst, src);
1593   emit_byte(0x0F);
1594   emit_byte(0x7F);
1595   emit_operand(src, dst);
1596 }
1597 
1598 void Assembler::movdqu(XMMRegister dst, Address src) {
1599   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1600   InstructionMark im(this);
1601   emit_byte(0xF3);
1602   prefix(src, dst);
1603   emit_byte(0x0F);
1604   emit_byte(0x6F);
1605   emit_operand(dst, src);
1606 }
1607 
1608 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
1609   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1610   emit_byte(0xF3);
1611   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
1612   emit_byte(0x0F);
1613   emit_byte(0x6F);
1614   emit_byte(0xC0 | encode);
1615 }
1616 
1617 void Assembler::movdqu(Address dst, XMMRegister src) {
1618   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1619   InstructionMark im(this);
1620   emit_byte(0xF3);
1621   prefix(dst, src);
1622   emit_byte(0x0F);
1623   emit_byte(0x7F);
1624   emit_operand(src, dst);
1625 }
1626 
1627 // Uses zero extension on 64bit
1628 
1629 void Assembler::movl(Register dst, int32_t imm32) {
1630   int encode = prefix_and_encode(dst->encoding());
1631   emit_byte(0xB8 | encode);
1632   emit_long(imm32);
1633 }
1634 
1635 void Assembler::movl(Register dst, Register src) {
1636   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1637   emit_byte(0x8B);
1638   emit_byte(0xC0 | encode);
1639 }
1640 
1641 void Assembler::movl(Register dst, Address src) {
1642   InstructionMark im(this);
1643   prefix(src, dst);
1644   emit_byte(0x8B);
1645   emit_operand(dst, src);
1646 }
1647 
1648 void Assembler::movl(Address dst, int32_t imm32) {
1649   InstructionMark im(this);
1650   prefix(dst);
1651   emit_byte(0xC7);
1652   emit_operand(rax, dst, 4);
1653   emit_long(imm32);
1654 }
1655 
1656 void Assembler::movl(Address dst, Register src) {
1657   InstructionMark im(this);
1658   prefix(dst, src);
1659   emit_byte(0x89);
1660   emit_operand(src, dst);
1661 }
1662 
1663 // New cpus require to use movsd and movss to avoid partial register stall
1664 // when loading from memory. But for old Opteron use movlpd instead of movsd.
1665 // The selection is done in MacroAssembler::movdbl() and movflt().
1666 void Assembler::movlpd(XMMRegister dst, Address src) {
1667   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1668   InstructionMark im(this);
1669   emit_byte(0x66);
1670   prefix(src, dst);
1671   emit_byte(0x0F);
1672   emit_byte(0x12);
1673   emit_operand(dst, src);
1674 }
1675 
1676 void Assembler::movq( MMXRegister dst, Address src ) {
1677   assert( VM_Version::supports_mmx(), "" );
1678   emit_byte(0x0F);
1679   emit_byte(0x6F);
1680   emit_operand(dst, src);
1681 }
1682 
1683 void Assembler::movq( Address dst, MMXRegister src ) {
1684   assert( VM_Version::supports_mmx(), "" );
1685   emit_byte(0x0F);
1686   emit_byte(0x7F);
1687   // workaround gcc (3.2.1-7a) bug
1688   // In that version of gcc with only an emit_operand(MMX, Address)
1689   // gcc will tail jump and try and reverse the parameters completely
1690   // obliterating dst in the process. By having a version available
1691   // that doesn't need to swap the args at the tail jump the bug is
1692   // avoided.
1693   emit_operand(dst, src);
1694 }
1695 
1696 void Assembler::movq(XMMRegister dst, Address src) {
1697   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1698   InstructionMark im(this);
1699   emit_byte(0xF3);
1700   prefix(src, dst);
1701   emit_byte(0x0F);
1702   emit_byte(0x7E);
1703   emit_operand(dst, src);
1704 }
1705 
1706 void Assembler::movq(Address dst, XMMRegister src) {
1707   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1708   InstructionMark im(this);
1709   emit_byte(0x66);
1710   prefix(dst, src);
1711   emit_byte(0x0F);
1712   emit_byte(0xD6);
1713   emit_operand(src, dst);
1714 }
1715 
1716 void Assembler::movsbl(Register dst, Address src) { // movsxb
1717   InstructionMark im(this);
1718   prefix(src, dst);
1719   emit_byte(0x0F);
1720   emit_byte(0xBE);
1721   emit_operand(dst, src);
1722 }
1723 
1724 void Assembler::movsbl(Register dst, Register src) { // movsxb
1725   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
1726   int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
1727   emit_byte(0x0F);
1728   emit_byte(0xBE);
1729   emit_byte(0xC0 | encode);
1730 }
1731 
1732 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
1733   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1734   emit_byte(0xF2);
1735   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1736   emit_byte(0x0F);
1737   emit_byte(0x10);
1738   emit_byte(0xC0 | encode);
1739 }
1740 
1741 void Assembler::movsd(XMMRegister dst, Address src) {
1742   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1743   InstructionMark im(this);
1744   emit_byte(0xF2);
1745   prefix(src, dst);
1746   emit_byte(0x0F);
1747   emit_byte(0x10);
1748   emit_operand(dst, src);
1749 }
1750 
1751 void Assembler::movsd(Address dst, XMMRegister src) {
1752   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1753   InstructionMark im(this);
1754   emit_byte(0xF2);
1755   prefix(dst, src);
1756   emit_byte(0x0F);
1757   emit_byte(0x11);
1758   emit_operand(src, dst);
1759 }
1760 
1761 void Assembler::movss(XMMRegister dst, XMMRegister src) {
1762   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1763   emit_byte(0xF3);
1764   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1765   emit_byte(0x0F);
1766   emit_byte(0x10);
1767   emit_byte(0xC0 | encode);
1768 }
1769 
1770 void Assembler::movss(XMMRegister dst, Address src) {
1771   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1772   InstructionMark im(this);
1773   emit_byte(0xF3);
1774   prefix(src, dst);
1775   emit_byte(0x0F);
1776   emit_byte(0x10);
1777   emit_operand(dst, src);
1778 }
1779 
1780 void Assembler::movss(Address dst, XMMRegister src) {
1781   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1782   InstructionMark im(this);
1783   emit_byte(0xF3);
1784   prefix(dst, src);
1785   emit_byte(0x0F);
1786   emit_byte(0x11);
1787   emit_operand(src, dst);
1788 }
1789 
1790 void Assembler::movswl(Register dst, Address src) { // movsxw
1791   InstructionMark im(this);
1792   prefix(src, dst);
1793   emit_byte(0x0F);
1794   emit_byte(0xBF);
1795   emit_operand(dst, src);
1796 }
1797 
1798 void Assembler::movswl(Register dst, Register src) { // movsxw
1799   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1800   emit_byte(0x0F);
1801   emit_byte(0xBF);
1802   emit_byte(0xC0 | encode);
1803 }
1804 
1805 void Assembler::movw(Address dst, int imm16) {
1806   InstructionMark im(this);
1807 
1808   emit_byte(0x66); // switch to 16-bit mode
1809   prefix(dst);
1810   emit_byte(0xC7);
1811   emit_operand(rax, dst, 2);
1812   emit_word(imm16);
1813 }
1814 
1815 void Assembler::movw(Register dst, Address src) {
1816   InstructionMark im(this);
1817   emit_byte(0x66);
1818   prefix(src, dst);
1819   emit_byte(0x8B);
1820   emit_operand(dst, src);
1821 }
1822 
1823 void Assembler::movw(Address dst, Register src) {
1824   InstructionMark im(this);
1825   emit_byte(0x66);
1826   prefix(dst, src);
1827   emit_byte(0x89);
1828   emit_operand(src, dst);
1829 }
1830 
1831 void Assembler::movzbl(Register dst, Address src) { // movzxb
1832   InstructionMark im(this);
1833   prefix(src, dst);
1834   emit_byte(0x0F);
1835   emit_byte(0xB6);
1836   emit_operand(dst, src);
1837 }
1838 
1839 void Assembler::movzbl(Register dst, Register src) { // movzxb
1840   NOT_LP64(assert(src->has_byte_register(), "must have byte register"));
1841   int encode = prefix_and_encode(dst->encoding(), src->encoding(), true);
1842   emit_byte(0x0F);
1843   emit_byte(0xB6);
1844   emit_byte(0xC0 | encode);
1845 }
1846 
1847 void Assembler::movzwl(Register dst, Address src) { // movzxw
1848   InstructionMark im(this);
1849   prefix(src, dst);
1850   emit_byte(0x0F);
1851   emit_byte(0xB7);
1852   emit_operand(dst, src);
1853 }
1854 
1855 void Assembler::movzwl(Register dst, Register src) { // movzxw
1856   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1857   emit_byte(0x0F);
1858   emit_byte(0xB7);
1859   emit_byte(0xC0 | encode);
1860 }
1861 
1862 void Assembler::mull(Address src) {
1863   InstructionMark im(this);
1864   prefix(src);
1865   emit_byte(0xF7);
1866   emit_operand(rsp, src);
1867 }
1868 
1869 void Assembler::mull(Register src) {
1870   int encode = prefix_and_encode(src->encoding());
1871   emit_byte(0xF7);
1872   emit_byte(0xE0 | encode);
1873 }
1874 
1875 void Assembler::mulsd(XMMRegister dst, Address src) {
1876   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1877   InstructionMark im(this);
1878   emit_byte(0xF2);
1879   prefix(src, dst);
1880   emit_byte(0x0F);
1881   emit_byte(0x59);
1882   emit_operand(dst, src);
1883 }
1884 
1885 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
1886   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
1887   emit_byte(0xF2);
1888   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1889   emit_byte(0x0F);
1890   emit_byte(0x59);
1891   emit_byte(0xC0 | encode);
1892 }
1893 
1894 void Assembler::mulss(XMMRegister dst, Address src) {
1895   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1896   InstructionMark im(this);
1897   emit_byte(0xF3);
1898   prefix(src, dst);
1899   emit_byte(0x0F);
1900   emit_byte(0x59);
1901   emit_operand(dst, src);
1902 }
1903 
1904 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
1905   NOT_LP64(assert(VM_Version::supports_sse(), ""));
1906   emit_byte(0xF3);
1907   int encode = prefix_and_encode(dst->encoding(), src->encoding());
1908   emit_byte(0x0F);
1909   emit_byte(0x59);
1910   emit_byte(0xC0 | encode);
1911 }
1912 
1913 void Assembler::negl(Register dst) {
1914   int encode = prefix_and_encode(dst->encoding());
1915   emit_byte(0xF7);
1916   emit_byte(0xD8 | encode);
1917 }
1918 
1919 void Assembler::nop(int i) {
1920 #ifdef ASSERT
1921   assert(i > 0, " ");
1922   // The fancy nops aren't currently recognized by debuggers making it a
1923   // pain to disassemble code while debugging. If asserts are on clearly
1924   // speed is not an issue so simply use the single byte traditional nop
1925   // to do alignment.
1926 
1927   for (; i > 0 ; i--) emit_byte(0x90);
1928   return;
1929 
1930 #endif // ASSERT
1931 
1932   if (UseAddressNop && VM_Version::is_intel()) {
1933     //
1934     // Using multi-bytes nops "0x0F 0x1F [address]" for Intel
1935     //  1: 0x90
1936     //  2: 0x66 0x90
1937     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
1938     //  4: 0x0F 0x1F 0x40 0x00
1939     //  5: 0x0F 0x1F 0x44 0x00 0x00
1940     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
1941     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
1942     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
1943     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
1944     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
1945     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
1946 
1947     // The rest coding is Intel specific - don't use consecutive address nops
1948 
1949     // 12: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
1950     // 13: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
1951     // 14: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
1952     // 15: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x66 0x66 0x66 0x90
1953 
1954     while(i >= 15) {
1955       // For Intel don't generate consecutive addess nops (mix with regular nops)
1956       i -= 15;
1957       emit_byte(0x66);   // size prefix
1958       emit_byte(0x66);   // size prefix
1959       emit_byte(0x66);   // size prefix
1960       addr_nop_8();
1961       emit_byte(0x66);   // size prefix
1962       emit_byte(0x66);   // size prefix
1963       emit_byte(0x66);   // size prefix
1964       emit_byte(0x90);   // nop
1965     }
1966     switch (i) {
1967       case 14:
1968         emit_byte(0x66); // size prefix
1969       case 13:
1970         emit_byte(0x66); // size prefix
1971       case 12:
1972         addr_nop_8();
1973         emit_byte(0x66); // size prefix
1974         emit_byte(0x66); // size prefix
1975         emit_byte(0x66); // size prefix
1976         emit_byte(0x90); // nop
1977         break;
1978       case 11:
1979         emit_byte(0x66); // size prefix
1980       case 10:
1981         emit_byte(0x66); // size prefix
1982       case 9:
1983         emit_byte(0x66); // size prefix
1984       case 8:
1985         addr_nop_8();
1986         break;
1987       case 7:
1988         addr_nop_7();
1989         break;
1990       case 6:
1991         emit_byte(0x66); // size prefix
1992       case 5:
1993         addr_nop_5();
1994         break;
1995       case 4:
1996         addr_nop_4();
1997         break;
1998       case 3:
1999         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2000         emit_byte(0x66); // size prefix
2001       case 2:
2002         emit_byte(0x66); // size prefix
2003       case 1:
2004         emit_byte(0x90); // nop
2005         break;
2006       default:
2007         assert(i == 0, " ");
2008     }
2009     return;
2010   }
2011   if (UseAddressNop && VM_Version::is_amd()) {
2012     //
2013     // Using multi-bytes nops "0x0F 0x1F [address]" for AMD.
2014     //  1: 0x90
2015     //  2: 0x66 0x90
2016     //  3: 0x66 0x66 0x90 (don't use "0x0F 0x1F 0x00" - need patching safe padding)
2017     //  4: 0x0F 0x1F 0x40 0x00
2018     //  5: 0x0F 0x1F 0x44 0x00 0x00
2019     //  6: 0x66 0x0F 0x1F 0x44 0x00 0x00
2020     //  7: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2021     //  8: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2022     //  9: 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2023     // 10: 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2024     // 11: 0x66 0x66 0x66 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2025 
2026     // The rest coding is AMD specific - use consecutive address nops
2027 
2028     // 12: 0x66 0x0F 0x1F 0x44 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2029     // 13: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x66 0x0F 0x1F 0x44 0x00 0x00
2030     // 14: 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2031     // 15: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x80 0x00 0x00 0x00 0x00
2032     // 16: 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00 0x0F 0x1F 0x84 0x00 0x00 0x00 0x00 0x00
2033     //     Size prefixes (0x66) are added for larger sizes
2034 
2035     while(i >= 22) {
2036       i -= 11;
2037       emit_byte(0x66); // size prefix
2038       emit_byte(0x66); // size prefix
2039       emit_byte(0x66); // size prefix
2040       addr_nop_8();
2041     }
2042     // Generate first nop for size between 21-12
2043     switch (i) {
2044       case 21:
2045         i -= 1;
2046         emit_byte(0x66); // size prefix
2047       case 20:
2048       case 19:
2049         i -= 1;
2050         emit_byte(0x66); // size prefix
2051       case 18:
2052       case 17:
2053         i -= 1;
2054         emit_byte(0x66); // size prefix
2055       case 16:
2056       case 15:
2057         i -= 8;
2058         addr_nop_8();
2059         break;
2060       case 14:
2061       case 13:
2062         i -= 7;
2063         addr_nop_7();
2064         break;
2065       case 12:
2066         i -= 6;
2067         emit_byte(0x66); // size prefix
2068         addr_nop_5();
2069         break;
2070       default:
2071         assert(i < 12, " ");
2072     }
2073 
2074     // Generate second nop for size between 11-1
2075     switch (i) {
2076       case 11:
2077         emit_byte(0x66); // size prefix
2078       case 10:
2079         emit_byte(0x66); // size prefix
2080       case 9:
2081         emit_byte(0x66); // size prefix
2082       case 8:
2083         addr_nop_8();
2084         break;
2085       case 7:
2086         addr_nop_7();
2087         break;
2088       case 6:
2089         emit_byte(0x66); // size prefix
2090       case 5:
2091         addr_nop_5();
2092         break;
2093       case 4:
2094         addr_nop_4();
2095         break;
2096       case 3:
2097         // Don't use "0x0F 0x1F 0x00" - need patching safe padding
2098         emit_byte(0x66); // size prefix
2099       case 2:
2100         emit_byte(0x66); // size prefix
2101       case 1:
2102         emit_byte(0x90); // nop
2103         break;
2104       default:
2105         assert(i == 0, " ");
2106     }
2107     return;
2108   }
2109 
2110   // Using nops with size prefixes "0x66 0x90".
2111   // From AMD Optimization Guide:
2112   //  1: 0x90
2113   //  2: 0x66 0x90
2114   //  3: 0x66 0x66 0x90
2115   //  4: 0x66 0x66 0x66 0x90
2116   //  5: 0x66 0x66 0x90 0x66 0x90
2117   //  6: 0x66 0x66 0x90 0x66 0x66 0x90
2118   //  7: 0x66 0x66 0x66 0x90 0x66 0x66 0x90
2119   //  8: 0x66 0x66 0x66 0x90 0x66 0x66 0x66 0x90
2120   //  9: 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2121   // 10: 0x66 0x66 0x66 0x90 0x66 0x66 0x90 0x66 0x66 0x90
2122   //
2123   while(i > 12) {
2124     i -= 4;
2125     emit_byte(0x66); // size prefix
2126     emit_byte(0x66);
2127     emit_byte(0x66);
2128     emit_byte(0x90); // nop
2129   }
2130   // 1 - 12 nops
2131   if(i > 8) {
2132     if(i > 9) {
2133       i -= 1;
2134       emit_byte(0x66);
2135     }
2136     i -= 3;
2137     emit_byte(0x66);
2138     emit_byte(0x66);
2139     emit_byte(0x90);
2140   }
2141   // 1 - 8 nops
2142   if(i > 4) {
2143     if(i > 6) {
2144       i -= 1;
2145       emit_byte(0x66);
2146     }
2147     i -= 3;
2148     emit_byte(0x66);
2149     emit_byte(0x66);
2150     emit_byte(0x90);
2151   }
2152   switch (i) {
2153     case 4:
2154       emit_byte(0x66);
2155     case 3:
2156       emit_byte(0x66);
2157     case 2:
2158       emit_byte(0x66);
2159     case 1:
2160       emit_byte(0x90);
2161       break;
2162     default:
2163       assert(i == 0, " ");
2164   }
2165 }
2166 
2167 void Assembler::notl(Register dst) {
2168   int encode = prefix_and_encode(dst->encoding());
2169   emit_byte(0xF7);
2170   emit_byte(0xD0 | encode );
2171 }
2172 
2173 void Assembler::orl(Address dst, int32_t imm32) {
2174   InstructionMark im(this);
2175   prefix(dst);
2176   emit_byte(0x81);
2177   emit_operand(rcx, dst, 4);
2178   emit_long(imm32);
2179 }
2180 
2181 void Assembler::orl(Register dst, int32_t imm32) {
2182   prefix(dst);
2183   emit_arith(0x81, 0xC8, dst, imm32);
2184 }
2185 
2186 
2187 void Assembler::orl(Register dst, Address src) {
2188   InstructionMark im(this);
2189   prefix(src, dst);
2190   emit_byte(0x0B);
2191   emit_operand(dst, src);
2192 }
2193 
2194 
2195 void Assembler::orl(Register dst, Register src) {
2196   (void) prefix_and_encode(dst->encoding(), src->encoding());
2197   emit_arith(0x0B, 0xC0, dst, src);
2198 }
2199 
2200 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
2201   assert(VM_Version::supports_sse4_2(), "");
2202 
2203   InstructionMark im(this);
2204   emit_byte(0x66);
2205   prefix(src, dst);
2206   emit_byte(0x0F);
2207   emit_byte(0x3A);
2208   emit_byte(0x61);
2209   emit_operand(dst, src);
2210   emit_byte(imm8);
2211 }
2212 
2213 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
2214   assert(VM_Version::supports_sse4_2(), "");
2215 
2216   emit_byte(0x66);
2217   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
2218   emit_byte(0x0F);
2219   emit_byte(0x3A);
2220   emit_byte(0x61);
2221   emit_byte(0xC0 | encode);
2222   emit_byte(imm8);
2223 }
2224 
2225 // generic
2226 void Assembler::pop(Register dst) {
2227   int encode = prefix_and_encode(dst->encoding());
2228   emit_byte(0x58 | encode);
2229 }
2230 
2231 void Assembler::popcntl(Register dst, Address src) {
2232   assert(VM_Version::supports_popcnt(), "must support");
2233   InstructionMark im(this);
2234   emit_byte(0xF3);
2235   prefix(src, dst);
2236   emit_byte(0x0F);
2237   emit_byte(0xB8);
2238   emit_operand(dst, src);
2239 }
2240 
2241 void Assembler::popcntl(Register dst, Register src) {
2242   assert(VM_Version::supports_popcnt(), "must support");
2243   emit_byte(0xF3);
2244   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2245   emit_byte(0x0F);
2246   emit_byte(0xB8);
2247   emit_byte(0xC0 | encode);
2248 }
2249 
2250 void Assembler::popf() {
2251   emit_byte(0x9D);
2252 }
2253 
2254 #ifndef _LP64 // no 32bit push/pop on amd64
2255 void Assembler::popl(Address dst) {
2256   // NOTE: this will adjust stack by 8byte on 64bits
2257   InstructionMark im(this);
2258   prefix(dst);
2259   emit_byte(0x8F);
2260   emit_operand(rax, dst);
2261 }
2262 #endif
2263 
2264 void Assembler::prefetch_prefix(Address src) {
2265   prefix(src);
2266   emit_byte(0x0F);
2267 }
2268 
2269 void Assembler::prefetchnta(Address src) {
2270   NOT_LP64(assert(VM_Version::supports_sse2(), "must support"));
2271   InstructionMark im(this);
2272   prefetch_prefix(src);
2273   emit_byte(0x18);
2274   emit_operand(rax, src); // 0, src
2275 }
2276 
2277 void Assembler::prefetchr(Address src) {
2278   NOT_LP64(assert(VM_Version::supports_3dnow(), "must support"));
2279   InstructionMark im(this);
2280   prefetch_prefix(src);
2281   emit_byte(0x0D);
2282   emit_operand(rax, src); // 0, src
2283 }
2284 
2285 void Assembler::prefetcht0(Address src) {
2286   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2287   InstructionMark im(this);
2288   prefetch_prefix(src);
2289   emit_byte(0x18);
2290   emit_operand(rcx, src); // 1, src
2291 }
2292 
2293 void Assembler::prefetcht1(Address src) {
2294   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2295   InstructionMark im(this);
2296   prefetch_prefix(src);
2297   emit_byte(0x18);
2298   emit_operand(rdx, src); // 2, src
2299 }
2300 
2301 void Assembler::prefetcht2(Address src) {
2302   NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
2303   InstructionMark im(this);
2304   prefetch_prefix(src);
2305   emit_byte(0x18);
2306   emit_operand(rbx, src); // 3, src
2307 }
2308 
2309 void Assembler::prefetchw(Address src) {
2310   NOT_LP64(assert(VM_Version::supports_3dnow(), "must support"));
2311   InstructionMark im(this);
2312   prefetch_prefix(src);
2313   emit_byte(0x0D);
2314   emit_operand(rcx, src); // 1, src
2315 }
2316 
2317 void Assembler::prefix(Prefix p) {
2318   a_byte(p);
2319 }
2320 
2321 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
2322   assert(isByte(mode), "invalid value");
2323   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2324 
2325   emit_byte(0x66);
2326   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2327   emit_byte(0x0F);
2328   emit_byte(0x70);
2329   emit_byte(0xC0 | encode);
2330   emit_byte(mode & 0xFF);
2331 
2332 }
2333 
2334 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
2335   assert(isByte(mode), "invalid value");
2336   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2337 
2338   InstructionMark im(this);
2339   emit_byte(0x66);
2340   prefix(src, dst);
2341   emit_byte(0x0F);
2342   emit_byte(0x70);
2343   emit_operand(dst, src);
2344   emit_byte(mode & 0xFF);
2345 }
2346 
2347 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
2348   assert(isByte(mode), "invalid value");
2349   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2350 
2351   emit_byte(0xF2);
2352   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2353   emit_byte(0x0F);
2354   emit_byte(0x70);
2355   emit_byte(0xC0 | encode);
2356   emit_byte(mode & 0xFF);
2357 }
2358 
2359 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
2360   assert(isByte(mode), "invalid value");
2361   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2362 
2363   InstructionMark im(this);
2364   emit_byte(0xF2);
2365   prefix(src, dst); // QQ new
2366   emit_byte(0x0F);
2367   emit_byte(0x70);
2368   emit_operand(dst, src);
2369   emit_byte(mode & 0xFF);
2370 }
2371 
2372 void Assembler::psrlq(XMMRegister dst, int shift) {
2373   // HMM Table D-1 says sse2 or mmx
2374   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2375 
2376   int encode = prefixq_and_encode(xmm2->encoding(), dst->encoding());
2377   emit_byte(0x66);
2378   emit_byte(0x0F);
2379   emit_byte(0x73);
2380   emit_byte(0xC0 | encode);
2381   emit_byte(shift);
2382 }
2383 
2384 void Assembler::ptest(XMMRegister dst, Address src) {
2385   assert(VM_Version::supports_sse4_1(), "");
2386 
2387   InstructionMark im(this);
2388   emit_byte(0x66);
2389   prefix(src, dst);
2390   emit_byte(0x0F);
2391   emit_byte(0x38);
2392   emit_byte(0x17);
2393   emit_operand(dst, src);
2394 }
2395 
2396 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
2397   assert(VM_Version::supports_sse4_1(), "");
2398 
2399   emit_byte(0x66);
2400   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
2401   emit_byte(0x0F);
2402   emit_byte(0x38);
2403   emit_byte(0x17);
2404   emit_byte(0xC0 | encode);
2405 }
2406 
2407 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
2408   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2409   emit_byte(0x66);
2410   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2411   emit_byte(0x0F);
2412   emit_byte(0x60);
2413   emit_byte(0xC0 | encode);
2414 }
2415 
2416 void Assembler::push(int32_t imm32) {
2417   // in 64bits we push 64bits onto the stack but only
2418   // take a 32bit immediate
2419   emit_byte(0x68);
2420   emit_long(imm32);
2421 }
2422 
2423 void Assembler::push(Register src) {
2424   int encode = prefix_and_encode(src->encoding());
2425 
2426   emit_byte(0x50 | encode);
2427 }
2428 
2429 void Assembler::pushf() {
2430   emit_byte(0x9C);
2431 }
2432 
2433 #ifndef _LP64 // no 32bit push/pop on amd64
2434 void Assembler::pushl(Address src) {
2435   // Note this will push 64bit on 64bit
2436   InstructionMark im(this);
2437   prefix(src);
2438   emit_byte(0xFF);
2439   emit_operand(rsi, src);
2440 }
2441 #endif
2442 
2443 void Assembler::pxor(XMMRegister dst, Address src) {
2444   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2445   InstructionMark im(this);
2446   emit_byte(0x66);
2447   prefix(src, dst);
2448   emit_byte(0x0F);
2449   emit_byte(0xEF);
2450   emit_operand(dst, src);
2451 }
2452 
2453 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
2454   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2455   InstructionMark im(this);
2456   emit_byte(0x66);
2457   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2458   emit_byte(0x0F);
2459   emit_byte(0xEF);
2460   emit_byte(0xC0 | encode);
2461 }
2462 
2463 void Assembler::rcll(Register dst, int imm8) {
2464   assert(isShiftCount(imm8), "illegal shift count");
2465   int encode = prefix_and_encode(dst->encoding());
2466   if (imm8 == 1) {
2467     emit_byte(0xD1);
2468     emit_byte(0xD0 | encode);
2469   } else {
2470     emit_byte(0xC1);
2471     emit_byte(0xD0 | encode);
2472     emit_byte(imm8);
2473   }
2474 }
2475 
2476 // copies data from [esi] to [edi] using rcx pointer sized words
2477 // generic
2478 void Assembler::rep_mov() {
2479   emit_byte(0xF3);
2480   // MOVSQ
2481   LP64_ONLY(prefix(REX_W));
2482   emit_byte(0xA5);
2483 }
2484 
2485 // sets rcx pointer sized words with rax, value at [edi]
2486 // generic
2487 void Assembler::rep_set() { // rep_set
2488   emit_byte(0xF3);
2489   // STOSQ
2490   LP64_ONLY(prefix(REX_W));
2491   emit_byte(0xAB);
2492 }
2493 
2494 // scans rcx pointer sized words at [edi] for occurance of rax,
2495 // generic
2496 void Assembler::repne_scan() { // repne_scan
2497   emit_byte(0xF2);
2498   // SCASQ
2499   LP64_ONLY(prefix(REX_W));
2500   emit_byte(0xAF);
2501 }
2502 
2503 #ifdef _LP64
2504 // scans rcx 4 byte words at [edi] for occurance of rax,
2505 // generic
2506 void Assembler::repne_scanl() { // repne_scan
2507   emit_byte(0xF2);
2508   // SCASL
2509   emit_byte(0xAF);
2510 }
2511 #endif
2512 
2513 void Assembler::ret(int imm16) {
2514   if (imm16 == 0) {
2515     emit_byte(0xC3);
2516   } else {
2517     emit_byte(0xC2);
2518     emit_word(imm16);
2519   }
2520 }
2521 
2522 void Assembler::sahf() {
2523 #ifdef _LP64
2524   // Not supported in 64bit mode
2525   ShouldNotReachHere();
2526 #endif
2527   emit_byte(0x9E);
2528 }
2529 
2530 void Assembler::sarl(Register dst, int imm8) {
2531   int encode = prefix_and_encode(dst->encoding());
2532   assert(isShiftCount(imm8), "illegal shift count");
2533   if (imm8 == 1) {
2534     emit_byte(0xD1);
2535     emit_byte(0xF8 | encode);
2536   } else {
2537     emit_byte(0xC1);
2538     emit_byte(0xF8 | encode);
2539     emit_byte(imm8);
2540   }
2541 }
2542 
2543 void Assembler::sarl(Register dst) {
2544   int encode = prefix_and_encode(dst->encoding());
2545   emit_byte(0xD3);
2546   emit_byte(0xF8 | encode);
2547 }
2548 
2549 void Assembler::sbbl(Address dst, int32_t imm32) {
2550   InstructionMark im(this);
2551   prefix(dst);
2552   emit_arith_operand(0x81, rbx, dst, imm32);
2553 }
2554 
2555 void Assembler::sbbl(Register dst, int32_t imm32) {
2556   prefix(dst);
2557   emit_arith(0x81, 0xD8, dst, imm32);
2558 }
2559 
2560 
2561 void Assembler::sbbl(Register dst, Address src) {
2562   InstructionMark im(this);
2563   prefix(src, dst);
2564   emit_byte(0x1B);
2565   emit_operand(dst, src);
2566 }
2567 
2568 void Assembler::sbbl(Register dst, Register src) {
2569   (void) prefix_and_encode(dst->encoding(), src->encoding());
2570   emit_arith(0x1B, 0xC0, dst, src);
2571 }
2572 
2573 void Assembler::setb(Condition cc, Register dst) {
2574   assert(0 <= cc && cc < 16, "illegal cc");
2575   int encode = prefix_and_encode(dst->encoding(), true);
2576   emit_byte(0x0F);
2577   emit_byte(0x90 | cc);
2578   emit_byte(0xC0 | encode);
2579 }
2580 
2581 void Assembler::shll(Register dst, int imm8) {
2582   assert(isShiftCount(imm8), "illegal shift count");
2583   int encode = prefix_and_encode(dst->encoding());
2584   if (imm8 == 1 ) {
2585     emit_byte(0xD1);
2586     emit_byte(0xE0 | encode);
2587   } else {
2588     emit_byte(0xC1);
2589     emit_byte(0xE0 | encode);
2590     emit_byte(imm8);
2591   }
2592 }
2593 
2594 void Assembler::shll(Register dst) {
2595   int encode = prefix_and_encode(dst->encoding());
2596   emit_byte(0xD3);
2597   emit_byte(0xE0 | encode);
2598 }
2599 
2600 void Assembler::shrl(Register dst, int imm8) {
2601   assert(isShiftCount(imm8), "illegal shift count");
2602   int encode = prefix_and_encode(dst->encoding());
2603   emit_byte(0xC1);
2604   emit_byte(0xE8 | encode);
2605   emit_byte(imm8);
2606 }
2607 
2608 void Assembler::shrl(Register dst) {
2609   int encode = prefix_and_encode(dst->encoding());
2610   emit_byte(0xD3);
2611   emit_byte(0xE8 | encode);
2612 }
2613 
2614 // copies a single word from [esi] to [edi]
2615 void Assembler::smovl() {
2616   emit_byte(0xA5);
2617 }
2618 
2619 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
2620   // HMM Table D-1 says sse2
2621   // NOT_LP64(assert(VM_Version::supports_sse(), ""));
2622   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2623   emit_byte(0xF2);
2624   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2625   emit_byte(0x0F);
2626   emit_byte(0x51);
2627   emit_byte(0xC0 | encode);
2628 }
2629 
2630 void Assembler::stmxcsr( Address dst) {
2631   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2632   InstructionMark im(this);
2633   prefix(dst);
2634   emit_byte(0x0F);
2635   emit_byte(0xAE);
2636   emit_operand(as_Register(3), dst);
2637 }
2638 
2639 void Assembler::subl(Address dst, int32_t imm32) {
2640   InstructionMark im(this);
2641   prefix(dst);
2642   if (is8bit(imm32)) {
2643     emit_byte(0x83);
2644     emit_operand(rbp, dst, 1);
2645     emit_byte(imm32 & 0xFF);
2646   } else {
2647     emit_byte(0x81);
2648     emit_operand(rbp, dst, 4);
2649     emit_long(imm32);
2650   }
2651 }
2652 
2653 void Assembler::subl(Register dst, int32_t imm32) {
2654   prefix(dst);
2655   emit_arith(0x81, 0xE8, dst, imm32);
2656 }
2657 
2658 void Assembler::subl(Address dst, Register src) {
2659   InstructionMark im(this);
2660   prefix(dst, src);
2661   emit_byte(0x29);
2662   emit_operand(src, dst);
2663 }
2664 
2665 void Assembler::subl(Register dst, Address src) {
2666   InstructionMark im(this);
2667   prefix(src, dst);
2668   emit_byte(0x2B);
2669   emit_operand(dst, src);
2670 }
2671 
2672 void Assembler::subl(Register dst, Register src) {
2673   (void) prefix_and_encode(dst->encoding(), src->encoding());
2674   emit_arith(0x2B, 0xC0, dst, src);
2675 }
2676 
2677 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
2678   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2679   emit_byte(0xF2);
2680   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2681   emit_byte(0x0F);
2682   emit_byte(0x5C);
2683   emit_byte(0xC0 | encode);
2684 }
2685 
2686 void Assembler::subsd(XMMRegister dst, Address src) {
2687   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2688   InstructionMark im(this);
2689   emit_byte(0xF2);
2690   prefix(src, dst);
2691   emit_byte(0x0F);
2692   emit_byte(0x5C);
2693   emit_operand(dst, src);
2694 }
2695 
2696 void Assembler::subss(XMMRegister dst, XMMRegister src) {
2697   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2698   emit_byte(0xF3);
2699   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2700   emit_byte(0x0F);
2701   emit_byte(0x5C);
2702   emit_byte(0xC0 | encode);
2703 }
2704 
2705 void Assembler::subss(XMMRegister dst, Address src) {
2706   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2707   InstructionMark im(this);
2708   emit_byte(0xF3);
2709   prefix(src, dst);
2710   emit_byte(0x0F);
2711   emit_byte(0x5C);
2712   emit_operand(dst, src);
2713 }
2714 
2715 void Assembler::testb(Register dst, int imm8) {
2716   NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
2717   (void) prefix_and_encode(dst->encoding(), true);
2718   emit_arith_b(0xF6, 0xC0, dst, imm8);
2719 }
2720 
2721 void Assembler::testl(Register dst, int32_t imm32) {
2722   // not using emit_arith because test
2723   // doesn't support sign-extension of
2724   // 8bit operands
2725   int encode = dst->encoding();
2726   if (encode == 0) {
2727     emit_byte(0xA9);
2728   } else {
2729     encode = prefix_and_encode(encode);
2730     emit_byte(0xF7);
2731     emit_byte(0xC0 | encode);
2732   }
2733   emit_long(imm32);
2734 }
2735 
2736 void Assembler::testl(Register dst, Register src) {
2737   (void) prefix_and_encode(dst->encoding(), src->encoding());
2738   emit_arith(0x85, 0xC0, dst, src);
2739 }
2740 
2741 void Assembler::testl(Register dst, Address  src) {
2742   InstructionMark im(this);
2743   prefix(src, dst);
2744   emit_byte(0x85);
2745   emit_operand(dst, src);
2746 }
2747 
2748 void Assembler::ucomisd(XMMRegister dst, Address src) {
2749   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2750   emit_byte(0x66);
2751   ucomiss(dst, src);
2752 }
2753 
2754 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
2755   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2756   emit_byte(0x66);
2757   ucomiss(dst, src);
2758 }
2759 
2760 void Assembler::ucomiss(XMMRegister dst, Address src) {
2761   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2762 
2763   InstructionMark im(this);
2764   prefix(src, dst);
2765   emit_byte(0x0F);
2766   emit_byte(0x2E);
2767   emit_operand(dst, src);
2768 }
2769 
2770 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
2771   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2772   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2773   emit_byte(0x0F);
2774   emit_byte(0x2E);
2775   emit_byte(0xC0 | encode);
2776 }
2777 
2778 
2779 void Assembler::xaddl(Address dst, Register src) {
2780   InstructionMark im(this);
2781   prefix(dst, src);
2782   emit_byte(0x0F);
2783   emit_byte(0xC1);
2784   emit_operand(src, dst);
2785 }
2786 
2787 void Assembler::xchgl(Register dst, Address src) { // xchg
2788   InstructionMark im(this);
2789   prefix(src, dst);
2790   emit_byte(0x87);
2791   emit_operand(dst, src);
2792 }
2793 
2794 void Assembler::xchgl(Register dst, Register src) {
2795   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2796   emit_byte(0x87);
2797   emit_byte(0xc0 | encode);
2798 }
2799 
2800 void Assembler::xorl(Register dst, int32_t imm32) {
2801   prefix(dst);
2802   emit_arith(0x81, 0xF0, dst, imm32);
2803 }
2804 
2805 void Assembler::xorl(Register dst, Address src) {
2806   InstructionMark im(this);
2807   prefix(src, dst);
2808   emit_byte(0x33);
2809   emit_operand(dst, src);
2810 }
2811 
2812 void Assembler::xorl(Register dst, Register src) {
2813   (void) prefix_and_encode(dst->encoding(), src->encoding());
2814   emit_arith(0x33, 0xC0, dst, src);
2815 }
2816 
2817 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
2818   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2819   emit_byte(0x66);
2820   xorps(dst, src);
2821 }
2822 
2823 void Assembler::xorpd(XMMRegister dst, Address src) {
2824   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
2825   InstructionMark im(this);
2826   emit_byte(0x66);
2827   prefix(src, dst);
2828   emit_byte(0x0F);
2829   emit_byte(0x57);
2830   emit_operand(dst, src);
2831 }
2832 
2833 
2834 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
2835   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2836   int encode = prefix_and_encode(dst->encoding(), src->encoding());
2837   emit_byte(0x0F);
2838   emit_byte(0x57);
2839   emit_byte(0xC0 | encode);
2840 }
2841 
2842 void Assembler::xorps(XMMRegister dst, Address src) {
2843   NOT_LP64(assert(VM_Version::supports_sse(), ""));
2844   InstructionMark im(this);
2845   prefix(src, dst);
2846   emit_byte(0x0F);
2847   emit_byte(0x57);
2848   emit_operand(dst, src);
2849 }
2850 
2851 #ifndef _LP64
2852 // 32bit only pieces of the assembler
2853 
2854 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
2855   // NO PREFIX AS NEVER 64BIT
2856   InstructionMark im(this);
2857   emit_byte(0x81);
2858   emit_byte(0xF8 | src1->encoding());
2859   emit_data(imm32, rspec, 0);
2860 }
2861 
2862 void Assembler::cmp_literal32(Address src1, int32_t imm32, RelocationHolder const& rspec) {
2863   // NO PREFIX AS NEVER 64BIT (not even 32bit versions of 64bit regs
2864   InstructionMark im(this);
2865   emit_byte(0x81);
2866   emit_operand(rdi, src1);
2867   emit_data(imm32, rspec, 0);
2868 }
2869 
2870 // The 64-bit (32bit platform) cmpxchg compares the value at adr with the contents of rdx:rax,
2871 // and stores rcx:rbx into adr if so; otherwise, the value at adr is loaded
2872 // into rdx:rax.  The ZF is set if the compared values were equal, and cleared otherwise.
2873 void Assembler::cmpxchg8(Address adr) {
2874   InstructionMark im(this);
2875   emit_byte(0x0F);
2876   emit_byte(0xc7);
2877   emit_operand(rcx, adr);
2878 }
2879 
2880 void Assembler::decl(Register dst) {
2881   // Don't use it directly. Use MacroAssembler::decrementl() instead.
2882  emit_byte(0x48 | dst->encoding());
2883 }
2884 
2885 #endif // _LP64
2886 
2887 // 64bit typically doesn't use the x87 but needs to for the trig funcs
2888 
2889 void Assembler::fabs() {
2890   emit_byte(0xD9);
2891   emit_byte(0xE1);
2892 }
2893 
2894 void Assembler::fadd(int i) {
2895   emit_farith(0xD8, 0xC0, i);
2896 }
2897 
2898 void Assembler::fadd_d(Address src) {
2899   InstructionMark im(this);
2900   emit_byte(0xDC);
2901   emit_operand32(rax, src);
2902 }
2903 
2904 void Assembler::fadd_s(Address src) {
2905   InstructionMark im(this);
2906   emit_byte(0xD8);
2907   emit_operand32(rax, src);
2908 }
2909 
2910 void Assembler::fadda(int i) {
2911   emit_farith(0xDC, 0xC0, i);
2912 }
2913 
2914 void Assembler::faddp(int i) {
2915   emit_farith(0xDE, 0xC0, i);
2916 }
2917 
2918 void Assembler::fchs() {
2919   emit_byte(0xD9);
2920   emit_byte(0xE0);
2921 }
2922 
2923 void Assembler::fcom(int i) {
2924   emit_farith(0xD8, 0xD0, i);
2925 }
2926 
2927 void Assembler::fcomp(int i) {
2928   emit_farith(0xD8, 0xD8, i);
2929 }
2930 
2931 void Assembler::fcomp_d(Address src) {
2932   InstructionMark im(this);
2933   emit_byte(0xDC);
2934   emit_operand32(rbx, src);
2935 }
2936 
2937 void Assembler::fcomp_s(Address src) {
2938   InstructionMark im(this);
2939   emit_byte(0xD8);
2940   emit_operand32(rbx, src);
2941 }
2942 
2943 void Assembler::fcompp() {
2944   emit_byte(0xDE);
2945   emit_byte(0xD9);
2946 }
2947 
2948 void Assembler::fcos() {
2949   emit_byte(0xD9);
2950   emit_byte(0xFF);
2951 }
2952 
2953 void Assembler::fdecstp() {
2954   emit_byte(0xD9);
2955   emit_byte(0xF6);
2956 }
2957 
2958 void Assembler::fdiv(int i) {
2959   emit_farith(0xD8, 0xF0, i);
2960 }
2961 
2962 void Assembler::fdiv_d(Address src) {
2963   InstructionMark im(this);
2964   emit_byte(0xDC);
2965   emit_operand32(rsi, src);
2966 }
2967 
2968 void Assembler::fdiv_s(Address src) {
2969   InstructionMark im(this);
2970   emit_byte(0xD8);
2971   emit_operand32(rsi, src);
2972 }
2973 
2974 void Assembler::fdiva(int i) {
2975   emit_farith(0xDC, 0xF8, i);
2976 }
2977 
2978 // Note: The Intel manual (Pentium Processor User's Manual, Vol.3, 1994)
2979 //       is erroneous for some of the floating-point instructions below.
2980 
2981 void Assembler::fdivp(int i) {
2982   emit_farith(0xDE, 0xF8, i);                    // ST(0) <- ST(0) / ST(1) and pop (Intel manual wrong)
2983 }
2984 
2985 void Assembler::fdivr(int i) {
2986   emit_farith(0xD8, 0xF8, i);
2987 }
2988 
2989 void Assembler::fdivr_d(Address src) {
2990   InstructionMark im(this);
2991   emit_byte(0xDC);
2992   emit_operand32(rdi, src);
2993 }
2994 
2995 void Assembler::fdivr_s(Address src) {
2996   InstructionMark im(this);
2997   emit_byte(0xD8);
2998   emit_operand32(rdi, src);
2999 }
3000 
3001 void Assembler::fdivra(int i) {
3002   emit_farith(0xDC, 0xF0, i);
3003 }
3004 
3005 void Assembler::fdivrp(int i) {
3006   emit_farith(0xDE, 0xF0, i);                    // ST(0) <- ST(1) / ST(0) and pop (Intel manual wrong)
3007 }
3008 
3009 void Assembler::ffree(int i) {
3010   emit_farith(0xDD, 0xC0, i);
3011 }
3012 
3013 void Assembler::fild_d(Address adr) {
3014   InstructionMark im(this);
3015   emit_byte(0xDF);
3016   emit_operand32(rbp, adr);
3017 }
3018 
3019 void Assembler::fild_s(Address adr) {
3020   InstructionMark im(this);
3021   emit_byte(0xDB);
3022   emit_operand32(rax, adr);
3023 }
3024 
3025 void Assembler::fincstp() {
3026   emit_byte(0xD9);
3027   emit_byte(0xF7);
3028 }
3029 
3030 void Assembler::finit() {
3031   emit_byte(0x9B);
3032   emit_byte(0xDB);
3033   emit_byte(0xE3);
3034 }
3035 
3036 void Assembler::fist_s(Address adr) {
3037   InstructionMark im(this);
3038   emit_byte(0xDB);
3039   emit_operand32(rdx, adr);
3040 }
3041 
3042 void Assembler::fistp_d(Address adr) {
3043   InstructionMark im(this);
3044   emit_byte(0xDF);
3045   emit_operand32(rdi, adr);
3046 }
3047 
3048 void Assembler::fistp_s(Address adr) {
3049   InstructionMark im(this);
3050   emit_byte(0xDB);
3051   emit_operand32(rbx, adr);
3052 }
3053 
3054 void Assembler::fld1() {
3055   emit_byte(0xD9);
3056   emit_byte(0xE8);
3057 }
3058 
3059 void Assembler::fld_d(Address adr) {
3060   InstructionMark im(this);
3061   emit_byte(0xDD);
3062   emit_operand32(rax, adr);
3063 }
3064 
3065 void Assembler::fld_s(Address adr) {
3066   InstructionMark im(this);
3067   emit_byte(0xD9);
3068   emit_operand32(rax, adr);
3069 }
3070 
3071 
3072 void Assembler::fld_s(int index) {
3073   emit_farith(0xD9, 0xC0, index);
3074 }
3075 
3076 void Assembler::fld_x(Address adr) {
3077   InstructionMark im(this);
3078   emit_byte(0xDB);
3079   emit_operand32(rbp, adr);
3080 }
3081 
3082 void Assembler::fldcw(Address src) {
3083   InstructionMark im(this);
3084   emit_byte(0xd9);
3085   emit_operand32(rbp, src);
3086 }
3087 
3088 void Assembler::fldenv(Address src) {
3089   InstructionMark im(this);
3090   emit_byte(0xD9);
3091   emit_operand32(rsp, src);
3092 }
3093 
3094 void Assembler::fldlg2() {
3095   emit_byte(0xD9);
3096   emit_byte(0xEC);
3097 }
3098 
3099 void Assembler::fldln2() {
3100   emit_byte(0xD9);
3101   emit_byte(0xED);
3102 }
3103 
3104 void Assembler::fldz() {
3105   emit_byte(0xD9);
3106   emit_byte(0xEE);
3107 }
3108 
3109 void Assembler::flog() {
3110   fldln2();
3111   fxch();
3112   fyl2x();
3113 }
3114 
3115 void Assembler::flog10() {
3116   fldlg2();
3117   fxch();
3118   fyl2x();
3119 }
3120 
3121 void Assembler::fmul(int i) {
3122   emit_farith(0xD8, 0xC8, i);
3123 }
3124 
3125 void Assembler::fmul_d(Address src) {
3126   InstructionMark im(this);
3127   emit_byte(0xDC);
3128   emit_operand32(rcx, src);
3129 }
3130 
3131 void Assembler::fmul_s(Address src) {
3132   InstructionMark im(this);
3133   emit_byte(0xD8);
3134   emit_operand32(rcx, src);
3135 }
3136 
3137 void Assembler::fmula(int i) {
3138   emit_farith(0xDC, 0xC8, i);
3139 }
3140 
3141 void Assembler::fmulp(int i) {
3142   emit_farith(0xDE, 0xC8, i);
3143 }
3144 
3145 void Assembler::fnsave(Address dst) {
3146   InstructionMark im(this);
3147   emit_byte(0xDD);
3148   emit_operand32(rsi, dst);
3149 }
3150 
3151 void Assembler::fnstcw(Address src) {
3152   InstructionMark im(this);
3153   emit_byte(0x9B);
3154   emit_byte(0xD9);
3155   emit_operand32(rdi, src);
3156 }
3157 
3158 void Assembler::fnstsw_ax() {
3159   emit_byte(0xdF);
3160   emit_byte(0xE0);
3161 }
3162 
3163 void Assembler::fprem() {
3164   emit_byte(0xD9);
3165   emit_byte(0xF8);
3166 }
3167 
3168 void Assembler::fprem1() {
3169   emit_byte(0xD9);
3170   emit_byte(0xF5);
3171 }
3172 
3173 void Assembler::frstor(Address src) {
3174   InstructionMark im(this);
3175   emit_byte(0xDD);
3176   emit_operand32(rsp, src);
3177 }
3178 
3179 void Assembler::fsin() {
3180   emit_byte(0xD9);
3181   emit_byte(0xFE);
3182 }
3183 
3184 void Assembler::fsqrt() {
3185   emit_byte(0xD9);
3186   emit_byte(0xFA);
3187 }
3188 
3189 void Assembler::fst_d(Address adr) {
3190   InstructionMark im(this);
3191   emit_byte(0xDD);
3192   emit_operand32(rdx, adr);
3193 }
3194 
3195 void Assembler::fst_s(Address adr) {
3196   InstructionMark im(this);
3197   emit_byte(0xD9);
3198   emit_operand32(rdx, adr);
3199 }
3200 
3201 void Assembler::fstp_d(Address adr) {
3202   InstructionMark im(this);
3203   emit_byte(0xDD);
3204   emit_operand32(rbx, adr);
3205 }
3206 
3207 void Assembler::fstp_d(int index) {
3208   emit_farith(0xDD, 0xD8, index);
3209 }
3210 
3211 void Assembler::fstp_s(Address adr) {
3212   InstructionMark im(this);
3213   emit_byte(0xD9);
3214   emit_operand32(rbx, adr);
3215 }
3216 
3217 void Assembler::fstp_x(Address adr) {
3218   InstructionMark im(this);
3219   emit_byte(0xDB);
3220   emit_operand32(rdi, adr);
3221 }
3222 
3223 void Assembler::fsub(int i) {
3224   emit_farith(0xD8, 0xE0, i);
3225 }
3226 
3227 void Assembler::fsub_d(Address src) {
3228   InstructionMark im(this);
3229   emit_byte(0xDC);
3230   emit_operand32(rsp, src);
3231 }
3232 
3233 void Assembler::fsub_s(Address src) {
3234   InstructionMark im(this);
3235   emit_byte(0xD8);
3236   emit_operand32(rsp, src);
3237 }
3238 
3239 void Assembler::fsuba(int i) {
3240   emit_farith(0xDC, 0xE8, i);
3241 }
3242 
3243 void Assembler::fsubp(int i) {
3244   emit_farith(0xDE, 0xE8, i);                    // ST(0) <- ST(0) - ST(1) and pop (Intel manual wrong)
3245 }
3246 
3247 void Assembler::fsubr(int i) {
3248   emit_farith(0xD8, 0xE8, i);
3249 }
3250 
3251 void Assembler::fsubr_d(Address src) {
3252   InstructionMark im(this);
3253   emit_byte(0xDC);
3254   emit_operand32(rbp, src);
3255 }
3256 
3257 void Assembler::fsubr_s(Address src) {
3258   InstructionMark im(this);
3259   emit_byte(0xD8);
3260   emit_operand32(rbp, src);
3261 }
3262 
3263 void Assembler::fsubra(int i) {
3264   emit_farith(0xDC, 0xE0, i);
3265 }
3266 
3267 void Assembler::fsubrp(int i) {
3268   emit_farith(0xDE, 0xE0, i);                    // ST(0) <- ST(1) - ST(0) and pop (Intel manual wrong)
3269 }
3270 
3271 void Assembler::ftan() {
3272   emit_byte(0xD9);
3273   emit_byte(0xF2);
3274   emit_byte(0xDD);
3275   emit_byte(0xD8);
3276 }
3277 
3278 void Assembler::ftst() {
3279   emit_byte(0xD9);
3280   emit_byte(0xE4);
3281 }
3282 
3283 void Assembler::fucomi(int i) {
3284   // make sure the instruction is supported (introduced for P6, together with cmov)
3285   guarantee(VM_Version::supports_cmov(), "illegal instruction");
3286   emit_farith(0xDB, 0xE8, i);
3287 }
3288 
3289 void Assembler::fucomip(int i) {
3290   // make sure the instruction is supported (introduced for P6, together with cmov)
3291   guarantee(VM_Version::supports_cmov(), "illegal instruction");
3292   emit_farith(0xDF, 0xE8, i);
3293 }
3294 
3295 void Assembler::fwait() {
3296   emit_byte(0x9B);
3297 }
3298 
3299 void Assembler::fxch(int i) {
3300   emit_farith(0xD9, 0xC8, i);
3301 }
3302 
3303 void Assembler::fyl2x() {
3304   emit_byte(0xD9);
3305   emit_byte(0xF1);
3306 }
3307 
3308 
3309 #ifndef _LP64
3310 
3311 void Assembler::incl(Register dst) {
3312   // Don't use it directly. Use MacroAssembler::incrementl() instead.
3313  emit_byte(0x40 | dst->encoding());
3314 }
3315 
3316 void Assembler::lea(Register dst, Address src) {
3317   leal(dst, src);
3318 }
3319 
3320 void Assembler::mov_literal32(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
3321   InstructionMark im(this);
3322   emit_byte(0xC7);
3323   emit_operand(rax, dst);
3324   emit_data((int)imm32, rspec, 0);
3325 }
3326 
3327 void Assembler::mov_literal32(Register dst, int32_t imm32, RelocationHolder const& rspec) {
3328   InstructionMark im(this);
3329   int encode = prefix_and_encode(dst->encoding());
3330   emit_byte(0xB8 | encode);
3331   emit_data((int)imm32, rspec, 0);
3332 }
3333 
3334 void Assembler::popa() { // 32bit
3335   emit_byte(0x61);
3336 }
3337 
3338 void Assembler::push_literal32(int32_t imm32, RelocationHolder const& rspec) {
3339   InstructionMark im(this);
3340   emit_byte(0x68);
3341   emit_data(imm32, rspec, 0);
3342 }
3343 
3344 void Assembler::pusha() { // 32bit
3345   emit_byte(0x60);
3346 }
3347 
3348 void Assembler::set_byte_if_not_zero(Register dst) {
3349   emit_byte(0x0F);
3350   emit_byte(0x95);
3351   emit_byte(0xE0 | dst->encoding());
3352 }
3353 
3354 void Assembler::shldl(Register dst, Register src) {
3355   emit_byte(0x0F);
3356   emit_byte(0xA5);
3357   emit_byte(0xC0 | src->encoding() << 3 | dst->encoding());
3358 }
3359 
3360 void Assembler::shrdl(Register dst, Register src) {
3361   emit_byte(0x0F);
3362   emit_byte(0xAD);
3363   emit_byte(0xC0 | src->encoding() << 3 | dst->encoding());
3364 }
3365 
3366 #else // LP64
3367 
3368 void Assembler::set_byte_if_not_zero(Register dst) {
3369   int enc = prefix_and_encode(dst->encoding(), true);
3370   emit_byte(0x0F);
3371   emit_byte(0x95);
3372   emit_byte(0xE0 | enc);
3373 }
3374 
3375 // 64bit only pieces of the assembler
3376 // This should only be used by 64bit instructions that can use rip-relative
3377 // it cannot be used by instructions that want an immediate value.
3378 
3379 bool Assembler::reachable(AddressLiteral adr) {
3380   int64_t disp;
3381   // None will force a 64bit literal to the code stream. Likely a placeholder
3382   // for something that will be patched later and we need to certain it will
3383   // always be reachable.
3384   if (adr.reloc() == relocInfo::none) {
3385     return false;
3386   }
3387   if (adr.reloc() == relocInfo::internal_word_type) {
3388     // This should be rip relative and easily reachable.
3389     return true;
3390   }
3391   if (adr.reloc() == relocInfo::virtual_call_type ||
3392       adr.reloc() == relocInfo::opt_virtual_call_type ||
3393       adr.reloc() == relocInfo::static_call_type ||
3394       adr.reloc() == relocInfo::static_stub_type ) {
3395     // This should be rip relative within the code cache and easily
3396     // reachable until we get huge code caches. (At which point
3397     // ic code is going to have issues).
3398     return true;
3399   }
3400   if (adr.reloc() != relocInfo::external_word_type &&
3401       adr.reloc() != relocInfo::poll_return_type &&  // these are really external_word but need special
3402       adr.reloc() != relocInfo::poll_type &&         // relocs to identify them
3403       adr.reloc() != relocInfo::runtime_call_type ) {
3404     return false;
3405   }
3406 
3407   // Stress the correction code
3408   if (ForceUnreachable) {
3409     // Must be runtimecall reloc, see if it is in the codecache
3410     // Flipping stuff in the codecache to be unreachable causes issues
3411     // with things like inline caches where the additional instructions
3412     // are not handled.
3413     if (CodeCache::find_blob(adr._target) == NULL) {
3414       return false;
3415     }
3416   }
3417   // For external_word_type/runtime_call_type if it is reachable from where we
3418   // are now (possibly a temp buffer) and where we might end up
3419   // anywhere in the codeCache then we are always reachable.
3420   // This would have to change if we ever save/restore shared code
3421   // to be more pessimistic.
3422 
3423   disp = (int64_t)adr._target - ((int64_t)CodeCache::low_bound() + sizeof(int));
3424   if (!is_simm32(disp)) return false;
3425   disp = (int64_t)adr._target - ((int64_t)CodeCache::high_bound() + sizeof(int));
3426   if (!is_simm32(disp)) return false;
3427 
3428   disp = (int64_t)adr._target - ((int64_t)_code_pos + sizeof(int));
3429 
3430   // Because rip relative is a disp + address_of_next_instruction and we
3431   // don't know the value of address_of_next_instruction we apply a fudge factor
3432   // to make sure we will be ok no matter the size of the instruction we get placed into.
3433   // We don't have to fudge the checks above here because they are already worst case.
3434 
3435   // 12 == override/rex byte, opcode byte, rm byte, sib byte, a 4-byte disp , 4-byte literal
3436   // + 4 because better safe than sorry.
3437   const int fudge = 12 + 4;
3438   if (disp < 0) {
3439     disp -= fudge;
3440   } else {
3441     disp += fudge;
3442   }
3443   return is_simm32(disp);
3444 }
3445 
3446 void Assembler::emit_data64(jlong data,
3447                             relocInfo::relocType rtype,
3448                             int format) {
3449   if (rtype == relocInfo::none) {
3450     emit_long64(data);
3451   } else {
3452     emit_data64(data, Relocation::spec_simple(rtype), format);
3453   }
3454 }
3455 
3456 void Assembler::emit_data64(jlong data,
3457                             RelocationHolder const& rspec,
3458                             int format) {
3459   assert(imm_operand == 0, "default format must be immediate in this file");
3460   assert(imm_operand == format, "must be immediate");
3461   assert(inst_mark() != NULL, "must be inside InstructionMark");
3462   // Do not use AbstractAssembler::relocate, which is not intended for
3463   // embedded words.  Instead, relocate to the enclosing instruction.
3464   code_section()->relocate(inst_mark(), rspec, format);
3465 #ifdef ASSERT
3466   check_relocation(rspec, format);
3467 #endif
3468   emit_long64(data);
3469 }
3470 
3471 int Assembler::prefix_and_encode(int reg_enc, bool byteinst) {
3472   if (reg_enc >= 8) {
3473     prefix(REX_B);
3474     reg_enc -= 8;
3475   } else if (byteinst && reg_enc >= 4) {
3476     prefix(REX);
3477   }
3478   return reg_enc;
3479 }
3480 
3481 int Assembler::prefixq_and_encode(int reg_enc) {
3482   if (reg_enc < 8) {
3483     prefix(REX_W);
3484   } else {
3485     prefix(REX_WB);
3486     reg_enc -= 8;
3487   }
3488   return reg_enc;
3489 }
3490 
3491 int Assembler::prefix_and_encode(int dst_enc, int src_enc, bool byteinst) {
3492   if (dst_enc < 8) {
3493     if (src_enc >= 8) {
3494       prefix(REX_B);
3495       src_enc -= 8;
3496     } else if (byteinst && src_enc >= 4) {
3497       prefix(REX);
3498     }
3499   } else {
3500     if (src_enc < 8) {
3501       prefix(REX_R);
3502     } else {
3503       prefix(REX_RB);
3504       src_enc -= 8;
3505     }
3506     dst_enc -= 8;
3507   }
3508   return dst_enc << 3 | src_enc;
3509 }
3510 
3511 int Assembler::prefixq_and_encode(int dst_enc, int src_enc) {
3512   if (dst_enc < 8) {
3513     if (src_enc < 8) {
3514       prefix(REX_W);
3515     } else {
3516       prefix(REX_WB);
3517       src_enc -= 8;
3518     }
3519   } else {
3520     if (src_enc < 8) {
3521       prefix(REX_WR);
3522     } else {
3523       prefix(REX_WRB);
3524       src_enc -= 8;
3525     }
3526     dst_enc -= 8;
3527   }
3528   return dst_enc << 3 | src_enc;
3529 }
3530 
3531 void Assembler::prefix(Register reg) {
3532   if (reg->encoding() >= 8) {
3533     prefix(REX_B);
3534   }
3535 }
3536 
3537 void Assembler::prefix(Address adr) {
3538   if (adr.base_needs_rex()) {
3539     if (adr.index_needs_rex()) {
3540       prefix(REX_XB);
3541     } else {
3542       prefix(REX_B);
3543     }
3544   } else {
3545     if (adr.index_needs_rex()) {
3546       prefix(REX_X);
3547     }
3548   }
3549 }
3550 
3551 void Assembler::prefixq(Address adr) {
3552   if (adr.base_needs_rex()) {
3553     if (adr.index_needs_rex()) {
3554       prefix(REX_WXB);
3555     } else {
3556       prefix(REX_WB);
3557     }
3558   } else {
3559     if (adr.index_needs_rex()) {
3560       prefix(REX_WX);
3561     } else {
3562       prefix(REX_W);
3563     }
3564   }
3565 }
3566 
3567 
3568 void Assembler::prefix(Address adr, Register reg, bool byteinst) {
3569   if (reg->encoding() < 8) {
3570     if (adr.base_needs_rex()) {
3571       if (adr.index_needs_rex()) {
3572         prefix(REX_XB);
3573       } else {
3574         prefix(REX_B);
3575       }
3576     } else {
3577       if (adr.index_needs_rex()) {
3578         prefix(REX_X);
3579       } else if (reg->encoding() >= 4 ) {
3580         prefix(REX);
3581       }
3582     }
3583   } else {
3584     if (adr.base_needs_rex()) {
3585       if (adr.index_needs_rex()) {
3586         prefix(REX_RXB);
3587       } else {
3588         prefix(REX_RB);
3589       }
3590     } else {
3591       if (adr.index_needs_rex()) {
3592         prefix(REX_RX);
3593       } else {
3594         prefix(REX_R);
3595       }
3596     }
3597   }
3598 }
3599 
3600 void Assembler::prefixq(Address adr, Register src) {
3601   if (src->encoding() < 8) {
3602     if (adr.base_needs_rex()) {
3603       if (adr.index_needs_rex()) {
3604         prefix(REX_WXB);
3605       } else {
3606         prefix(REX_WB);
3607       }
3608     } else {
3609       if (adr.index_needs_rex()) {
3610         prefix(REX_WX);
3611       } else {
3612         prefix(REX_W);
3613       }
3614     }
3615   } else {
3616     if (adr.base_needs_rex()) {
3617       if (adr.index_needs_rex()) {
3618         prefix(REX_WRXB);
3619       } else {
3620         prefix(REX_WRB);
3621       }
3622     } else {
3623       if (adr.index_needs_rex()) {
3624         prefix(REX_WRX);
3625       } else {
3626         prefix(REX_WR);
3627       }
3628     }
3629   }
3630 }
3631 
3632 void Assembler::prefix(Address adr, XMMRegister reg) {
3633   if (reg->encoding() < 8) {
3634     if (adr.base_needs_rex()) {
3635       if (adr.index_needs_rex()) {
3636         prefix(REX_XB);
3637       } else {
3638         prefix(REX_B);
3639       }
3640     } else {
3641       if (adr.index_needs_rex()) {
3642         prefix(REX_X);
3643       }
3644     }
3645   } else {
3646     if (adr.base_needs_rex()) {
3647       if (adr.index_needs_rex()) {
3648         prefix(REX_RXB);
3649       } else {
3650         prefix(REX_RB);
3651       }
3652     } else {
3653       if (adr.index_needs_rex()) {
3654         prefix(REX_RX);
3655       } else {
3656         prefix(REX_R);
3657       }
3658     }
3659   }
3660 }
3661 
3662 void Assembler::adcq(Register dst, int32_t imm32) {
3663   (void) prefixq_and_encode(dst->encoding());
3664   emit_arith(0x81, 0xD0, dst, imm32);
3665 }
3666 
3667 void Assembler::adcq(Register dst, Address src) {
3668   InstructionMark im(this);
3669   prefixq(src, dst);
3670   emit_byte(0x13);
3671   emit_operand(dst, src);
3672 }
3673 
3674 void Assembler::adcq(Register dst, Register src) {
3675   (int) prefixq_and_encode(dst->encoding(), src->encoding());
3676   emit_arith(0x13, 0xC0, dst, src);
3677 }
3678 
3679 void Assembler::addq(Address dst, int32_t imm32) {
3680   InstructionMark im(this);
3681   prefixq(dst);
3682   emit_arith_operand(0x81, rax, dst,imm32);
3683 }
3684 
3685 void Assembler::addq(Address dst, Register src) {
3686   InstructionMark im(this);
3687   prefixq(dst, src);
3688   emit_byte(0x01);
3689   emit_operand(src, dst);
3690 }
3691 
3692 void Assembler::addq(Register dst, int32_t imm32) {
3693   (void) prefixq_and_encode(dst->encoding());
3694   emit_arith(0x81, 0xC0, dst, imm32);
3695 }
3696 
3697 void Assembler::addq(Register dst, Address src) {
3698   InstructionMark im(this);
3699   prefixq(src, dst);
3700   emit_byte(0x03);
3701   emit_operand(dst, src);
3702 }
3703 
3704 void Assembler::addq(Register dst, Register src) {
3705   (void) prefixq_and_encode(dst->encoding(), src->encoding());
3706   emit_arith(0x03, 0xC0, dst, src);
3707 }
3708 
3709 void Assembler::andq(Register dst, int32_t imm32) {
3710   (void) prefixq_and_encode(dst->encoding());
3711   emit_arith(0x81, 0xE0, dst, imm32);
3712 }
3713 
3714 void Assembler::andq(Register dst, Address src) {
3715   InstructionMark im(this);
3716   prefixq(src, dst);
3717   emit_byte(0x23);
3718   emit_operand(dst, src);
3719 }
3720 
3721 void Assembler::andq(Register dst, Register src) {
3722   (int) prefixq_and_encode(dst->encoding(), src->encoding());
3723   emit_arith(0x23, 0xC0, dst, src);
3724 }
3725 
3726 void Assembler::bsfq(Register dst, Register src) {
3727   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3728   emit_byte(0x0F);
3729   emit_byte(0xBC);
3730   emit_byte(0xC0 | encode);
3731 }
3732 
3733 void Assembler::bsrq(Register dst, Register src) {
3734   assert(!VM_Version::supports_lzcnt(), "encoding is treated as LZCNT");
3735   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3736   emit_byte(0x0F);
3737   emit_byte(0xBD);
3738   emit_byte(0xC0 | encode);
3739 }
3740 
3741 void Assembler::bswapq(Register reg) {
3742   int encode = prefixq_and_encode(reg->encoding());
3743   emit_byte(0x0F);
3744   emit_byte(0xC8 | encode);
3745 }
3746 
3747 void Assembler::cdqq() {
3748   prefix(REX_W);
3749   emit_byte(0x99);
3750 }
3751 
3752 void Assembler::clflush(Address adr) {
3753   prefix(adr);
3754   emit_byte(0x0F);
3755   emit_byte(0xAE);
3756   emit_operand(rdi, adr);
3757 }
3758 
3759 void Assembler::cmovq(Condition cc, Register dst, Register src) {
3760   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3761   emit_byte(0x0F);
3762   emit_byte(0x40 | cc);
3763   emit_byte(0xC0 | encode);
3764 }
3765 
3766 void Assembler::cmovq(Condition cc, Register dst, Address src) {
3767   InstructionMark im(this);
3768   prefixq(src, dst);
3769   emit_byte(0x0F);
3770   emit_byte(0x40 | cc);
3771   emit_operand(dst, src);
3772 }
3773 
3774 void Assembler::cmpq(Address dst, int32_t imm32) {
3775   InstructionMark im(this);
3776   prefixq(dst);
3777   emit_byte(0x81);
3778   emit_operand(rdi, dst, 4);
3779   emit_long(imm32);
3780 }
3781 
3782 void Assembler::cmpq(Register dst, int32_t imm32) {
3783   (void) prefixq_and_encode(dst->encoding());
3784   emit_arith(0x81, 0xF8, dst, imm32);
3785 }
3786 
3787 void Assembler::cmpq(Address dst, Register src) {
3788   InstructionMark im(this);
3789   prefixq(dst, src);
3790   emit_byte(0x3B);
3791   emit_operand(src, dst);
3792 }
3793 
3794 void Assembler::cmpq(Register dst, Register src) {
3795   (void) prefixq_and_encode(dst->encoding(), src->encoding());
3796   emit_arith(0x3B, 0xC0, dst, src);
3797 }
3798 
3799 void Assembler::cmpq(Register dst, Address  src) {
3800   InstructionMark im(this);
3801   prefixq(src, dst);
3802   emit_byte(0x3B);
3803   emit_operand(dst, src);
3804 }
3805 
3806 void Assembler::cmpxchgq(Register reg, Address adr) {
3807   InstructionMark im(this);
3808   prefixq(adr, reg);
3809   emit_byte(0x0F);
3810   emit_byte(0xB1);
3811   emit_operand(reg, adr);
3812 }
3813 
3814 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
3815   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3816   emit_byte(0xF2);
3817   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3818   emit_byte(0x0F);
3819   emit_byte(0x2A);
3820   emit_byte(0xC0 | encode);
3821 }
3822 
3823 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
3824   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3825   emit_byte(0xF3);
3826   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3827   emit_byte(0x0F);
3828   emit_byte(0x2A);
3829   emit_byte(0xC0 | encode);
3830 }
3831 
3832 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
3833   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
3834   emit_byte(0xF2);
3835   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3836   emit_byte(0x0F);
3837   emit_byte(0x2C);
3838   emit_byte(0xC0 | encode);
3839 }
3840 
3841 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
3842   NOT_LP64(assert(VM_Version::supports_sse(), ""));
3843   emit_byte(0xF3);
3844   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3845   emit_byte(0x0F);
3846   emit_byte(0x2C);
3847   emit_byte(0xC0 | encode);
3848 }
3849 
3850 void Assembler::decl(Register dst) {
3851   // Don't use it directly. Use MacroAssembler::decrementl() instead.
3852   // Use two-byte form (one-byte form is a REX prefix in 64-bit mode)
3853   int encode = prefix_and_encode(dst->encoding());
3854   emit_byte(0xFF);
3855   emit_byte(0xC8 | encode);
3856 }
3857 
3858 void Assembler::decq(Register dst) {
3859   // Don't use it directly. Use MacroAssembler::decrementq() instead.
3860   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
3861   int encode = prefixq_and_encode(dst->encoding());
3862   emit_byte(0xFF);
3863   emit_byte(0xC8 | encode);
3864 }
3865 
3866 void Assembler::decq(Address dst) {
3867   // Don't use it directly. Use MacroAssembler::decrementq() instead.
3868   InstructionMark im(this);
3869   prefixq(dst);
3870   emit_byte(0xFF);
3871   emit_operand(rcx, dst);
3872 }
3873 
3874 void Assembler::fxrstor(Address src) {
3875   prefixq(src);
3876   emit_byte(0x0F);
3877   emit_byte(0xAE);
3878   emit_operand(as_Register(1), src);
3879 }
3880 
3881 void Assembler::fxsave(Address dst) {
3882   prefixq(dst);
3883   emit_byte(0x0F);
3884   emit_byte(0xAE);
3885   emit_operand(as_Register(0), dst);
3886 }
3887 
3888 void Assembler::idivq(Register src) {
3889   int encode = prefixq_and_encode(src->encoding());
3890   emit_byte(0xF7);
3891   emit_byte(0xF8 | encode);
3892 }
3893 
3894 void Assembler::imulq(Register dst, Register src) {
3895   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3896   emit_byte(0x0F);
3897   emit_byte(0xAF);
3898   emit_byte(0xC0 | encode);
3899 }
3900 
3901 void Assembler::imulq(Register dst, Register src, int value) {
3902   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3903   if (is8bit(value)) {
3904     emit_byte(0x6B);
3905     emit_byte(0xC0 | encode);
3906     emit_byte(value);
3907   } else {
3908     emit_byte(0x69);
3909     emit_byte(0xC0 | encode);
3910     emit_long(value);
3911   }
3912 }
3913 
3914 void Assembler::incl(Register dst) {
3915   // Don't use it directly. Use MacroAssembler::incrementl() instead.
3916   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
3917   int encode = prefix_and_encode(dst->encoding());
3918   emit_byte(0xFF);
3919   emit_byte(0xC0 | encode);
3920 }
3921 
3922 void Assembler::incq(Register dst) {
3923   // Don't use it directly. Use MacroAssembler::incrementq() instead.
3924   // Use two-byte form (one-byte from is a REX prefix in 64-bit mode)
3925   int encode = prefixq_and_encode(dst->encoding());
3926   emit_byte(0xFF);
3927   emit_byte(0xC0 | encode);
3928 }
3929 
3930 void Assembler::incq(Address dst) {
3931   // Don't use it directly. Use MacroAssembler::incrementq() instead.
3932   InstructionMark im(this);
3933   prefixq(dst);
3934   emit_byte(0xFF);
3935   emit_operand(rax, dst);
3936 }
3937 
3938 void Assembler::lea(Register dst, Address src) {
3939   leaq(dst, src);
3940 }
3941 
3942 void Assembler::leaq(Register dst, Address src) {
3943   InstructionMark im(this);
3944   prefixq(src, dst);
3945   emit_byte(0x8D);
3946   emit_operand(dst, src);
3947 }
3948 
3949 void Assembler::mov64(Register dst, int64_t imm64) {
3950   InstructionMark im(this);
3951   int encode = prefixq_and_encode(dst->encoding());
3952   emit_byte(0xB8 | encode);
3953   emit_long64(imm64);
3954 }
3955 
3956 void Assembler::mov_literal64(Register dst, intptr_t imm64, RelocationHolder const& rspec) {
3957   InstructionMark im(this);
3958   int encode = prefixq_and_encode(dst->encoding());
3959   emit_byte(0xB8 | encode);
3960   emit_data64(imm64, rspec);
3961 }
3962 
3963 void Assembler::mov_narrow_oop(Register dst, int32_t imm32, RelocationHolder const& rspec) {
3964   InstructionMark im(this);
3965   int encode = prefix_and_encode(dst->encoding());
3966   emit_byte(0xB8 | encode);
3967   emit_data((int)imm32, rspec, narrow_oop_operand);
3968 }
3969 
3970 void Assembler::mov_narrow_oop(Address dst, int32_t imm32,  RelocationHolder const& rspec) {
3971   InstructionMark im(this);
3972   prefix(dst);
3973   emit_byte(0xC7);
3974   emit_operand(rax, dst, 4);
3975   emit_data((int)imm32, rspec, narrow_oop_operand);
3976 }
3977 
3978 void Assembler::cmp_narrow_oop(Register src1, int32_t imm32, RelocationHolder const& rspec) {
3979   InstructionMark im(this);
3980   int encode = prefix_and_encode(src1->encoding());
3981   emit_byte(0x81);
3982   emit_byte(0xF8 | encode);
3983   emit_data((int)imm32, rspec, narrow_oop_operand);
3984 }
3985 
3986 void Assembler::cmp_narrow_oop(Address src1, int32_t imm32, RelocationHolder const& rspec) {
3987   InstructionMark im(this);
3988   prefix(src1);
3989   emit_byte(0x81);
3990   emit_operand(rax, src1, 4);
3991   emit_data((int)imm32, rspec, narrow_oop_operand);
3992 }
3993 
3994 void Assembler::lzcntq(Register dst, Register src) {
3995   assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
3996   emit_byte(0xF3);
3997   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
3998   emit_byte(0x0F);
3999   emit_byte(0xBD);
4000   emit_byte(0xC0 | encode);
4001 }
4002 
4003 void Assembler::movdq(XMMRegister dst, Register src) {
4004   // table D-1 says MMX/SSE2
4005   NOT_LP64(assert(VM_Version::supports_sse2() || VM_Version::supports_mmx(), ""));
4006   emit_byte(0x66);
4007   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4008   emit_byte(0x0F);
4009   emit_byte(0x6E);
4010   emit_byte(0xC0 | encode);
4011 }
4012 
4013 void Assembler::movdq(Register dst, XMMRegister src) {
4014   // table D-1 says MMX/SSE2
4015   NOT_LP64(assert(VM_Version::supports_sse2() || VM_Version::supports_mmx(), ""));
4016   emit_byte(0x66);
4017   // swap src/dst to get correct prefix
4018   int encode = prefixq_and_encode(src->encoding(), dst->encoding());
4019   emit_byte(0x0F);
4020   emit_byte(0x7E);
4021   emit_byte(0xC0 | encode);
4022 }
4023 
4024 void Assembler::movq(Register dst, Register src) {
4025   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4026   emit_byte(0x8B);
4027   emit_byte(0xC0 | encode);
4028 }
4029 
4030 void Assembler::movq(Register dst, Address src) {
4031   InstructionMark im(this);
4032   prefixq(src, dst);
4033   emit_byte(0x8B);
4034   emit_operand(dst, src);
4035 }
4036 
4037 void Assembler::movq(Address dst, Register src) {
4038   InstructionMark im(this);
4039   prefixq(dst, src);
4040   emit_byte(0x89);
4041   emit_operand(src, dst);
4042 }
4043 
4044 void Assembler::movsbq(Register dst, Address src) {
4045   InstructionMark im(this);
4046   prefixq(src, dst);
4047   emit_byte(0x0F);
4048   emit_byte(0xBE);
4049   emit_operand(dst, src);
4050 }
4051 
4052 void Assembler::movsbq(Register dst, Register src) {
4053   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4054   emit_byte(0x0F);
4055   emit_byte(0xBE);
4056   emit_byte(0xC0 | encode);
4057 }
4058 
4059 void Assembler::movslq(Register dst, int32_t imm32) {
4060   // dbx shows movslq(rcx, 3) as movq     $0x0000000049000000,(%rbx)
4061   // and movslq(r8, 3); as movl     $0x0000000048000000,(%rbx)
4062   // as a result we shouldn't use until tested at runtime...
4063   ShouldNotReachHere();
4064   InstructionMark im(this);
4065   int encode = prefixq_and_encode(dst->encoding());
4066   emit_byte(0xC7 | encode);
4067   emit_long(imm32);
4068 }
4069 
4070 void Assembler::movslq(Address dst, int32_t imm32) {
4071   assert(is_simm32(imm32), "lost bits");
4072   InstructionMark im(this);
4073   prefixq(dst);
4074   emit_byte(0xC7);
4075   emit_operand(rax, dst, 4);
4076   emit_long(imm32);
4077 }
4078 
4079 void Assembler::movslq(Register dst, Address src) {
4080   InstructionMark im(this);
4081   prefixq(src, dst);
4082   emit_byte(0x63);
4083   emit_operand(dst, src);
4084 }
4085 
4086 void Assembler::movslq(Register dst, Register src) {
4087   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4088   emit_byte(0x63);
4089   emit_byte(0xC0 | encode);
4090 }
4091 
4092 void Assembler::movswq(Register dst, Address src) {
4093   InstructionMark im(this);
4094   prefixq(src, dst);
4095   emit_byte(0x0F);
4096   emit_byte(0xBF);
4097   emit_operand(dst, src);
4098 }
4099 
4100 void Assembler::movswq(Register dst, Register src) {
4101   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4102   emit_byte(0x0F);
4103   emit_byte(0xBF);
4104   emit_byte(0xC0 | encode);
4105 }
4106 
4107 void Assembler::movzbq(Register dst, Address src) {
4108   InstructionMark im(this);
4109   prefixq(src, dst);
4110   emit_byte(0x0F);
4111   emit_byte(0xB6);
4112   emit_operand(dst, src);
4113 }
4114 
4115 void Assembler::movzbq(Register dst, Register src) {
4116   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4117   emit_byte(0x0F);
4118   emit_byte(0xB6);
4119   emit_byte(0xC0 | encode);
4120 }
4121 
4122 void Assembler::movzwq(Register dst, Address src) {
4123   InstructionMark im(this);
4124   prefixq(src, dst);
4125   emit_byte(0x0F);
4126   emit_byte(0xB7);
4127   emit_operand(dst, src);
4128 }
4129 
4130 void Assembler::movzwq(Register dst, Register src) {
4131   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4132   emit_byte(0x0F);
4133   emit_byte(0xB7);
4134   emit_byte(0xC0 | encode);
4135 }
4136 
4137 void Assembler::negq(Register dst) {
4138   int encode = prefixq_and_encode(dst->encoding());
4139   emit_byte(0xF7);
4140   emit_byte(0xD8 | encode);
4141 }
4142 
4143 void Assembler::notq(Register dst) {
4144   int encode = prefixq_and_encode(dst->encoding());
4145   emit_byte(0xF7);
4146   emit_byte(0xD0 | encode);
4147 }
4148 
4149 void Assembler::orq(Address dst, int32_t imm32) {
4150   InstructionMark im(this);
4151   prefixq(dst);
4152   emit_byte(0x81);
4153   emit_operand(rcx, dst, 4);
4154   emit_long(imm32);
4155 }
4156 
4157 void Assembler::orq(Register dst, int32_t imm32) {
4158   (void) prefixq_and_encode(dst->encoding());
4159   emit_arith(0x81, 0xC8, dst, imm32);
4160 }
4161 
4162 void Assembler::orq(Register dst, Address src) {
4163   InstructionMark im(this);
4164   prefixq(src, dst);
4165   emit_byte(0x0B);
4166   emit_operand(dst, src);
4167 }
4168 
4169 void Assembler::orq(Register dst, Register src) {
4170   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4171   emit_arith(0x0B, 0xC0, dst, src);
4172 }
4173 
4174 void Assembler::popa() { // 64bit
4175   movq(r15, Address(rsp, 0));
4176   movq(r14, Address(rsp, wordSize));
4177   movq(r13, Address(rsp, 2 * wordSize));
4178   movq(r12, Address(rsp, 3 * wordSize));
4179   movq(r11, Address(rsp, 4 * wordSize));
4180   movq(r10, Address(rsp, 5 * wordSize));
4181   movq(r9,  Address(rsp, 6 * wordSize));
4182   movq(r8,  Address(rsp, 7 * wordSize));
4183   movq(rdi, Address(rsp, 8 * wordSize));
4184   movq(rsi, Address(rsp, 9 * wordSize));
4185   movq(rbp, Address(rsp, 10 * wordSize));
4186   // skip rsp
4187   movq(rbx, Address(rsp, 12 * wordSize));
4188   movq(rdx, Address(rsp, 13 * wordSize));
4189   movq(rcx, Address(rsp, 14 * wordSize));
4190   movq(rax, Address(rsp, 15 * wordSize));
4191 
4192   addq(rsp, 16 * wordSize);
4193 }
4194 
4195 void Assembler::popcntq(Register dst, Address src) {
4196   assert(VM_Version::supports_popcnt(), "must support");
4197   InstructionMark im(this);
4198   emit_byte(0xF3);
4199   prefixq(src, dst);
4200   emit_byte(0x0F);
4201   emit_byte(0xB8);
4202   emit_operand(dst, src);
4203 }
4204 
4205 void Assembler::popcntq(Register dst, Register src) {
4206   assert(VM_Version::supports_popcnt(), "must support");
4207   emit_byte(0xF3);
4208   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4209   emit_byte(0x0F);
4210   emit_byte(0xB8);
4211   emit_byte(0xC0 | encode);
4212 }
4213 
4214 void Assembler::popq(Address dst) {
4215   InstructionMark im(this);
4216   prefixq(dst);
4217   emit_byte(0x8F);
4218   emit_operand(rax, dst);
4219 }
4220 
4221 void Assembler::pusha() { // 64bit
4222   // we have to store original rsp.  ABI says that 128 bytes
4223   // below rsp are local scratch.
4224   movq(Address(rsp, -5 * wordSize), rsp);
4225 
4226   subq(rsp, 16 * wordSize);
4227 
4228   movq(Address(rsp, 15 * wordSize), rax);
4229   movq(Address(rsp, 14 * wordSize), rcx);
4230   movq(Address(rsp, 13 * wordSize), rdx);
4231   movq(Address(rsp, 12 * wordSize), rbx);
4232   // skip rsp
4233   movq(Address(rsp, 10 * wordSize), rbp);
4234   movq(Address(rsp, 9 * wordSize), rsi);
4235   movq(Address(rsp, 8 * wordSize), rdi);
4236   movq(Address(rsp, 7 * wordSize), r8);
4237   movq(Address(rsp, 6 * wordSize), r9);
4238   movq(Address(rsp, 5 * wordSize), r10);
4239   movq(Address(rsp, 4 * wordSize), r11);
4240   movq(Address(rsp, 3 * wordSize), r12);
4241   movq(Address(rsp, 2 * wordSize), r13);
4242   movq(Address(rsp, wordSize), r14);
4243   movq(Address(rsp, 0), r15);
4244 }
4245 
4246 void Assembler::pushq(Address src) {
4247   InstructionMark im(this);
4248   prefixq(src);
4249   emit_byte(0xFF);
4250   emit_operand(rsi, src);
4251 }
4252 
4253 void Assembler::rclq(Register dst, int imm8) {
4254   assert(isShiftCount(imm8 >> 1), "illegal shift count");
4255   int encode = prefixq_and_encode(dst->encoding());
4256   if (imm8 == 1) {
4257     emit_byte(0xD1);
4258     emit_byte(0xD0 | encode);
4259   } else {
4260     emit_byte(0xC1);
4261     emit_byte(0xD0 | encode);
4262     emit_byte(imm8);
4263   }
4264 }
4265 void Assembler::sarq(Register dst, int imm8) {
4266   assert(isShiftCount(imm8 >> 1), "illegal shift count");
4267   int encode = prefixq_and_encode(dst->encoding());
4268   if (imm8 == 1) {
4269     emit_byte(0xD1);
4270     emit_byte(0xF8 | encode);
4271   } else {
4272     emit_byte(0xC1);
4273     emit_byte(0xF8 | encode);
4274     emit_byte(imm8);
4275   }
4276 }
4277 
4278 void Assembler::sarq(Register dst) {
4279   int encode = prefixq_and_encode(dst->encoding());
4280   emit_byte(0xD3);
4281   emit_byte(0xF8 | encode);
4282 }
4283 void Assembler::sbbq(Address dst, int32_t imm32) {
4284   InstructionMark im(this);
4285   prefixq(dst);
4286   emit_arith_operand(0x81, rbx, dst, imm32);
4287 }
4288 
4289 void Assembler::sbbq(Register dst, int32_t imm32) {
4290   (void) prefixq_and_encode(dst->encoding());
4291   emit_arith(0x81, 0xD8, dst, imm32);
4292 }
4293 
4294 void Assembler::sbbq(Register dst, Address src) {
4295   InstructionMark im(this);
4296   prefixq(src, dst);
4297   emit_byte(0x1B);
4298   emit_operand(dst, src);
4299 }
4300 
4301 void Assembler::sbbq(Register dst, Register src) {
4302   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4303   emit_arith(0x1B, 0xC0, dst, src);
4304 }
4305 
4306 void Assembler::shlq(Register dst, int imm8) {
4307   assert(isShiftCount(imm8 >> 1), "illegal shift count");
4308   int encode = prefixq_and_encode(dst->encoding());
4309   if (imm8 == 1) {
4310     emit_byte(0xD1);
4311     emit_byte(0xE0 | encode);
4312   } else {
4313     emit_byte(0xC1);
4314     emit_byte(0xE0 | encode);
4315     emit_byte(imm8);
4316   }
4317 }
4318 
4319 void Assembler::shlq(Register dst) {
4320   int encode = prefixq_and_encode(dst->encoding());
4321   emit_byte(0xD3);
4322   emit_byte(0xE0 | encode);
4323 }
4324 
4325 void Assembler::shrq(Register dst, int imm8) {
4326   assert(isShiftCount(imm8 >> 1), "illegal shift count");
4327   int encode = prefixq_and_encode(dst->encoding());
4328   emit_byte(0xC1);
4329   emit_byte(0xE8 | encode);
4330   emit_byte(imm8);
4331 }
4332 
4333 void Assembler::shrq(Register dst) {
4334   int encode = prefixq_and_encode(dst->encoding());
4335   emit_byte(0xD3);
4336   emit_byte(0xE8 | encode);
4337 }
4338 
4339 void Assembler::sqrtsd(XMMRegister dst, Address src) {
4340   NOT_LP64(assert(VM_Version::supports_sse2(), ""));
4341   InstructionMark im(this);
4342   emit_byte(0xF2);
4343   prefix(src, dst);
4344   emit_byte(0x0F);
4345   emit_byte(0x51);
4346   emit_operand(dst, src);
4347 }
4348 
4349 void Assembler::subq(Address dst, int32_t imm32) {
4350   InstructionMark im(this);
4351   prefixq(dst);
4352   if (is8bit(imm32)) {
4353     emit_byte(0x83);
4354     emit_operand(rbp, dst, 1);
4355     emit_byte(imm32 & 0xFF);
4356   } else {
4357     emit_byte(0x81);
4358     emit_operand(rbp, dst, 4);
4359     emit_long(imm32);
4360   }
4361 }
4362 
4363 void Assembler::subq(Register dst, int32_t imm32) {
4364   (void) prefixq_and_encode(dst->encoding());
4365   emit_arith(0x81, 0xE8, dst, imm32);
4366 }
4367 
4368 void Assembler::subq(Address dst, Register src) {
4369   InstructionMark im(this);
4370   prefixq(dst, src);
4371   emit_byte(0x29);
4372   emit_operand(src, dst);
4373 }
4374 
4375 void Assembler::subq(Register dst, Address src) {
4376   InstructionMark im(this);
4377   prefixq(src, dst);
4378   emit_byte(0x2B);
4379   emit_operand(dst, src);
4380 }
4381 
4382 void Assembler::subq(Register dst, Register src) {
4383   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4384   emit_arith(0x2B, 0xC0, dst, src);
4385 }
4386 
4387 void Assembler::testq(Register dst, int32_t imm32) {
4388   // not using emit_arith because test
4389   // doesn't support sign-extension of
4390   // 8bit operands
4391   int encode = dst->encoding();
4392   if (encode == 0) {
4393     prefix(REX_W);
4394     emit_byte(0xA9);
4395   } else {
4396     encode = prefixq_and_encode(encode);
4397     emit_byte(0xF7);
4398     emit_byte(0xC0 | encode);
4399   }
4400   emit_long(imm32);
4401 }
4402 
4403 void Assembler::testq(Register dst, Register src) {
4404   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4405   emit_arith(0x85, 0xC0, dst, src);
4406 }
4407 
4408 void Assembler::xaddq(Address dst, Register src) {
4409   InstructionMark im(this);
4410   prefixq(dst, src);
4411   emit_byte(0x0F);
4412   emit_byte(0xC1);
4413   emit_operand(src, dst);
4414 }
4415 
4416 void Assembler::xchgq(Register dst, Address src) {
4417   InstructionMark im(this);
4418   prefixq(src, dst);
4419   emit_byte(0x87);
4420   emit_operand(dst, src);
4421 }
4422 
4423 void Assembler::xchgq(Register dst, Register src) {
4424   int encode = prefixq_and_encode(dst->encoding(), src->encoding());
4425   emit_byte(0x87);
4426   emit_byte(0xc0 | encode);
4427 }
4428 
4429 void Assembler::xorq(Register dst, Register src) {
4430   (void) prefixq_and_encode(dst->encoding(), src->encoding());
4431   emit_arith(0x33, 0xC0, dst, src);
4432 }
4433 
4434 void Assembler::xorq(Register dst, Address src) {
4435   InstructionMark im(this);
4436   prefixq(src, dst);
4437   emit_byte(0x33);
4438   emit_operand(dst, src);
4439 }
4440 
4441 #endif // !LP64
4442 
4443 static Assembler::Condition reverse[] = {
4444     Assembler::noOverflow     /* overflow      = 0x0 */ ,
4445     Assembler::overflow       /* noOverflow    = 0x1 */ ,
4446     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
4447     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
4448     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
4449     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
4450     Assembler::above          /* belowEqual    = 0x6 */ ,
4451     Assembler::belowEqual     /* above         = 0x7 */ ,
4452     Assembler::positive       /* negative      = 0x8 */ ,
4453     Assembler::negative       /* positive      = 0x9 */ ,
4454     Assembler::noParity       /* parity        = 0xa */ ,
4455     Assembler::parity         /* noParity      = 0xb */ ,
4456     Assembler::greaterEqual   /* less          = 0xc */ ,
4457     Assembler::less           /* greaterEqual  = 0xd */ ,
4458     Assembler::greater        /* lessEqual     = 0xe */ ,
4459     Assembler::lessEqual      /* greater       = 0xf, */
4460 
4461 };
4462 
4463 
4464 // Implementation of MacroAssembler
4465 
4466 // First all the versions that have distinct versions depending on 32/64 bit
4467 // Unless the difference is trivial (1 line or so).
4468 
4469 #ifndef _LP64
4470 
4471 // 32bit versions
4472 
4473 Address MacroAssembler::as_Address(AddressLiteral adr) {
4474   return Address(adr.target(), adr.rspec());
4475 }
4476 
4477 Address MacroAssembler::as_Address(ArrayAddress adr) {
4478   return Address::make_array(adr);
4479 }
4480 
4481 int MacroAssembler::biased_locking_enter(Register lock_reg,
4482                                          Register obj_reg,
4483                                          Register swap_reg,
4484                                          Register tmp_reg,
4485                                          bool swap_reg_contains_mark,
4486                                          Label& done,
4487                                          Label* slow_case,
4488                                          BiasedLockingCounters* counters) {
4489   assert(UseBiasedLocking, "why call this otherwise?");
4490   assert(swap_reg == rax, "swap_reg must be rax, for cmpxchg");
4491   assert_different_registers(lock_reg, obj_reg, swap_reg);
4492 
4493   if (PrintBiasedLockingStatistics && counters == NULL)
4494     counters = BiasedLocking::counters();
4495 
4496   bool need_tmp_reg = false;
4497   if (tmp_reg == noreg) {
4498     need_tmp_reg = true;
4499     tmp_reg = lock_reg;
4500   } else {
4501     assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
4502   }
4503   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
4504   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
4505   Address klass_addr     (obj_reg, oopDesc::klass_offset_in_bytes());
4506   Address saved_mark_addr(lock_reg, 0);
4507 
4508   // Biased locking
4509   // See whether the lock is currently biased toward our thread and
4510   // whether the epoch is still valid
4511   // Note that the runtime guarantees sufficient alignment of JavaThread
4512   // pointers to allow age to be placed into low bits
4513   // First check to see whether biasing is even enabled for this object
4514   Label cas_label;
4515   int null_check_offset = -1;
4516   if (!swap_reg_contains_mark) {
4517     null_check_offset = offset();
4518     movl(swap_reg, mark_addr);
4519   }
4520   if (need_tmp_reg) {
4521     push(tmp_reg);
4522   }
4523   movl(tmp_reg, swap_reg);
4524   andl(tmp_reg, markOopDesc::biased_lock_mask_in_place);
4525   cmpl(tmp_reg, markOopDesc::biased_lock_pattern);
4526   if (need_tmp_reg) {
4527     pop(tmp_reg);
4528   }
4529   jcc(Assembler::notEqual, cas_label);
4530   // The bias pattern is present in the object's header. Need to check
4531   // whether the bias owner and the epoch are both still current.
4532   // Note that because there is no current thread register on x86 we
4533   // need to store off the mark word we read out of the object to
4534   // avoid reloading it and needing to recheck invariants below. This
4535   // store is unfortunate but it makes the overall code shorter and
4536   // simpler.
4537   movl(saved_mark_addr, swap_reg);
4538   if (need_tmp_reg) {
4539     push(tmp_reg);
4540   }
4541   get_thread(tmp_reg);
4542   xorl(swap_reg, tmp_reg);
4543   if (swap_reg_contains_mark) {
4544     null_check_offset = offset();
4545   }
4546   movl(tmp_reg, klass_addr);
4547   xorl(swap_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
4548   andl(swap_reg, ~((int) markOopDesc::age_mask_in_place));
4549   if (need_tmp_reg) {
4550     pop(tmp_reg);
4551   }
4552   if (counters != NULL) {
4553     cond_inc32(Assembler::zero,
4554                ExternalAddress((address)counters->biased_lock_entry_count_addr()));
4555   }
4556   jcc(Assembler::equal, done);
4557 
4558   Label try_revoke_bias;
4559   Label try_rebias;
4560 
4561   // At this point we know that the header has the bias pattern and
4562   // that we are not the bias owner in the current epoch. We need to
4563   // figure out more details about the state of the header in order to
4564   // know what operations can be legally performed on the object's
4565   // header.
4566 
4567   // If the low three bits in the xor result aren't clear, that means
4568   // the prototype header is no longer biased and we have to revoke
4569   // the bias on this object.
4570   testl(swap_reg, markOopDesc::biased_lock_mask_in_place);
4571   jcc(Assembler::notZero, try_revoke_bias);
4572 
4573   // Biasing is still enabled for this data type. See whether the
4574   // epoch of the current bias is still valid, meaning that the epoch
4575   // bits of the mark word are equal to the epoch bits of the
4576   // prototype header. (Note that the prototype header's epoch bits
4577   // only change at a safepoint.) If not, attempt to rebias the object
4578   // toward the current thread. Note that we must be absolutely sure
4579   // that the current epoch is invalid in order to do this because
4580   // otherwise the manipulations it performs on the mark word are
4581   // illegal.
4582   testl(swap_reg, markOopDesc::epoch_mask_in_place);
4583   jcc(Assembler::notZero, try_rebias);
4584 
4585   // The epoch of the current bias is still valid but we know nothing
4586   // about the owner; it might be set or it might be clear. Try to
4587   // acquire the bias of the object using an atomic operation. If this
4588   // fails we will go in to the runtime to revoke the object's bias.
4589   // Note that we first construct the presumed unbiased header so we
4590   // don't accidentally blow away another thread's valid bias.
4591   movl(swap_reg, saved_mark_addr);
4592   andl(swap_reg,
4593        markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
4594   if (need_tmp_reg) {
4595     push(tmp_reg);
4596   }
4597   get_thread(tmp_reg);
4598   orl(tmp_reg, swap_reg);
4599   if (os::is_MP()) {
4600     lock();
4601   }
4602   cmpxchgptr(tmp_reg, Address(obj_reg, 0));
4603   if (need_tmp_reg) {
4604     pop(tmp_reg);
4605   }
4606   // If the biasing toward our thread failed, this means that
4607   // another thread succeeded in biasing it toward itself and we
4608   // need to revoke that bias. The revocation will occur in the
4609   // interpreter runtime in the slow case.
4610   if (counters != NULL) {
4611     cond_inc32(Assembler::zero,
4612                ExternalAddress((address)counters->anonymously_biased_lock_entry_count_addr()));
4613   }
4614   if (slow_case != NULL) {
4615     jcc(Assembler::notZero, *slow_case);
4616   }
4617   jmp(done);
4618 
4619   bind(try_rebias);
4620   // At this point we know the epoch has expired, meaning that the
4621   // current "bias owner", if any, is actually invalid. Under these
4622   // circumstances _only_, we are allowed to use the current header's
4623   // value as the comparison value when doing the cas to acquire the
4624   // bias in the current epoch. In other words, we allow transfer of
4625   // the bias from one thread to another directly in this situation.
4626   //
4627   // FIXME: due to a lack of registers we currently blow away the age
4628   // bits in this situation. Should attempt to preserve them.
4629   if (need_tmp_reg) {
4630     push(tmp_reg);
4631   }
4632   get_thread(tmp_reg);
4633   movl(swap_reg, klass_addr);
4634   orl(tmp_reg, Address(swap_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
4635   movl(swap_reg, saved_mark_addr);
4636   if (os::is_MP()) {
4637     lock();
4638   }
4639   cmpxchgptr(tmp_reg, Address(obj_reg, 0));
4640   if (need_tmp_reg) {
4641     pop(tmp_reg);
4642   }
4643   // If the biasing toward our thread failed, then another thread
4644   // succeeded in biasing it toward itself and we need to revoke that
4645   // bias. The revocation will occur in the runtime in the slow case.
4646   if (counters != NULL) {
4647     cond_inc32(Assembler::zero,
4648                ExternalAddress((address)counters->rebiased_lock_entry_count_addr()));
4649   }
4650   if (slow_case != NULL) {
4651     jcc(Assembler::notZero, *slow_case);
4652   }
4653   jmp(done);
4654 
4655   bind(try_revoke_bias);
4656   // The prototype mark in the klass doesn't have the bias bit set any
4657   // more, indicating that objects of this data type are not supposed
4658   // to be biased any more. We are going to try to reset the mark of
4659   // this object to the prototype value and fall through to the
4660   // CAS-based locking scheme. Note that if our CAS fails, it means
4661   // that another thread raced us for the privilege of revoking the
4662   // bias of this particular object, so it's okay to continue in the
4663   // normal locking code.
4664   //
4665   // FIXME: due to a lack of registers we currently blow away the age
4666   // bits in this situation. Should attempt to preserve them.
4667   movl(swap_reg, saved_mark_addr);
4668   if (need_tmp_reg) {
4669     push(tmp_reg);
4670   }
4671   movl(tmp_reg, klass_addr);
4672   movl(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
4673   if (os::is_MP()) {
4674     lock();
4675   }
4676   cmpxchgptr(tmp_reg, Address(obj_reg, 0));
4677   if (need_tmp_reg) {
4678     pop(tmp_reg);
4679   }
4680   // Fall through to the normal CAS-based lock, because no matter what
4681   // the result of the above CAS, some thread must have succeeded in
4682   // removing the bias bit from the object's header.
4683   if (counters != NULL) {
4684     cond_inc32(Assembler::zero,
4685                ExternalAddress((address)counters->revoked_lock_entry_count_addr()));
4686   }
4687 
4688   bind(cas_label);
4689 
4690   return null_check_offset;
4691 }
4692 void MacroAssembler::call_VM_leaf_base(address entry_point,
4693                                        int number_of_arguments) {
4694   call(RuntimeAddress(entry_point));
4695   increment(rsp, number_of_arguments * wordSize);
4696 }
4697 
4698 void MacroAssembler::cmpoop(Address src1, jobject obj) {
4699   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
4700 }
4701 
4702 void MacroAssembler::cmpoop(Register src1, jobject obj) {
4703   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
4704 }
4705 
4706 void MacroAssembler::extend_sign(Register hi, Register lo) {
4707   // According to Intel Doc. AP-526, "Integer Divide", p.18.
4708   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
4709     cdql();
4710   } else {
4711     movl(hi, lo);
4712     sarl(hi, 31);
4713   }
4714 }
4715 
4716 void MacroAssembler::fat_nop() {
4717   // A 5 byte nop that is safe for patching (see patch_verified_entry)
4718   emit_byte(0x26); // es:
4719   emit_byte(0x2e); // cs:
4720   emit_byte(0x64); // fs:
4721   emit_byte(0x65); // gs:
4722   emit_byte(0x90);
4723 }
4724 
4725 void MacroAssembler::jC2(Register tmp, Label& L) {
4726   // set parity bit if FPU flag C2 is set (via rax)
4727   save_rax(tmp);
4728   fwait(); fnstsw_ax();
4729   sahf();
4730   restore_rax(tmp);
4731   // branch
4732   jcc(Assembler::parity, L);
4733 }
4734 
4735 void MacroAssembler::jnC2(Register tmp, Label& L) {
4736   // set parity bit if FPU flag C2 is set (via rax)
4737   save_rax(tmp);
4738   fwait(); fnstsw_ax();
4739   sahf();
4740   restore_rax(tmp);
4741   // branch
4742   jcc(Assembler::noParity, L);
4743 }
4744 
4745 // 32bit can do a case table jump in one instruction but we no longer allow the base
4746 // to be installed in the Address class
4747 void MacroAssembler::jump(ArrayAddress entry) {
4748   jmp(as_Address(entry));
4749 }
4750 
4751 // Note: y_lo will be destroyed
4752 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
4753   // Long compare for Java (semantics as described in JVM spec.)
4754   Label high, low, done;
4755 
4756   cmpl(x_hi, y_hi);
4757   jcc(Assembler::less, low);
4758   jcc(Assembler::greater, high);
4759   // x_hi is the return register
4760   xorl(x_hi, x_hi);
4761   cmpl(x_lo, y_lo);
4762   jcc(Assembler::below, low);
4763   jcc(Assembler::equal, done);
4764 
4765   bind(high);
4766   xorl(x_hi, x_hi);
4767   increment(x_hi);
4768   jmp(done);
4769 
4770   bind(low);
4771   xorl(x_hi, x_hi);
4772   decrementl(x_hi);
4773 
4774   bind(done);
4775 }
4776 
4777 void MacroAssembler::lea(Register dst, AddressLiteral src) {
4778     mov_literal32(dst, (int32_t)src.target(), src.rspec());
4779 }
4780 
4781 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
4782   // leal(dst, as_Address(adr));
4783   // see note in movl as to why we must use a move
4784   mov_literal32(dst, (int32_t) adr.target(), adr.rspec());
4785 }
4786 
4787 void MacroAssembler::leave() {
4788   mov(rsp, rbp);
4789   pop(rbp);
4790 }
4791 
4792 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
4793   // Multiplication of two Java long values stored on the stack
4794   // as illustrated below. Result is in rdx:rax.
4795   //
4796   // rsp ---> [  ??  ] \               \
4797   //            ....    | y_rsp_offset  |
4798   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
4799   //          [ y_hi ]                  | (in bytes)
4800   //            ....                    |
4801   //          [ x_lo ]                 /
4802   //          [ x_hi ]
4803   //            ....
4804   //
4805   // Basic idea: lo(result) = lo(x_lo * y_lo)
4806   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
4807   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
4808   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
4809   Label quick;
4810   // load x_hi, y_hi and check if quick
4811   // multiplication is possible
4812   movl(rbx, x_hi);
4813   movl(rcx, y_hi);
4814   movl(rax, rbx);
4815   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
4816   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
4817   // do full multiplication
4818   // 1st step
4819   mull(y_lo);                                    // x_hi * y_lo
4820   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
4821   // 2nd step
4822   movl(rax, x_lo);
4823   mull(rcx);                                     // x_lo * y_hi
4824   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
4825   // 3rd step
4826   bind(quick);                                   // note: rbx, = 0 if quick multiply!
4827   movl(rax, x_lo);
4828   mull(y_lo);                                    // x_lo * y_lo
4829   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
4830 }
4831 
4832 void MacroAssembler::lneg(Register hi, Register lo) {
4833   negl(lo);
4834   adcl(hi, 0);
4835   negl(hi);
4836 }
4837 
4838 void MacroAssembler::lshl(Register hi, Register lo) {
4839   // Java shift left long support (semantics as described in JVM spec., p.305)
4840   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
4841   // shift value is in rcx !
4842   assert(hi != rcx, "must not use rcx");
4843   assert(lo != rcx, "must not use rcx");
4844   const Register s = rcx;                        // shift count
4845   const int      n = BitsPerWord;
4846   Label L;
4847   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
4848   cmpl(s, n);                                    // if (s < n)
4849   jcc(Assembler::less, L);                       // else (s >= n)
4850   movl(hi, lo);                                  // x := x << n
4851   xorl(lo, lo);
4852   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
4853   bind(L);                                       // s (mod n) < n
4854   shldl(hi, lo);                                 // x := x << s
4855   shll(lo);
4856 }
4857 
4858 
4859 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
4860   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
4861   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
4862   assert(hi != rcx, "must not use rcx");
4863   assert(lo != rcx, "must not use rcx");
4864   const Register s = rcx;                        // shift count
4865   const int      n = BitsPerWord;
4866   Label L;
4867   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
4868   cmpl(s, n);                                    // if (s < n)
4869   jcc(Assembler::less, L);                       // else (s >= n)
4870   movl(lo, hi);                                  // x := x >> n
4871   if (sign_extension) sarl(hi, 31);
4872   else                xorl(hi, hi);
4873   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
4874   bind(L);                                       // s (mod n) < n
4875   shrdl(lo, hi);                                 // x := x >> s
4876   if (sign_extension) sarl(hi);
4877   else                shrl(hi);
4878 }
4879 
4880 void MacroAssembler::movoop(Register dst, jobject obj) {
4881   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
4882 }
4883 
4884 void MacroAssembler::movoop(Address dst, jobject obj) {
4885   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
4886 }
4887 
4888 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
4889   if (src.is_lval()) {
4890     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
4891   } else {
4892     movl(dst, as_Address(src));
4893   }
4894 }
4895 
4896 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
4897   movl(as_Address(dst), src);
4898 }
4899 
4900 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
4901   movl(dst, as_Address(src));
4902 }
4903 
4904 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
4905 void MacroAssembler::movptr(Address dst, intptr_t src) {
4906   movl(dst, src);
4907 }
4908 
4909 
4910 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
4911   movsd(dst, as_Address(src));
4912 }
4913 
4914 void MacroAssembler::pop_callee_saved_registers() {
4915   pop(rcx);
4916   pop(rdx);
4917   pop(rdi);
4918   pop(rsi);
4919 }
4920 
4921 void MacroAssembler::pop_fTOS() {
4922   fld_d(Address(rsp, 0));
4923   addl(rsp, 2 * wordSize);
4924 }
4925 
4926 void MacroAssembler::push_callee_saved_registers() {
4927   push(rsi);
4928   push(rdi);
4929   push(rdx);
4930   push(rcx);
4931 }
4932 
4933 void MacroAssembler::push_fTOS() {
4934   subl(rsp, 2 * wordSize);
4935   fstp_d(Address(rsp, 0));
4936 }
4937 
4938 
4939 void MacroAssembler::pushoop(jobject obj) {
4940   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
4941 }
4942 
4943 
4944 void MacroAssembler::pushptr(AddressLiteral src) {
4945   if (src.is_lval()) {
4946     push_literal32((int32_t)src.target(), src.rspec());
4947   } else {
4948     pushl(as_Address(src));
4949   }
4950 }
4951 
4952 void MacroAssembler::set_word_if_not_zero(Register dst) {
4953   xorl(dst, dst);
4954   set_byte_if_not_zero(dst);
4955 }
4956 
4957 static void pass_arg0(MacroAssembler* masm, Register arg) {
4958   masm->push(arg);
4959 }
4960 
4961 static void pass_arg1(MacroAssembler* masm, Register arg) {
4962   masm->push(arg);
4963 }
4964 
4965 static void pass_arg2(MacroAssembler* masm, Register arg) {
4966   masm->push(arg);
4967 }
4968 
4969 static void pass_arg3(MacroAssembler* masm, Register arg) {
4970   masm->push(arg);
4971 }
4972 
4973 #ifndef PRODUCT
4974 extern "C" void findpc(intptr_t x);
4975 #endif
4976 
4977 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
4978   // In order to get locks to work, we need to fake a in_VM state
4979   JavaThread* thread = JavaThread::current();
4980   JavaThreadState saved_state = thread->thread_state();
4981   thread->set_thread_state(_thread_in_vm);
4982   if (ShowMessageBoxOnError) {
4983     JavaThread* thread = JavaThread::current();
4984     JavaThreadState saved_state = thread->thread_state();
4985     thread->set_thread_state(_thread_in_vm);
4986     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
4987       ttyLocker ttyl;
4988       BytecodeCounter::print();
4989     }
4990     // To see where a verify_oop failed, get $ebx+40/X for this frame.
4991     // This is the value of eip which points to where verify_oop will return.
4992     if (os::message_box(msg, "Execution stopped, print registers?")) {
4993       ttyLocker ttyl;
4994       tty->print_cr("eip = 0x%08x", eip);
4995 #ifndef PRODUCT
4996       tty->cr();
4997       findpc(eip);
4998       tty->cr();
4999 #endif
5000       tty->print_cr("rax, = 0x%08x", rax);
5001       tty->print_cr("rbx, = 0x%08x", rbx);
5002       tty->print_cr("rcx = 0x%08x", rcx);
5003       tty->print_cr("rdx = 0x%08x", rdx);
5004       tty->print_cr("rdi = 0x%08x", rdi);
5005       tty->print_cr("rsi = 0x%08x", rsi);
5006       tty->print_cr("rbp, = 0x%08x", rbp);
5007       tty->print_cr("rsp = 0x%08x", rsp);
5008       BREAKPOINT;
5009     }
5010   } else {
5011     ttyLocker ttyl;
5012     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
5013     assert(false, "DEBUG MESSAGE");
5014   }
5015   ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
5016 }
5017 
5018 void MacroAssembler::stop(const char* msg) {
5019   ExternalAddress message((address)msg);
5020   // push address of message
5021   pushptr(message.addr());
5022   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
5023   pusha();                                           // push registers
5024   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
5025   hlt();
5026 }
5027 
5028 void MacroAssembler::warn(const char* msg) {
5029   push_CPU_state();
5030 
5031   ExternalAddress message((address) msg);
5032   // push address of message
5033   pushptr(message.addr());
5034 
5035   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
5036   addl(rsp, wordSize);       // discard argument
5037   pop_CPU_state();
5038 }
5039 
5040 #else // _LP64
5041 
5042 // 64 bit versions
5043 
5044 Address MacroAssembler::as_Address(AddressLiteral adr) {
5045   // amd64 always does this as a pc-rel
5046   // we can be absolute or disp based on the instruction type
5047   // jmp/call are displacements others are absolute
5048   assert(!adr.is_lval(), "must be rval");
5049   assert(reachable(adr), "must be");
5050   return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
5051 
5052 }
5053 
5054 Address MacroAssembler::as_Address(ArrayAddress adr) {
5055   AddressLiteral base = adr.base();
5056   lea(rscratch1, base);
5057   Address index = adr.index();
5058   assert(index._disp == 0, "must not have disp"); // maybe it can?
5059   Address array(rscratch1, index._index, index._scale, index._disp);
5060   return array;
5061 }
5062 
5063 int MacroAssembler::biased_locking_enter(Register lock_reg,
5064                                          Register obj_reg,
5065                                          Register swap_reg,
5066                                          Register tmp_reg,
5067                                          bool swap_reg_contains_mark,
5068                                          Label& done,
5069                                          Label* slow_case,
5070                                          BiasedLockingCounters* counters) {
5071   assert(UseBiasedLocking, "why call this otherwise?");
5072   assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
5073   assert(tmp_reg != noreg, "tmp_reg must be supplied");
5074   assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
5075   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
5076   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
5077   Address saved_mark_addr(lock_reg, 0);
5078 
5079   if (PrintBiasedLockingStatistics && counters == NULL)
5080     counters = BiasedLocking::counters();
5081 
5082   // Biased locking
5083   // See whether the lock is currently biased toward our thread and
5084   // whether the epoch is still valid
5085   // Note that the runtime guarantees sufficient alignment of JavaThread
5086   // pointers to allow age to be placed into low bits
5087   // First check to see whether biasing is even enabled for this object
5088   Label cas_label;
5089   int null_check_offset = -1;
5090   if (!swap_reg_contains_mark) {
5091     null_check_offset = offset();
5092     movq(swap_reg, mark_addr);
5093   }
5094   movq(tmp_reg, swap_reg);
5095   andq(tmp_reg, markOopDesc::biased_lock_mask_in_place);
5096   cmpq(tmp_reg, markOopDesc::biased_lock_pattern);
5097   jcc(Assembler::notEqual, cas_label);
5098   // The bias pattern is present in the object's header. Need to check
5099   // whether the bias owner and the epoch are both still current.
5100   load_prototype_header(tmp_reg, obj_reg);
5101   orq(tmp_reg, r15_thread);
5102   xorq(tmp_reg, swap_reg);
5103   andq(tmp_reg, ~((int) markOopDesc::age_mask_in_place));
5104   if (counters != NULL) {
5105     cond_inc32(Assembler::zero,
5106                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
5107   }
5108   jcc(Assembler::equal, done);
5109 
5110   Label try_revoke_bias;
5111   Label try_rebias;
5112 
5113   // At this point we know that the header has the bias pattern and
5114   // that we are not the bias owner in the current epoch. We need to
5115   // figure out more details about the state of the header in order to
5116   // know what operations can be legally performed on the object's
5117   // header.
5118 
5119   // If the low three bits in the xor result aren't clear, that means
5120   // the prototype header is no longer biased and we have to revoke
5121   // the bias on this object.
5122   testq(tmp_reg, markOopDesc::biased_lock_mask_in_place);
5123   jcc(Assembler::notZero, try_revoke_bias);
5124 
5125   // Biasing is still enabled for this data type. See whether the
5126   // epoch of the current bias is still valid, meaning that the epoch
5127   // bits of the mark word are equal to the epoch bits of the
5128   // prototype header. (Note that the prototype header's epoch bits
5129   // only change at a safepoint.) If not, attempt to rebias the object
5130   // toward the current thread. Note that we must be absolutely sure
5131   // that the current epoch is invalid in order to do this because
5132   // otherwise the manipulations it performs on the mark word are
5133   // illegal.
5134   testq(tmp_reg, markOopDesc::epoch_mask_in_place);
5135   jcc(Assembler::notZero, try_rebias);
5136 
5137   // The epoch of the current bias is still valid but we know nothing
5138   // about the owner; it might be set or it might be clear. Try to
5139   // acquire the bias of the object using an atomic operation. If this
5140   // fails we will go in to the runtime to revoke the object's bias.
5141   // Note that we first construct the presumed unbiased header so we
5142   // don't accidentally blow away another thread's valid bias.
5143   andq(swap_reg,
5144        markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
5145   movq(tmp_reg, swap_reg);
5146   orq(tmp_reg, r15_thread);
5147   if (os::is_MP()) {
5148     lock();
5149   }
5150   cmpxchgq(tmp_reg, Address(obj_reg, 0));
5151   // If the biasing toward our thread failed, this means that
5152   // another thread succeeded in biasing it toward itself and we
5153   // need to revoke that bias. The revocation will occur in the
5154   // interpreter runtime in the slow case.
5155   if (counters != NULL) {
5156     cond_inc32(Assembler::zero,
5157                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
5158   }
5159   if (slow_case != NULL) {
5160     jcc(Assembler::notZero, *slow_case);
5161   }
5162   jmp(done);
5163 
5164   bind(try_rebias);
5165   // At this point we know the epoch has expired, meaning that the
5166   // current "bias owner", if any, is actually invalid. Under these
5167   // circumstances _only_, we are allowed to use the current header's
5168   // value as the comparison value when doing the cas to acquire the
5169   // bias in the current epoch. In other words, we allow transfer of
5170   // the bias from one thread to another directly in this situation.
5171   //
5172   // FIXME: due to a lack of registers we currently blow away the age
5173   // bits in this situation. Should attempt to preserve them.
5174   load_prototype_header(tmp_reg, obj_reg);
5175   orq(tmp_reg, r15_thread);
5176   if (os::is_MP()) {
5177     lock();
5178   }
5179   cmpxchgq(tmp_reg, Address(obj_reg, 0));
5180   // If the biasing toward our thread failed, then another thread
5181   // succeeded in biasing it toward itself and we need to revoke that
5182   // bias. The revocation will occur in the runtime in the slow case.
5183   if (counters != NULL) {
5184     cond_inc32(Assembler::zero,
5185                ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
5186   }
5187   if (slow_case != NULL) {
5188     jcc(Assembler::notZero, *slow_case);
5189   }
5190   jmp(done);
5191 
5192   bind(try_revoke_bias);
5193   // The prototype mark in the klass doesn't have the bias bit set any
5194   // more, indicating that objects of this data type are not supposed
5195   // to be biased any more. We are going to try to reset the mark of
5196   // this object to the prototype value and fall through to the
5197   // CAS-based locking scheme. Note that if our CAS fails, it means
5198   // that another thread raced us for the privilege of revoking the
5199   // bias of this particular object, so it's okay to continue in the
5200   // normal locking code.
5201   //
5202   // FIXME: due to a lack of registers we currently blow away the age
5203   // bits in this situation. Should attempt to preserve them.
5204   load_prototype_header(tmp_reg, obj_reg);
5205   if (os::is_MP()) {
5206     lock();
5207   }
5208   cmpxchgq(tmp_reg, Address(obj_reg, 0));
5209   // Fall through to the normal CAS-based lock, because no matter what
5210   // the result of the above CAS, some thread must have succeeded in
5211   // removing the bias bit from the object's header.
5212   if (counters != NULL) {
5213     cond_inc32(Assembler::zero,
5214                ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
5215   }
5216 
5217   bind(cas_label);
5218 
5219   return null_check_offset;
5220 }
5221 
5222 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
5223   Label L, E;
5224 
5225 #ifdef _WIN64
5226   // Windows always allocates space for it's register args
5227   assert(num_args <= 4, "only register arguments supported");
5228   subq(rsp,  frame::arg_reg_save_area_bytes);
5229 #endif
5230 
5231   // Align stack if necessary
5232   testl(rsp, 15);
5233   jcc(Assembler::zero, L);
5234 
5235   subq(rsp, 8);
5236   {
5237     call(RuntimeAddress(entry_point));
5238   }
5239   addq(rsp, 8);
5240   jmp(E);
5241 
5242   bind(L);
5243   {
5244     call(RuntimeAddress(entry_point));
5245   }
5246 
5247   bind(E);
5248 
5249 #ifdef _WIN64
5250   // restore stack pointer
5251   addq(rsp, frame::arg_reg_save_area_bytes);
5252 #endif
5253 
5254 }
5255 
5256 void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
5257   assert(!src2.is_lval(), "should use cmpptr");
5258 
5259   if (reachable(src2)) {
5260     cmpq(src1, as_Address(src2));
5261   } else {
5262     lea(rscratch1, src2);
5263     Assembler::cmpq(src1, Address(rscratch1, 0));
5264   }
5265 }
5266 
5267 int MacroAssembler::corrected_idivq(Register reg) {
5268   // Full implementation of Java ldiv and lrem; checks for special
5269   // case as described in JVM spec., p.243 & p.271.  The function
5270   // returns the (pc) offset of the idivl instruction - may be needed
5271   // for implicit exceptions.
5272   //
5273   //         normal case                           special case
5274   //
5275   // input : rax: dividend                         min_long
5276   //         reg: divisor   (may not be eax/edx)   -1
5277   //
5278   // output: rax: quotient  (= rax idiv reg)       min_long
5279   //         rdx: remainder (= rax irem reg)       0
5280   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
5281   static const int64_t min_long = 0x8000000000000000;
5282   Label normal_case, special_case;
5283 
5284   // check for special case
5285   cmp64(rax, ExternalAddress((address) &min_long));
5286   jcc(Assembler::notEqual, normal_case);
5287   xorl(rdx, rdx); // prepare rdx for possible special case (where
5288                   // remainder = 0)
5289   cmpq(reg, -1);
5290   jcc(Assembler::equal, special_case);
5291 
5292   // handle normal case
5293   bind(normal_case);
5294   cdqq();
5295   int idivq_offset = offset();
5296   idivq(reg);
5297 
5298   // normal and special case exit
5299   bind(special_case);
5300 
5301   return idivq_offset;
5302 }
5303 
5304 void MacroAssembler::decrementq(Register reg, int value) {
5305   if (value == min_jint) { subq(reg, value); return; }
5306   if (value <  0) { incrementq(reg, -value); return; }
5307   if (value == 0) {                        ; return; }
5308   if (value == 1 && UseIncDec) { decq(reg) ; return; }
5309   /* else */      { subq(reg, value)       ; return; }
5310 }
5311 
5312 void MacroAssembler::decrementq(Address dst, int value) {
5313   if (value == min_jint) { subq(dst, value); return; }
5314   if (value <  0) { incrementq(dst, -value); return; }
5315   if (value == 0) {                        ; return; }
5316   if (value == 1 && UseIncDec) { decq(dst) ; return; }
5317   /* else */      { subq(dst, value)       ; return; }
5318 }
5319 
5320 void MacroAssembler::fat_nop() {
5321   // A 5 byte nop that is safe for patching (see patch_verified_entry)
5322   // Recommened sequence from 'Software Optimization Guide for the AMD
5323   // Hammer Processor'
5324   emit_byte(0x66);
5325   emit_byte(0x66);
5326   emit_byte(0x90);
5327   emit_byte(0x66);
5328   emit_byte(0x90);
5329 }
5330 
5331 void MacroAssembler::incrementq(Register reg, int value) {
5332   if (value == min_jint) { addq(reg, value); return; }
5333   if (value <  0) { decrementq(reg, -value); return; }
5334   if (value == 0) {                        ; return; }
5335   if (value == 1 && UseIncDec) { incq(reg) ; return; }
5336   /* else */      { addq(reg, value)       ; return; }
5337 }
5338 
5339 void MacroAssembler::incrementq(Address dst, int value) {
5340   if (value == min_jint) { addq(dst, value); return; }
5341   if (value <  0) { decrementq(dst, -value); return; }
5342   if (value == 0) {                        ; return; }
5343   if (value == 1 && UseIncDec) { incq(dst) ; return; }
5344   /* else */      { addq(dst, value)       ; return; }
5345 }
5346 
5347 // 32bit can do a case table jump in one instruction but we no longer allow the base
5348 // to be installed in the Address class
5349 void MacroAssembler::jump(ArrayAddress entry) {
5350   lea(rscratch1, entry.base());
5351   Address dispatch = entry.index();
5352   assert(dispatch._base == noreg, "must be");
5353   dispatch._base = rscratch1;
5354   jmp(dispatch);
5355 }
5356 
5357 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
5358   ShouldNotReachHere(); // 64bit doesn't use two regs
5359   cmpq(x_lo, y_lo);
5360 }
5361 
5362 void MacroAssembler::lea(Register dst, AddressLiteral src) {
5363     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
5364 }
5365 
5366 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
5367   mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());
5368   movptr(dst, rscratch1);
5369 }
5370 
5371 void MacroAssembler::leave() {
5372   // %%% is this really better? Why not on 32bit too?
5373   emit_byte(0xC9); // LEAVE
5374 }
5375 
5376 void MacroAssembler::lneg(Register hi, Register lo) {
5377   ShouldNotReachHere(); // 64bit doesn't use two regs
5378   negq(lo);
5379 }
5380 
5381 void MacroAssembler::movoop(Register dst, jobject obj) {
5382   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
5383 }
5384 
5385 void MacroAssembler::movoop(Address dst, jobject obj) {
5386   mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
5387   movq(dst, rscratch1);
5388 }
5389 
5390 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
5391   if (src.is_lval()) {
5392     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
5393   } else {
5394     if (reachable(src)) {
5395       movq(dst, as_Address(src));
5396     } else {
5397       lea(rscratch1, src);
5398       movq(dst, Address(rscratch1,0));
5399     }
5400   }
5401 }
5402 
5403 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
5404   movq(as_Address(dst), src);
5405 }
5406 
5407 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
5408   movq(dst, as_Address(src));
5409 }
5410 
5411 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
5412 void MacroAssembler::movptr(Address dst, intptr_t src) {
5413   mov64(rscratch1, src);
5414   movq(dst, rscratch1);
5415 }
5416 
5417 // These are mostly for initializing NULL
5418 void MacroAssembler::movptr(Address dst, int32_t src) {
5419   movslq(dst, src);
5420 }
5421 
5422 void MacroAssembler::movptr(Register dst, int32_t src) {
5423   mov64(dst, (intptr_t)src);
5424 }
5425 
5426 void MacroAssembler::pushoop(jobject obj) {
5427   movoop(rscratch1, obj);
5428   push(rscratch1);
5429 }
5430 
5431 void MacroAssembler::pushptr(AddressLiteral src) {
5432   lea(rscratch1, src);
5433   if (src.is_lval()) {
5434     push(rscratch1);
5435   } else {
5436     pushq(Address(rscratch1, 0));
5437   }
5438 }
5439 
5440 void MacroAssembler::reset_last_Java_frame(bool clear_fp,
5441                                            bool clear_pc) {
5442   // we must set sp to zero to clear frame
5443   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
5444   // must clear fp, so that compiled frames are not confused; it is
5445   // possible that we need it only for debugging
5446   if (clear_fp) {
5447     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
5448   }
5449 
5450   if (clear_pc) {
5451     movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
5452   }
5453 }
5454 
5455 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
5456                                          Register last_java_fp,
5457                                          address  last_java_pc) {
5458   // determine last_java_sp register
5459   if (!last_java_sp->is_valid()) {
5460     last_java_sp = rsp;
5461   }
5462 
5463   // last_java_fp is optional
5464   if (last_java_fp->is_valid()) {
5465     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),
5466            last_java_fp);
5467   }
5468 
5469   // last_java_pc is optional
5470   if (last_java_pc != NULL) {
5471     Address java_pc(r15_thread,
5472                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
5473     lea(rscratch1, InternalAddress(last_java_pc));
5474     movptr(java_pc, rscratch1);
5475   }
5476 
5477   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
5478 }
5479 
5480 static void pass_arg0(MacroAssembler* masm, Register arg) {
5481   if (c_rarg0 != arg ) {
5482     masm->mov(c_rarg0, arg);
5483   }
5484 }
5485 
5486 static void pass_arg1(MacroAssembler* masm, Register arg) {
5487   if (c_rarg1 != arg ) {
5488     masm->mov(c_rarg1, arg);
5489   }
5490 }
5491 
5492 static void pass_arg2(MacroAssembler* masm, Register arg) {
5493   if (c_rarg2 != arg ) {
5494     masm->mov(c_rarg2, arg);
5495   }
5496 }
5497 
5498 static void pass_arg3(MacroAssembler* masm, Register arg) {
5499   if (c_rarg3 != arg ) {
5500     masm->mov(c_rarg3, arg);
5501   }
5502 }
5503 
5504 void MacroAssembler::stop(const char* msg) {
5505   address rip = pc();
5506   pusha(); // get regs on stack
5507   lea(c_rarg0, ExternalAddress((address) msg));
5508   lea(c_rarg1, InternalAddress(rip));
5509   movq(c_rarg2, rsp); // pass pointer to regs array
5510   andq(rsp, -16); // align stack as required by ABI
5511   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
5512   hlt();
5513 }
5514 
5515 void MacroAssembler::warn(const char* msg) {
5516   push(r12);
5517   movq(r12, rsp);
5518   andq(rsp, -16);     // align stack as required by push_CPU_state and call
5519 
5520   push_CPU_state();   // keeps alignment at 16 bytes
5521   lea(c_rarg0, ExternalAddress((address) msg));
5522   call_VM_leaf(CAST_FROM_FN_PTR(address, warning), c_rarg0);
5523   pop_CPU_state();
5524 
5525   movq(rsp, r12);
5526   pop(r12);
5527 }
5528 
5529 #ifndef PRODUCT
5530 extern "C" void findpc(intptr_t x);
5531 #endif
5532 
5533 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
5534   // In order to get locks to work, we need to fake a in_VM state
5535   if (ShowMessageBoxOnError ) {
5536     JavaThread* thread = JavaThread::current();
5537     JavaThreadState saved_state = thread->thread_state();
5538     thread->set_thread_state(_thread_in_vm);
5539 #ifndef PRODUCT
5540     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
5541       ttyLocker ttyl;
5542       BytecodeCounter::print();
5543     }
5544 #endif
5545     // To see where a verify_oop failed, get $ebx+40/X for this frame.
5546     // XXX correct this offset for amd64
5547     // This is the value of eip which points to where verify_oop will return.
5548     if (os::message_box(msg, "Execution stopped, print registers?")) {
5549       ttyLocker ttyl;
5550       tty->print_cr("rip = 0x%016lx", pc);
5551 #ifndef PRODUCT
5552       tty->cr();
5553       findpc(pc);
5554       tty->cr();
5555 #endif
5556       tty->print_cr("rax = 0x%016lx", regs[15]);
5557       tty->print_cr("rbx = 0x%016lx", regs[12]);
5558       tty->print_cr("rcx = 0x%016lx", regs[14]);
5559       tty->print_cr("rdx = 0x%016lx", regs[13]);
5560       tty->print_cr("rdi = 0x%016lx", regs[8]);
5561       tty->print_cr("rsi = 0x%016lx", regs[9]);
5562       tty->print_cr("rbp = 0x%016lx", regs[10]);
5563       tty->print_cr("rsp = 0x%016lx", regs[11]);
5564       tty->print_cr("r8  = 0x%016lx", regs[7]);
5565       tty->print_cr("r9  = 0x%016lx", regs[6]);
5566       tty->print_cr("r10 = 0x%016lx", regs[5]);
5567       tty->print_cr("r11 = 0x%016lx", regs[4]);
5568       tty->print_cr("r12 = 0x%016lx", regs[3]);
5569       tty->print_cr("r13 = 0x%016lx", regs[2]);
5570       tty->print_cr("r14 = 0x%016lx", regs[1]);
5571       tty->print_cr("r15 = 0x%016lx", regs[0]);
5572       BREAKPOINT;
5573     }
5574     ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
5575   } else {
5576     ttyLocker ttyl;
5577     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
5578                     msg);
5579   }
5580 }
5581 
5582 #endif // _LP64
5583 
5584 // Now versions that are common to 32/64 bit
5585 
5586 void MacroAssembler::addptr(Register dst, int32_t imm32) {
5587   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
5588 }
5589 
5590 void MacroAssembler::addptr(Register dst, Register src) {
5591   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
5592 }
5593 
5594 void MacroAssembler::addptr(Address dst, Register src) {
5595   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
5596 }
5597 
5598 void MacroAssembler::align(int modulus) {
5599   if (offset() % modulus != 0) {
5600     nop(modulus - (offset() % modulus));
5601   }
5602 }
5603 
5604 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {
5605   if (reachable(src)) {
5606     andpd(dst, as_Address(src));
5607   } else {
5608     lea(rscratch1, src);
5609     andpd(dst, Address(rscratch1, 0));
5610   }
5611 }
5612 
5613 void MacroAssembler::andptr(Register dst, int32_t imm32) {
5614   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
5615 }
5616 
5617 void MacroAssembler::atomic_incl(AddressLiteral counter_addr) {
5618   pushf();
5619   if (os::is_MP())
5620     lock();
5621   incrementl(counter_addr);
5622   popf();
5623 }
5624 
5625 // Writes to stack successive pages until offset reached to check for
5626 // stack overflow + shadow pages.  This clobbers tmp.
5627 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
5628   movptr(tmp, rsp);
5629   // Bang stack for total size given plus shadow page size.
5630   // Bang one page at a time because large size can bang beyond yellow and
5631   // red zones.
5632   Label loop;
5633   bind(loop);
5634   movl(Address(tmp, (-os::vm_page_size())), size );
5635   subptr(tmp, os::vm_page_size());
5636   subl(size, os::vm_page_size());
5637   jcc(Assembler::greater, loop);
5638 
5639   // Bang down shadow pages too.
5640   // The -1 because we already subtracted 1 page.
5641   for (int i = 0; i< StackShadowPages-1; i++) {
5642     // this could be any sized move but this is can be a debugging crumb
5643     // so the bigger the better.
5644     movptr(Address(tmp, (-i*os::vm_page_size())), size );
5645   }
5646 }
5647 
5648 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
5649   assert(UseBiasedLocking, "why call this otherwise?");
5650 
5651   // Check for biased locking unlock case, which is a no-op
5652   // Note: we do not have to check the thread ID for two reasons.
5653   // First, the interpreter checks for IllegalMonitorStateException at
5654   // a higher level. Second, if the bias was revoked while we held the
5655   // lock, the object could not be rebiased toward another thread, so
5656   // the bias bit would be clear.
5657   movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
5658   andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);
5659   cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
5660   jcc(Assembler::equal, done);
5661 }
5662 
5663 void MacroAssembler::c2bool(Register x) {
5664   // implements x == 0 ? 0 : 1
5665   // note: must only look at least-significant byte of x
5666   //       since C-style booleans are stored in one byte
5667   //       only! (was bug)
5668   andl(x, 0xFF);
5669   setb(Assembler::notZero, x);
5670 }
5671 
5672 // Wouldn't need if AddressLiteral version had new name
5673 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
5674   Assembler::call(L, rtype);
5675 }
5676 
5677 void MacroAssembler::call(Register entry) {
5678   Assembler::call(entry);
5679 }
5680 
5681 void MacroAssembler::call(AddressLiteral entry) {
5682   if (reachable(entry)) {
5683     Assembler::call_literal(entry.target(), entry.rspec());
5684   } else {
5685     lea(rscratch1, entry);
5686     Assembler::call(rscratch1);
5687   }
5688 }
5689 
5690 // Implementation of call_VM versions
5691 
5692 void MacroAssembler::call_VM(Register oop_result,
5693                              address entry_point,
5694                              bool check_exceptions) {
5695   Label C, E;
5696   call(C, relocInfo::none);
5697   jmp(E);
5698 
5699   bind(C);
5700   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
5701   ret(0);
5702 
5703   bind(E);
5704 }
5705 
5706 void MacroAssembler::call_VM(Register oop_result,
5707                              address entry_point,
5708                              Register arg_1,
5709                              bool check_exceptions) {
5710   Label C, E;
5711   call(C, relocInfo::none);
5712   jmp(E);
5713 
5714   bind(C);
5715   pass_arg1(this, arg_1);
5716   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
5717   ret(0);
5718 
5719   bind(E);
5720 }
5721 
5722 void MacroAssembler::call_VM(Register oop_result,
5723                              address entry_point,
5724                              Register arg_1,
5725                              Register arg_2,
5726                              bool check_exceptions) {
5727   Label C, E;
5728   call(C, relocInfo::none);
5729   jmp(E);
5730 
5731   bind(C);
5732 
5733   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
5734 
5735   pass_arg2(this, arg_2);
5736   pass_arg1(this, arg_1);
5737   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
5738   ret(0);
5739 
5740   bind(E);
5741 }
5742 
5743 void MacroAssembler::call_VM(Register oop_result,
5744                              address entry_point,
5745                              Register arg_1,
5746                              Register arg_2,
5747                              Register arg_3,
5748                              bool check_exceptions) {
5749   Label C, E;
5750   call(C, relocInfo::none);
5751   jmp(E);
5752 
5753   bind(C);
5754 
5755   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
5756   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
5757   pass_arg3(this, arg_3);
5758 
5759   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
5760   pass_arg2(this, arg_2);
5761 
5762   pass_arg1(this, arg_1);
5763   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
5764   ret(0);
5765 
5766   bind(E);
5767 }
5768 
5769 void MacroAssembler::call_VM(Register oop_result,
5770                              Register last_java_sp,
5771                              address entry_point,
5772                              int number_of_arguments,
5773                              bool check_exceptions) {
5774   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
5775   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
5776 }
5777 
5778 void MacroAssembler::call_VM(Register oop_result,
5779                              Register last_java_sp,
5780                              address entry_point,
5781                              Register arg_1,
5782                              bool check_exceptions) {
5783   pass_arg1(this, arg_1);
5784   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
5785 }
5786 
5787 void MacroAssembler::call_VM(Register oop_result,
5788                              Register last_java_sp,
5789                              address entry_point,
5790                              Register arg_1,
5791                              Register arg_2,
5792                              bool check_exceptions) {
5793 
5794   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
5795   pass_arg2(this, arg_2);
5796   pass_arg1(this, arg_1);
5797   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
5798 }
5799 
5800 void MacroAssembler::call_VM(Register oop_result,
5801                              Register last_java_sp,
5802                              address entry_point,
5803                              Register arg_1,
5804                              Register arg_2,
5805                              Register arg_3,
5806                              bool check_exceptions) {
5807   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
5808   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
5809   pass_arg3(this, arg_3);
5810   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
5811   pass_arg2(this, arg_2);
5812   pass_arg1(this, arg_1);
5813   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
5814 }
5815 
5816 void MacroAssembler::call_VM_base(Register oop_result,
5817                                   Register java_thread,
5818                                   Register last_java_sp,
5819                                   address  entry_point,
5820                                   int      number_of_arguments,
5821                                   bool     check_exceptions) {
5822   // determine java_thread register
5823   if (!java_thread->is_valid()) {
5824 #ifdef _LP64
5825     java_thread = r15_thread;
5826 #else
5827     java_thread = rdi;
5828     get_thread(java_thread);
5829 #endif // LP64
5830   }
5831   // determine last_java_sp register
5832   if (!last_java_sp->is_valid()) {
5833     last_java_sp = rsp;
5834   }
5835   // debugging support
5836   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
5837   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
5838   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
5839   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
5840 
5841   // push java thread (becomes first argument of C function)
5842 
5843   NOT_LP64(push(java_thread); number_of_arguments++);
5844   LP64_ONLY(mov(c_rarg0, r15_thread));
5845 
5846   // set last Java frame before call
5847   assert(last_java_sp != rbp, "can't use ebp/rbp");
5848 
5849   // Only interpreter should have to set fp
5850   set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);
5851 
5852   // do the call, remove parameters
5853   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
5854 
5855   // restore the thread (cannot use the pushed argument since arguments
5856   // may be overwritten by C code generated by an optimizing compiler);
5857   // however can use the register value directly if it is callee saved.
5858   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
5859     // rdi & rsi (also r15) are callee saved -> nothing to do
5860 #ifdef ASSERT
5861     guarantee(java_thread != rax, "change this code");
5862     push(rax);
5863     { Label L;
5864       get_thread(rax);
5865       cmpptr(java_thread, rax);
5866       jcc(Assembler::equal, L);
5867       stop("MacroAssembler::call_VM_base: rdi not callee saved?");
5868       bind(L);
5869     }
5870     pop(rax);
5871 #endif
5872   } else {
5873     get_thread(java_thread);
5874   }
5875   // reset last Java frame
5876   // Only interpreter should have to clear fp
5877   reset_last_Java_frame(java_thread, true, false);
5878 
5879 #ifndef CC_INTERP
5880    // C++ interp handles this in the interpreter
5881   check_and_handle_popframe(java_thread);
5882   check_and_handle_earlyret(java_thread);
5883 #endif /* CC_INTERP */
5884 
5885   if (check_exceptions) {
5886     // check for pending exceptions (java_thread is set upon return)
5887     cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
5888 #ifndef _LP64
5889     jump_cc(Assembler::notEqual,
5890             RuntimeAddress(StubRoutines::forward_exception_entry()));
5891 #else
5892     // This used to conditionally jump to forward_exception however it is
5893     // possible if we relocate that the branch will not reach. So we must jump
5894     // around so we can always reach
5895 
5896     Label ok;
5897     jcc(Assembler::equal, ok);
5898     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
5899     bind(ok);
5900 #endif // LP64
5901   }
5902 
5903   // get oop result if there is one and reset the value in the thread
5904   if (oop_result->is_valid()) {
5905     movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
5906     movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
5907     verify_oop(oop_result, "broken oop in call_VM_base");
5908   }
5909 }
5910 
5911 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
5912 
5913   // Calculate the value for last_Java_sp
5914   // somewhat subtle. call_VM does an intermediate call
5915   // which places a return address on the stack just under the
5916   // stack pointer as the user finsihed with it. This allows
5917   // use to retrieve last_Java_pc from last_Java_sp[-1].
5918   // On 32bit we then have to push additional args on the stack to accomplish
5919   // the actual requested call. On 64bit call_VM only can use register args
5920   // so the only extra space is the return address that call_VM created.
5921   // This hopefully explains the calculations here.
5922 
5923 #ifdef _LP64
5924   // We've pushed one address, correct last_Java_sp
5925   lea(rax, Address(rsp, wordSize));
5926 #else
5927   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
5928 #endif // LP64
5929 
5930   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
5931 
5932 }
5933 
5934 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
5935   call_VM_leaf_base(entry_point, number_of_arguments);
5936 }
5937 
5938 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
5939   pass_arg0(this, arg_0);
5940   call_VM_leaf(entry_point, 1);
5941 }
5942 
5943 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
5944 
5945   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
5946   pass_arg1(this, arg_1);
5947   pass_arg0(this, arg_0);
5948   call_VM_leaf(entry_point, 2);
5949 }
5950 
5951 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
5952   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
5953   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
5954   pass_arg2(this, arg_2);
5955   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
5956   pass_arg1(this, arg_1);
5957   pass_arg0(this, arg_0);
5958   call_VM_leaf(entry_point, 3);
5959 }
5960 
5961 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
5962 }
5963 
5964 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
5965 }
5966 
5967 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) {
5968   if (reachable(src1)) {
5969     cmpl(as_Address(src1), imm);
5970   } else {
5971     lea(rscratch1, src1);
5972     cmpl(Address(rscratch1, 0), imm);
5973   }
5974 }
5975 
5976 void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
5977   assert(!src2.is_lval(), "use cmpptr");
5978   if (reachable(src2)) {
5979     cmpl(src1, as_Address(src2));
5980   } else {
5981     lea(rscratch1, src2);
5982     cmpl(src1, Address(rscratch1, 0));
5983   }
5984 }
5985 
5986 void MacroAssembler::cmp32(Register src1, int32_t imm) {
5987   Assembler::cmpl(src1, imm);
5988 }
5989 
5990 void MacroAssembler::cmp32(Register src1, Address src2) {
5991   Assembler::cmpl(src1, src2);
5992 }
5993 
5994 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
5995   ucomisd(opr1, opr2);
5996 
5997   Label L;
5998   if (unordered_is_less) {
5999     movl(dst, -1);
6000     jcc(Assembler::parity, L);
6001     jcc(Assembler::below , L);
6002     movl(dst, 0);
6003     jcc(Assembler::equal , L);
6004     increment(dst);
6005   } else { // unordered is greater
6006     movl(dst, 1);
6007     jcc(Assembler::parity, L);
6008     jcc(Assembler::above , L);
6009     movl(dst, 0);
6010     jcc(Assembler::equal , L);
6011     decrementl(dst);
6012   }
6013   bind(L);
6014 }
6015 
6016 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
6017   ucomiss(opr1, opr2);
6018 
6019   Label L;
6020   if (unordered_is_less) {
6021     movl(dst, -1);
6022     jcc(Assembler::parity, L);
6023     jcc(Assembler::below , L);
6024     movl(dst, 0);
6025     jcc(Assembler::equal , L);
6026     increment(dst);
6027   } else { // unordered is greater
6028     movl(dst, 1);
6029     jcc(Assembler::parity, L);
6030     jcc(Assembler::above , L);
6031     movl(dst, 0);
6032     jcc(Assembler::equal , L);
6033     decrementl(dst);
6034   }
6035   bind(L);
6036 }
6037 
6038 
6039 void MacroAssembler::cmp8(AddressLiteral src1, int imm) {
6040   if (reachable(src1)) {
6041     cmpb(as_Address(src1), imm);
6042   } else {
6043     lea(rscratch1, src1);
6044     cmpb(Address(rscratch1, 0), imm);
6045   }
6046 }
6047 
6048 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
6049 #ifdef _LP64
6050   if (src2.is_lval()) {
6051     movptr(rscratch1, src2);
6052     Assembler::cmpq(src1, rscratch1);
6053   } else if (reachable(src2)) {
6054     cmpq(src1, as_Address(src2));
6055   } else {
6056     lea(rscratch1, src2);
6057     Assembler::cmpq(src1, Address(rscratch1, 0));
6058   }
6059 #else
6060   if (src2.is_lval()) {
6061     cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
6062   } else {
6063     cmpl(src1, as_Address(src2));
6064   }
6065 #endif // _LP64
6066 }
6067 
6068 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
6069   assert(src2.is_lval(), "not a mem-mem compare");
6070 #ifdef _LP64
6071   // moves src2's literal address
6072   movptr(rscratch1, src2);
6073   Assembler::cmpq(src1, rscratch1);
6074 #else
6075   cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
6076 #endif // _LP64
6077 }
6078 
6079 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) {
6080   if (reachable(adr)) {
6081     if (os::is_MP())
6082       lock();
6083     cmpxchgptr(reg, as_Address(adr));
6084   } else {
6085     lea(rscratch1, adr);
6086     if (os::is_MP())
6087       lock();
6088     cmpxchgptr(reg, Address(rscratch1, 0));
6089   }
6090 }
6091 
6092 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
6093   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
6094 }
6095 
6096 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {
6097   if (reachable(src)) {
6098     comisd(dst, as_Address(src));
6099   } else {
6100     lea(rscratch1, src);
6101     comisd(dst, Address(rscratch1, 0));
6102   }
6103 }
6104 
6105 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {
6106   if (reachable(src)) {
6107     comiss(dst, as_Address(src));
6108   } else {
6109     lea(rscratch1, src);
6110     comiss(dst, Address(rscratch1, 0));
6111   }
6112 }
6113 
6114 
6115 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
6116   Condition negated_cond = negate_condition(cond);
6117   Label L;
6118   jcc(negated_cond, L);
6119   atomic_incl(counter_addr);
6120   bind(L);
6121 }
6122 
6123 int MacroAssembler::corrected_idivl(Register reg) {
6124   // Full implementation of Java idiv and irem; checks for
6125   // special case as described in JVM spec., p.243 & p.271.
6126   // The function returns the (pc) offset of the idivl
6127   // instruction - may be needed for implicit exceptions.
6128   //
6129   //         normal case                           special case
6130   //
6131   // input : rax,: dividend                         min_int
6132   //         reg: divisor   (may not be rax,/rdx)   -1
6133   //
6134   // output: rax,: quotient  (= rax, idiv reg)       min_int
6135   //         rdx: remainder (= rax, irem reg)       0
6136   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
6137   const int min_int = 0x80000000;
6138   Label normal_case, special_case;
6139 
6140   // check for special case
6141   cmpl(rax, min_int);
6142   jcc(Assembler::notEqual, normal_case);
6143   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
6144   cmpl(reg, -1);
6145   jcc(Assembler::equal, special_case);
6146 
6147   // handle normal case
6148   bind(normal_case);
6149   cdql();
6150   int idivl_offset = offset();
6151   idivl(reg);
6152 
6153   // normal and special case exit
6154   bind(special_case);
6155 
6156   return idivl_offset;
6157 }
6158 
6159 
6160 
6161 void MacroAssembler::decrementl(Register reg, int value) {
6162   if (value == min_jint) {subl(reg, value) ; return; }
6163   if (value <  0) { incrementl(reg, -value); return; }
6164   if (value == 0) {                        ; return; }
6165   if (value == 1 && UseIncDec) { decl(reg) ; return; }
6166   /* else */      { subl(reg, value)       ; return; }
6167 }
6168 
6169 void MacroAssembler::decrementl(Address dst, int value) {
6170   if (value == min_jint) {subl(dst, value) ; return; }
6171   if (value <  0) { incrementl(dst, -value); return; }
6172   if (value == 0) {                        ; return; }
6173   if (value == 1 && UseIncDec) { decl(dst) ; return; }
6174   /* else */      { subl(dst, value)       ; return; }
6175 }
6176 
6177 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
6178   assert (shift_value > 0, "illegal shift value");
6179   Label _is_positive;
6180   testl (reg, reg);
6181   jcc (Assembler::positive, _is_positive);
6182   int offset = (1 << shift_value) - 1 ;
6183 
6184   if (offset == 1) {
6185     incrementl(reg);
6186   } else {
6187     addl(reg, offset);
6188   }
6189 
6190   bind (_is_positive);
6191   sarl(reg, shift_value);
6192 }
6193 
6194 // !defined(COMPILER2) is because of stupid core builds
6195 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2)
6196 void MacroAssembler::empty_FPU_stack() {
6197   if (VM_Version::supports_mmx()) {
6198     emms();
6199   } else {
6200     for (int i = 8; i-- > 0; ) ffree(i);
6201   }
6202 }
6203 #endif // !LP64 || C1 || !C2
6204 
6205 
6206 // Defines obj, preserves var_size_in_bytes
6207 void MacroAssembler::eden_allocate(Register obj,
6208                                    Register var_size_in_bytes,
6209                                    int con_size_in_bytes,
6210                                    Register t1,
6211                                    Label& slow_case) {
6212   assert(obj == rax, "obj must be in rax, for cmpxchg");
6213   assert_different_registers(obj, var_size_in_bytes, t1);
6214   if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
6215     jmp(slow_case);
6216   } else {
6217     Register end = t1;
6218     Label retry;
6219     bind(retry);
6220     ExternalAddress heap_top((address) Universe::heap()->top_addr());
6221     movptr(obj, heap_top);
6222     if (var_size_in_bytes == noreg) {
6223       lea(end, Address(obj, con_size_in_bytes));
6224     } else {
6225       lea(end, Address(obj, var_size_in_bytes, Address::times_1));
6226     }
6227     // if end < obj then we wrapped around => object too long => slow case
6228     cmpptr(end, obj);
6229     jcc(Assembler::below, slow_case);
6230     cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
6231     jcc(Assembler::above, slow_case);
6232     // Compare obj with the top addr, and if still equal, store the new top addr in
6233     // end at the address of the top addr pointer. Sets ZF if was equal, and clears
6234     // it otherwise. Use lock prefix for atomicity on MPs.
6235     locked_cmpxchgptr(end, heap_top);
6236     jcc(Assembler::notEqual, retry);
6237   }
6238 }
6239 
6240 void MacroAssembler::enter() {
6241   push(rbp);
6242   mov(rbp, rsp);
6243 }
6244 
6245 void MacroAssembler::fcmp(Register tmp) {
6246   fcmp(tmp, 1, true, true);
6247 }
6248 
6249 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
6250   assert(!pop_right || pop_left, "usage error");
6251   if (VM_Version::supports_cmov()) {
6252     assert(tmp == noreg, "unneeded temp");
6253     if (pop_left) {
6254       fucomip(index);
6255     } else {
6256       fucomi(index);
6257     }
6258     if (pop_right) {
6259       fpop();
6260     }
6261   } else {
6262     assert(tmp != noreg, "need temp");
6263     if (pop_left) {
6264       if (pop_right) {
6265         fcompp();
6266       } else {
6267         fcomp(index);
6268       }
6269     } else {
6270       fcom(index);
6271     }
6272     // convert FPU condition into eflags condition via rax,
6273     save_rax(tmp);
6274     fwait(); fnstsw_ax();
6275     sahf();
6276     restore_rax(tmp);
6277   }
6278   // condition codes set as follows:
6279   //
6280   // CF (corresponds to C0) if x < y
6281   // PF (corresponds to C2) if unordered
6282   // ZF (corresponds to C3) if x = y
6283 }
6284 
6285 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
6286   fcmp2int(dst, unordered_is_less, 1, true, true);
6287 }
6288 
6289 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
6290   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
6291   Label L;
6292   if (unordered_is_less) {
6293     movl(dst, -1);
6294     jcc(Assembler::parity, L);
6295     jcc(Assembler::below , L);
6296     movl(dst, 0);
6297     jcc(Assembler::equal , L);
6298     increment(dst);
6299   } else { // unordered is greater
6300     movl(dst, 1);
6301     jcc(Assembler::parity, L);
6302     jcc(Assembler::above , L);
6303     movl(dst, 0);
6304     jcc(Assembler::equal , L);
6305     decrementl(dst);
6306   }
6307   bind(L);
6308 }
6309 
6310 void MacroAssembler::fld_d(AddressLiteral src) {
6311   fld_d(as_Address(src));
6312 }
6313 
6314 void MacroAssembler::fld_s(AddressLiteral src) {
6315   fld_s(as_Address(src));
6316 }
6317 
6318 void MacroAssembler::fld_x(AddressLiteral src) {
6319   Assembler::fld_x(as_Address(src));
6320 }
6321 
6322 void MacroAssembler::fldcw(AddressLiteral src) {
6323   Assembler::fldcw(as_Address(src));
6324 }
6325 
6326 void MacroAssembler::fpop() {
6327   ffree();
6328   fincstp();
6329 }
6330 
6331 void MacroAssembler::fremr(Register tmp) {
6332   save_rax(tmp);
6333   { Label L;
6334     bind(L);
6335     fprem();
6336     fwait(); fnstsw_ax();
6337 #ifdef _LP64
6338     testl(rax, 0x400);
6339     jcc(Assembler::notEqual, L);
6340 #else
6341     sahf();
6342     jcc(Assembler::parity, L);
6343 #endif // _LP64
6344   }
6345   restore_rax(tmp);
6346   // Result is in ST0.
6347   // Note: fxch & fpop to get rid of ST1
6348   // (otherwise FPU stack could overflow eventually)
6349   fxch(1);
6350   fpop();
6351 }
6352 
6353 
6354 void MacroAssembler::incrementl(AddressLiteral dst) {
6355   if (reachable(dst)) {
6356     incrementl(as_Address(dst));
6357   } else {
6358     lea(rscratch1, dst);
6359     incrementl(Address(rscratch1, 0));
6360   }
6361 }
6362 
6363 void MacroAssembler::incrementl(ArrayAddress dst) {
6364   incrementl(as_Address(dst));
6365 }
6366 
6367 void MacroAssembler::incrementl(Register reg, int value) {
6368   if (value == min_jint) {addl(reg, value) ; return; }
6369   if (value <  0) { decrementl(reg, -value); return; }
6370   if (value == 0) {                        ; return; }
6371   if (value == 1 && UseIncDec) { incl(reg) ; return; }
6372   /* else */      { addl(reg, value)       ; return; }
6373 }
6374 
6375 void MacroAssembler::incrementl(Address dst, int value) {
6376   if (value == min_jint) {addl(dst, value) ; return; }
6377   if (value <  0) { decrementl(dst, -value); return; }
6378   if (value == 0) {                        ; return; }
6379   if (value == 1 && UseIncDec) { incl(dst) ; return; }
6380   /* else */      { addl(dst, value)       ; return; }
6381 }
6382 
6383 void MacroAssembler::jump(AddressLiteral dst) {
6384   if (reachable(dst)) {
6385     jmp_literal(dst.target(), dst.rspec());
6386   } else {
6387     lea(rscratch1, dst);
6388     jmp(rscratch1);
6389   }
6390 }
6391 
6392 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
6393   if (reachable(dst)) {
6394     InstructionMark im(this);
6395     relocate(dst.reloc());
6396     const int short_size = 2;
6397     const int long_size = 6;
6398     int offs = (intptr_t)dst.target() - ((intptr_t)_code_pos);
6399     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
6400       // 0111 tttn #8-bit disp
6401       emit_byte(0x70 | cc);
6402       emit_byte((offs - short_size) & 0xFF);
6403     } else {
6404       // 0000 1111 1000 tttn #32-bit disp
6405       emit_byte(0x0F);
6406       emit_byte(0x80 | cc);
6407       emit_long(offs - long_size);
6408     }
6409   } else {
6410 #ifdef ASSERT
6411     warning("reversing conditional branch");
6412 #endif /* ASSERT */
6413     Label skip;
6414     jccb(reverse[cc], skip);
6415     lea(rscratch1, dst);
6416     Assembler::jmp(rscratch1);
6417     bind(skip);
6418   }
6419 }
6420 
6421 void MacroAssembler::ldmxcsr(AddressLiteral src) {
6422   if (reachable(src)) {
6423     Assembler::ldmxcsr(as_Address(src));
6424   } else {
6425     lea(rscratch1, src);
6426     Assembler::ldmxcsr(Address(rscratch1, 0));
6427   }
6428 }
6429 
6430 int MacroAssembler::load_signed_byte(Register dst, Address src) {
6431   int off;
6432   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
6433     off = offset();
6434     movsbl(dst, src); // movsxb
6435   } else {
6436     off = load_unsigned_byte(dst, src);
6437     shll(dst, 24);
6438     sarl(dst, 24);
6439   }
6440   return off;
6441 }
6442 
6443 // Note: load_signed_short used to be called load_signed_word.
6444 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
6445 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
6446 // The term "word" in HotSpot means a 32- or 64-bit machine word.
6447 int MacroAssembler::load_signed_short(Register dst, Address src) {
6448   int off;
6449   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
6450     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
6451     // version but this is what 64bit has always done. This seems to imply
6452     // that users are only using 32bits worth.
6453     off = offset();
6454     movswl(dst, src); // movsxw
6455   } else {
6456     off = load_unsigned_short(dst, src);
6457     shll(dst, 16);
6458     sarl(dst, 16);
6459   }
6460   return off;
6461 }
6462 
6463 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
6464   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
6465   // and "3.9 Partial Register Penalties", p. 22).
6466   int off;
6467   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
6468     off = offset();
6469     movzbl(dst, src); // movzxb
6470   } else {
6471     xorl(dst, dst);
6472     off = offset();
6473     movb(dst, src);
6474   }
6475   return off;
6476 }
6477 
6478 // Note: load_unsigned_short used to be called load_unsigned_word.
6479 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
6480   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
6481   // and "3.9 Partial Register Penalties", p. 22).
6482   int off;
6483   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
6484     off = offset();
6485     movzwl(dst, src); // movzxw
6486   } else {
6487     xorl(dst, dst);
6488     off = offset();
6489     movw(dst, src);
6490   }
6491   return off;
6492 }
6493 
6494 void MacroAssembler::load_sized_value(Register dst, Address src,
6495                                       int size_in_bytes, bool is_signed) {
6496   switch (size_in_bytes ^ (is_signed ? -1 : 0)) {
6497 #ifndef _LP64
6498   // For case 8, caller is responsible for manually loading
6499   // the second word into another register.
6500   case ~8:  // fall through:
6501   case  8:  movl(                dst, src ); break;
6502 #else
6503   case ~8:  // fall through:
6504   case  8:  movq(                dst, src ); break;
6505 #endif
6506   case ~4:  // fall through:
6507   case  4:  movl(                dst, src ); break;
6508   case ~2:  load_signed_short(   dst, src ); break;
6509   case  2:  load_unsigned_short( dst, src ); break;
6510   case ~1:  load_signed_byte(    dst, src ); break;
6511   case  1:  load_unsigned_byte(  dst, src ); break;
6512   default:  ShouldNotReachHere();
6513   }
6514 }
6515 
6516 void MacroAssembler::mov32(AddressLiteral dst, Register src) {
6517   if (reachable(dst)) {
6518     movl(as_Address(dst), src);
6519   } else {
6520     lea(rscratch1, dst);
6521     movl(Address(rscratch1, 0), src);
6522   }
6523 }
6524 
6525 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
6526   if (reachable(src)) {
6527     movl(dst, as_Address(src));
6528   } else {
6529     lea(rscratch1, src);
6530     movl(dst, Address(rscratch1, 0));
6531   }
6532 }
6533 
6534 // C++ bool manipulation
6535 
6536 void MacroAssembler::movbool(Register dst, Address src) {
6537   if(sizeof(bool) == 1)
6538     movb(dst, src);
6539   else if(sizeof(bool) == 2)
6540     movw(dst, src);
6541   else if(sizeof(bool) == 4)
6542     movl(dst, src);
6543   else
6544     // unsupported
6545     ShouldNotReachHere();
6546 }
6547 
6548 void MacroAssembler::movbool(Address dst, bool boolconst) {
6549   if(sizeof(bool) == 1)
6550     movb(dst, (int) boolconst);
6551   else if(sizeof(bool) == 2)
6552     movw(dst, (int) boolconst);
6553   else if(sizeof(bool) == 4)
6554     movl(dst, (int) boolconst);
6555   else
6556     // unsupported
6557     ShouldNotReachHere();
6558 }
6559 
6560 void MacroAssembler::movbool(Address dst, Register src) {
6561   if(sizeof(bool) == 1)
6562     movb(dst, src);
6563   else if(sizeof(bool) == 2)
6564     movw(dst, src);
6565   else if(sizeof(bool) == 4)
6566     movl(dst, src);
6567   else
6568     // unsupported
6569     ShouldNotReachHere();
6570 }
6571 
6572 void MacroAssembler::movbyte(ArrayAddress dst, int src) {
6573   movb(as_Address(dst), src);
6574 }
6575 
6576 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
6577   if (reachable(src)) {
6578     if (UseXmmLoadAndClearUpper) {
6579       movsd (dst, as_Address(src));
6580     } else {
6581       movlpd(dst, as_Address(src));
6582     }
6583   } else {
6584     lea(rscratch1, src);
6585     if (UseXmmLoadAndClearUpper) {
6586       movsd (dst, Address(rscratch1, 0));
6587     } else {
6588       movlpd(dst, Address(rscratch1, 0));
6589     }
6590   }
6591 }
6592 
6593 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
6594   if (reachable(src)) {
6595     movss(dst, as_Address(src));
6596   } else {
6597     lea(rscratch1, src);
6598     movss(dst, Address(rscratch1, 0));
6599   }
6600 }
6601 
6602 void MacroAssembler::movptr(Register dst, Register src) {
6603   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
6604 }
6605 
6606 void MacroAssembler::movptr(Register dst, Address src) {
6607   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
6608 }
6609 
6610 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
6611 void MacroAssembler::movptr(Register dst, intptr_t src) {
6612   LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src));
6613 }
6614 
6615 void MacroAssembler::movptr(Address dst, Register src) {
6616   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
6617 }
6618 
6619 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
6620   if (reachable(src)) {
6621     movss(dst, as_Address(src));
6622   } else {
6623     lea(rscratch1, src);
6624     movss(dst, Address(rscratch1, 0));
6625   }
6626 }
6627 
6628 void MacroAssembler::null_check(Register reg, int offset) {
6629   if (needs_explicit_null_check(offset)) {
6630     // provoke OS NULL exception if reg = NULL by
6631     // accessing M[reg] w/o changing any (non-CC) registers
6632     // NOTE: cmpl is plenty here to provoke a segv
6633     cmpptr(rax, Address(reg, 0));
6634     // Note: should probably use testl(rax, Address(reg, 0));
6635     //       may be shorter code (however, this version of
6636     //       testl needs to be implemented first)
6637   } else {
6638     // nothing to do, (later) access of M[reg + offset]
6639     // will provoke OS NULL exception if reg = NULL
6640   }
6641 }
6642 
6643 void MacroAssembler::os_breakpoint() {
6644   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
6645   // (e.g., MSVC can't call ps() otherwise)
6646   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
6647 }
6648 
6649 void MacroAssembler::pop_CPU_state() {
6650   pop_FPU_state();
6651   pop_IU_state();
6652 }
6653 
6654 void MacroAssembler::pop_FPU_state() {
6655   NOT_LP64(frstor(Address(rsp, 0));)
6656   LP64_ONLY(fxrstor(Address(rsp, 0));)
6657   addptr(rsp, FPUStateSizeInWords * wordSize);
6658 }
6659 
6660 void MacroAssembler::pop_IU_state() {
6661   popa();
6662   LP64_ONLY(addq(rsp, 8));
6663   popf();
6664 }
6665 
6666 // Save Integer and Float state
6667 // Warning: Stack must be 16 byte aligned (64bit)
6668 void MacroAssembler::push_CPU_state() {
6669   push_IU_state();
6670   push_FPU_state();
6671 }
6672 
6673 void MacroAssembler::push_FPU_state() {
6674   subptr(rsp, FPUStateSizeInWords * wordSize);
6675 #ifndef _LP64
6676   fnsave(Address(rsp, 0));
6677   fwait();
6678 #else
6679   fxsave(Address(rsp, 0));
6680 #endif // LP64
6681 }
6682 
6683 void MacroAssembler::push_IU_state() {
6684   // Push flags first because pusha kills them
6685   pushf();
6686   // Make sure rsp stays 16-byte aligned
6687   LP64_ONLY(subq(rsp, 8));
6688   pusha();
6689 }
6690 
6691 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp, bool clear_pc) {
6692   // determine java_thread register
6693   if (!java_thread->is_valid()) {
6694     java_thread = rdi;
6695     get_thread(java_thread);
6696   }
6697   // we must set sp to zero to clear frame
6698   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
6699   if (clear_fp) {
6700     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
6701   }
6702 
6703   if (clear_pc)
6704     movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
6705 
6706 }
6707 
6708 void MacroAssembler::restore_rax(Register tmp) {
6709   if (tmp == noreg) pop(rax);
6710   else if (tmp != rax) mov(rax, tmp);
6711 }
6712 
6713 void MacroAssembler::round_to(Register reg, int modulus) {
6714   addptr(reg, modulus - 1);
6715   andptr(reg, -modulus);
6716 }
6717 
6718 void MacroAssembler::save_rax(Register tmp) {
6719   if (tmp == noreg) push(rax);
6720   else if (tmp != rax) mov(tmp, rax);
6721 }
6722 
6723 // Write serialization page so VM thread can do a pseudo remote membar.
6724 // We use the current thread pointer to calculate a thread specific
6725 // offset to write to within the page. This minimizes bus traffic
6726 // due to cache line collision.
6727 void MacroAssembler::serialize_memory(Register thread, Register tmp) {
6728   movl(tmp, thread);
6729   shrl(tmp, os::get_serialize_page_shift_count());
6730   andl(tmp, (os::vm_page_size() - sizeof(int)));
6731 
6732   Address index(noreg, tmp, Address::times_1);
6733   ExternalAddress page(os::get_memory_serialize_page());
6734 
6735   // Size of store must match masking code above
6736   movl(as_Address(ArrayAddress(page, index)), tmp);
6737 }
6738 
6739 // Calls to C land
6740 //
6741 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
6742 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
6743 // has to be reset to 0. This is required to allow proper stack traversal.
6744 void MacroAssembler::set_last_Java_frame(Register java_thread,
6745                                          Register last_java_sp,
6746                                          Register last_java_fp,
6747                                          address  last_java_pc) {
6748   // determine java_thread register
6749   if (!java_thread->is_valid()) {
6750     java_thread = rdi;
6751     get_thread(java_thread);
6752   }
6753   // determine last_java_sp register
6754   if (!last_java_sp->is_valid()) {
6755     last_java_sp = rsp;
6756   }
6757 
6758   // last_java_fp is optional
6759 
6760   if (last_java_fp->is_valid()) {
6761     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
6762   }
6763 
6764   // last_java_pc is optional
6765 
6766   if (last_java_pc != NULL) {
6767     lea(Address(java_thread,
6768                  JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()),
6769         InternalAddress(last_java_pc));
6770 
6771   }
6772   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
6773 }
6774 
6775 void MacroAssembler::shlptr(Register dst, int imm8) {
6776   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
6777 }
6778 
6779 void MacroAssembler::shrptr(Register dst, int imm8) {
6780   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
6781 }
6782 
6783 void MacroAssembler::sign_extend_byte(Register reg) {
6784   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
6785     movsbl(reg, reg); // movsxb
6786   } else {
6787     shll(reg, 24);
6788     sarl(reg, 24);
6789   }
6790 }
6791 
6792 void MacroAssembler::sign_extend_short(Register reg) {
6793   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
6794     movswl(reg, reg); // movsxw
6795   } else {
6796     shll(reg, 16);
6797     sarl(reg, 16);
6798   }
6799 }
6800 
6801 //////////////////////////////////////////////////////////////////////////////////
6802 #ifndef SERIALGC
6803 
6804 void MacroAssembler::g1_write_barrier_pre(Register obj,
6805 #ifndef _LP64
6806                                           Register thread,
6807 #endif
6808                                           Register tmp,
6809                                           Register tmp2,
6810                                           bool tosca_live) {
6811   LP64_ONLY(Register thread = r15_thread;)
6812   Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
6813                                        PtrQueue::byte_offset_of_active()));
6814 
6815   Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
6816                                        PtrQueue::byte_offset_of_index()));
6817   Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
6818                                        PtrQueue::byte_offset_of_buf()));
6819 
6820 
6821   Label done;
6822   Label runtime;
6823 
6824   // if (!marking_in_progress) goto done;
6825   if (in_bytes(PtrQueue::byte_width_of_active()) == 4) {
6826     cmpl(in_progress, 0);
6827   } else {
6828     assert(in_bytes(PtrQueue::byte_width_of_active()) == 1, "Assumption");
6829     cmpb(in_progress, 0);
6830   }
6831   jcc(Assembler::equal, done);
6832 
6833   // if (x.f == NULL) goto done;
6834 #ifdef _LP64
6835   load_heap_oop(tmp2, Address(obj, 0));
6836 #else
6837   movptr(tmp2, Address(obj, 0));
6838 #endif
6839   cmpptr(tmp2, (int32_t) NULL_WORD);
6840   jcc(Assembler::equal, done);
6841 
6842   // Can we store original value in the thread's buffer?
6843 
6844 #ifdef _LP64
6845   movslq(tmp, index);
6846   cmpq(tmp, 0);
6847 #else
6848   cmpl(index, 0);
6849 #endif
6850   jcc(Assembler::equal, runtime);
6851 #ifdef _LP64
6852   subq(tmp, wordSize);
6853   movl(index, tmp);
6854   addq(tmp, buffer);
6855 #else
6856   subl(index, wordSize);
6857   movl(tmp, buffer);
6858   addl(tmp, index);
6859 #endif
6860   movptr(Address(tmp, 0), tmp2);
6861   jmp(done);
6862   bind(runtime);
6863   // save the live input values
6864   if(tosca_live) push(rax);
6865   push(obj);
6866 #ifdef _LP64
6867   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), tmp2, r15_thread);
6868 #else
6869   push(thread);
6870   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), tmp2, thread);
6871   pop(thread);
6872 #endif
6873   pop(obj);
6874   if(tosca_live) pop(rax);
6875   bind(done);
6876 
6877 }
6878 
6879 void MacroAssembler::g1_write_barrier_post(Register store_addr,
6880                                            Register new_val,
6881 #ifndef _LP64
6882                                            Register thread,
6883 #endif
6884                                            Register tmp,
6885                                            Register tmp2) {
6886 
6887   LP64_ONLY(Register thread = r15_thread;)
6888   Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
6889                                        PtrQueue::byte_offset_of_index()));
6890   Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
6891                                        PtrQueue::byte_offset_of_buf()));
6892   BarrierSet* bs = Universe::heap()->barrier_set();
6893   CardTableModRefBS* ct = (CardTableModRefBS*)bs;
6894   Label done;
6895   Label runtime;
6896 
6897   // Does store cross heap regions?
6898 
6899   movptr(tmp, store_addr);
6900   xorptr(tmp, new_val);
6901   shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
6902   jcc(Assembler::equal, done);
6903 
6904   // crosses regions, storing NULL?
6905 
6906   cmpptr(new_val, (int32_t) NULL_WORD);
6907   jcc(Assembler::equal, done);
6908 
6909   // storing region crossing non-NULL, is card already dirty?
6910 
6911   ExternalAddress cardtable((address) ct->byte_map_base);
6912   assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
6913 #ifdef _LP64
6914   const Register card_addr = tmp;
6915 
6916   movq(card_addr, store_addr);
6917   shrq(card_addr, CardTableModRefBS::card_shift);
6918 
6919   lea(tmp2, cardtable);
6920 
6921   // get the address of the card
6922   addq(card_addr, tmp2);
6923 #else
6924   const Register card_index = tmp;
6925 
6926   movl(card_index, store_addr);
6927   shrl(card_index, CardTableModRefBS::card_shift);
6928 
6929   Address index(noreg, card_index, Address::times_1);
6930   const Register card_addr = tmp;
6931   lea(card_addr, as_Address(ArrayAddress(cardtable, index)));
6932 #endif
6933   cmpb(Address(card_addr, 0), 0);
6934   jcc(Assembler::equal, done);
6935 
6936   // storing a region crossing, non-NULL oop, card is clean.
6937   // dirty card and log.
6938 
6939   movb(Address(card_addr, 0), 0);
6940 
6941   cmpl(queue_index, 0);
6942   jcc(Assembler::equal, runtime);
6943   subl(queue_index, wordSize);
6944   movptr(tmp2, buffer);
6945 #ifdef _LP64
6946   movslq(rscratch1, queue_index);
6947   addq(tmp2, rscratch1);
6948   movq(Address(tmp2, 0), card_addr);
6949 #else
6950   addl(tmp2, queue_index);
6951   movl(Address(tmp2, 0), card_index);
6952 #endif
6953   jmp(done);
6954 
6955   bind(runtime);
6956   // save the live input values
6957   push(store_addr);
6958   push(new_val);
6959 #ifdef _LP64
6960   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread);
6961 #else
6962   push(thread);
6963   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
6964   pop(thread);
6965 #endif
6966   pop(new_val);
6967   pop(store_addr);
6968 
6969   bind(done);
6970 
6971 }
6972 
6973 #endif // SERIALGC
6974 //////////////////////////////////////////////////////////////////////////////////
6975 
6976 
6977 void MacroAssembler::store_check(Register obj) {
6978   // Does a store check for the oop in register obj. The content of
6979   // register obj is destroyed afterwards.
6980   store_check_part_1(obj);
6981   store_check_part_2(obj);
6982 }
6983 
6984 void MacroAssembler::store_check(Register obj, Address dst) {
6985   store_check(obj);
6986 }
6987 
6988 
6989 // split the store check operation so that other instructions can be scheduled inbetween
6990 void MacroAssembler::store_check_part_1(Register obj) {
6991   BarrierSet* bs = Universe::heap()->barrier_set();
6992   assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
6993   shrptr(obj, CardTableModRefBS::card_shift);
6994 }
6995 
6996 void MacroAssembler::store_check_part_2(Register obj) {
6997   BarrierSet* bs = Universe::heap()->barrier_set();
6998   assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
6999   CardTableModRefBS* ct = (CardTableModRefBS*)bs;
7000   assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
7001 
7002   // The calculation for byte_map_base is as follows:
7003   // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
7004   // So this essentially converts an address to a displacement and
7005   // it will never need to be relocated. On 64bit however the value may be too
7006   // large for a 32bit displacement
7007 
7008   intptr_t disp = (intptr_t) ct->byte_map_base;
7009   if (is_simm32(disp)) {
7010     Address cardtable(noreg, obj, Address::times_1, disp);
7011     movb(cardtable, 0);
7012   } else {
7013     // By doing it as an ExternalAddress disp could be converted to a rip-relative
7014     // displacement and done in a single instruction given favorable mapping and
7015     // a smarter version of as_Address. Worst case it is two instructions which
7016     // is no worse off then loading disp into a register and doing as a simple
7017     // Address() as above.
7018     // We can't do as ExternalAddress as the only style since if disp == 0 we'll
7019     // assert since NULL isn't acceptable in a reloci (see 6644928). In any case
7020     // in some cases we'll get a single instruction version.
7021 
7022     ExternalAddress cardtable((address)disp);
7023     Address index(noreg, obj, Address::times_1);
7024     movb(as_Address(ArrayAddress(cardtable, index)), 0);
7025   }
7026 }
7027 
7028 void MacroAssembler::subptr(Register dst, int32_t imm32) {
7029   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
7030 }
7031 
7032 void MacroAssembler::subptr(Register dst, Register src) {
7033   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
7034 }
7035 
7036 void MacroAssembler::test32(Register src1, AddressLiteral src2) {
7037   // src2 must be rval
7038 
7039   if (reachable(src2)) {
7040     testl(src1, as_Address(src2));
7041   } else {
7042     lea(rscratch1, src2);
7043     testl(src1, Address(rscratch1, 0));
7044   }
7045 }
7046 
7047 // C++ bool manipulation
7048 void MacroAssembler::testbool(Register dst) {
7049   if(sizeof(bool) == 1)
7050     testb(dst, 0xff);
7051   else if(sizeof(bool) == 2) {
7052     // testw implementation needed for two byte bools
7053     ShouldNotReachHere();
7054   } else if(sizeof(bool) == 4)
7055     testl(dst, dst);
7056   else
7057     // unsupported
7058     ShouldNotReachHere();
7059 }
7060 
7061 void MacroAssembler::testptr(Register dst, Register src) {
7062   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
7063 }
7064 
7065 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
7066 void MacroAssembler::tlab_allocate(Register obj,
7067                                    Register var_size_in_bytes,
7068                                    int con_size_in_bytes,
7069                                    Register t1,
7070                                    Register t2,
7071                                    Label& slow_case) {
7072   assert_different_registers(obj, t1, t2);
7073   assert_different_registers(obj, var_size_in_bytes, t1);
7074   Register end = t2;
7075   Register thread = NOT_LP64(t1) LP64_ONLY(r15_thread);
7076 
7077   verify_tlab();
7078 
7079   NOT_LP64(get_thread(thread));
7080 
7081   movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
7082   if (var_size_in_bytes == noreg) {
7083     lea(end, Address(obj, con_size_in_bytes));
7084   } else {
7085     lea(end, Address(obj, var_size_in_bytes, Address::times_1));
7086   }
7087   cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
7088   jcc(Assembler::above, slow_case);
7089 
7090   // update the tlab top pointer
7091   movptr(Address(thread, JavaThread::tlab_top_offset()), end);
7092 
7093   // recover var_size_in_bytes if necessary
7094   if (var_size_in_bytes == end) {
7095     subptr(var_size_in_bytes, obj);
7096   }
7097   verify_tlab();
7098 }
7099 
7100 // Preserves rbx, and rdx.
7101 void MacroAssembler::tlab_refill(Label& retry,
7102                                  Label& try_eden,
7103                                  Label& slow_case) {
7104   Register top = rax;
7105   Register t1  = rcx;
7106   Register t2  = rsi;
7107   Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
7108   assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
7109   Label do_refill, discard_tlab;
7110 
7111   if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
7112     // No allocation in the shared eden.
7113     jmp(slow_case);
7114   }
7115 
7116   NOT_LP64(get_thread(thread_reg));
7117 
7118   movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
7119   movptr(t1,  Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
7120 
7121   // calculate amount of free space
7122   subptr(t1, top);
7123   shrptr(t1, LogHeapWordSize);
7124 
7125   // Retain tlab and allocate object in shared space if
7126   // the amount free in the tlab is too large to discard.
7127   cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
7128   jcc(Assembler::lessEqual, discard_tlab);
7129 
7130   // Retain
7131   // %%% yuck as movptr...
7132   movptr(t2, (int32_t) ThreadLocalAllocBuffer::refill_waste_limit_increment());
7133   addptr(Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())), t2);
7134   if (TLABStats) {
7135     // increment number of slow_allocations
7136     addl(Address(thread_reg, in_bytes(JavaThread::tlab_slow_allocations_offset())), 1);
7137   }
7138   jmp(try_eden);
7139 
7140   bind(discard_tlab);
7141   if (TLABStats) {
7142     // increment number of refills
7143     addl(Address(thread_reg, in_bytes(JavaThread::tlab_number_of_refills_offset())), 1);
7144     // accumulate wastage -- t1 is amount free in tlab
7145     addl(Address(thread_reg, in_bytes(JavaThread::tlab_fast_refill_waste_offset())), t1);
7146   }
7147 
7148   // if tlab is currently allocated (top or end != null) then
7149   // fill [top, end + alignment_reserve) with array object
7150   testptr (top, top);
7151   jcc(Assembler::zero, do_refill);
7152 
7153   // set up the mark word
7154   movptr(Address(top, oopDesc::mark_offset_in_bytes()), (intptr_t)markOopDesc::prototype()->copy_set_hash(0x2));
7155   // set the length to the remaining space
7156   subptr(t1, typeArrayOopDesc::header_size(T_INT));
7157   addptr(t1, (int32_t)ThreadLocalAllocBuffer::alignment_reserve());
7158   shlptr(t1, log2_intptr(HeapWordSize/sizeof(jint)));
7159   movptr(Address(top, arrayOopDesc::length_offset_in_bytes()), t1);
7160   // set klass to intArrayKlass
7161   // dubious reloc why not an oop reloc?
7162   movptr(t1, ExternalAddress((address) Universe::intArrayKlassObj_addr()));
7163   // store klass last.  concurrent gcs assumes klass length is valid if
7164   // klass field is not null.
7165   store_klass(top, t1);
7166 
7167   // refill the tlab with an eden allocation
7168   bind(do_refill);
7169   movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
7170   shlptr(t1, LogHeapWordSize);
7171   // add object_size ??
7172   eden_allocate(top, t1, 0, t2, slow_case);
7173 
7174   // Check that t1 was preserved in eden_allocate.
7175 #ifdef ASSERT
7176   if (UseTLAB) {
7177     Label ok;
7178     Register tsize = rsi;
7179     assert_different_registers(tsize, thread_reg, t1);
7180     push(tsize);
7181     movptr(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
7182     shlptr(tsize, LogHeapWordSize);
7183     cmpptr(t1, tsize);
7184     jcc(Assembler::equal, ok);
7185     stop("assert(t1 != tlab size)");
7186     should_not_reach_here();
7187 
7188     bind(ok);
7189     pop(tsize);
7190   }
7191 #endif
7192   movptr(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top);
7193   movptr(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top);
7194   addptr(top, t1);
7195   subptr(top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
7196   movptr(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top);
7197   verify_tlab();
7198   jmp(retry);
7199 }
7200 
7201 static const double     pi_4 =  0.7853981633974483;
7202 
7203 void MacroAssembler::trigfunc(char trig, int num_fpu_regs_in_use) {
7204   // A hand-coded argument reduction for values in fabs(pi/4, pi/2)
7205   // was attempted in this code; unfortunately it appears that the
7206   // switch to 80-bit precision and back causes this to be
7207   // unprofitable compared with simply performing a runtime call if
7208   // the argument is out of the (-pi/4, pi/4) range.
7209 
7210   Register tmp = noreg;
7211   if (!VM_Version::supports_cmov()) {
7212     // fcmp needs a temporary so preserve rbx,
7213     tmp = rbx;
7214     push(tmp);
7215   }
7216 
7217   Label slow_case, done;
7218 
7219   ExternalAddress pi4_adr = (address)&pi_4;
7220   if (reachable(pi4_adr)) {
7221     // x ?<= pi/4
7222     fld_d(pi4_adr);
7223     fld_s(1);                // Stack:  X  PI/4  X
7224     fabs();                  // Stack: |X| PI/4  X
7225     fcmp(tmp);
7226     jcc(Assembler::above, slow_case);
7227 
7228     // fastest case: -pi/4 <= x <= pi/4
7229     switch(trig) {
7230     case 's':
7231       fsin();
7232       break;
7233     case 'c':
7234       fcos();
7235       break;
7236     case 't':
7237       ftan();
7238       break;
7239     default:
7240       assert(false, "bad intrinsic");
7241       break;
7242     }
7243     jmp(done);
7244   }
7245 
7246   // slow case: runtime call
7247   bind(slow_case);
7248   // Preserve registers across runtime call
7249   pusha();
7250   int incoming_argument_and_return_value_offset = -1;
7251   if (num_fpu_regs_in_use > 1) {
7252     // Must preserve all other FPU regs (could alternatively convert
7253     // SharedRuntime::dsin and dcos into assembly routines known not to trash
7254     // FPU state, but can not trust C compiler)
7255     NEEDS_CLEANUP;
7256     // NOTE that in this case we also push the incoming argument to
7257     // the stack and restore it later; we also use this stack slot to
7258     // hold the return value from dsin or dcos.
7259     for (int i = 0; i < num_fpu_regs_in_use; i++) {
7260       subptr(rsp, sizeof(jdouble));
7261       fstp_d(Address(rsp, 0));
7262     }
7263     incoming_argument_and_return_value_offset = sizeof(jdouble)*(num_fpu_regs_in_use-1);
7264     fld_d(Address(rsp, incoming_argument_and_return_value_offset));
7265   }
7266   subptr(rsp, sizeof(jdouble));
7267   fstp_d(Address(rsp, 0));
7268 #ifdef _LP64
7269   movdbl(xmm0, Address(rsp, 0));
7270 #endif // _LP64
7271 
7272   // NOTE: we must not use call_VM_leaf here because that requires a
7273   // complete interpreter frame in debug mode -- same bug as 4387334
7274   // MacroAssembler::call_VM_leaf_base is perfectly safe and will
7275   // do proper 64bit abi
7276 
7277   NEEDS_CLEANUP;
7278   // Need to add stack banging before this runtime call if it needs to
7279   // be taken; however, there is no generic stack banging routine at
7280   // the MacroAssembler level
7281   switch(trig) {
7282   case 's':
7283     {
7284       MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::dsin), 0);
7285     }
7286     break;
7287   case 'c':
7288     {
7289       MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::dcos), 0);
7290     }
7291     break;
7292   case 't':
7293     {
7294       MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::dtan), 0);
7295     }
7296     break;
7297   default:
7298     assert(false, "bad intrinsic");
7299     break;
7300   }
7301 #ifdef _LP64
7302     movsd(Address(rsp, 0), xmm0);
7303     fld_d(Address(rsp, 0));
7304 #endif // _LP64
7305   addptr(rsp, sizeof(jdouble));
7306   if (num_fpu_regs_in_use > 1) {
7307     // Must save return value to stack and then restore entire FPU stack
7308     fstp_d(Address(rsp, incoming_argument_and_return_value_offset));
7309     for (int i = 0; i < num_fpu_regs_in_use; i++) {
7310       fld_d(Address(rsp, 0));
7311       addptr(rsp, sizeof(jdouble));
7312     }
7313   }
7314   popa();
7315 
7316   // Come here with result in F-TOS
7317   bind(done);
7318 
7319   if (tmp != noreg) {
7320     pop(tmp);
7321   }
7322 }
7323 
7324 
7325 // Look up the method for a megamorphic invokeinterface call.
7326 // The target method is determined by <intf_klass, itable_index>.
7327 // The receiver klass is in recv_klass.
7328 // On success, the result will be in method_result, and execution falls through.
7329 // On failure, execution transfers to the given label.
7330 void MacroAssembler::lookup_interface_method(Register recv_klass,
7331                                              Register intf_klass,
7332                                              RegisterOrConstant itable_index,
7333                                              Register method_result,
7334                                              Register scan_temp,
7335                                              Label& L_no_such_interface) {
7336   assert_different_registers(recv_klass, intf_klass, method_result, scan_temp);
7337   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
7338          "caller must use same register for non-constant itable index as for method");
7339 
7340   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
7341   int vtable_base = instanceKlass::vtable_start_offset() * wordSize;
7342   int itentry_off = itableMethodEntry::method_offset_in_bytes();
7343   int scan_step   = itableOffsetEntry::size() * wordSize;
7344   int vte_size    = vtableEntry::size() * wordSize;
7345   Address::ScaleFactor times_vte_scale = Address::times_ptr;
7346   assert(vte_size == wordSize, "else adjust times_vte_scale");
7347 
7348   movl(scan_temp, Address(recv_klass, instanceKlass::vtable_length_offset() * wordSize));
7349 
7350   // %%% Could store the aligned, prescaled offset in the klassoop.
7351   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
7352   if (HeapWordsPerLong > 1) {
7353     // Round up to align_object_offset boundary
7354     // see code for instanceKlass::start_of_itable!
7355     round_to(scan_temp, BytesPerLong);
7356   }
7357 
7358   // Adjust recv_klass by scaled itable_index, so we can free itable_index.
7359   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
7360   lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
7361 
7362   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
7363   //   if (scan->interface() == intf) {
7364   //     result = (klass + scan->offset() + itable_index);
7365   //   }
7366   // }
7367   Label search, found_method;
7368 
7369   for (int peel = 1; peel >= 0; peel--) {
7370     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
7371     cmpptr(intf_klass, method_result);
7372 
7373     if (peel) {
7374       jccb(Assembler::equal, found_method);
7375     } else {
7376       jccb(Assembler::notEqual, search);
7377       // (invert the test to fall through to found_method...)
7378     }
7379 
7380     if (!peel)  break;
7381 
7382     bind(search);
7383 
7384     // Check that the previous entry is non-null.  A null entry means that
7385     // the receiver class doesn't implement the interface, and wasn't the
7386     // same as when the caller was compiled.
7387     testptr(method_result, method_result);
7388     jcc(Assembler::zero, L_no_such_interface);
7389     addptr(scan_temp, scan_step);
7390   }
7391 
7392   bind(found_method);
7393 
7394   // Got a hit.
7395   movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
7396   movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
7397 }
7398 
7399 
7400 void MacroAssembler::check_klass_subtype(Register sub_klass,
7401                            Register super_klass,
7402                            Register temp_reg,
7403                            Label& L_success) {
7404   Label L_failure;
7405   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, NULL);
7406   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
7407   bind(L_failure);
7408 }
7409 
7410 
7411 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
7412                                                    Register super_klass,
7413                                                    Register temp_reg,
7414                                                    Label* L_success,
7415                                                    Label* L_failure,
7416                                                    Label* L_slow_path,
7417                                         RegisterOrConstant super_check_offset) {
7418   assert_different_registers(sub_klass, super_klass, temp_reg);
7419   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
7420   if (super_check_offset.is_register()) {
7421     assert_different_registers(sub_klass, super_klass,
7422                                super_check_offset.as_register());
7423   } else if (must_load_sco) {
7424     assert(temp_reg != noreg, "supply either a temp or a register offset");
7425   }
7426 
7427   Label L_fallthrough;
7428   int label_nulls = 0;
7429   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
7430   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
7431   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
7432   assert(label_nulls <= 1, "at most one NULL in the batch");
7433 
7434   int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
7435                    Klass::secondary_super_cache_offset_in_bytes());
7436   int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
7437                     Klass::super_check_offset_offset_in_bytes());
7438   Address super_check_offset_addr(super_klass, sco_offset);
7439 
7440   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
7441   // range of a jccb.  If this routine grows larger, reconsider at
7442   // least some of these.
7443 #define local_jcc(assembler_cond, label)                                \
7444   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
7445   else                             jcc( assembler_cond, label) /*omit semi*/
7446 
7447   // Hacked jmp, which may only be used just before L_fallthrough.
7448 #define final_jmp(label)                                                \
7449   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
7450   else                            jmp(label)                /*omit semi*/
7451 
7452   // If the pointers are equal, we are done (e.g., String[] elements).
7453   // This self-check enables sharing of secondary supertype arrays among
7454   // non-primary types such as array-of-interface.  Otherwise, each such
7455   // type would need its own customized SSA.
7456   // We move this check to the front of the fast path because many
7457   // type checks are in fact trivially successful in this manner,
7458   // so we get a nicely predicted branch right at the start of the check.
7459   cmpptr(sub_klass, super_klass);
7460   local_jcc(Assembler::equal, *L_success);
7461 
7462   // Check the supertype display:
7463   if (must_load_sco) {
7464     // Positive movl does right thing on LP64.
7465     movl(temp_reg, super_check_offset_addr);
7466     super_check_offset = RegisterOrConstant(temp_reg);
7467   }
7468   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
7469   cmpptr(super_klass, super_check_addr); // load displayed supertype
7470 
7471   // This check has worked decisively for primary supers.
7472   // Secondary supers are sought in the super_cache ('super_cache_addr').
7473   // (Secondary supers are interfaces and very deeply nested subtypes.)
7474   // This works in the same check above because of a tricky aliasing
7475   // between the super_cache and the primary super display elements.
7476   // (The 'super_check_addr' can address either, as the case requires.)
7477   // Note that the cache is updated below if it does not help us find
7478   // what we need immediately.
7479   // So if it was a primary super, we can just fail immediately.
7480   // Otherwise, it's the slow path for us (no success at this point).
7481 
7482   if (super_check_offset.is_register()) {
7483     local_jcc(Assembler::equal, *L_success);
7484     cmpl(super_check_offset.as_register(), sc_offset);
7485     if (L_failure == &L_fallthrough) {
7486       local_jcc(Assembler::equal, *L_slow_path);
7487     } else {
7488       local_jcc(Assembler::notEqual, *L_failure);
7489       final_jmp(*L_slow_path);
7490     }
7491   } else if (super_check_offset.as_constant() == sc_offset) {
7492     // Need a slow path; fast failure is impossible.
7493     if (L_slow_path == &L_fallthrough) {
7494       local_jcc(Assembler::equal, *L_success);
7495     } else {
7496       local_jcc(Assembler::notEqual, *L_slow_path);
7497       final_jmp(*L_success);
7498     }
7499   } else {
7500     // No slow path; it's a fast decision.
7501     if (L_failure == &L_fallthrough) {
7502       local_jcc(Assembler::equal, *L_success);
7503     } else {
7504       local_jcc(Assembler::notEqual, *L_failure);
7505       final_jmp(*L_success);
7506     }
7507   }
7508 
7509   bind(L_fallthrough);
7510 
7511 #undef local_jcc
7512 #undef final_jmp
7513 }
7514 
7515 
7516 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
7517                                                    Register super_klass,
7518                                                    Register temp_reg,
7519                                                    Register temp2_reg,
7520                                                    Label* L_success,
7521                                                    Label* L_failure,
7522                                                    bool set_cond_codes) {
7523   assert_different_registers(sub_klass, super_klass, temp_reg);
7524   if (temp2_reg != noreg)
7525     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
7526 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
7527 
7528   Label L_fallthrough;
7529   int label_nulls = 0;
7530   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
7531   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
7532   assert(label_nulls <= 1, "at most one NULL in the batch");
7533 
7534   // a couple of useful fields in sub_klass:
7535   int ss_offset = (klassOopDesc::header_size() * HeapWordSize +
7536                    Klass::secondary_supers_offset_in_bytes());
7537   int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
7538                    Klass::secondary_super_cache_offset_in_bytes());
7539   Address secondary_supers_addr(sub_klass, ss_offset);
7540   Address super_cache_addr(     sub_klass, sc_offset);
7541 
7542   // Do a linear scan of the secondary super-klass chain.
7543   // This code is rarely used, so simplicity is a virtue here.
7544   // The repne_scan instruction uses fixed registers, which we must spill.
7545   // Don't worry too much about pre-existing connections with the input regs.
7546 
7547   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
7548   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
7549 
7550   // Get super_klass value into rax (even if it was in rdi or rcx).
7551   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
7552   if (super_klass != rax || UseCompressedOops) {
7553     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
7554     mov(rax, super_klass);
7555   }
7556   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
7557   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
7558 
7559 #ifndef PRODUCT
7560   int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
7561   ExternalAddress pst_counter_addr((address) pst_counter);
7562   NOT_LP64(  incrementl(pst_counter_addr) );
7563   LP64_ONLY( lea(rcx, pst_counter_addr) );
7564   LP64_ONLY( incrementl(Address(rcx, 0)) );
7565 #endif //PRODUCT
7566 
7567   // We will consult the secondary-super array.
7568   movptr(rdi, secondary_supers_addr);
7569   // Load the array length.  (Positive movl does right thing on LP64.)
7570   movl(rcx, Address(rdi, arrayOopDesc::length_offset_in_bytes()));
7571   // Skip to start of data.
7572   addptr(rdi, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
7573 
7574   // Scan RCX words at [RDI] for an occurrence of RAX.
7575   // Set NZ/Z based on last compare.
7576 #ifdef _LP64
7577   // This part is tricky, as values in supers array could be 32 or 64 bit wide
7578   // and we store values in objArrays always encoded, thus we need to encode
7579   // the value of rax before repne.  Note that rax is dead after the repne.
7580   if (UseCompressedOops) {
7581     encode_heap_oop_not_null(rax);
7582     // The superclass is never null; it would be a basic system error if a null
7583     // pointer were to sneak in here.  Note that we have already loaded the
7584     // Klass::super_check_offset from the super_klass in the fast path,
7585     // so if there is a null in that register, we are already in the afterlife.
7586     repne_scanl();
7587   } else
7588 #endif // _LP64
7589     repne_scan();
7590 
7591   // Unspill the temp. registers:
7592   if (pushed_rdi)  pop(rdi);
7593   if (pushed_rcx)  pop(rcx);
7594   if (pushed_rax)  pop(rax);
7595 
7596   if (set_cond_codes) {
7597     // Special hack for the AD files:  rdi is guaranteed non-zero.
7598     assert(!pushed_rdi, "rdi must be left non-NULL");
7599     // Also, the condition codes are properly set Z/NZ on succeed/failure.
7600   }
7601 
7602   if (L_failure == &L_fallthrough)
7603         jccb(Assembler::notEqual, *L_failure);
7604   else  jcc(Assembler::notEqual, *L_failure);
7605 
7606   // Success.  Cache the super we found and proceed in triumph.
7607   movptr(super_cache_addr, super_klass);
7608 
7609   if (L_success != &L_fallthrough) {
7610     jmp(*L_success);
7611   }
7612 
7613 #undef IS_A_TEMP
7614 
7615   bind(L_fallthrough);
7616 }
7617 
7618 
7619 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
7620   ucomisd(dst, as_Address(src));
7621 }
7622 
7623 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
7624   ucomiss(dst, as_Address(src));
7625 }
7626 
7627 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
7628   if (reachable(src)) {
7629     xorpd(dst, as_Address(src));
7630   } else {
7631     lea(rscratch1, src);
7632     xorpd(dst, Address(rscratch1, 0));
7633   }
7634 }
7635 
7636 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
7637   if (reachable(src)) {
7638     xorps(dst, as_Address(src));
7639   } else {
7640     lea(rscratch1, src);
7641     xorps(dst, Address(rscratch1, 0));
7642   }
7643 }
7644 
7645 void MacroAssembler::verify_oop(Register reg, const char* s) {
7646   if (!VerifyOops) return;
7647 
7648   // Pass register number to verify_oop_subroutine
7649   char* b = new char[strlen(s) + 50];
7650   sprintf(b, "verify_oop: %s: %s", reg->name(), s);
7651   push(rax);                          // save rax,
7652   push(reg);                          // pass register argument
7653   ExternalAddress buffer((address) b);
7654   // avoid using pushptr, as it modifies scratch registers
7655   // and our contract is not to modify anything
7656   movptr(rax, buffer.addr());
7657   push(rax);
7658   // call indirectly to solve generation ordering problem
7659   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
7660   call(rax);
7661 }
7662 
7663 
7664 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
7665                                                       Register tmp,
7666                                                       int offset) {
7667   intptr_t value = *delayed_value_addr;
7668   if (value != 0)
7669     return RegisterOrConstant(value + offset);
7670 
7671   // load indirectly to solve generation ordering problem
7672   movptr(tmp, ExternalAddress((address) delayed_value_addr));
7673 
7674 #ifdef ASSERT
7675   Label L;
7676   testptr(tmp, tmp);
7677   jccb(Assembler::notZero, L);
7678   hlt();
7679   bind(L);
7680 #endif
7681 
7682   if (offset != 0)
7683     addptr(tmp, offset);
7684 
7685   return RegisterOrConstant(tmp);
7686 }
7687 
7688 
7689 // registers on entry:
7690 //  - rax ('check' register): required MethodType
7691 //  - rcx: method handle
7692 //  - rdx, rsi, or ?: killable temp
7693 void MacroAssembler::check_method_handle_type(Register mtype_reg, Register mh_reg,
7694                                               Register temp_reg,
7695                                               Label& wrong_method_type) {
7696   if (UseCompressedOops)  unimplemented();  // field accesses must decode
7697   // compare method type against that of the receiver
7698   cmpptr(mtype_reg, Address(mh_reg, delayed_value(java_dyn_MethodHandle::type_offset_in_bytes, temp_reg)));
7699   jcc(Assembler::notEqual, wrong_method_type);
7700 }
7701 
7702 
7703 // A method handle has a "vmslots" field which gives the size of its
7704 // argument list in JVM stack slots.  This field is either located directly
7705 // in every method handle, or else is indirectly accessed through the
7706 // method handle's MethodType.  This macro hides the distinction.
7707 void MacroAssembler::load_method_handle_vmslots(Register vmslots_reg, Register mh_reg,
7708                                                 Register temp_reg) {
7709   if (UseCompressedOops)  unimplemented();  // field accesses must decode
7710   // load mh.type.form.vmslots
7711   if (java_dyn_MethodHandle::vmslots_offset_in_bytes() != 0) {
7712     // hoist vmslots into every mh to avoid dependent load chain
7713     movl(vmslots_reg, Address(mh_reg, delayed_value(java_dyn_MethodHandle::vmslots_offset_in_bytes, temp_reg)));
7714   } else {
7715     Register temp2_reg = vmslots_reg;
7716     movptr(temp2_reg, Address(mh_reg,    delayed_value(java_dyn_MethodHandle::type_offset_in_bytes, temp_reg)));
7717     movptr(temp2_reg, Address(temp2_reg, delayed_value(java_dyn_MethodType::form_offset_in_bytes, temp_reg)));
7718     movl(vmslots_reg, Address(temp2_reg, delayed_value(java_dyn_MethodTypeForm::vmslots_offset_in_bytes, temp_reg)));
7719   }
7720 }
7721 
7722 
7723 // registers on entry:
7724 //  - rcx: method handle
7725 //  - rdx: killable temp (interpreted only)
7726 //  - rax: killable temp (compiled only)
7727 void MacroAssembler::jump_to_method_handle_entry(Register mh_reg, Register temp_reg) {
7728   assert(mh_reg == rcx, "caller must put MH object in rcx");
7729   assert_different_registers(mh_reg, temp_reg);
7730 
7731   if (UseCompressedOops)  unimplemented();  // field accesses must decode
7732 
7733   // pick out the interpreted side of the handler
7734   movptr(temp_reg, Address(mh_reg, delayed_value(java_dyn_MethodHandle::vmentry_offset_in_bytes, temp_reg)));
7735 
7736   // off we go...
7737   jmp(Address(temp_reg, MethodHandleEntry::from_interpreted_entry_offset_in_bytes()));
7738 
7739   // for the various stubs which take control at this point,
7740   // see MethodHandles::generate_method_handle_stub
7741 }
7742 
7743 
7744 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
7745                                          int extra_slot_offset) {
7746   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
7747   int stackElementSize = Interpreter::stackElementSize();
7748   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
7749 #ifdef ASSERT
7750   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
7751   assert(offset1 - offset == stackElementSize, "correct arithmetic");
7752 #endif
7753   Register             scale_reg    = noreg;
7754   Address::ScaleFactor scale_factor = Address::no_scale;
7755   if (arg_slot.is_constant()) {
7756     offset += arg_slot.as_constant() * stackElementSize;
7757   } else {
7758     scale_reg    = arg_slot.as_register();
7759     scale_factor = Address::times(stackElementSize);
7760   }
7761   offset += wordSize;           // return PC is on stack
7762   return Address(rsp, scale_reg, scale_factor, offset);
7763 }
7764 
7765 
7766 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
7767   if (!VerifyOops) return;
7768 
7769   // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
7770   // Pass register number to verify_oop_subroutine
7771   char* b = new char[strlen(s) + 50];
7772   sprintf(b, "verify_oop_addr: %s", s);
7773 
7774   push(rax);                          // save rax,
7775   // addr may contain rsp so we will have to adjust it based on the push
7776   // we just did
7777   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
7778   // stores rax into addr which is backwards of what was intended.
7779   if (addr.uses(rsp)) {
7780     lea(rax, addr);
7781     pushptr(Address(rax, BytesPerWord));
7782   } else {
7783     pushptr(addr);
7784   }
7785 
7786   ExternalAddress buffer((address) b);
7787   // pass msg argument
7788   // avoid using pushptr, as it modifies scratch registers
7789   // and our contract is not to modify anything
7790   movptr(rax, buffer.addr());
7791   push(rax);
7792 
7793   // call indirectly to solve generation ordering problem
7794   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
7795   call(rax);
7796   // Caller pops the arguments and restores rax, from the stack
7797 }
7798 
7799 void MacroAssembler::verify_tlab() {
7800 #ifdef ASSERT
7801   if (UseTLAB && VerifyOops) {
7802     Label next, ok;
7803     Register t1 = rsi;
7804     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
7805 
7806     push(t1);
7807     NOT_LP64(push(thread_reg));
7808     NOT_LP64(get_thread(thread_reg));
7809 
7810     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
7811     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
7812     jcc(Assembler::aboveEqual, next);
7813     stop("assert(top >= start)");
7814     should_not_reach_here();
7815 
7816     bind(next);
7817     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
7818     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
7819     jcc(Assembler::aboveEqual, ok);
7820     stop("assert(top <= end)");
7821     should_not_reach_here();
7822 
7823     bind(ok);
7824     NOT_LP64(pop(thread_reg));
7825     pop(t1);
7826   }
7827 #endif
7828 }
7829 
7830 class ControlWord {
7831  public:
7832   int32_t _value;
7833 
7834   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
7835   int  precision_control() const       { return  (_value >>  8) & 3      ; }
7836   bool precision() const               { return ((_value >>  5) & 1) != 0; }
7837   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
7838   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
7839   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
7840   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
7841   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
7842 
7843   void print() const {
7844     // rounding control
7845     const char* rc;
7846     switch (rounding_control()) {
7847       case 0: rc = "round near"; break;
7848       case 1: rc = "round down"; break;
7849       case 2: rc = "round up  "; break;
7850       case 3: rc = "chop      "; break;
7851     };
7852     // precision control
7853     const char* pc;
7854     switch (precision_control()) {
7855       case 0: pc = "24 bits "; break;
7856       case 1: pc = "reserved"; break;
7857       case 2: pc = "53 bits "; break;
7858       case 3: pc = "64 bits "; break;
7859     };
7860     // flags
7861     char f[9];
7862     f[0] = ' ';
7863     f[1] = ' ';
7864     f[2] = (precision   ()) ? 'P' : 'p';
7865     f[3] = (underflow   ()) ? 'U' : 'u';
7866     f[4] = (overflow    ()) ? 'O' : 'o';
7867     f[5] = (zero_divide ()) ? 'Z' : 'z';
7868     f[6] = (denormalized()) ? 'D' : 'd';
7869     f[7] = (invalid     ()) ? 'I' : 'i';
7870     f[8] = '\x0';
7871     // output
7872     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
7873   }
7874 
7875 };
7876 
7877 class StatusWord {
7878  public:
7879   int32_t _value;
7880 
7881   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
7882   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
7883   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
7884   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
7885   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
7886   int  top() const                     { return  (_value >> 11) & 7      ; }
7887   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
7888   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
7889   bool precision() const               { return ((_value >>  5) & 1) != 0; }
7890   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
7891   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
7892   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
7893   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
7894   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
7895 
7896   void print() const {
7897     // condition codes
7898     char c[5];
7899     c[0] = (C3()) ? '3' : '-';
7900     c[1] = (C2()) ? '2' : '-';
7901     c[2] = (C1()) ? '1' : '-';
7902     c[3] = (C0()) ? '0' : '-';
7903     c[4] = '\x0';
7904     // flags
7905     char f[9];
7906     f[0] = (error_status()) ? 'E' : '-';
7907     f[1] = (stack_fault ()) ? 'S' : '-';
7908     f[2] = (precision   ()) ? 'P' : '-';
7909     f[3] = (underflow   ()) ? 'U' : '-';
7910     f[4] = (overflow    ()) ? 'O' : '-';
7911     f[5] = (zero_divide ()) ? 'Z' : '-';
7912     f[6] = (denormalized()) ? 'D' : '-';
7913     f[7] = (invalid     ()) ? 'I' : '-';
7914     f[8] = '\x0';
7915     // output
7916     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
7917   }
7918 
7919 };
7920 
7921 class TagWord {
7922  public:
7923   int32_t _value;
7924 
7925   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
7926 
7927   void print() const {
7928     printf("%04x", _value & 0xFFFF);
7929   }
7930 
7931 };
7932 
7933 class FPU_Register {
7934  public:
7935   int32_t _m0;
7936   int32_t _m1;
7937   int16_t _ex;
7938 
7939   bool is_indefinite() const           {
7940     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
7941   }
7942 
7943   void print() const {
7944     char  sign = (_ex < 0) ? '-' : '+';
7945     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
7946     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
7947   };
7948 
7949 };
7950 
7951 class FPU_State {
7952  public:
7953   enum {
7954     register_size       = 10,
7955     number_of_registers =  8,
7956     register_mask       =  7
7957   };
7958 
7959   ControlWord  _control_word;
7960   StatusWord   _status_word;
7961   TagWord      _tag_word;
7962   int32_t      _error_offset;
7963   int32_t      _error_selector;
7964   int32_t      _data_offset;
7965   int32_t      _data_selector;
7966   int8_t       _register[register_size * number_of_registers];
7967 
7968   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
7969   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
7970 
7971   const char* tag_as_string(int tag) const {
7972     switch (tag) {
7973       case 0: return "valid";
7974       case 1: return "zero";
7975       case 2: return "special";
7976       case 3: return "empty";
7977     }
7978     ShouldNotReachHere()
7979     return NULL;
7980   }
7981 
7982   void print() const {
7983     // print computation registers
7984     { int t = _status_word.top();
7985       for (int i = 0; i < number_of_registers; i++) {
7986         int j = (i - t) & register_mask;
7987         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
7988         st(j)->print();
7989         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
7990       }
7991     }
7992     printf("\n");
7993     // print control registers
7994     printf("ctrl = "); _control_word.print(); printf("\n");
7995     printf("stat = "); _status_word .print(); printf("\n");
7996     printf("tags = "); _tag_word    .print(); printf("\n");
7997   }
7998 
7999 };
8000 
8001 class Flag_Register {
8002  public:
8003   int32_t _value;
8004 
8005   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
8006   bool direction() const               { return ((_value >> 10) & 1) != 0; }
8007   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
8008   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
8009   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
8010   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
8011   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
8012 
8013   void print() const {
8014     // flags
8015     char f[8];
8016     f[0] = (overflow       ()) ? 'O' : '-';
8017     f[1] = (direction      ()) ? 'D' : '-';
8018     f[2] = (sign           ()) ? 'S' : '-';
8019     f[3] = (zero           ()) ? 'Z' : '-';
8020     f[4] = (auxiliary_carry()) ? 'A' : '-';
8021     f[5] = (parity         ()) ? 'P' : '-';
8022     f[6] = (carry          ()) ? 'C' : '-';
8023     f[7] = '\x0';
8024     // output
8025     printf("%08x  flags = %s", _value, f);
8026   }
8027 
8028 };
8029 
8030 class IU_Register {
8031  public:
8032   int32_t _value;
8033 
8034   void print() const {
8035     printf("%08x  %11d", _value, _value);
8036   }
8037 
8038 };
8039 
8040 class IU_State {
8041  public:
8042   Flag_Register _eflags;
8043   IU_Register   _rdi;
8044   IU_Register   _rsi;
8045   IU_Register   _rbp;
8046   IU_Register   _rsp;
8047   IU_Register   _rbx;
8048   IU_Register   _rdx;
8049   IU_Register   _rcx;
8050   IU_Register   _rax;
8051 
8052   void print() const {
8053     // computation registers
8054     printf("rax,  = "); _rax.print(); printf("\n");
8055     printf("rbx,  = "); _rbx.print(); printf("\n");
8056     printf("rcx  = "); _rcx.print(); printf("\n");
8057     printf("rdx  = "); _rdx.print(); printf("\n");
8058     printf("rdi  = "); _rdi.print(); printf("\n");
8059     printf("rsi  = "); _rsi.print(); printf("\n");
8060     printf("rbp,  = "); _rbp.print(); printf("\n");
8061     printf("rsp  = "); _rsp.print(); printf("\n");
8062     printf("\n");
8063     // control registers
8064     printf("flgs = "); _eflags.print(); printf("\n");
8065   }
8066 };
8067 
8068 
8069 class CPU_State {
8070  public:
8071   FPU_State _fpu_state;
8072   IU_State  _iu_state;
8073 
8074   void print() const {
8075     printf("--------------------------------------------------\n");
8076     _iu_state .print();
8077     printf("\n");
8078     _fpu_state.print();
8079     printf("--------------------------------------------------\n");
8080   }
8081 
8082 };
8083 
8084 
8085 static void _print_CPU_state(CPU_State* state) {
8086   state->print();
8087 };
8088 
8089 
8090 void MacroAssembler::print_CPU_state() {
8091   push_CPU_state();
8092   push(rsp);                // pass CPU state
8093   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
8094   addptr(rsp, wordSize);       // discard argument
8095   pop_CPU_state();
8096 }
8097 
8098 
8099 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
8100   static int counter = 0;
8101   FPU_State* fs = &state->_fpu_state;
8102   counter++;
8103   // For leaf calls, only verify that the top few elements remain empty.
8104   // We only need 1 empty at the top for C2 code.
8105   if( stack_depth < 0 ) {
8106     if( fs->tag_for_st(7) != 3 ) {
8107       printf("FPR7 not empty\n");
8108       state->print();
8109       assert(false, "error");
8110       return false;
8111     }
8112     return true;                // All other stack states do not matter
8113   }
8114 
8115   assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std,
8116          "bad FPU control word");
8117 
8118   // compute stack depth
8119   int i = 0;
8120   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
8121   int d = i;
8122   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
8123   // verify findings
8124   if (i != FPU_State::number_of_registers) {
8125     // stack not contiguous
8126     printf("%s: stack not contiguous at ST%d\n", s, i);
8127     state->print();
8128     assert(false, "error");
8129     return false;
8130   }
8131   // check if computed stack depth corresponds to expected stack depth
8132   if (stack_depth < 0) {
8133     // expected stack depth is -stack_depth or less
8134     if (d > -stack_depth) {
8135       // too many elements on the stack
8136       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
8137       state->print();
8138       assert(false, "error");
8139       return false;
8140     }
8141   } else {
8142     // expected stack depth is stack_depth
8143     if (d != stack_depth) {
8144       // wrong stack depth
8145       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
8146       state->print();
8147       assert(false, "error");
8148       return false;
8149     }
8150   }
8151   // everything is cool
8152   return true;
8153 }
8154 
8155 
8156 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
8157   if (!VerifyFPU) return;
8158   push_CPU_state();
8159   push(rsp);                // pass CPU state
8160   ExternalAddress msg((address) s);
8161   // pass message string s
8162   pushptr(msg.addr());
8163   push(stack_depth);        // pass stack depth
8164   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
8165   addptr(rsp, 3 * wordSize);   // discard arguments
8166   // check for error
8167   { Label L;
8168     testl(rax, rax);
8169     jcc(Assembler::notZero, L);
8170     int3();                  // break if error condition
8171     bind(L);
8172   }
8173   pop_CPU_state();
8174 }
8175 
8176 void MacroAssembler::load_klass(Register dst, Register src) {
8177 #ifdef _LP64
8178   if (UseCompressedOops) {
8179     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
8180     decode_heap_oop_not_null(dst);
8181   } else
8182 #endif
8183     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
8184 }
8185 
8186 void MacroAssembler::load_prototype_header(Register dst, Register src) {
8187 #ifdef _LP64
8188   if (UseCompressedOops) {
8189     assert (Universe::heap() != NULL, "java heap should be initialized");
8190     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
8191     if (Universe::narrow_oop_shift() != 0) {
8192       assert(Address::times_8 == LogMinObjAlignmentInBytes &&
8193              Address::times_8 == Universe::narrow_oop_shift(), "decode alg wrong");
8194       movq(dst, Address(r12_heapbase, dst, Address::times_8, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
8195     } else {
8196       movq(dst, Address(dst, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
8197     }
8198   } else
8199 #endif
8200   {
8201     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
8202     movptr(dst, Address(dst, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
8203   }
8204 }
8205 
8206 void MacroAssembler::store_klass(Register dst, Register src) {
8207 #ifdef _LP64
8208   if (UseCompressedOops) {
8209     encode_heap_oop_not_null(src);
8210     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
8211   } else
8212 #endif
8213     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
8214 }
8215 
8216 #ifdef _LP64
8217 void MacroAssembler::store_klass_gap(Register dst, Register src) {
8218   if (UseCompressedOops) {
8219     // Store to klass gap in destination
8220     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
8221   }
8222 }
8223 
8224 void MacroAssembler::load_heap_oop(Register dst, Address src) {
8225   if (UseCompressedOops) {
8226     movl(dst, src);
8227     decode_heap_oop(dst);
8228   } else {
8229     movq(dst, src);
8230   }
8231 }
8232 
8233 void MacroAssembler::store_heap_oop(Address dst, Register src) {
8234   if (UseCompressedOops) {
8235     assert(!dst.uses(src), "not enough registers");
8236     encode_heap_oop(src);
8237     movl(dst, src);
8238   } else {
8239     movq(dst, src);
8240   }
8241 }
8242 
8243 // Used for storing NULLs.
8244 void MacroAssembler::store_heap_oop_null(Address dst) {
8245   if (UseCompressedOops) {
8246     movl(dst, (int32_t)NULL_WORD);
8247   } else {
8248     movslq(dst, (int32_t)NULL_WORD);
8249   }
8250 }
8251 
8252 // Algorithm must match oop.inline.hpp encode_heap_oop.
8253 void MacroAssembler::encode_heap_oop(Register r) {
8254   assert (UseCompressedOops, "should be compressed");
8255   assert (Universe::heap() != NULL, "java heap should be initialized");
8256   if (Universe::narrow_oop_base() == NULL) {
8257     verify_oop(r, "broken oop in encode_heap_oop");
8258     if (Universe::narrow_oop_shift() != 0) {
8259       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
8260       shrq(r, LogMinObjAlignmentInBytes);
8261     }
8262     return;
8263   }
8264 #ifdef ASSERT
8265   if (CheckCompressedOops) {
8266     Label ok;
8267     push(rscratch1); // cmpptr trashes rscratch1
8268     cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_oop_base_addr()));
8269     jcc(Assembler::equal, ok);
8270     stop("MacroAssembler::encode_heap_oop: heap base corrupted?");
8271     bind(ok);
8272     pop(rscratch1);
8273   }
8274 #endif
8275   verify_oop(r, "broken oop in encode_heap_oop");
8276   testq(r, r);
8277   cmovq(Assembler::equal, r, r12_heapbase);
8278   subq(r, r12_heapbase);
8279   shrq(r, LogMinObjAlignmentInBytes);
8280 }
8281 
8282 void MacroAssembler::encode_heap_oop_not_null(Register r) {
8283   assert (UseCompressedOops, "should be compressed");
8284   assert (Universe::heap() != NULL, "java heap should be initialized");
8285 #ifdef ASSERT
8286   if (CheckCompressedOops) {
8287     Label ok;
8288     testq(r, r);
8289     jcc(Assembler::notEqual, ok);
8290     stop("null oop passed to encode_heap_oop_not_null");
8291     bind(ok);
8292   }
8293 #endif
8294   verify_oop(r, "broken oop in encode_heap_oop_not_null");
8295   if (Universe::narrow_oop_base() != NULL) {
8296     subq(r, r12_heapbase);
8297   }
8298   if (Universe::narrow_oop_shift() != 0) {
8299     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
8300     shrq(r, LogMinObjAlignmentInBytes);
8301   }
8302 }
8303 
8304 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
8305   assert (UseCompressedOops, "should be compressed");
8306   assert (Universe::heap() != NULL, "java heap should be initialized");
8307 #ifdef ASSERT
8308   if (CheckCompressedOops) {
8309     Label ok;
8310     testq(src, src);
8311     jcc(Assembler::notEqual, ok);
8312     stop("null oop passed to encode_heap_oop_not_null2");
8313     bind(ok);
8314   }
8315 #endif
8316   verify_oop(src, "broken oop in encode_heap_oop_not_null2");
8317   if (dst != src) {
8318     movq(dst, src);
8319   }
8320   if (Universe::narrow_oop_base() != NULL) {
8321     subq(dst, r12_heapbase);
8322   }
8323   if (Universe::narrow_oop_shift() != 0) {
8324     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
8325     shrq(dst, LogMinObjAlignmentInBytes);
8326   }
8327 }
8328 
8329 void  MacroAssembler::decode_heap_oop(Register r) {
8330   assert (UseCompressedOops, "should be compressed");
8331   assert (Universe::heap() != NULL, "java heap should be initialized");
8332   if (Universe::narrow_oop_base() == NULL) {
8333     if (Universe::narrow_oop_shift() != 0) {
8334       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
8335       shlq(r, LogMinObjAlignmentInBytes);
8336     }
8337     verify_oop(r, "broken oop in decode_heap_oop");
8338     return;
8339   }
8340 #ifdef ASSERT
8341   if (CheckCompressedOops) {
8342     Label ok;
8343     push(rscratch1);
8344     cmpptr(r12_heapbase,
8345            ExternalAddress((address)Universe::narrow_oop_base_addr()));
8346     jcc(Assembler::equal, ok);
8347     stop("MacroAssembler::decode_heap_oop: heap base corrupted?");
8348     bind(ok);
8349     pop(rscratch1);
8350   }
8351 #endif
8352 
8353   Label done;
8354   shlq(r, LogMinObjAlignmentInBytes);
8355   jccb(Assembler::equal, done);
8356   addq(r, r12_heapbase);
8357 #if 0
8358    // alternate decoding probably a wash.
8359    testq(r, r);
8360    jccb(Assembler::equal, done);
8361    leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
8362 #endif
8363   bind(done);
8364   verify_oop(r, "broken oop in decode_heap_oop");
8365 }
8366 
8367 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
8368   assert (UseCompressedOops, "should only be used for compressed headers");
8369   assert (Universe::heap() != NULL, "java heap should be initialized");
8370   // Cannot assert, unverified entry point counts instructions (see .ad file)
8371   // vtableStubs also counts instructions in pd_code_size_limit.
8372   // Also do not verify_oop as this is called by verify_oop.
8373   if (Universe::narrow_oop_shift() != 0) {
8374     assert (Address::times_8 == LogMinObjAlignmentInBytes &&
8375             Address::times_8 == Universe::narrow_oop_shift(), "decode alg wrong");
8376     // Don't use Shift since it modifies flags.
8377     leaq(r, Address(r12_heapbase, r, Address::times_8, 0));
8378   } else {
8379     assert (Universe::narrow_oop_base() == NULL, "sanity");
8380   }
8381 }
8382 
8383 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
8384   assert (UseCompressedOops, "should only be used for compressed headers");
8385   assert (Universe::heap() != NULL, "java heap should be initialized");
8386   // Cannot assert, unverified entry point counts instructions (see .ad file)
8387   // vtableStubs also counts instructions in pd_code_size_limit.
8388   // Also do not verify_oop as this is called by verify_oop.
8389   if (Universe::narrow_oop_shift() != 0) {
8390     assert (Address::times_8 == LogMinObjAlignmentInBytes &&
8391             Address::times_8 == Universe::narrow_oop_shift(), "decode alg wrong");
8392     leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
8393   } else if (dst != src) {
8394     assert (Universe::narrow_oop_base() == NULL, "sanity");
8395     movq(dst, src);
8396   }
8397 }
8398 
8399 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
8400   assert (UseCompressedOops, "should only be used for compressed headers");
8401   assert (Universe::heap() != NULL, "java heap should be initialized");
8402   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
8403   int oop_index = oop_recorder()->find_index(obj);
8404   RelocationHolder rspec = oop_Relocation::spec(oop_index);
8405   mov_narrow_oop(dst, oop_index, rspec);
8406 }
8407 
8408 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
8409   assert (UseCompressedOops, "should only be used for compressed headers");
8410   assert (Universe::heap() != NULL, "java heap should be initialized");
8411   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
8412   int oop_index = oop_recorder()->find_index(obj);
8413   RelocationHolder rspec = oop_Relocation::spec(oop_index);
8414   mov_narrow_oop(dst, oop_index, rspec);
8415 }
8416 
8417 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
8418   assert (UseCompressedOops, "should only be used for compressed headers");
8419   assert (Universe::heap() != NULL, "java heap should be initialized");
8420   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
8421   int oop_index = oop_recorder()->find_index(obj);
8422   RelocationHolder rspec = oop_Relocation::spec(oop_index);
8423   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
8424 }
8425 
8426 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
8427   assert (UseCompressedOops, "should only be used for compressed headers");
8428   assert (Universe::heap() != NULL, "java heap should be initialized");
8429   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
8430   int oop_index = oop_recorder()->find_index(obj);
8431   RelocationHolder rspec = oop_Relocation::spec(oop_index);
8432   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
8433 }
8434 
8435 void MacroAssembler::reinit_heapbase() {
8436   if (UseCompressedOops) {
8437     movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_oop_base_addr()));
8438   }
8439 }
8440 #endif // _LP64
8441 
8442 // IndexOf substring.
8443 void MacroAssembler::string_indexof(Register str1, Register str2,
8444                                     Register cnt1, Register cnt2, Register result,
8445                                     XMMRegister vec, Register tmp) {
8446   assert(UseSSE42Intrinsics, "SSE4.2 is required");
8447 
8448   Label RELOAD_SUBSTR, PREP_FOR_SCAN, SCAN_TO_SUBSTR,
8449         SCAN_SUBSTR, RET_NOT_FOUND, CLEANUP;
8450 
8451   push(str1); // string addr
8452   push(str2); // substr addr
8453   push(cnt2); // substr count
8454   jmpb(PREP_FOR_SCAN);
8455 
8456   // Substr count saved at sp
8457   // Substr saved at sp+1*wordSize
8458   // String saved at sp+2*wordSize
8459 
8460   // Reload substr for rescan
8461   bind(RELOAD_SUBSTR);
8462   movl(cnt2, Address(rsp, 0));
8463   movptr(str2, Address(rsp, wordSize));
8464   // We came here after the beginninig of the substring was
8465   // matched but the rest of it was not so we need to search
8466   // again. Start from the next element after the previous match.
8467   subptr(str1, result); // Restore counter
8468   shrl(str1, 1);
8469   addl(cnt1, str1);
8470   decrementl(cnt1);
8471   lea(str1, Address(result, 2)); // Reload string
8472 
8473   // Load substr
8474   bind(PREP_FOR_SCAN);
8475   movdqu(vec, Address(str2, 0));
8476   addl(cnt1, 8);  // prime the loop
8477   subptr(str1, 16);
8478 
8479   // Scan string for substr in 16-byte vectors
8480   bind(SCAN_TO_SUBSTR);
8481   subl(cnt1, 8);
8482   addptr(str1, 16);
8483 
8484   // pcmpestri
8485   //   inputs:
8486   //     xmm - substring
8487   //     rax - substring length (elements count)
8488   //     mem - scaned string
8489   //     rdx - string length (elements count)
8490   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
8491   //   outputs:
8492   //     rcx - matched index in string
8493   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
8494 
8495   pcmpestri(vec, Address(str1, 0), 0x0d);
8496   jcc(Assembler::above, SCAN_TO_SUBSTR);      // CF == 0 && ZF == 0
8497   jccb(Assembler::aboveEqual, RET_NOT_FOUND); // CF == 0
8498 
8499   // Fallthrough: found a potential substr
8500 
8501   // Make sure string is still long enough
8502   subl(cnt1, tmp);
8503   cmpl(cnt1, cnt2);
8504   jccb(Assembler::negative, RET_NOT_FOUND);
8505   // Compute start addr of substr
8506   lea(str1, Address(str1, tmp, Address::times_2));
8507   movptr(result, str1); // save
8508 
8509   // Compare potential substr
8510   addl(cnt1, 8);     // prime the loop
8511   addl(cnt2, 8);
8512   subptr(str1, 16);
8513   subptr(str2, 16);
8514 
8515   // Scan 16-byte vectors of string and substr
8516   bind(SCAN_SUBSTR);
8517   subl(cnt1, 8);
8518   subl(cnt2, 8);
8519   addptr(str1, 16);
8520   addptr(str2, 16);
8521   movdqu(vec, Address(str2, 0));
8522   pcmpestri(vec, Address(str1, 0), 0x0d);
8523   jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
8524   jcc(Assembler::positive, SCAN_SUBSTR);     // SF == 0
8525 
8526   // Compute substr offset
8527   subptr(result, Address(rsp, 2*wordSize));
8528   shrl(result, 1); // index
8529   jmpb(CLEANUP);
8530 
8531   bind(RET_NOT_FOUND);
8532   movl(result, -1);
8533 
8534   bind(CLEANUP);
8535   addptr(rsp, 3*wordSize);
8536 }
8537 
8538 // Compare strings.
8539 void MacroAssembler::string_compare(Register str1, Register str2,
8540                                     Register cnt1, Register cnt2, Register result,
8541                                     XMMRegister vec1, XMMRegister vec2) {
8542   Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
8543 
8544   // Compute the minimum of the string lengths and the
8545   // difference of the string lengths (stack).
8546   // Do the conditional move stuff
8547   movl(result, cnt1);
8548   subl(cnt1, cnt2);
8549   push(cnt1);
8550   if (VM_Version::supports_cmov()) {
8551     cmovl(Assembler::lessEqual, cnt2, result);
8552   } else {
8553     Label GT_LABEL;
8554     jccb(Assembler::greater, GT_LABEL);
8555     movl(cnt2, result);
8556     bind(GT_LABEL);
8557   }
8558 
8559   // Is the minimum length zero?
8560   testl(cnt2, cnt2);
8561   jcc(Assembler::zero, LENGTH_DIFF_LABEL);
8562 
8563   // Load first characters
8564   load_unsigned_short(result, Address(str1, 0));
8565   load_unsigned_short(cnt1, Address(str2, 0));
8566 
8567   // Compare first characters
8568   subl(result, cnt1);
8569   jcc(Assembler::notZero,  POP_LABEL);
8570   decrementl(cnt2);
8571   jcc(Assembler::zero, LENGTH_DIFF_LABEL);
8572 
8573   {
8574     // Check after comparing first character to see if strings are equivalent
8575     Label LSkip2;
8576     // Check if the strings start at same location
8577     cmpptr(str1, str2);
8578     jccb(Assembler::notEqual, LSkip2);
8579 
8580     // Check if the length difference is zero (from stack)
8581     cmpl(Address(rsp, 0), 0x0);
8582     jcc(Assembler::equal,  LENGTH_DIFF_LABEL);
8583 
8584     // Strings might not be equivalent
8585     bind(LSkip2);
8586   }
8587 
8588   // Advance to next character
8589   addptr(str1, 2);
8590   addptr(str2, 2);
8591 
8592   if (UseSSE42Intrinsics) {
8593     // With SSE4.2, use double quad vector compare
8594     Label COMPARE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
8595     // Setup to compare 16-byte vectors
8596     movl(cnt1, cnt2);
8597     andl(cnt2, 0xfffffff8); // cnt2 holds the vector count
8598     andl(cnt1, 0x00000007); // cnt1 holds the tail count
8599     testl(cnt2, cnt2);
8600     jccb(Assembler::zero, COMPARE_TAIL);
8601 
8602     lea(str2, Address(str2, cnt2, Address::times_2));
8603     lea(str1, Address(str1, cnt2, Address::times_2));
8604     negptr(cnt2);
8605 
8606     bind(COMPARE_VECTORS);
8607     movdqu(vec1, Address(str1, cnt2, Address::times_2));
8608     movdqu(vec2, Address(str2, cnt2, Address::times_2));
8609     pxor(vec1, vec2);
8610     ptest(vec1, vec1);
8611     jccb(Assembler::notZero, VECTOR_NOT_EQUAL);
8612     addptr(cnt2, 8);
8613     jcc(Assembler::notZero, COMPARE_VECTORS);
8614     jmpb(COMPARE_TAIL);
8615 
8616     // Mismatched characters in the vectors
8617     bind(VECTOR_NOT_EQUAL);
8618     lea(str1, Address(str1, cnt2, Address::times_2));
8619     lea(str2, Address(str2, cnt2, Address::times_2));
8620     movl(cnt1, 8);
8621 
8622     // Compare tail (< 8 chars), or rescan last vectors to
8623     // find 1st mismatched characters
8624     bind(COMPARE_TAIL);
8625     testl(cnt1, cnt1);
8626     jccb(Assembler::zero, LENGTH_DIFF_LABEL);
8627     movl(cnt2, cnt1);
8628     // Fallthru to tail compare
8629   }
8630 
8631   // Shift str2 and str1 to the end of the arrays, negate min
8632   lea(str1, Address(str1, cnt2, Address::times_2, 0));
8633   lea(str2, Address(str2, cnt2, Address::times_2, 0));
8634   negptr(cnt2);
8635 
8636     // Compare the rest of the characters
8637   bind(WHILE_HEAD_LABEL);
8638   load_unsigned_short(result, Address(str1, cnt2, Address::times_2, 0));
8639   load_unsigned_short(cnt1, Address(str2, cnt2, Address::times_2, 0));
8640   subl(result, cnt1);
8641   jccb(Assembler::notZero, POP_LABEL);
8642   increment(cnt2);
8643   jcc(Assembler::notZero, WHILE_HEAD_LABEL);
8644 
8645   // Strings are equal up to min length.  Return the length difference.
8646   bind(LENGTH_DIFF_LABEL);
8647   pop(result);
8648   jmpb(DONE_LABEL);
8649 
8650   // Discard the stored length difference
8651   bind(POP_LABEL);
8652   addptr(rsp, wordSize);
8653 
8654   // That's it
8655   bind(DONE_LABEL);
8656 }
8657 
8658 // Compare char[] arrays aligned to 4 bytes or substrings.
8659 void MacroAssembler::char_arrays_equals(bool is_array_equ, Register ary1, Register ary2,
8660                                         Register limit, Register result, Register chr,
8661                                         XMMRegister vec1, XMMRegister vec2) {
8662   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR;
8663 
8664   int length_offset  = arrayOopDesc::length_offset_in_bytes();
8665   int base_offset    = arrayOopDesc::base_offset_in_bytes(T_CHAR);
8666 
8667   // Check the input args
8668   cmpptr(ary1, ary2);
8669   jcc(Assembler::equal, TRUE_LABEL);
8670 
8671   if (is_array_equ) {
8672     // Need additional checks for arrays_equals.
8673     testptr(ary1, ary1);
8674     jcc(Assembler::zero, FALSE_LABEL);
8675     testptr(ary2, ary2);
8676     jcc(Assembler::zero, FALSE_LABEL);
8677 
8678     // Check the lengths
8679     movl(limit, Address(ary1, length_offset));
8680     cmpl(limit, Address(ary2, length_offset));
8681     jcc(Assembler::notEqual, FALSE_LABEL);
8682   }
8683 
8684   // count == 0
8685   testl(limit, limit);
8686   jcc(Assembler::zero, TRUE_LABEL);
8687 
8688   if (is_array_equ) {
8689     // Load array address
8690     lea(ary1, Address(ary1, base_offset));
8691     lea(ary2, Address(ary2, base_offset));
8692   }
8693 
8694   shll(limit, 1);      // byte count != 0
8695   movl(result, limit); // copy
8696 
8697   if (UseSSE42Intrinsics) {
8698     // With SSE4.2, use double quad vector compare
8699     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8700     // Compare 16-byte vectors
8701     andl(result, 0x0000000e);  //   tail count (in bytes)
8702     andl(limit, 0xfffffff0);   // vector count (in bytes)
8703     jccb(Assembler::zero, COMPARE_TAIL);
8704 
8705     lea(ary1, Address(ary1, limit, Address::times_1));
8706     lea(ary2, Address(ary2, limit, Address::times_1));
8707     negptr(limit);
8708 
8709     bind(COMPARE_WIDE_VECTORS);
8710     movdqu(vec1, Address(ary1, limit, Address::times_1));
8711     movdqu(vec2, Address(ary2, limit, Address::times_1));
8712     pxor(vec1, vec2);
8713     ptest(vec1, vec1);
8714     jccb(Assembler::notZero, FALSE_LABEL);
8715     addptr(limit, 16);
8716     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8717 
8718     bind(COMPARE_TAIL); // limit is zero
8719     movl(limit, result);
8720     // Fallthru to tail compare
8721   }
8722 
8723   // Compare 4-byte vectors
8724   andl(limit, 0xfffffffc); // vector count (in bytes)
8725   jccb(Assembler::zero, COMPARE_CHAR);
8726 
8727   lea(ary1, Address(ary1, limit, Address::times_1));
8728   lea(ary2, Address(ary2, limit, Address::times_1));
8729   negptr(limit);
8730 
8731   bind(COMPARE_VECTORS);
8732   movl(chr, Address(ary1, limit, Address::times_1));
8733   cmpl(chr, Address(ary2, limit, Address::times_1));
8734   jccb(Assembler::notEqual, FALSE_LABEL);
8735   addptr(limit, 4);
8736   jcc(Assembler::notZero, COMPARE_VECTORS);
8737 
8738   // Compare trailing char (final 2 bytes), if any
8739   bind(COMPARE_CHAR);
8740   testl(result, 0x2);   // tail  char
8741   jccb(Assembler::zero, TRUE_LABEL);
8742   load_unsigned_short(chr, Address(ary1, 0));
8743   load_unsigned_short(limit, Address(ary2, 0));
8744   cmpl(chr, limit);
8745   jccb(Assembler::notEqual, FALSE_LABEL);
8746 
8747   bind(TRUE_LABEL);
8748   movl(result, 1);   // return true
8749   jmpb(DONE);
8750 
8751   bind(FALSE_LABEL);
8752   xorl(result, result); // return false
8753 
8754   // That's it
8755   bind(DONE);
8756 }
8757 
8758 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
8759   switch (cond) {
8760     // Note some conditions are synonyms for others
8761     case Assembler::zero:         return Assembler::notZero;
8762     case Assembler::notZero:      return Assembler::zero;
8763     case Assembler::less:         return Assembler::greaterEqual;
8764     case Assembler::lessEqual:    return Assembler::greater;
8765     case Assembler::greater:      return Assembler::lessEqual;
8766     case Assembler::greaterEqual: return Assembler::less;
8767     case Assembler::below:        return Assembler::aboveEqual;
8768     case Assembler::belowEqual:   return Assembler::above;
8769     case Assembler::above:        return Assembler::belowEqual;
8770     case Assembler::aboveEqual:   return Assembler::below;
8771     case Assembler::overflow:     return Assembler::noOverflow;
8772     case Assembler::noOverflow:   return Assembler::overflow;
8773     case Assembler::negative:     return Assembler::positive;
8774     case Assembler::positive:     return Assembler::negative;
8775     case Assembler::parity:       return Assembler::noParity;
8776     case Assembler::noParity:     return Assembler::parity;
8777   }
8778   ShouldNotReachHere(); return Assembler::overflow;
8779 }
8780 
8781 SkipIfEqual::SkipIfEqual(
8782     MacroAssembler* masm, const bool* flag_addr, bool value) {
8783   _masm = masm;
8784   _masm->cmp8(ExternalAddress((address)flag_addr), value);
8785   _masm->jcc(Assembler::equal, _label);
8786 }
8787 
8788 SkipIfEqual::~SkipIfEqual() {
8789   _masm->bind(_label);
8790 }