1 /*
   2  * Copyright (c) 1997, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "asm/assembler.inline.hpp"
  28 #include "compiler/disassembler.hpp"
  29 #include "gc_interface/collectedHeap.inline.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "memory/cardTableModRefBS.hpp"
  32 #include "memory/resourceArea.hpp"
  33 #include "memory/universe.hpp"
  34 #include "prims/methodHandles.hpp"
  35 #include "runtime/biasedLocking.hpp"
  36 #include "runtime/interfaceSupport.hpp"
  37 #include "runtime/objectMonitor.hpp"
  38 #include "runtime/os.hpp"
  39 #include "runtime/sharedRuntime.hpp"
  40 #include "runtime/stubRoutines.hpp"
  41 #include "utilities/macros.hpp"
  42 #if INCLUDE_ALL_GCS
  43 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
  44 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
  45 #include "gc_implementation/g1/heapRegion.hpp"
  46 #endif // INCLUDE_ALL_GCS
  47 
  48 #ifdef PRODUCT
  49 #define BLOCK_COMMENT(str) /* nothing */
  50 #define STOP(error) stop(error)
  51 #else
  52 #define BLOCK_COMMENT(str) block_comment(str)
  53 #define STOP(error) block_comment(error); stop(error)
  54 #endif
  55 
  56 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  57 
  58 
  59 #ifdef ASSERT
  60 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  61 #endif
  62 
  63 static Assembler::Condition reverse[] = {
  64     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  65     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  66     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  67     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  68     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
  69     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
  70     Assembler::above          /* belowEqual    = 0x6 */ ,
  71     Assembler::belowEqual     /* above         = 0x7 */ ,
  72     Assembler::positive       /* negative      = 0x8 */ ,
  73     Assembler::negative       /* positive      = 0x9 */ ,
  74     Assembler::noParity       /* parity        = 0xa */ ,
  75     Assembler::parity         /* noParity      = 0xb */ ,
  76     Assembler::greaterEqual   /* less          = 0xc */ ,
  77     Assembler::less           /* greaterEqual  = 0xd */ ,
  78     Assembler::greater        /* lessEqual     = 0xe */ ,
  79     Assembler::lessEqual      /* greater       = 0xf, */
  80 
  81 };
  82 
  83 
  84 // Implementation of MacroAssembler
  85 
  86 // First all the versions that have distinct versions depending on 32/64 bit
  87 // Unless the difference is trivial (1 line or so).
  88 
  89 #ifndef _LP64
  90 
  91 // 32bit versions
  92 
  93 Address MacroAssembler::as_Address(AddressLiteral adr) {
  94   return Address(adr.target(), adr.rspec());
  95 }
  96 
  97 Address MacroAssembler::as_Address(ArrayAddress adr) {
  98   return Address::make_array(adr);
  99 }
 100 
 101 void MacroAssembler::call_VM_leaf_base(address entry_point,
 102                                        int number_of_arguments) {
 103   call(RuntimeAddress(entry_point));
 104   increment(rsp, number_of_arguments * wordSize);
 105 }
 106 
 107 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
 108   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 109 }
 110 
 111 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
 112   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 113 }
 114 
 115 void MacroAssembler::cmpoop(Address src1, jobject obj) {
 116   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 117 }
 118 
 119 void MacroAssembler::cmpoop(Register src1, jobject obj) {
 120   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 121 }
 122 
 123 void MacroAssembler::extend_sign(Register hi, Register lo) {
 124   // According to Intel Doc. AP-526, "Integer Divide", p.18.
 125   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
 126     cdql();
 127   } else {
 128     movl(hi, lo);
 129     sarl(hi, 31);
 130   }
 131 }
 132 
 133 void MacroAssembler::jC2(Register tmp, Label& L) {
 134   // set parity bit if FPU flag C2 is set (via rax)
 135   save_rax(tmp);
 136   fwait(); fnstsw_ax();
 137   sahf();
 138   restore_rax(tmp);
 139   // branch
 140   jcc(Assembler::parity, L);
 141 }
 142 
 143 void MacroAssembler::jnC2(Register tmp, Label& L) {
 144   // set parity bit if FPU flag C2 is set (via rax)
 145   save_rax(tmp);
 146   fwait(); fnstsw_ax();
 147   sahf();
 148   restore_rax(tmp);
 149   // branch
 150   jcc(Assembler::noParity, L);
 151 }
 152 
 153 // 32bit can do a case table jump in one instruction but we no longer allow the base
 154 // to be installed in the Address class
 155 void MacroAssembler::jump(ArrayAddress entry) {
 156   jmp(as_Address(entry));
 157 }
 158 
 159 // Note: y_lo will be destroyed
 160 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 161   // Long compare for Java (semantics as described in JVM spec.)
 162   Label high, low, done;
 163 
 164   cmpl(x_hi, y_hi);
 165   jcc(Assembler::less, low);
 166   jcc(Assembler::greater, high);
 167   // x_hi is the return register
 168   xorl(x_hi, x_hi);
 169   cmpl(x_lo, y_lo);
 170   jcc(Assembler::below, low);
 171   jcc(Assembler::equal, done);
 172 
 173   bind(high);
 174   xorl(x_hi, x_hi);
 175   increment(x_hi);
 176   jmp(done);
 177 
 178   bind(low);
 179   xorl(x_hi, x_hi);
 180   decrementl(x_hi);
 181 
 182   bind(done);
 183 }
 184 
 185 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 186     mov_literal32(dst, (int32_t)src.target(), src.rspec());
 187 }
 188 
 189 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 190   // leal(dst, as_Address(adr));
 191   // see note in movl as to why we must use a move
 192   mov_literal32(dst, (int32_t) adr.target(), adr.rspec());
 193 }
 194 
 195 void MacroAssembler::leave() {
 196   mov(rsp, rbp);
 197   pop(rbp);
 198 }
 199 
 200 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
 201   // Multiplication of two Java long values stored on the stack
 202   // as illustrated below. Result is in rdx:rax.
 203   //
 204   // rsp ---> [  ??  ] \               \
 205   //            ....    | y_rsp_offset  |
 206   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
 207   //          [ y_hi ]                  | (in bytes)
 208   //            ....                    |
 209   //          [ x_lo ]                 /
 210   //          [ x_hi ]
 211   //            ....
 212   //
 213   // Basic idea: lo(result) = lo(x_lo * y_lo)
 214   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
 215   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
 216   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
 217   Label quick;
 218   // load x_hi, y_hi and check if quick
 219   // multiplication is possible
 220   movl(rbx, x_hi);
 221   movl(rcx, y_hi);
 222   movl(rax, rbx);
 223   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
 224   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
 225   // do full multiplication
 226   // 1st step
 227   mull(y_lo);                                    // x_hi * y_lo
 228   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
 229   // 2nd step
 230   movl(rax, x_lo);
 231   mull(rcx);                                     // x_lo * y_hi
 232   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
 233   // 3rd step
 234   bind(quick);                                   // note: rbx, = 0 if quick multiply!
 235   movl(rax, x_lo);
 236   mull(y_lo);                                    // x_lo * y_lo
 237   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
 238 }
 239 
 240 void MacroAssembler::lneg(Register hi, Register lo) {
 241   negl(lo);
 242   adcl(hi, 0);
 243   negl(hi);
 244 }
 245 
 246 void MacroAssembler::lshl(Register hi, Register lo) {
 247   // Java shift left long support (semantics as described in JVM spec., p.305)
 248   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
 249   // shift value is in rcx !
 250   assert(hi != rcx, "must not use rcx");
 251   assert(lo != rcx, "must not use rcx");
 252   const Register s = rcx;                        // shift count
 253   const int      n = BitsPerWord;
 254   Label L;
 255   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 256   cmpl(s, n);                                    // if (s < n)
 257   jcc(Assembler::less, L);                       // else (s >= n)
 258   movl(hi, lo);                                  // x := x << n
 259   xorl(lo, lo);
 260   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 261   bind(L);                                       // s (mod n) < n
 262   shldl(hi, lo);                                 // x := x << s
 263   shll(lo);
 264 }
 265 
 266 
 267 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
 268   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
 269   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
 270   assert(hi != rcx, "must not use rcx");
 271   assert(lo != rcx, "must not use rcx");
 272   const Register s = rcx;                        // shift count
 273   const int      n = BitsPerWord;
 274   Label L;
 275   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 276   cmpl(s, n);                                    // if (s < n)
 277   jcc(Assembler::less, L);                       // else (s >= n)
 278   movl(lo, hi);                                  // x := x >> n
 279   if (sign_extension) sarl(hi, 31);
 280   else                xorl(hi, hi);
 281   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 282   bind(L);                                       // s (mod n) < n
 283   shrdl(lo, hi);                                 // x := x >> s
 284   if (sign_extension) sarl(hi);
 285   else                shrl(hi);
 286 }
 287 
 288 void MacroAssembler::movoop(Register dst, jobject obj) {
 289   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 290 }
 291 
 292 void MacroAssembler::movoop(Address dst, jobject obj) {
 293   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 294 }
 295 
 296 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 297   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 298 }
 299 
 300 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 301   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 302 }
 303 
 304 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
 305   if (src.is_lval()) {
 306     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
 307   } else {
 308     movl(dst, as_Address(src));
 309   }
 310 }
 311 
 312 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 313   movl(as_Address(dst), src);
 314 }
 315 
 316 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 317   movl(dst, as_Address(src));
 318 }
 319 
 320 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 321 void MacroAssembler::movptr(Address dst, intptr_t src) {
 322   movl(dst, src);
 323 }
 324 
 325 
 326 void MacroAssembler::pop_callee_saved_registers() {
 327   pop(rcx);
 328   pop(rdx);
 329   pop(rdi);
 330   pop(rsi);
 331 }
 332 
 333 void MacroAssembler::pop_fTOS() {
 334   fld_d(Address(rsp, 0));
 335   addl(rsp, 2 * wordSize);
 336 }
 337 
 338 void MacroAssembler::push_callee_saved_registers() {
 339   push(rsi);
 340   push(rdi);
 341   push(rdx);
 342   push(rcx);
 343 }
 344 
 345 void MacroAssembler::push_fTOS() {
 346   subl(rsp, 2 * wordSize);
 347   fstp_d(Address(rsp, 0));
 348 }
 349 
 350 
 351 void MacroAssembler::pushoop(jobject obj) {
 352   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
 353 }
 354 
 355 void MacroAssembler::pushklass(Metadata* obj) {
 356   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
 357 }
 358 
 359 void MacroAssembler::pushptr(AddressLiteral src) {
 360   if (src.is_lval()) {
 361     push_literal32((int32_t)src.target(), src.rspec());
 362   } else {
 363     pushl(as_Address(src));
 364   }
 365 }
 366 
 367 void MacroAssembler::set_word_if_not_zero(Register dst) {
 368   xorl(dst, dst);
 369   set_byte_if_not_zero(dst);
 370 }
 371 
 372 static void pass_arg0(MacroAssembler* masm, Register arg) {
 373   masm->push(arg);
 374 }
 375 
 376 static void pass_arg1(MacroAssembler* masm, Register arg) {
 377   masm->push(arg);
 378 }
 379 
 380 static void pass_arg2(MacroAssembler* masm, Register arg) {
 381   masm->push(arg);
 382 }
 383 
 384 static void pass_arg3(MacroAssembler* masm, Register arg) {
 385   masm->push(arg);
 386 }
 387 
 388 #ifndef PRODUCT
 389 extern "C" void findpc(intptr_t x);
 390 #endif
 391 
 392 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
 393   // In order to get locks to work, we need to fake a in_VM state
 394   JavaThread* thread = JavaThread::current();
 395   JavaThreadState saved_state = thread->thread_state();
 396   thread->set_thread_state(_thread_in_vm);
 397   if (ShowMessageBoxOnError) {
 398     JavaThread* thread = JavaThread::current();
 399     JavaThreadState saved_state = thread->thread_state();
 400     thread->set_thread_state(_thread_in_vm);
 401     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 402       ttyLocker ttyl;
 403       BytecodeCounter::print();
 404     }
 405     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 406     // This is the value of eip which points to where verify_oop will return.
 407     if (os::message_box(msg, "Execution stopped, print registers?")) {
 408       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
 409       BREAKPOINT;
 410     }
 411   } else {
 412     ttyLocker ttyl;
 413     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
 414   }
 415   // Don't assert holding the ttyLock
 416     assert(false, err_msg("DEBUG MESSAGE: %s", msg));
 417   ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 418 }
 419 
 420 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
 421   ttyLocker ttyl;
 422   FlagSetting fs(Debugging, true);
 423   tty->print_cr("eip = 0x%08x", eip);
 424 #ifndef PRODUCT
 425   if ((WizardMode || Verbose) && PrintMiscellaneous) {
 426     tty->cr();
 427     findpc(eip);
 428     tty->cr();
 429   }
 430 #endif
 431 #define PRINT_REG(rax) \
 432   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
 433   PRINT_REG(rax);
 434   PRINT_REG(rbx);
 435   PRINT_REG(rcx);
 436   PRINT_REG(rdx);
 437   PRINT_REG(rdi);
 438   PRINT_REG(rsi);
 439   PRINT_REG(rbp);
 440   PRINT_REG(rsp);
 441 #undef PRINT_REG
 442   // Print some words near top of staack.
 443   int* dump_sp = (int*) rsp;
 444   for (int col1 = 0; col1 < 8; col1++) {
 445     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 446     os::print_location(tty, *dump_sp++);
 447   }
 448   for (int row = 0; row < 16; row++) {
 449     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 450     for (int col = 0; col < 8; col++) {
 451       tty->print(" 0x%08x", *dump_sp++);
 452     }
 453     tty->cr();
 454   }
 455   // Print some instructions around pc:
 456   Disassembler::decode((address)eip-64, (address)eip);
 457   tty->print_cr("--------");
 458   Disassembler::decode((address)eip, (address)eip+32);
 459 }
 460 
 461 void MacroAssembler::stop(const char* msg) {
 462   ExternalAddress message((address)msg);
 463   // push address of message
 464   pushptr(message.addr());
 465   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 466   pusha();                                            // push registers
 467   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 468   hlt();
 469 }
 470 
 471 void MacroAssembler::warn(const char* msg) {
 472   push_CPU_state();
 473 
 474   ExternalAddress message((address) msg);
 475   // push address of message
 476   pushptr(message.addr());
 477 
 478   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
 479   addl(rsp, wordSize);       // discard argument
 480   pop_CPU_state();
 481 }
 482 
 483 void MacroAssembler::print_state() {
 484   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 485   pusha();                                            // push registers
 486 
 487   push_CPU_state();
 488   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
 489   pop_CPU_state();
 490 
 491   popa();
 492   addl(rsp, wordSize);
 493 }
 494 
 495 #else // _LP64
 496 
 497 // 64 bit versions
 498 
 499 Address MacroAssembler::as_Address(AddressLiteral adr) {
 500   // amd64 always does this as a pc-rel
 501   // we can be absolute or disp based on the instruction type
 502   // jmp/call are displacements others are absolute
 503   assert(!adr.is_lval(), "must be rval");
 504   assert(reachable(adr), "must be");
 505   return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
 506 
 507 }
 508 
 509 Address MacroAssembler::as_Address(ArrayAddress adr) {
 510   AddressLiteral base = adr.base();
 511   lea(rscratch1, base);
 512   Address index = adr.index();
 513   assert(index._disp == 0, "must not have disp"); // maybe it can?
 514   Address array(rscratch1, index._index, index._scale, index._disp);
 515   return array;
 516 }
 517 
 518 
 519 
 520 
 521 
 522 
 523 
 524 
 525 
 526 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
 527   Label L, E;
 528 
 529 #ifdef _WIN64
 530   // Windows always allocates space for it's register args
 531   assert(num_args <= 4, "only register arguments supported");
 532   subq(rsp,  frame::arg_reg_save_area_bytes);
 533 #endif
 534 
 535   // Align stack if necessary
 536   testl(rsp, 15);
 537   jcc(Assembler::zero, L);
 538 
 539   subq(rsp, 8);
 540   {
 541     call(RuntimeAddress(entry_point));
 542   }
 543   addq(rsp, 8);
 544   jmp(E);
 545 
 546   bind(L);
 547   {
 548     call(RuntimeAddress(entry_point));
 549   }
 550 
 551   bind(E);
 552 
 553 #ifdef _WIN64
 554   // restore stack pointer
 555   addq(rsp, frame::arg_reg_save_area_bytes);
 556 #endif
 557 
 558 }
 559 
 560 void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
 561   assert(!src2.is_lval(), "should use cmpptr");
 562 
 563   if (reachable(src2)) {
 564     cmpq(src1, as_Address(src2));
 565   } else {
 566     lea(rscratch1, src2);
 567     Assembler::cmpq(src1, Address(rscratch1, 0));
 568   }
 569 }
 570 
 571 int MacroAssembler::corrected_idivq(Register reg) {
 572   // Full implementation of Java ldiv and lrem; checks for special
 573   // case as described in JVM spec., p.243 & p.271.  The function
 574   // returns the (pc) offset of the idivl instruction - may be needed
 575   // for implicit exceptions.
 576   //
 577   //         normal case                           special case
 578   //
 579   // input : rax: dividend                         min_long
 580   //         reg: divisor   (may not be eax/edx)   -1
 581   //
 582   // output: rax: quotient  (= rax idiv reg)       min_long
 583   //         rdx: remainder (= rax irem reg)       0
 584   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
 585   static const int64_t min_long = 0x8000000000000000;
 586   Label normal_case, special_case;
 587 
 588   // check for special case
 589   cmp64(rax, ExternalAddress((address) &min_long));
 590   jcc(Assembler::notEqual, normal_case);
 591   xorl(rdx, rdx); // prepare rdx for possible special case (where
 592                   // remainder = 0)
 593   cmpq(reg, -1);
 594   jcc(Assembler::equal, special_case);
 595 
 596   // handle normal case
 597   bind(normal_case);
 598   cdqq();
 599   int idivq_offset = offset();
 600   idivq(reg);
 601 
 602   // normal and special case exit
 603   bind(special_case);
 604 
 605   return idivq_offset;
 606 }
 607 
 608 void MacroAssembler::decrementq(Register reg, int value) {
 609   if (value == min_jint) { subq(reg, value); return; }
 610   if (value <  0) { incrementq(reg, -value); return; }
 611   if (value == 0) {                        ; return; }
 612   if (value == 1 && UseIncDec) { decq(reg) ; return; }
 613   /* else */      { subq(reg, value)       ; return; }
 614 }
 615 
 616 void MacroAssembler::decrementq(Address dst, int value) {
 617   if (value == min_jint) { subq(dst, value); return; }
 618   if (value <  0) { incrementq(dst, -value); return; }
 619   if (value == 0) {                        ; return; }
 620   if (value == 1 && UseIncDec) { decq(dst) ; return; }
 621   /* else */      { subq(dst, value)       ; return; }
 622 }
 623 
 624 void MacroAssembler::incrementq(Register reg, int value) {
 625   if (value == min_jint) { addq(reg, value); return; }
 626   if (value <  0) { decrementq(reg, -value); return; }
 627   if (value == 0) {                        ; return; }
 628   if (value == 1 && UseIncDec) { incq(reg) ; return; }
 629   /* else */      { addq(reg, value)       ; return; }
 630 }
 631 
 632 void MacroAssembler::incrementq(Address dst, int value) {
 633   if (value == min_jint) { addq(dst, value); return; }
 634   if (value <  0) { decrementq(dst, -value); return; }
 635   if (value == 0) {                        ; return; }
 636   if (value == 1 && UseIncDec) { incq(dst) ; return; }
 637   /* else */      { addq(dst, value)       ; return; }
 638 }
 639 
 640 // 32bit can do a case table jump in one instruction but we no longer allow the base
 641 // to be installed in the Address class
 642 void MacroAssembler::jump(ArrayAddress entry) {
 643   lea(rscratch1, entry.base());
 644   Address dispatch = entry.index();
 645   assert(dispatch._base == noreg, "must be");
 646   dispatch._base = rscratch1;
 647   jmp(dispatch);
 648 }
 649 
 650 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 651   ShouldNotReachHere(); // 64bit doesn't use two regs
 652   cmpq(x_lo, y_lo);
 653 }
 654 
 655 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 656     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 657 }
 658 
 659 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 660   mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());
 661   movptr(dst, rscratch1);
 662 }
 663 
 664 void MacroAssembler::leave() {
 665   // %%% is this really better? Why not on 32bit too?
 666   emit_int8((unsigned char)0xC9); // LEAVE
 667 }
 668 
 669 void MacroAssembler::lneg(Register hi, Register lo) {
 670   ShouldNotReachHere(); // 64bit doesn't use two regs
 671   negq(lo);
 672 }
 673 
 674 void MacroAssembler::movoop(Register dst, jobject obj) {
 675   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 676 }
 677 
 678 void MacroAssembler::movoop(Address dst, jobject obj) {
 679   mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 680   movq(dst, rscratch1);
 681 }
 682 
 683 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 684   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 685 }
 686 
 687 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 688   mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 689   movq(dst, rscratch1);
 690 }
 691 
 692 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
 693   if (src.is_lval()) {
 694     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 695   } else {
 696     if (reachable(src)) {
 697       movq(dst, as_Address(src));
 698     } else {
 699       lea(rscratch1, src);
 700       movq(dst, Address(rscratch1,0));
 701     }
 702   }
 703 }
 704 
 705 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 706   movq(as_Address(dst), src);
 707 }
 708 
 709 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 710   movq(dst, as_Address(src));
 711 }
 712 
 713 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 714 void MacroAssembler::movptr(Address dst, intptr_t src) {
 715   mov64(rscratch1, src);
 716   movq(dst, rscratch1);
 717 }
 718 
 719 // These are mostly for initializing NULL
 720 void MacroAssembler::movptr(Address dst, int32_t src) {
 721   movslq(dst, src);
 722 }
 723 
 724 void MacroAssembler::movptr(Register dst, int32_t src) {
 725   mov64(dst, (intptr_t)src);
 726 }
 727 
 728 void MacroAssembler::pushoop(jobject obj) {
 729   movoop(rscratch1, obj);
 730   push(rscratch1);
 731 }
 732 
 733 void MacroAssembler::pushklass(Metadata* obj) {
 734   mov_metadata(rscratch1, obj);
 735   push(rscratch1);
 736 }
 737 
 738 void MacroAssembler::pushptr(AddressLiteral src) {
 739   lea(rscratch1, src);
 740   if (src.is_lval()) {
 741     push(rscratch1);
 742   } else {
 743     pushq(Address(rscratch1, 0));
 744   }
 745 }
 746 
 747 void MacroAssembler::reset_last_Java_frame(bool clear_fp,
 748                                            bool clear_pc) {
 749   // we must set sp to zero to clear frame
 750   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 751   // must clear fp, so that compiled frames are not confused; it is
 752   // possible that we need it only for debugging
 753   if (clear_fp) {
 754     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 755   }
 756 
 757   if (clear_pc) {
 758     movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 759   }
 760 }
 761 
 762 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 763                                          Register last_java_fp,
 764                                          address  last_java_pc) {
 765   // determine last_java_sp register
 766   if (!last_java_sp->is_valid()) {
 767     last_java_sp = rsp;
 768   }
 769 
 770   // last_java_fp is optional
 771   if (last_java_fp->is_valid()) {
 772     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),
 773            last_java_fp);
 774   }
 775 
 776   // last_java_pc is optional
 777   if (last_java_pc != NULL) {
 778     Address java_pc(r15_thread,
 779                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 780     lea(rscratch1, InternalAddress(last_java_pc));
 781     movptr(java_pc, rscratch1);
 782   }
 783 
 784   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 785 }
 786 
 787 static void pass_arg0(MacroAssembler* masm, Register arg) {
 788   if (c_rarg0 != arg ) {
 789     masm->mov(c_rarg0, arg);
 790   }
 791 }
 792 
 793 static void pass_arg1(MacroAssembler* masm, Register arg) {
 794   if (c_rarg1 != arg ) {
 795     masm->mov(c_rarg1, arg);
 796   }
 797 }
 798 
 799 static void pass_arg2(MacroAssembler* masm, Register arg) {
 800   if (c_rarg2 != arg ) {
 801     masm->mov(c_rarg2, arg);
 802   }
 803 }
 804 
 805 static void pass_arg3(MacroAssembler* masm, Register arg) {
 806   if (c_rarg3 != arg ) {
 807     masm->mov(c_rarg3, arg);
 808   }
 809 }
 810 
 811 void MacroAssembler::stop(const char* msg) {
 812   address rip = pc();
 813   pusha(); // get regs on stack
 814   lea(c_rarg0, ExternalAddress((address) msg));
 815   lea(c_rarg1, InternalAddress(rip));
 816   movq(c_rarg2, rsp); // pass pointer to regs array
 817   andq(rsp, -16); // align stack as required by ABI
 818   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
 819   hlt();
 820 }
 821 
 822 void MacroAssembler::warn(const char* msg) {
 823   push(rbp);
 824   movq(rbp, rsp);
 825   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 826   push_CPU_state();   // keeps alignment at 16 bytes
 827   lea(c_rarg0, ExternalAddress((address) msg));
 828   call_VM_leaf(CAST_FROM_FN_PTR(address, warning), c_rarg0);
 829   pop_CPU_state();
 830   mov(rsp, rbp);
 831   pop(rbp);
 832 }
 833 
 834 void MacroAssembler::print_state() {
 835   address rip = pc();
 836   pusha();            // get regs on stack
 837   push(rbp);
 838   movq(rbp, rsp);
 839   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 840   push_CPU_state();   // keeps alignment at 16 bytes
 841 
 842   lea(c_rarg0, InternalAddress(rip));
 843   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
 844   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
 845 
 846   pop_CPU_state();
 847   mov(rsp, rbp);
 848   pop(rbp);
 849   popa();
 850 }
 851 
 852 #ifndef PRODUCT
 853 extern "C" void findpc(intptr_t x);
 854 #endif
 855 
 856 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
 857   // In order to get locks to work, we need to fake a in_VM state
 858   if (ShowMessageBoxOnError) {
 859     JavaThread* thread = JavaThread::current();
 860     JavaThreadState saved_state = thread->thread_state();
 861     thread->set_thread_state(_thread_in_vm);
 862 #ifndef PRODUCT
 863     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 864       ttyLocker ttyl;
 865       BytecodeCounter::print();
 866     }
 867 #endif
 868     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 869     // XXX correct this offset for amd64
 870     // This is the value of eip which points to where verify_oop will return.
 871     if (os::message_box(msg, "Execution stopped, print registers?")) {
 872       print_state64(pc, regs);
 873       BREAKPOINT;
 874       assert(false, "start up GDB");
 875     }
 876     ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 877   } else {
 878     ttyLocker ttyl;
 879     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
 880                     msg);
 881     assert(false, err_msg("DEBUG MESSAGE: %s", msg));
 882   }
 883 }
 884 
 885 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
 886   ttyLocker ttyl;
 887   FlagSetting fs(Debugging, true);
 888   tty->print_cr("rip = 0x%016lx", pc);
 889 #ifndef PRODUCT
 890   tty->cr();
 891   findpc(pc);
 892   tty->cr();
 893 #endif
 894 #define PRINT_REG(rax, value) \
 895   { tty->print("%s = ", #rax); os::print_location(tty, value); }
 896   PRINT_REG(rax, regs[15]);
 897   PRINT_REG(rbx, regs[12]);
 898   PRINT_REG(rcx, regs[14]);
 899   PRINT_REG(rdx, regs[13]);
 900   PRINT_REG(rdi, regs[8]);
 901   PRINT_REG(rsi, regs[9]);
 902   PRINT_REG(rbp, regs[10]);
 903   PRINT_REG(rsp, regs[11]);
 904   PRINT_REG(r8 , regs[7]);
 905   PRINT_REG(r9 , regs[6]);
 906   PRINT_REG(r10, regs[5]);
 907   PRINT_REG(r11, regs[4]);
 908   PRINT_REG(r12, regs[3]);
 909   PRINT_REG(r13, regs[2]);
 910   PRINT_REG(r14, regs[1]);
 911   PRINT_REG(r15, regs[0]);
 912 #undef PRINT_REG
 913   // Print some words near top of staack.
 914   int64_t* rsp = (int64_t*) regs[11];
 915   int64_t* dump_sp = rsp;
 916   for (int col1 = 0; col1 < 8; col1++) {
 917     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp);
 918     os::print_location(tty, *dump_sp++);
 919   }
 920   for (int row = 0; row < 25; row++) {
 921     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (int64_t)dump_sp);
 922     for (int col = 0; col < 4; col++) {
 923       tty->print(" 0x%016lx", *dump_sp++);
 924     }
 925     tty->cr();
 926   }
 927   // Print some instructions around pc:
 928   Disassembler::decode((address)pc-64, (address)pc);
 929   tty->print_cr("--------");
 930   Disassembler::decode((address)pc, (address)pc+32);
 931 }
 932 
 933 #endif // _LP64
 934 
 935 // Now versions that are common to 32/64 bit
 936 
 937 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 938   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 939 }
 940 
 941 void MacroAssembler::addptr(Register dst, Register src) {
 942   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 943 }
 944 
 945 void MacroAssembler::addptr(Address dst, Register src) {
 946   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 947 }
 948 
 949 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {
 950   if (reachable(src)) {
 951     Assembler::addsd(dst, as_Address(src));
 952   } else {
 953     lea(rscratch1, src);
 954     Assembler::addsd(dst, Address(rscratch1, 0));
 955   }
 956 }
 957 
 958 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {
 959   if (reachable(src)) {
 960     addss(dst, as_Address(src));
 961   } else {
 962     lea(rscratch1, src);
 963     addss(dst, Address(rscratch1, 0));
 964   }
 965 }
 966 
 967 void MacroAssembler::align(int modulus) {
 968   if (offset() % modulus != 0) {
 969     nop(modulus - (offset() % modulus));
 970   }
 971 }
 972 
 973 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {
 974   // Used in sign-masking with aligned address.
 975   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 976   if (reachable(src)) {
 977     Assembler::andpd(dst, as_Address(src));
 978   } else {
 979     lea(rscratch1, src);
 980     Assembler::andpd(dst, Address(rscratch1, 0));
 981   }
 982 }
 983 
 984 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {
 985   // Used in sign-masking with aligned address.
 986   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 987   if (reachable(src)) {
 988     Assembler::andps(dst, as_Address(src));
 989   } else {
 990     lea(rscratch1, src);
 991     Assembler::andps(dst, Address(rscratch1, 0));
 992   }
 993 }
 994 
 995 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 996   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
 997 }
 998 
 999 void MacroAssembler::atomic_incl(AddressLiteral counter_addr) {
1000   pushf();
1001   if (reachable(counter_addr)) {
1002     if (os::is_MP())
1003       lock();
1004     incrementl(as_Address(counter_addr));
1005   } else {
1006     lea(rscratch1, counter_addr);
1007     if (os::is_MP())
1008       lock();
1009     incrementl(Address(rscratch1, 0));
1010   }
1011   popf();
1012 }
1013 
1014 // Writes to stack successive pages until offset reached to check for
1015 // stack overflow + shadow pages.  This clobbers tmp.
1016 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
1017   movptr(tmp, rsp);
1018   // Bang stack for total size given plus shadow page size.
1019   // Bang one page at a time because large size can bang beyond yellow and
1020   // red zones.
1021   Label loop;
1022   bind(loop);
1023   movl(Address(tmp, (-os::vm_page_size())), size );
1024   subptr(tmp, os::vm_page_size());
1025   subl(size, os::vm_page_size());
1026   jcc(Assembler::greater, loop);
1027 
1028   // Bang down shadow pages too.
1029   // At this point, (tmp-0) is the last address touched, so don't
1030   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
1031   // was post-decremented.)  Skip this address by starting at i=1, and
1032   // touch a few more pages below.  N.B.  It is important to touch all
1033   // the way down to and including i=StackShadowPages.
1034   for (int i = 1; i <= StackShadowPages; i++) {
1035     // this could be any sized move but this is can be a debugging crumb
1036     // so the bigger the better.
1037     movptr(Address(tmp, (-i*os::vm_page_size())), size );
1038   }
1039 }
1040 
1041 int MacroAssembler::biased_locking_enter(Register lock_reg,
1042                                          Register obj_reg,
1043                                          Register swap_reg,
1044                                          Register tmp_reg,
1045                                          bool swap_reg_contains_mark,
1046                                          Label& done,
1047                                          Label* slow_case,
1048                                          BiasedLockingCounters* counters) {
1049   assert(UseBiasedLocking, "why call this otherwise?");
1050   assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
1051   LP64_ONLY( assert(tmp_reg != noreg, "tmp_reg must be supplied"); )
1052   bool need_tmp_reg = false;
1053   if (tmp_reg == noreg) {
1054     need_tmp_reg = true;
1055     tmp_reg = lock_reg;
1056     assert_different_registers(lock_reg, obj_reg, swap_reg);
1057   } else {
1058     assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
1059   }
1060   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
1061   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
1062   Address saved_mark_addr(lock_reg, 0);
1063 
1064   if (PrintBiasedLockingStatistics && counters == NULL) {
1065     counters = BiasedLocking::counters();
1066   }
1067   // Biased locking
1068   // See whether the lock is currently biased toward our thread and
1069   // whether the epoch is still valid
1070   // Note that the runtime guarantees sufficient alignment of JavaThread
1071   // pointers to allow age to be placed into low bits
1072   // First check to see whether biasing is even enabled for this object
1073   Label cas_label;
1074   int null_check_offset = -1;
1075   if (!swap_reg_contains_mark) {
1076     null_check_offset = offset();
1077     movptr(swap_reg, mark_addr);
1078   }
1079   if (need_tmp_reg) {
1080     push(tmp_reg);
1081   }
1082   movptr(tmp_reg, swap_reg);
1083   andptr(tmp_reg, markOopDesc::biased_lock_mask_in_place);
1084   cmpptr(tmp_reg, markOopDesc::biased_lock_pattern);
1085   if (need_tmp_reg) {
1086     pop(tmp_reg);
1087   }
1088   jcc(Assembler::notEqual, cas_label);
1089   // The bias pattern is present in the object's header. Need to check
1090   // whether the bias owner and the epoch are both still current.
1091 #ifndef _LP64
1092   // Note that because there is no current thread register on x86_32 we
1093   // need to store off the mark word we read out of the object to
1094   // avoid reloading it and needing to recheck invariants below. This
1095   // store is unfortunate but it makes the overall code shorter and
1096   // simpler.
1097   movptr(saved_mark_addr, swap_reg);
1098 #endif
1099   if (need_tmp_reg) {
1100     push(tmp_reg);
1101   }
1102   if (swap_reg_contains_mark) {
1103     null_check_offset = offset();
1104   }
1105   load_prototype_header(tmp_reg, obj_reg);
1106 #ifdef _LP64
1107   orptr(tmp_reg, r15_thread);
1108   xorptr(tmp_reg, swap_reg);
1109   Register header_reg = tmp_reg;
1110 #else
1111   xorptr(tmp_reg, swap_reg);
1112   get_thread(swap_reg);
1113   xorptr(swap_reg, tmp_reg);
1114   Register header_reg = swap_reg;
1115 #endif
1116   andptr(header_reg, ~((int) markOopDesc::age_mask_in_place));
1117   if (need_tmp_reg) {
1118     pop(tmp_reg);
1119   }
1120   if (counters != NULL) {
1121     cond_inc32(Assembler::zero,
1122                ExternalAddress((address) counters->biased_lock_entry_count_addr()));
1123   }
1124   jcc(Assembler::equal, done);
1125 
1126   Label try_revoke_bias;
1127   Label try_rebias;
1128 
1129   // At this point we know that the header has the bias pattern and
1130   // that we are not the bias owner in the current epoch. We need to
1131   // figure out more details about the state of the header in order to
1132   // know what operations can be legally performed on the object's
1133   // header.
1134 
1135   // If the low three bits in the xor result aren't clear, that means
1136   // the prototype header is no longer biased and we have to revoke
1137   // the bias on this object.
1138   testptr(header_reg, markOopDesc::biased_lock_mask_in_place);
1139   jccb(Assembler::notZero, try_revoke_bias);
1140 
1141   // Biasing is still enabled for this data type. See whether the
1142   // epoch of the current bias is still valid, meaning that the epoch
1143   // bits of the mark word are equal to the epoch bits of the
1144   // prototype header. (Note that the prototype header's epoch bits
1145   // only change at a safepoint.) If not, attempt to rebias the object
1146   // toward the current thread. Note that we must be absolutely sure
1147   // that the current epoch is invalid in order to do this because
1148   // otherwise the manipulations it performs on the mark word are
1149   // illegal.
1150   testptr(header_reg, markOopDesc::epoch_mask_in_place);
1151   jccb(Assembler::notZero, try_rebias);
1152 
1153   // The epoch of the current bias is still valid but we know nothing
1154   // about the owner; it might be set or it might be clear. Try to
1155   // acquire the bias of the object using an atomic operation. If this
1156   // fails we will go in to the runtime to revoke the object's bias.
1157   // Note that we first construct the presumed unbiased header so we
1158   // don't accidentally blow away another thread's valid bias.
1159   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1160   andptr(swap_reg,
1161          markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
1162   if (need_tmp_reg) {
1163     push(tmp_reg);
1164   }
1165 #ifdef _LP64
1166   movptr(tmp_reg, swap_reg);
1167   orptr(tmp_reg, r15_thread);
1168 #else
1169   get_thread(tmp_reg);
1170   orptr(tmp_reg, swap_reg);
1171 #endif
1172   if (os::is_MP()) {
1173     lock();
1174   }
1175   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1176   if (need_tmp_reg) {
1177     pop(tmp_reg);
1178   }
1179   // If the biasing toward our thread failed, this means that
1180   // another thread succeeded in biasing it toward itself and we
1181   // need to revoke that bias. The revocation will occur in the
1182   // interpreter runtime in the slow case.
1183   if (counters != NULL) {
1184     cond_inc32(Assembler::zero,
1185                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
1186   }
1187   if (slow_case != NULL) {
1188     jcc(Assembler::notZero, *slow_case);
1189   }
1190   jmp(done);
1191 
1192   bind(try_rebias);
1193   // At this point we know the epoch has expired, meaning that the
1194   // current "bias owner", if any, is actually invalid. Under these
1195   // circumstances _only_, we are allowed to use the current header's
1196   // value as the comparison value when doing the cas to acquire the
1197   // bias in the current epoch. In other words, we allow transfer of
1198   // the bias from one thread to another directly in this situation.
1199   //
1200   // FIXME: due to a lack of registers we currently blow away the age
1201   // bits in this situation. Should attempt to preserve them.
1202   if (need_tmp_reg) {
1203     push(tmp_reg);
1204   }
1205   load_prototype_header(tmp_reg, obj_reg);
1206 #ifdef _LP64
1207   orptr(tmp_reg, r15_thread);
1208 #else
1209   get_thread(swap_reg);
1210   orptr(tmp_reg, swap_reg);
1211   movptr(swap_reg, saved_mark_addr);
1212 #endif
1213   if (os::is_MP()) {
1214     lock();
1215   }
1216   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1217   if (need_tmp_reg) {
1218     pop(tmp_reg);
1219   }
1220   // If the biasing toward our thread failed, then another thread
1221   // succeeded in biasing it toward itself and we need to revoke that
1222   // bias. The revocation will occur in the runtime in the slow case.
1223   if (counters != NULL) {
1224     cond_inc32(Assembler::zero,
1225                ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
1226   }
1227   if (slow_case != NULL) {
1228     jcc(Assembler::notZero, *slow_case);
1229   }
1230   jmp(done);
1231 
1232   bind(try_revoke_bias);
1233   // The prototype mark in the klass doesn't have the bias bit set any
1234   // more, indicating that objects of this data type are not supposed
1235   // to be biased any more. We are going to try to reset the mark of
1236   // this object to the prototype value and fall through to the
1237   // CAS-based locking scheme. Note that if our CAS fails, it means
1238   // that another thread raced us for the privilege of revoking the
1239   // bias of this particular object, so it's okay to continue in the
1240   // normal locking code.
1241   //
1242   // FIXME: due to a lack of registers we currently blow away the age
1243   // bits in this situation. Should attempt to preserve them.
1244   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1245   if (need_tmp_reg) {
1246     push(tmp_reg);
1247   }
1248   load_prototype_header(tmp_reg, obj_reg);
1249   if (os::is_MP()) {
1250     lock();
1251   }
1252   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1253   if (need_tmp_reg) {
1254     pop(tmp_reg);
1255   }
1256   // Fall through to the normal CAS-based lock, because no matter what
1257   // the result of the above CAS, some thread must have succeeded in
1258   // removing the bias bit from the object's header.
1259   if (counters != NULL) {
1260     cond_inc32(Assembler::zero,
1261                ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
1262   }
1263 
1264   bind(cas_label);
1265 
1266   return null_check_offset;
1267 }
1268 
1269 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
1270   assert(UseBiasedLocking, "why call this otherwise?");
1271 
1272   // Check for biased locking unlock case, which is a no-op
1273   // Note: we do not have to check the thread ID for two reasons.
1274   // First, the interpreter checks for IllegalMonitorStateException at
1275   // a higher level. Second, if the bias was revoked while we held the
1276   // lock, the object could not be rebiased toward another thread, so
1277   // the bias bit would be clear.
1278   movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1279   andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);
1280   cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
1281   jcc(Assembler::equal, done);
1282 }
1283 
1284 #ifdef COMPILER2
1285 // Fast_Lock and Fast_Unlock used by C2
1286 
1287 // Because the transitions from emitted code to the runtime
1288 // monitorenter/exit helper stubs are so slow it's critical that
1289 // we inline both the stack-locking fast-path and the inflated fast path.
1290 //
1291 // See also: cmpFastLock and cmpFastUnlock.
1292 //
1293 // What follows is a specialized inline transliteration of the code
1294 // in slow_enter() and slow_exit().  If we're concerned about I$ bloat
1295 // another option would be to emit TrySlowEnter and TrySlowExit methods
1296 // at startup-time.  These methods would accept arguments as
1297 // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure
1298 // indications in the icc.ZFlag.  Fast_Lock and Fast_Unlock would simply
1299 // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.
1300 // In practice, however, the # of lock sites is bounded and is usually small.
1301 // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer
1302 // if the processor uses simple bimodal branch predictors keyed by EIP
1303 // Since the helper routines would be called from multiple synchronization
1304 // sites.
1305 //
1306 // An even better approach would be write "MonitorEnter()" and "MonitorExit()"
1307 // in java - using j.u.c and unsafe - and just bind the lock and unlock sites
1308 // to those specialized methods.  That'd give us a mostly platform-independent
1309 // implementation that the JITs could optimize and inline at their pleasure.
1310 // Done correctly, the only time we'd need to cross to native could would be
1311 // to park() or unpark() threads.  We'd also need a few more unsafe operators
1312 // to (a) prevent compiler-JIT reordering of non-volatile accesses, and
1313 // (b) explicit barriers or fence operations.
1314 //
1315 // TODO:
1316 //
1317 // *  Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr).
1318 //    This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals.
1319 //    Given TLAB allocation, Self is usually manifested in a register, so passing it into
1320 //    the lock operators would typically be faster than reifying Self.
1321 //
1322 // *  Ideally I'd define the primitives as:
1323 //       fast_lock   (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.
1324 //       fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED
1325 //    Unfortunately ADLC bugs prevent us from expressing the ideal form.
1326 //    Instead, we're stuck with a rather awkward and brittle register assignments below.
1327 //    Furthermore the register assignments are overconstrained, possibly resulting in
1328 //    sub-optimal code near the synchronization site.
1329 //
1330 // *  Eliminate the sp-proximity tests and just use "== Self" tests instead.
1331 //    Alternately, use a better sp-proximity test.
1332 //
1333 // *  Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.
1334 //    Either one is sufficient to uniquely identify a thread.
1335 //    TODO: eliminate use of sp in _owner and use get_thread(tr) instead.
1336 //
1337 // *  Intrinsify notify() and notifyAll() for the common cases where the
1338 //    object is locked by the calling thread but the waitlist is empty.
1339 //    avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().
1340 //
1341 // *  use jccb and jmpb instead of jcc and jmp to improve code density.
1342 //    But beware of excessive branch density on AMD Opterons.
1343 //
1344 // *  Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success
1345 //    or failure of the fast-path.  If the fast-path fails then we pass
1346 //    control to the slow-path, typically in C.  In Fast_Lock and
1347 //    Fast_Unlock we often branch to DONE_LABEL, just to find that C2
1348 //    will emit a conditional branch immediately after the node.
1349 //    So we have branches to branches and lots of ICC.ZF games.
1350 //    Instead, it might be better to have C2 pass a "FailureLabel"
1351 //    into Fast_Lock and Fast_Unlock.  In the case of success, control
1352 //    will drop through the node.  ICC.ZF is undefined at exit.
1353 //    In the case of failure, the node will branch directly to the
1354 //    FailureLabel
1355 
1356 
1357 // obj: object to lock
1358 // box: on-stack box address (displaced header location) - KILLED
1359 // rax,: tmp -- KILLED
1360 // scr: tmp -- KILLED
1361 void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg, Register scrReg, BiasedLockingCounters* counters) {
1362   // Ensure the register assignents are disjoint
1363   guarantee (objReg != boxReg, "");
1364   guarantee (objReg != tmpReg, "");
1365   guarantee (objReg != scrReg, "");
1366   guarantee (boxReg != tmpReg, "");
1367   guarantee (boxReg != scrReg, "");
1368   guarantee (tmpReg == rax, "");
1369 
1370   if (counters != NULL) {
1371     atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()));
1372   }
1373   if (EmitSync & 1) {
1374       // set box->dhw = unused_mark (3)
1375       // Force all sync thru slow-path: slow_enter() and slow_exit()
1376       movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1377       cmpptr (rsp, (int32_t)NULL_WORD);
1378   } else
1379   if (EmitSync & 2) {
1380       Label DONE_LABEL ;
1381       if (UseBiasedLocking) {
1382          // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
1383          biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters);
1384       }
1385 
1386       movptr(tmpReg, Address(objReg, 0));           // fetch markword
1387       orptr (tmpReg, 0x1);
1388       movptr(Address(boxReg, 0), tmpReg);           // Anticipate successful CAS
1389       if (os::is_MP()) {
1390         lock();
1391       }
1392       cmpxchgptr(boxReg, Address(objReg, 0));       // Updates tmpReg
1393       jccb(Assembler::equal, DONE_LABEL);
1394       // Recursive locking
1395       subptr(tmpReg, rsp);
1396       andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
1397       movptr(Address(boxReg, 0), tmpReg);
1398       bind(DONE_LABEL);
1399   } else {
1400     // Possible cases that we'll encounter in fast_lock
1401     // ------------------------------------------------
1402     // * Inflated
1403     //    -- unlocked
1404     //    -- Locked
1405     //       = by self
1406     //       = by other
1407     // * biased
1408     //    -- by Self
1409     //    -- by other
1410     // * neutral
1411     // * stack-locked
1412     //    -- by self
1413     //       = sp-proximity test hits
1414     //       = sp-proximity test generates false-negative
1415     //    -- by other
1416     //
1417 
1418     Label IsInflated, DONE_LABEL;
1419 
1420     // it's stack-locked, biased or neutral
1421     // TODO: optimize away redundant LDs of obj->mark and improve the markword triage
1422     // order to reduce the number of conditional branches in the most common cases.
1423     // Beware -- there's a subtle invariant that fetch of the markword
1424     // at [FETCH], below, will never observe a biased encoding (*101b).
1425     // If this invariant is not held we risk exclusion (safety) failure.
1426     if (UseBiasedLocking && !UseOptoBiasInlining) {
1427       biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, counters);
1428     }
1429 
1430     movptr(tmpReg, Address(objReg, 0));          // [FETCH]
1431     testl (tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased
1432     jccb  (Assembler::notZero, IsInflated);
1433 
1434     // Attempt stack-locking ...
1435     orptr (tmpReg, 0x1);
1436     movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
1437     if (os::is_MP()) {
1438       lock();
1439     }
1440     cmpxchgptr(boxReg, Address(objReg, 0));      // Updates tmpReg
1441     if (counters != NULL) {
1442       cond_inc32(Assembler::equal,
1443                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
1444     }
1445     jccb(Assembler::equal, DONE_LABEL);
1446 
1447     // Recursive locking
1448     subptr(tmpReg, rsp);
1449     andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
1450     movptr(Address(boxReg, 0), tmpReg);
1451     if (counters != NULL) {
1452       cond_inc32(Assembler::equal,
1453                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
1454     }
1455     jmpb(DONE_LABEL);
1456 
1457     bind(IsInflated);
1458 #ifndef _LP64
1459     // The object is inflated.
1460     //
1461     // TODO-FIXME: eliminate the ugly use of manifest constants:
1462     //   Use markOopDesc::monitor_value instead of "2".
1463     //   use markOop::unused_mark() instead of "3".
1464     // The tmpReg value is an objectMonitor reference ORed with
1465     // markOopDesc::monitor_value (2).   We can either convert tmpReg to an
1466     // objectmonitor pointer by masking off the "2" bit or we can just
1467     // use tmpReg as an objectmonitor pointer but bias the objectmonitor
1468     // field offsets with "-2" to compensate for and annul the low-order tag bit.
1469     //
1470     // I use the latter as it avoids AGI stalls.
1471     // As such, we write "mov r, [tmpReg+OFFSETOF(Owner)-2]"
1472     // instead of "mov r, [tmpReg+OFFSETOF(Owner)]".
1473     //
1474     #define OFFSET_SKEWED(f) ((ObjectMonitor::f ## _offset_in_bytes())-2)
1475 
1476     // boxReg refers to the on-stack BasicLock in the current frame.
1477     // We'd like to write:
1478     //   set box->_displaced_header = markOop::unused_mark().  Any non-0 value suffices.
1479     // This is convenient but results a ST-before-CAS penalty.  The following CAS suffers
1480     // additional latency as we have another ST in the store buffer that must drain.
1481 
1482     if (EmitSync & 8192) {
1483        movptr(Address(boxReg, 0), 3);            // results in ST-before-CAS penalty
1484        get_thread (scrReg);
1485        movptr(boxReg, tmpReg);                    // consider: LEA box, [tmp-2]
1486        movptr(tmpReg, NULL_WORD);                 // consider: xor vs mov
1487        if (os::is_MP()) {
1488          lock();
1489        }
1490        cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
1491     } else
1492     if ((EmitSync & 128) == 0) {                      // avoid ST-before-CAS
1493        movptr(scrReg, boxReg);
1494        movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2]
1495 
1496        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
1497        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
1498           // prefetchw [eax + Offset(_owner)-2]
1499           prefetchw(Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
1500        }
1501 
1502        if ((EmitSync & 64) == 0) {
1503          // Optimistic form: consider XORL tmpReg,tmpReg
1504          movptr(tmpReg, NULL_WORD);
1505        } else {
1506          // Can suffer RTS->RTO upgrades on shared or cold $ lines
1507          // Test-And-CAS instead of CAS
1508          movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));   // rax, = m->_owner
1509          testptr(tmpReg, tmpReg);                   // Locked ?
1510          jccb  (Assembler::notZero, DONE_LABEL);
1511        }
1512 
1513        // Appears unlocked - try to swing _owner from null to non-null.
1514        // Ideally, I'd manifest "Self" with get_thread and then attempt
1515        // to CAS the register containing Self into m->Owner.
1516        // But we don't have enough registers, so instead we can either try to CAS
1517        // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
1518        // we later store "Self" into m->Owner.  Transiently storing a stack address
1519        // (rsp or the address of the box) into  m->owner is harmless.
1520        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1521        if (os::is_MP()) {
1522          lock();
1523        }
1524        cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
1525        movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3
1526        jccb  (Assembler::notZero, DONE_LABEL);
1527        get_thread (scrReg);                    // beware: clobbers ICCs
1528        movptr(Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2), scrReg);
1529        xorptr(boxReg, boxReg);                 // set icc.ZFlag = 1 to indicate success
1530 
1531        // If the CAS fails we can either retry or pass control to the slow-path.
1532        // We use the latter tactic.
1533        // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1534        // If the CAS was successful ...
1535        //   Self has acquired the lock
1536        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1537        // Intentional fall-through into DONE_LABEL ...
1538     } else {
1539        movptr(Address(boxReg, 0), intptr_t(markOopDesc::unused_mark()));  // results in ST-before-CAS penalty
1540        movptr(boxReg, tmpReg);
1541 
1542        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
1543        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
1544           // prefetchw [eax + Offset(_owner)-2]
1545           prefetchw(Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
1546        }
1547 
1548        if ((EmitSync & 64) == 0) {
1549          // Optimistic form
1550          xorptr  (tmpReg, tmpReg);
1551        } else {
1552          // Can suffer RTS->RTO upgrades on shared or cold $ lines
1553          movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));   // rax, = m->_owner
1554          testptr(tmpReg, tmpReg);                   // Locked ?
1555          jccb  (Assembler::notZero, DONE_LABEL);
1556        }
1557 
1558        // Appears unlocked - try to swing _owner from null to non-null.
1559        // Use either "Self" (in scr) or rsp as thread identity in _owner.
1560        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1561        get_thread (scrReg);
1562        if (os::is_MP()) {
1563          lock();
1564        }
1565        cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
1566 
1567        // If the CAS fails we can either retry or pass control to the slow-path.
1568        // We use the latter tactic.
1569        // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1570        // If the CAS was successful ...
1571        //   Self has acquired the lock
1572        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1573        // Intentional fall-through into DONE_LABEL ...
1574     }
1575 #else // _LP64
1576     // It's inflated
1577 
1578     // TODO: someday avoid the ST-before-CAS penalty by
1579     // relocating (deferring) the following ST.
1580     // We should also think about trying a CAS without having
1581     // fetched _owner.  If the CAS is successful we may
1582     // avoid an RTO->RTS upgrade on the $line.
1583 
1584     // Without cast to int32_t a movptr will destroy r10 which is typically obj
1585     movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1586 
1587     mov    (boxReg, tmpReg);
1588     movptr (tmpReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
1589     testptr(tmpReg, tmpReg);
1590     jccb   (Assembler::notZero, DONE_LABEL);
1591 
1592     // It's inflated and appears unlocked
1593     if (os::is_MP()) {
1594       lock();
1595     }
1596     cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
1597     // Intentional fall-through into DONE_LABEL ...
1598 
1599 #endif
1600 
1601     // DONE_LABEL is a hot target - we'd really like to place it at the
1602     // start of cache line by padding with NOPs.
1603     // See the AMD and Intel software optimization manuals for the
1604     // most efficient "long" NOP encodings.
1605     // Unfortunately none of our alignment mechanisms suffice.
1606     bind(DONE_LABEL);
1607 
1608     // At DONE_LABEL the icc ZFlag is set as follows ...
1609     // Fast_Unlock uses the same protocol.
1610     // ZFlag == 1 -> Success
1611     // ZFlag == 0 -> Failure - force control through the slow-path
1612   }
1613 }
1614 
1615 // obj: object to unlock
1616 // box: box address (displaced header location), killed.  Must be EAX.
1617 // tmp: killed, cannot be obj nor box.
1618 //
1619 // Some commentary on balanced locking:
1620 //
1621 // Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites.
1622 // Methods that don't have provably balanced locking are forced to run in the
1623 // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
1624 // The interpreter provides two properties:
1625 // I1:  At return-time the interpreter automatically and quietly unlocks any
1626 //      objects acquired the current activation (frame).  Recall that the
1627 //      interpreter maintains an on-stack list of locks currently held by
1628 //      a frame.
1629 // I2:  If a method attempts to unlock an object that is not held by the
1630 //      the frame the interpreter throws IMSX.
1631 //
1632 // Lets say A(), which has provably balanced locking, acquires O and then calls B().
1633 // B() doesn't have provably balanced locking so it runs in the interpreter.
1634 // Control returns to A() and A() unlocks O.  By I1 and I2, above, we know that O
1635 // is still locked by A().
1636 //
1637 // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:
1638 // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
1639 // should not be unlocked by "normal" java-level locking and vice-versa.  The specification
1640 // doesn't specify what will occur if a program engages in such mixed-mode locking, however.
1641 
1642 void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg) {
1643   guarantee (objReg != boxReg, "");
1644   guarantee (objReg != tmpReg, "");
1645   guarantee (boxReg != tmpReg, "");
1646   guarantee (boxReg == rax, "");
1647 
1648   if (EmitSync & 4) {
1649     // Disable - inhibit all inlining.  Force control through the slow-path
1650     cmpptr (rsp, 0);
1651   } else
1652   if (EmitSync & 8) {
1653     Label DONE_LABEL;
1654     if (UseBiasedLocking) {
1655        biased_locking_exit(objReg, tmpReg, DONE_LABEL);
1656     }
1657     // Classic stack-locking code ...
1658     // Check whether the displaced header is 0
1659     //(=> recursive unlock)
1660     movptr(tmpReg, Address(boxReg, 0));
1661     testptr(tmpReg, tmpReg);
1662     jccb(Assembler::zero, DONE_LABEL);
1663     // If not recursive lock, reset the header to displaced header
1664     if (os::is_MP()) {
1665       lock();
1666     }
1667     cmpxchgptr(tmpReg, Address(objReg, 0));   // Uses RAX which is box
1668     bind(DONE_LABEL);
1669   } else {
1670     Label DONE_LABEL, Stacked, CheckSucc;
1671 
1672     // Critically, the biased locking test must have precedence over
1673     // and appear before the (box->dhw == 0) recursive stack-lock test.
1674     if (UseBiasedLocking && !UseOptoBiasInlining) {
1675        biased_locking_exit(objReg, tmpReg, DONE_LABEL);
1676     }
1677 
1678     cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
1679     movptr(tmpReg, Address(objReg, 0));             // Examine the object's markword
1680     jccb  (Assembler::zero, DONE_LABEL);            // 0 indicates recursive stack-lock
1681 
1682     testptr(tmpReg, 0x02);                          // Inflated?
1683     jccb  (Assembler::zero, Stacked);
1684 
1685     // It's inflated.
1686     // Despite our balanced locking property we still check that m->_owner == Self
1687     // as java routines or native JNI code called by this thread might
1688     // have released the lock.
1689     // Refer to the comments in synchronizer.cpp for how we might encode extra
1690     // state in _succ so we can avoid fetching EntryList|cxq.
1691     //
1692     // I'd like to add more cases in fast_lock() and fast_unlock() --
1693     // such as recursive enter and exit -- but we have to be wary of
1694     // I$ bloat, T$ effects and BP$ effects.
1695     //
1696     // If there's no contention try a 1-0 exit.  That is, exit without
1697     // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how
1698     // we detect and recover from the race that the 1-0 exit admits.
1699     //
1700     // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier
1701     // before it STs null into _owner, releasing the lock.  Updates
1702     // to data protected by the critical section must be visible before
1703     // we drop the lock (and thus before any other thread could acquire
1704     // the lock and observe the fields protected by the lock).
1705     // IA32's memory-model is SPO, so STs are ordered with respect to
1706     // each other and there's no need for an explicit barrier (fence).
1707     // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
1708 #ifndef _LP64
1709     get_thread (boxReg);
1710     if ((EmitSync & 4096) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
1711       // prefetchw [ebx + Offset(_owner)-2]
1712       prefetchw(Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
1713     }
1714 
1715     // Note that we could employ various encoding schemes to reduce
1716     // the number of loads below (currently 4) to just 2 or 3.
1717     // Refer to the comments in synchronizer.cpp.
1718     // In practice the chain of fetches doesn't seem to impact performance, however.
1719     if ((EmitSync & 65536) == 0 && (EmitSync & 256)) {
1720        // Attempt to reduce branch density - AMD's branch predictor.
1721        xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
1722        orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2));
1723        orptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2));
1724        orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2));
1725        jccb  (Assembler::notZero, DONE_LABEL);
1726        movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD);
1727        jmpb  (DONE_LABEL);
1728     } else {
1729        xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
1730        orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2));
1731        jccb  (Assembler::notZero, DONE_LABEL);
1732        movptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2));
1733        orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2));
1734        jccb  (Assembler::notZero, CheckSucc);
1735        movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD);
1736        jmpb  (DONE_LABEL);
1737     }
1738 
1739     // The Following code fragment (EmitSync & 65536) improves the performance of
1740     // contended applications and contended synchronization microbenchmarks.
1741     // Unfortunately the emission of the code - even though not executed - causes regressions
1742     // in scimark and jetstream, evidently because of $ effects.  Replacing the code
1743     // with an equal number of never-executed NOPs results in the same regression.
1744     // We leave it off by default.
1745 
1746     if ((EmitSync & 65536) != 0) {
1747        Label LSuccess, LGoSlowPath ;
1748 
1749        bind  (CheckSucc);
1750 
1751        // Optional pre-test ... it's safe to elide this
1752        if ((EmitSync & 16) == 0) {
1753           cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD);
1754           jccb  (Assembler::zero, LGoSlowPath);
1755        }
1756 
1757        // We have a classic Dekker-style idiom:
1758        //    ST m->_owner = 0 ; MEMBAR; LD m->_succ
1759        // There are a number of ways to implement the barrier:
1760        // (1) lock:andl &m->_owner, 0
1761        //     is fast, but mask doesn't currently support the "ANDL M,IMM32" form.
1762        //     LOCK: ANDL [ebx+Offset(_Owner)-2], 0
1763        //     Encodes as 81 31 OFF32 IMM32 or 83 63 OFF8 IMM8
1764        // (2) If supported, an explicit MFENCE is appealing.
1765        //     In older IA32 processors MFENCE is slower than lock:add or xchg
1766        //     particularly if the write-buffer is full as might be the case if
1767        //     if stores closely precede the fence or fence-equivalent instruction.
1768        //     In more modern implementations MFENCE appears faster, however.
1769        // (3) In lieu of an explicit fence, use lock:addl to the top-of-stack
1770        //     The $lines underlying the top-of-stack should be in M-state.
1771        //     The locked add instruction is serializing, of course.
1772        // (4) Use xchg, which is serializing
1773        //     mov boxReg, 0; xchgl boxReg, [tmpReg + Offset(_owner)-2] also works
1774        // (5) ST m->_owner = 0 and then execute lock:orl &m->_succ, 0.
1775        //     The integer condition codes will tell us if succ was 0.
1776        //     Since _succ and _owner should reside in the same $line and
1777        //     we just stored into _owner, it's likely that the $line
1778        //     remains in M-state for the lock:orl.
1779        //
1780        // We currently use (3), although it's likely that switching to (2)
1781        // is correct for the future.
1782 
1783        movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), NULL_WORD);
1784        if (os::is_MP()) {
1785           if (VM_Version::supports_sse2() && 1 == FenceInstruction) {
1786             mfence();
1787           } else {
1788             lock (); addptr(Address(rsp, 0), 0);
1789           }
1790        }
1791        // Ratify _succ remains non-null
1792        cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0);
1793        jccb  (Assembler::notZero, LSuccess);
1794 
1795        xorptr(boxReg, boxReg);                  // box is really EAX
1796        if (os::is_MP()) { lock(); }
1797        cmpxchgptr(rsp, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
1798        jccb  (Assembler::notEqual, LSuccess);
1799        // Since we're low on registers we installed rsp as a placeholding in _owner.
1800        // Now install Self over rsp.  This is safe as we're transitioning from
1801        // non-null to non=null
1802        get_thread (boxReg);
1803        movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), boxReg);
1804        // Intentional fall-through into LGoSlowPath ...
1805 
1806        bind  (LGoSlowPath);
1807        orptr(boxReg, 1);                      // set ICC.ZF=0 to indicate failure
1808        jmpb  (DONE_LABEL);
1809 
1810        bind  (LSuccess);
1811        xorptr(boxReg, boxReg);                 // set ICC.ZF=1 to indicate success
1812        jmpb  (DONE_LABEL);
1813     }
1814 
1815     bind (Stacked);
1816     // It's not inflated and it's not recursively stack-locked and it's not biased.
1817     // It must be stack-locked.
1818     // Try to reset the header to displaced header.
1819     // The "box" value on the stack is stable, so we can reload
1820     // and be assured we observe the same value as above.
1821     movptr(tmpReg, Address(boxReg, 0));
1822     if (os::is_MP()) {
1823       lock();
1824     }
1825     cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
1826     // Intention fall-thru into DONE_LABEL
1827 
1828     // DONE_LABEL is a hot target - we'd really like to place it at the
1829     // start of cache line by padding with NOPs.
1830     // See the AMD and Intel software optimization manuals for the
1831     // most efficient "long" NOP encodings.
1832     // Unfortunately none of our alignment mechanisms suffice.
1833     if ((EmitSync & 65536) == 0) {
1834        bind (CheckSucc);
1835     }
1836 #else // _LP64
1837     // It's inflated
1838     movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
1839     xorptr(boxReg, r15_thread);
1840     orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2));
1841     jccb  (Assembler::notZero, DONE_LABEL);
1842     movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2));
1843     orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2));
1844     jccb  (Assembler::notZero, CheckSucc);
1845     movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD);
1846     jmpb  (DONE_LABEL);
1847 
1848     if ((EmitSync & 65536) == 0) {
1849       Label LSuccess, LGoSlowPath ;
1850       bind  (CheckSucc);
1851       cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD);
1852       jccb  (Assembler::zero, LGoSlowPath);
1853 
1854       // I'd much rather use lock:andl m->_owner, 0 as it's faster than the
1855       // the explicit ST;MEMBAR combination, but masm doesn't currently support
1856       // "ANDQ M,IMM".  Don't use MFENCE here.  lock:add to TOS, xchg, etc
1857       // are all faster when the write buffer is populated.
1858       movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD);
1859       if (os::is_MP()) {
1860          lock (); addl (Address(rsp, 0), 0);
1861       }
1862       cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD);
1863       jccb  (Assembler::notZero, LSuccess);
1864 
1865       movptr (boxReg, (int32_t)NULL_WORD);                   // box is really EAX
1866       if (os::is_MP()) { lock(); }
1867       cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
1868       jccb  (Assembler::notEqual, LSuccess);
1869       // Intentional fall-through into slow-path
1870 
1871       bind  (LGoSlowPath);
1872       orl   (boxReg, 1);                      // set ICC.ZF=0 to indicate failure
1873       jmpb  (DONE_LABEL);
1874 
1875       bind  (LSuccess);
1876       testl (boxReg, 0);                      // set ICC.ZF=1 to indicate success
1877       jmpb  (DONE_LABEL);
1878     }
1879 
1880     bind  (Stacked);
1881     movptr(tmpReg, Address (boxReg, 0));      // re-fetch
1882     if (os::is_MP()) { lock(); }
1883     cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
1884 
1885     if (EmitSync & 65536) {
1886        bind (CheckSucc);
1887     }
1888 #endif
1889     bind(DONE_LABEL);
1890     // Avoid branch to branch on AMD processors
1891     if (EmitSync & 32768) {
1892        nop();
1893     }
1894   }
1895 }
1896 #endif // COMPILER2
1897 
1898 void MacroAssembler::c2bool(Register x) {
1899   // implements x == 0 ? 0 : 1
1900   // note: must only look at least-significant byte of x
1901   //       since C-style booleans are stored in one byte
1902   //       only! (was bug)
1903   andl(x, 0xFF);
1904   setb(Assembler::notZero, x);
1905 }
1906 
1907 // Wouldn't need if AddressLiteral version had new name
1908 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
1909   Assembler::call(L, rtype);
1910 }
1911 
1912 void MacroAssembler::call(Register entry) {
1913   Assembler::call(entry);
1914 }
1915 
1916 void MacroAssembler::call(AddressLiteral entry) {
1917   if (reachable(entry)) {
1918     Assembler::call_literal(entry.target(), entry.rspec());
1919   } else {
1920     lea(rscratch1, entry);
1921     Assembler::call(rscratch1);
1922   }
1923 }
1924 
1925 void MacroAssembler::ic_call(address entry) {
1926   RelocationHolder rh = virtual_call_Relocation::spec(pc());
1927   movptr(rax, (intptr_t)Universe::non_oop_word());
1928   call(AddressLiteral(entry, rh));
1929 }
1930 
1931 // Implementation of call_VM versions
1932 
1933 void MacroAssembler::call_VM(Register oop_result,
1934                              address entry_point,
1935                              bool check_exceptions) {
1936   Label C, E;
1937   call(C, relocInfo::none);
1938   jmp(E);
1939 
1940   bind(C);
1941   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
1942   ret(0);
1943 
1944   bind(E);
1945 }
1946 
1947 void MacroAssembler::call_VM(Register oop_result,
1948                              address entry_point,
1949                              Register arg_1,
1950                              bool check_exceptions) {
1951   Label C, E;
1952   call(C, relocInfo::none);
1953   jmp(E);
1954 
1955   bind(C);
1956   pass_arg1(this, arg_1);
1957   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
1958   ret(0);
1959 
1960   bind(E);
1961 }
1962 
1963 void MacroAssembler::call_VM(Register oop_result,
1964                              address entry_point,
1965                              Register arg_1,
1966                              Register arg_2,
1967                              bool check_exceptions) {
1968   Label C, E;
1969   call(C, relocInfo::none);
1970   jmp(E);
1971 
1972   bind(C);
1973 
1974   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1975 
1976   pass_arg2(this, arg_2);
1977   pass_arg1(this, arg_1);
1978   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
1979   ret(0);
1980 
1981   bind(E);
1982 }
1983 
1984 void MacroAssembler::call_VM(Register oop_result,
1985                              address entry_point,
1986                              Register arg_1,
1987                              Register arg_2,
1988                              Register arg_3,
1989                              bool check_exceptions) {
1990   Label C, E;
1991   call(C, relocInfo::none);
1992   jmp(E);
1993 
1994   bind(C);
1995 
1996   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
1997   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
1998   pass_arg3(this, arg_3);
1999 
2000   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2001   pass_arg2(this, arg_2);
2002 
2003   pass_arg1(this, arg_1);
2004   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
2005   ret(0);
2006 
2007   bind(E);
2008 }
2009 
2010 void MacroAssembler::call_VM(Register oop_result,
2011                              Register last_java_sp,
2012                              address entry_point,
2013                              int number_of_arguments,
2014                              bool check_exceptions) {
2015   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2016   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2017 }
2018 
2019 void MacroAssembler::call_VM(Register oop_result,
2020                              Register last_java_sp,
2021                              address entry_point,
2022                              Register arg_1,
2023                              bool check_exceptions) {
2024   pass_arg1(this, arg_1);
2025   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2026 }
2027 
2028 void MacroAssembler::call_VM(Register oop_result,
2029                              Register last_java_sp,
2030                              address entry_point,
2031                              Register arg_1,
2032                              Register arg_2,
2033                              bool check_exceptions) {
2034 
2035   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2036   pass_arg2(this, arg_2);
2037   pass_arg1(this, arg_1);
2038   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2039 }
2040 
2041 void MacroAssembler::call_VM(Register oop_result,
2042                              Register last_java_sp,
2043                              address entry_point,
2044                              Register arg_1,
2045                              Register arg_2,
2046                              Register arg_3,
2047                              bool check_exceptions) {
2048   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2049   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2050   pass_arg3(this, arg_3);
2051   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2052   pass_arg2(this, arg_2);
2053   pass_arg1(this, arg_1);
2054   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2055 }
2056 
2057 void MacroAssembler::super_call_VM(Register oop_result,
2058                                    Register last_java_sp,
2059                                    address entry_point,
2060                                    int number_of_arguments,
2061                                    bool check_exceptions) {
2062   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2063   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2064 }
2065 
2066 void MacroAssembler::super_call_VM(Register oop_result,
2067                                    Register last_java_sp,
2068                                    address entry_point,
2069                                    Register arg_1,
2070                                    bool check_exceptions) {
2071   pass_arg1(this, arg_1);
2072   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2073 }
2074 
2075 void MacroAssembler::super_call_VM(Register oop_result,
2076                                    Register last_java_sp,
2077                                    address entry_point,
2078                                    Register arg_1,
2079                                    Register arg_2,
2080                                    bool check_exceptions) {
2081 
2082   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2083   pass_arg2(this, arg_2);
2084   pass_arg1(this, arg_1);
2085   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2086 }
2087 
2088 void MacroAssembler::super_call_VM(Register oop_result,
2089                                    Register last_java_sp,
2090                                    address entry_point,
2091                                    Register arg_1,
2092                                    Register arg_2,
2093                                    Register arg_3,
2094                                    bool check_exceptions) {
2095   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2096   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2097   pass_arg3(this, arg_3);
2098   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2099   pass_arg2(this, arg_2);
2100   pass_arg1(this, arg_1);
2101   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2102 }
2103 
2104 void MacroAssembler::call_VM_base(Register oop_result,
2105                                   Register java_thread,
2106                                   Register last_java_sp,
2107                                   address  entry_point,
2108                                   int      number_of_arguments,
2109                                   bool     check_exceptions) {
2110   // determine java_thread register
2111   if (!java_thread->is_valid()) {
2112 #ifdef _LP64
2113     java_thread = r15_thread;
2114 #else
2115     java_thread = rdi;
2116     get_thread(java_thread);
2117 #endif // LP64
2118   }
2119   // determine last_java_sp register
2120   if (!last_java_sp->is_valid()) {
2121     last_java_sp = rsp;
2122   }
2123   // debugging support
2124   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
2125   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
2126 #ifdef ASSERT
2127   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
2128   // r12 is the heapbase.
2129   LP64_ONLY(if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
2130 #endif // ASSERT
2131 
2132   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
2133   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
2134 
2135   // push java thread (becomes first argument of C function)
2136 
2137   NOT_LP64(push(java_thread); number_of_arguments++);
2138   LP64_ONLY(mov(c_rarg0, r15_thread));
2139 
2140   // set last Java frame before call
2141   assert(last_java_sp != rbp, "can't use ebp/rbp");
2142 
2143   // Only interpreter should have to set fp
2144   set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);
2145 
2146   // do the call, remove parameters
2147   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
2148 
2149   // restore the thread (cannot use the pushed argument since arguments
2150   // may be overwritten by C code generated by an optimizing compiler);
2151   // however can use the register value directly if it is callee saved.
2152   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
2153     // rdi & rsi (also r15) are callee saved -> nothing to do
2154 #ifdef ASSERT
2155     guarantee(java_thread != rax, "change this code");
2156     push(rax);
2157     { Label L;
2158       get_thread(rax);
2159       cmpptr(java_thread, rax);
2160       jcc(Assembler::equal, L);
2161       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
2162       bind(L);
2163     }
2164     pop(rax);
2165 #endif
2166   } else {
2167     get_thread(java_thread);
2168   }
2169   // reset last Java frame
2170   // Only interpreter should have to clear fp
2171   reset_last_Java_frame(java_thread, true, false);
2172 
2173 #ifndef CC_INTERP
2174    // C++ interp handles this in the interpreter
2175   check_and_handle_popframe(java_thread);
2176   check_and_handle_earlyret(java_thread);
2177 #endif /* CC_INTERP */
2178 
2179   if (check_exceptions) {
2180     // check for pending exceptions (java_thread is set upon return)
2181     cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
2182 #ifndef _LP64
2183     jump_cc(Assembler::notEqual,
2184             RuntimeAddress(StubRoutines::forward_exception_entry()));
2185 #else
2186     // This used to conditionally jump to forward_exception however it is
2187     // possible if we relocate that the branch will not reach. So we must jump
2188     // around so we can always reach
2189 
2190     Label ok;
2191     jcc(Assembler::equal, ok);
2192     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2193     bind(ok);
2194 #endif // LP64
2195   }
2196 
2197   // get oop result if there is one and reset the value in the thread
2198   if (oop_result->is_valid()) {
2199     get_vm_result(oop_result, java_thread);
2200   }
2201 }
2202 
2203 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
2204 
2205   // Calculate the value for last_Java_sp
2206   // somewhat subtle. call_VM does an intermediate call
2207   // which places a return address on the stack just under the
2208   // stack pointer as the user finsihed with it. This allows
2209   // use to retrieve last_Java_pc from last_Java_sp[-1].
2210   // On 32bit we then have to push additional args on the stack to accomplish
2211   // the actual requested call. On 64bit call_VM only can use register args
2212   // so the only extra space is the return address that call_VM created.
2213   // This hopefully explains the calculations here.
2214 
2215 #ifdef _LP64
2216   // We've pushed one address, correct last_Java_sp
2217   lea(rax, Address(rsp, wordSize));
2218 #else
2219   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
2220 #endif // LP64
2221 
2222   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
2223 
2224 }
2225 
2226 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
2227   call_VM_leaf_base(entry_point, number_of_arguments);
2228 }
2229 
2230 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
2231   pass_arg0(this, arg_0);
2232   call_VM_leaf(entry_point, 1);
2233 }
2234 
2235 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2236 
2237   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2238   pass_arg1(this, arg_1);
2239   pass_arg0(this, arg_0);
2240   call_VM_leaf(entry_point, 2);
2241 }
2242 
2243 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2244   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2245   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2246   pass_arg2(this, arg_2);
2247   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2248   pass_arg1(this, arg_1);
2249   pass_arg0(this, arg_0);
2250   call_VM_leaf(entry_point, 3);
2251 }
2252 
2253 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
2254   pass_arg0(this, arg_0);
2255   MacroAssembler::call_VM_leaf_base(entry_point, 1);
2256 }
2257 
2258 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2259 
2260   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2261   pass_arg1(this, arg_1);
2262   pass_arg0(this, arg_0);
2263   MacroAssembler::call_VM_leaf_base(entry_point, 2);
2264 }
2265 
2266 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2267   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2268   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2269   pass_arg2(this, arg_2);
2270   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2271   pass_arg1(this, arg_1);
2272   pass_arg0(this, arg_0);
2273   MacroAssembler::call_VM_leaf_base(entry_point, 3);
2274 }
2275 
2276 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
2277   LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg"));
2278   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2279   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2280   pass_arg3(this, arg_3);
2281   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2282   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2283   pass_arg2(this, arg_2);
2284   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2285   pass_arg1(this, arg_1);
2286   pass_arg0(this, arg_0);
2287   MacroAssembler::call_VM_leaf_base(entry_point, 4);
2288 }
2289 
2290 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
2291   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
2292   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
2293   verify_oop(oop_result, "broken oop in call_VM_base");
2294 }
2295 
2296 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
2297   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
2298   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
2299 }
2300 
2301 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
2302 }
2303 
2304 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
2305 }
2306 
2307 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) {
2308   if (reachable(src1)) {
2309     cmpl(as_Address(src1), imm);
2310   } else {
2311     lea(rscratch1, src1);
2312     cmpl(Address(rscratch1, 0), imm);
2313   }
2314 }
2315 
2316 void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
2317   assert(!src2.is_lval(), "use cmpptr");
2318   if (reachable(src2)) {
2319     cmpl(src1, as_Address(src2));
2320   } else {
2321     lea(rscratch1, src2);
2322     cmpl(src1, Address(rscratch1, 0));
2323   }
2324 }
2325 
2326 void MacroAssembler::cmp32(Register src1, int32_t imm) {
2327   Assembler::cmpl(src1, imm);
2328 }
2329 
2330 void MacroAssembler::cmp32(Register src1, Address src2) {
2331   Assembler::cmpl(src1, src2);
2332 }
2333 
2334 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2335   ucomisd(opr1, opr2);
2336 
2337   Label L;
2338   if (unordered_is_less) {
2339     movl(dst, -1);
2340     jcc(Assembler::parity, L);
2341     jcc(Assembler::below , L);
2342     movl(dst, 0);
2343     jcc(Assembler::equal , L);
2344     increment(dst);
2345   } else { // unordered is greater
2346     movl(dst, 1);
2347     jcc(Assembler::parity, L);
2348     jcc(Assembler::above , L);
2349     movl(dst, 0);
2350     jcc(Assembler::equal , L);
2351     decrementl(dst);
2352   }
2353   bind(L);
2354 }
2355 
2356 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2357   ucomiss(opr1, opr2);
2358 
2359   Label L;
2360   if (unordered_is_less) {
2361     movl(dst, -1);
2362     jcc(Assembler::parity, L);
2363     jcc(Assembler::below , L);
2364     movl(dst, 0);
2365     jcc(Assembler::equal , L);
2366     increment(dst);
2367   } else { // unordered is greater
2368     movl(dst, 1);
2369     jcc(Assembler::parity, L);
2370     jcc(Assembler::above , L);
2371     movl(dst, 0);
2372     jcc(Assembler::equal , L);
2373     decrementl(dst);
2374   }
2375   bind(L);
2376 }
2377 
2378 
2379 void MacroAssembler::cmp8(AddressLiteral src1, int imm) {
2380   if (reachable(src1)) {
2381     cmpb(as_Address(src1), imm);
2382   } else {
2383     lea(rscratch1, src1);
2384     cmpb(Address(rscratch1, 0), imm);
2385   }
2386 }
2387 
2388 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
2389 #ifdef _LP64
2390   if (src2.is_lval()) {
2391     movptr(rscratch1, src2);
2392     Assembler::cmpq(src1, rscratch1);
2393   } else if (reachable(src2)) {
2394     cmpq(src1, as_Address(src2));
2395   } else {
2396     lea(rscratch1, src2);
2397     Assembler::cmpq(src1, Address(rscratch1, 0));
2398   }
2399 #else
2400   if (src2.is_lval()) {
2401     cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2402   } else {
2403     cmpl(src1, as_Address(src2));
2404   }
2405 #endif // _LP64
2406 }
2407 
2408 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
2409   assert(src2.is_lval(), "not a mem-mem compare");
2410 #ifdef _LP64
2411   // moves src2's literal address
2412   movptr(rscratch1, src2);
2413   Assembler::cmpq(src1, rscratch1);
2414 #else
2415   cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2416 #endif // _LP64
2417 }
2418 
2419 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) {
2420   if (reachable(adr)) {
2421     if (os::is_MP())
2422       lock();
2423     cmpxchgptr(reg, as_Address(adr));
2424   } else {
2425     lea(rscratch1, adr);
2426     if (os::is_MP())
2427       lock();
2428     cmpxchgptr(reg, Address(rscratch1, 0));
2429   }
2430 }
2431 
2432 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
2433   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
2434 }
2435 
2436 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {
2437   if (reachable(src)) {
2438     Assembler::comisd(dst, as_Address(src));
2439   } else {
2440     lea(rscratch1, src);
2441     Assembler::comisd(dst, Address(rscratch1, 0));
2442   }
2443 }
2444 
2445 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {
2446   if (reachable(src)) {
2447     Assembler::comiss(dst, as_Address(src));
2448   } else {
2449     lea(rscratch1, src);
2450     Assembler::comiss(dst, Address(rscratch1, 0));
2451   }
2452 }
2453 
2454 
2455 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
2456   Condition negated_cond = negate_condition(cond);
2457   Label L;
2458   jcc(negated_cond, L);
2459   atomic_incl(counter_addr);
2460   bind(L);
2461 }
2462 
2463 int MacroAssembler::corrected_idivl(Register reg) {
2464   // Full implementation of Java idiv and irem; checks for
2465   // special case as described in JVM spec., p.243 & p.271.
2466   // The function returns the (pc) offset of the idivl
2467   // instruction - may be needed for implicit exceptions.
2468   //
2469   //         normal case                           special case
2470   //
2471   // input : rax,: dividend                         min_int
2472   //         reg: divisor   (may not be rax,/rdx)   -1
2473   //
2474   // output: rax,: quotient  (= rax, idiv reg)       min_int
2475   //         rdx: remainder (= rax, irem reg)       0
2476   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
2477   const int min_int = 0x80000000;
2478   Label normal_case, special_case;
2479 
2480   // check for special case
2481   cmpl(rax, min_int);
2482   jcc(Assembler::notEqual, normal_case);
2483   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
2484   cmpl(reg, -1);
2485   jcc(Assembler::equal, special_case);
2486 
2487   // handle normal case
2488   bind(normal_case);
2489   cdql();
2490   int idivl_offset = offset();
2491   idivl(reg);
2492 
2493   // normal and special case exit
2494   bind(special_case);
2495 
2496   return idivl_offset;
2497 }
2498 
2499 
2500 
2501 void MacroAssembler::decrementl(Register reg, int value) {
2502   if (value == min_jint) {subl(reg, value) ; return; }
2503   if (value <  0) { incrementl(reg, -value); return; }
2504   if (value == 0) {                        ; return; }
2505   if (value == 1 && UseIncDec) { decl(reg) ; return; }
2506   /* else */      { subl(reg, value)       ; return; }
2507 }
2508 
2509 void MacroAssembler::decrementl(Address dst, int value) {
2510   if (value == min_jint) {subl(dst, value) ; return; }
2511   if (value <  0) { incrementl(dst, -value); return; }
2512   if (value == 0) {                        ; return; }
2513   if (value == 1 && UseIncDec) { decl(dst) ; return; }
2514   /* else */      { subl(dst, value)       ; return; }
2515 }
2516 
2517 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
2518   assert (shift_value > 0, "illegal shift value");
2519   Label _is_positive;
2520   testl (reg, reg);
2521   jcc (Assembler::positive, _is_positive);
2522   int offset = (1 << shift_value) - 1 ;
2523 
2524   if (offset == 1) {
2525     incrementl(reg);
2526   } else {
2527     addl(reg, offset);
2528   }
2529 
2530   bind (_is_positive);
2531   sarl(reg, shift_value);
2532 }
2533 
2534 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
2535   if (reachable(src)) {
2536     Assembler::divsd(dst, as_Address(src));
2537   } else {
2538     lea(rscratch1, src);
2539     Assembler::divsd(dst, Address(rscratch1, 0));
2540   }
2541 }
2542 
2543 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
2544   if (reachable(src)) {
2545     Assembler::divss(dst, as_Address(src));
2546   } else {
2547     lea(rscratch1, src);
2548     Assembler::divss(dst, Address(rscratch1, 0));
2549   }
2550 }
2551 
2552 // !defined(COMPILER2) is because of stupid core builds
2553 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2)
2554 void MacroAssembler::empty_FPU_stack() {
2555   if (VM_Version::supports_mmx()) {
2556     emms();
2557   } else {
2558     for (int i = 8; i-- > 0; ) ffree(i);
2559   }
2560 }
2561 #endif // !LP64 || C1 || !C2
2562 
2563 
2564 // Defines obj, preserves var_size_in_bytes
2565 void MacroAssembler::eden_allocate(Register obj,
2566                                    Register var_size_in_bytes,
2567                                    int con_size_in_bytes,
2568                                    Register t1,
2569                                    Label& slow_case) {
2570   assert(obj == rax, "obj must be in rax, for cmpxchg");
2571   assert_different_registers(obj, var_size_in_bytes, t1);
2572   if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
2573     jmp(slow_case);
2574   } else {
2575     Register end = t1;
2576     Label retry;
2577     bind(retry);
2578     ExternalAddress heap_top((address) Universe::heap()->top_addr());
2579     movptr(obj, heap_top);
2580     if (var_size_in_bytes == noreg) {
2581       lea(end, Address(obj, con_size_in_bytes));
2582     } else {
2583       lea(end, Address(obj, var_size_in_bytes, Address::times_1));
2584     }
2585     // if end < obj then we wrapped around => object too long => slow case
2586     cmpptr(end, obj);
2587     jcc(Assembler::below, slow_case);
2588     cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
2589     jcc(Assembler::above, slow_case);
2590     // Compare obj with the top addr, and if still equal, store the new top addr in
2591     // end at the address of the top addr pointer. Sets ZF if was equal, and clears
2592     // it otherwise. Use lock prefix for atomicity on MPs.
2593     locked_cmpxchgptr(end, heap_top);
2594     jcc(Assembler::notEqual, retry);
2595   }
2596 }
2597 
2598 void MacroAssembler::enter() {
2599   push(rbp);
2600   mov(rbp, rsp);
2601 }
2602 
2603 // A 5 byte nop that is safe for patching (see patch_verified_entry)
2604 void MacroAssembler::fat_nop() {
2605   if (UseAddressNop) {
2606     addr_nop_5();
2607   } else {
2608     emit_int8(0x26); // es:
2609     emit_int8(0x2e); // cs:
2610     emit_int8(0x64); // fs:
2611     emit_int8(0x65); // gs:
2612     emit_int8((unsigned char)0x90);
2613   }
2614 }
2615 
2616 void MacroAssembler::fcmp(Register tmp) {
2617   fcmp(tmp, 1, true, true);
2618 }
2619 
2620 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
2621   assert(!pop_right || pop_left, "usage error");
2622   if (VM_Version::supports_cmov()) {
2623     assert(tmp == noreg, "unneeded temp");
2624     if (pop_left) {
2625       fucomip(index);
2626     } else {
2627       fucomi(index);
2628     }
2629     if (pop_right) {
2630       fpop();
2631     }
2632   } else {
2633     assert(tmp != noreg, "need temp");
2634     if (pop_left) {
2635       if (pop_right) {
2636         fcompp();
2637       } else {
2638         fcomp(index);
2639       }
2640     } else {
2641       fcom(index);
2642     }
2643     // convert FPU condition into eflags condition via rax,
2644     save_rax(tmp);
2645     fwait(); fnstsw_ax();
2646     sahf();
2647     restore_rax(tmp);
2648   }
2649   // condition codes set as follows:
2650   //
2651   // CF (corresponds to C0) if x < y
2652   // PF (corresponds to C2) if unordered
2653   // ZF (corresponds to C3) if x = y
2654 }
2655 
2656 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
2657   fcmp2int(dst, unordered_is_less, 1, true, true);
2658 }
2659 
2660 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
2661   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
2662   Label L;
2663   if (unordered_is_less) {
2664     movl(dst, -1);
2665     jcc(Assembler::parity, L);
2666     jcc(Assembler::below , L);
2667     movl(dst, 0);
2668     jcc(Assembler::equal , L);
2669     increment(dst);
2670   } else { // unordered is greater
2671     movl(dst, 1);
2672     jcc(Assembler::parity, L);
2673     jcc(Assembler::above , L);
2674     movl(dst, 0);
2675     jcc(Assembler::equal , L);
2676     decrementl(dst);
2677   }
2678   bind(L);
2679 }
2680 
2681 void MacroAssembler::fld_d(AddressLiteral src) {
2682   fld_d(as_Address(src));
2683 }
2684 
2685 void MacroAssembler::fld_s(AddressLiteral src) {
2686   fld_s(as_Address(src));
2687 }
2688 
2689 void MacroAssembler::fld_x(AddressLiteral src) {
2690   Assembler::fld_x(as_Address(src));
2691 }
2692 
2693 void MacroAssembler::fldcw(AddressLiteral src) {
2694   Assembler::fldcw(as_Address(src));
2695 }
2696 
2697 void MacroAssembler::pow_exp_core_encoding() {
2698   // kills rax, rcx, rdx
2699   subptr(rsp,sizeof(jdouble));
2700   // computes 2^X. Stack: X ...
2701   // f2xm1 computes 2^X-1 but only operates on -1<=X<=1. Get int(X) and
2702   // keep it on the thread's stack to compute 2^int(X) later
2703   // then compute 2^(X-int(X)) as (2^(X-int(X)-1+1)
2704   // final result is obtained with: 2^X = 2^int(X) * 2^(X-int(X))
2705   fld_s(0);                 // Stack: X X ...
2706   frndint();                // Stack: int(X) X ...
2707   fsuba(1);                 // Stack: int(X) X-int(X) ...
2708   fistp_s(Address(rsp,0));  // move int(X) as integer to thread's stack. Stack: X-int(X) ...
2709   f2xm1();                  // Stack: 2^(X-int(X))-1 ...
2710   fld1();                   // Stack: 1 2^(X-int(X))-1 ...
2711   faddp(1);                 // Stack: 2^(X-int(X))
2712   // computes 2^(int(X)): add exponent bias (1023) to int(X), then
2713   // shift int(X)+1023 to exponent position.
2714   // Exponent is limited to 11 bits if int(X)+1023 does not fit in 11
2715   // bits, set result to NaN. 0x000 and 0x7FF are reserved exponent
2716   // values so detect them and set result to NaN.
2717   movl(rax,Address(rsp,0));
2718   movl(rcx, -2048); // 11 bit mask and valid NaN binary encoding
2719   addl(rax, 1023);
2720   movl(rdx,rax);
2721   shll(rax,20);
2722   // Check that 0 < int(X)+1023 < 2047. Otherwise set rax to NaN.
2723   addl(rdx,1);
2724   // Check that 1 < int(X)+1023+1 < 2048
2725   // in 3 steps:
2726   // 1- (int(X)+1023+1)&-2048 == 0 => 0 <= int(X)+1023+1 < 2048
2727   // 2- (int(X)+1023+1)&-2048 != 0
2728   // 3- (int(X)+1023+1)&-2048 != 1
2729   // Do 2- first because addl just updated the flags.
2730   cmov32(Assembler::equal,rax,rcx);
2731   cmpl(rdx,1);
2732   cmov32(Assembler::equal,rax,rcx);
2733   testl(rdx,rcx);
2734   cmov32(Assembler::notEqual,rax,rcx);
2735   movl(Address(rsp,4),rax);
2736   movl(Address(rsp,0),0);
2737   fmul_d(Address(rsp,0));   // Stack: 2^X ...
2738   addptr(rsp,sizeof(jdouble));
2739 }
2740 
2741 void MacroAssembler::increase_precision() {
2742   subptr(rsp, BytesPerWord);
2743   fnstcw(Address(rsp, 0));
2744   movl(rax, Address(rsp, 0));
2745   orl(rax, 0x300);
2746   push(rax);
2747   fldcw(Address(rsp, 0));
2748   pop(rax);
2749 }
2750 
2751 void MacroAssembler::restore_precision() {
2752   fldcw(Address(rsp, 0));
2753   addptr(rsp, BytesPerWord);
2754 }
2755 
2756 void MacroAssembler::fast_pow() {
2757   // computes X^Y = 2^(Y * log2(X))
2758   // if fast computation is not possible, result is NaN. Requires
2759   // fallback from user of this macro.
2760   // increase precision for intermediate steps of the computation
2761   increase_precision();
2762   fyl2x();                 // Stack: (Y*log2(X)) ...
2763   pow_exp_core_encoding(); // Stack: exp(X) ...
2764   restore_precision();
2765 }
2766 
2767 void MacroAssembler::fast_exp() {
2768   // computes exp(X) = 2^(X * log2(e))
2769   // if fast computation is not possible, result is NaN. Requires
2770   // fallback from user of this macro.
2771   // increase precision for intermediate steps of the computation
2772   increase_precision();
2773   fldl2e();                // Stack: log2(e) X ...
2774   fmulp(1);                // Stack: (X*log2(e)) ...
2775   pow_exp_core_encoding(); // Stack: exp(X) ...
2776   restore_precision();
2777 }
2778 
2779 void MacroAssembler::pow_or_exp(bool is_exp, int num_fpu_regs_in_use) {
2780   // kills rax, rcx, rdx
2781   // pow and exp needs 2 extra registers on the fpu stack.
2782   Label slow_case, done;
2783   Register tmp = noreg;
2784   if (!VM_Version::supports_cmov()) {
2785     // fcmp needs a temporary so preserve rdx,
2786     tmp = rdx;
2787   }
2788   Register tmp2 = rax;
2789   Register tmp3 = rcx;
2790 
2791   if (is_exp) {
2792     // Stack: X
2793     fld_s(0);                   // duplicate argument for runtime call. Stack: X X
2794     fast_exp();                 // Stack: exp(X) X
2795     fcmp(tmp, 0, false, false); // Stack: exp(X) X
2796     // exp(X) not equal to itself: exp(X) is NaN go to slow case.
2797     jcc(Assembler::parity, slow_case);
2798     // get rid of duplicate argument. Stack: exp(X)
2799     if (num_fpu_regs_in_use > 0) {
2800       fxch();
2801       fpop();
2802     } else {
2803       ffree(1);
2804     }
2805     jmp(done);
2806   } else {
2807     // Stack: X Y
2808     Label x_negative, y_odd;
2809 
2810     fldz();                     // Stack: 0 X Y
2811     fcmp(tmp, 1, true, false);  // Stack: X Y
2812     jcc(Assembler::above, x_negative);
2813 
2814     // X >= 0
2815 
2816     fld_s(1);                   // duplicate arguments for runtime call. Stack: Y X Y
2817     fld_s(1);                   // Stack: X Y X Y
2818     fast_pow();                 // Stack: X^Y X Y
2819     fcmp(tmp, 0, false, false); // Stack: X^Y X Y
2820     // X^Y not equal to itself: X^Y is NaN go to slow case.
2821     jcc(Assembler::parity, slow_case);
2822     // get rid of duplicate arguments. Stack: X^Y
2823     if (num_fpu_regs_in_use > 0) {
2824       fxch(); fpop();
2825       fxch(); fpop();
2826     } else {
2827       ffree(2);
2828       ffree(1);
2829     }
2830     jmp(done);
2831 
2832     // X <= 0
2833     bind(x_negative);
2834 
2835     fld_s(1);                   // Stack: Y X Y
2836     frndint();                  // Stack: int(Y) X Y
2837     fcmp(tmp, 2, false, false); // Stack: int(Y) X Y
2838     jcc(Assembler::notEqual, slow_case);
2839 
2840     subptr(rsp, 8);
2841 
2842     // For X^Y, when X < 0, Y has to be an integer and the final
2843     // result depends on whether it's odd or even. We just checked
2844     // that int(Y) == Y.  We move int(Y) to gp registers as a 64 bit
2845     // integer to test its parity. If int(Y) is huge and doesn't fit
2846     // in the 64 bit integer range, the integer indefinite value will
2847     // end up in the gp registers. Huge numbers are all even, the
2848     // integer indefinite number is even so it's fine.
2849 
2850 #ifdef ASSERT
2851     // Let's check we don't end up with an integer indefinite number
2852     // when not expected. First test for huge numbers: check whether
2853     // int(Y)+1 == int(Y) which is true for very large numbers and
2854     // those are all even. A 64 bit integer is guaranteed to not
2855     // overflow for numbers where y+1 != y (when precision is set to
2856     // double precision).
2857     Label y_not_huge;
2858 
2859     fld1();                     // Stack: 1 int(Y) X Y
2860     fadd(1);                    // Stack: 1+int(Y) int(Y) X Y
2861 
2862 #ifdef _LP64
2863     // trip to memory to force the precision down from double extended
2864     // precision
2865     fstp_d(Address(rsp, 0));
2866     fld_d(Address(rsp, 0));
2867 #endif
2868 
2869     fcmp(tmp, 1, true, false);  // Stack: int(Y) X Y
2870 #endif
2871 
2872     // move int(Y) as 64 bit integer to thread's stack
2873     fistp_d(Address(rsp,0));    // Stack: X Y
2874 
2875 #ifdef ASSERT
2876     jcc(Assembler::notEqual, y_not_huge);
2877 
2878     // Y is huge so we know it's even. It may not fit in a 64 bit
2879     // integer and we don't want the debug code below to see the
2880     // integer indefinite value so overwrite int(Y) on the thread's
2881     // stack with 0.
2882     movl(Address(rsp, 0), 0);
2883     movl(Address(rsp, 4), 0);
2884 
2885     bind(y_not_huge);
2886 #endif
2887 
2888     fld_s(1);                   // duplicate arguments for runtime call. Stack: Y X Y
2889     fld_s(1);                   // Stack: X Y X Y
2890     fabs();                     // Stack: abs(X) Y X Y
2891     fast_pow();                 // Stack: abs(X)^Y X Y
2892     fcmp(tmp, 0, false, false); // Stack: abs(X)^Y X Y
2893     // abs(X)^Y not equal to itself: abs(X)^Y is NaN go to slow case.
2894 
2895     pop(tmp2);
2896     NOT_LP64(pop(tmp3));
2897     jcc(Assembler::parity, slow_case);
2898 
2899 #ifdef ASSERT
2900     // Check that int(Y) is not integer indefinite value (int
2901     // overflow). Shouldn't happen because for values that would
2902     // overflow, 1+int(Y)==Y which was tested earlier.
2903 #ifndef _LP64
2904     {
2905       Label integer;
2906       testl(tmp2, tmp2);
2907       jcc(Assembler::notZero, integer);
2908       cmpl(tmp3, 0x80000000);
2909       jcc(Assembler::notZero, integer);
2910       STOP("integer indefinite value shouldn't be seen here");
2911       bind(integer);
2912     }
2913 #else
2914     {
2915       Label integer;
2916       mov(tmp3, tmp2); // preserve tmp2 for parity check below
2917       shlq(tmp3, 1);
2918       jcc(Assembler::carryClear, integer);
2919       jcc(Assembler::notZero, integer);
2920       STOP("integer indefinite value shouldn't be seen here");
2921       bind(integer);
2922     }
2923 #endif
2924 #endif
2925 
2926     // get rid of duplicate arguments. Stack: X^Y
2927     if (num_fpu_regs_in_use > 0) {
2928       fxch(); fpop();
2929       fxch(); fpop();
2930     } else {
2931       ffree(2);
2932       ffree(1);
2933     }
2934 
2935     testl(tmp2, 1);
2936     jcc(Assembler::zero, done); // X <= 0, Y even: X^Y = abs(X)^Y
2937     // X <= 0, Y even: X^Y = -abs(X)^Y
2938 
2939     fchs();                     // Stack: -abs(X)^Y Y
2940     jmp(done);
2941   }
2942 
2943   // slow case: runtime call
2944   bind(slow_case);
2945 
2946   fpop();                       // pop incorrect result or int(Y)
2947 
2948   fp_runtime_fallback(is_exp ? CAST_FROM_FN_PTR(address, SharedRuntime::dexp) : CAST_FROM_FN_PTR(address, SharedRuntime::dpow),
2949                       is_exp ? 1 : 2, num_fpu_regs_in_use);
2950 
2951   // Come here with result in F-TOS
2952   bind(done);
2953 }
2954 
2955 void MacroAssembler::fpop() {
2956   ffree();
2957   fincstp();
2958 }
2959 
2960 void MacroAssembler::fremr(Register tmp) {
2961   save_rax(tmp);
2962   { Label L;
2963     bind(L);
2964     fprem();
2965     fwait(); fnstsw_ax();
2966 #ifdef _LP64
2967     testl(rax, 0x400);
2968     jcc(Assembler::notEqual, L);
2969 #else
2970     sahf();
2971     jcc(Assembler::parity, L);
2972 #endif // _LP64
2973   }
2974   restore_rax(tmp);
2975   // Result is in ST0.
2976   // Note: fxch & fpop to get rid of ST1
2977   // (otherwise FPU stack could overflow eventually)
2978   fxch(1);
2979   fpop();
2980 }
2981 
2982 
2983 void MacroAssembler::incrementl(AddressLiteral dst) {
2984   if (reachable(dst)) {
2985     incrementl(as_Address(dst));
2986   } else {
2987     lea(rscratch1, dst);
2988     incrementl(Address(rscratch1, 0));
2989   }
2990 }
2991 
2992 void MacroAssembler::incrementl(ArrayAddress dst) {
2993   incrementl(as_Address(dst));
2994 }
2995 
2996 void MacroAssembler::incrementl(Register reg, int value) {
2997   if (value == min_jint) {addl(reg, value) ; return; }
2998   if (value <  0) { decrementl(reg, -value); return; }
2999   if (value == 0) {                        ; return; }
3000   if (value == 1 && UseIncDec) { incl(reg) ; return; }
3001   /* else */      { addl(reg, value)       ; return; }
3002 }
3003 
3004 void MacroAssembler::incrementl(Address dst, int value) {
3005   if (value == min_jint) {addl(dst, value) ; return; }
3006   if (value <  0) { decrementl(dst, -value); return; }
3007   if (value == 0) {                        ; return; }
3008   if (value == 1 && UseIncDec) { incl(dst) ; return; }
3009   /* else */      { addl(dst, value)       ; return; }
3010 }
3011 
3012 void MacroAssembler::jump(AddressLiteral dst) {
3013   if (reachable(dst)) {
3014     jmp_literal(dst.target(), dst.rspec());
3015   } else {
3016     lea(rscratch1, dst);
3017     jmp(rscratch1);
3018   }
3019 }
3020 
3021 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
3022   if (reachable(dst)) {
3023     InstructionMark im(this);
3024     relocate(dst.reloc());
3025     const int short_size = 2;
3026     const int long_size = 6;
3027     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
3028     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
3029       // 0111 tttn #8-bit disp
3030       emit_int8(0x70 | cc);
3031       emit_int8((offs - short_size) & 0xFF);
3032     } else {
3033       // 0000 1111 1000 tttn #32-bit disp
3034       emit_int8(0x0F);
3035       emit_int8((unsigned char)(0x80 | cc));
3036       emit_int32(offs - long_size);
3037     }
3038   } else {
3039 #ifdef ASSERT
3040     warning("reversing conditional branch");
3041 #endif /* ASSERT */
3042     Label skip;
3043     jccb(reverse[cc], skip);
3044     lea(rscratch1, dst);
3045     Assembler::jmp(rscratch1);
3046     bind(skip);
3047   }
3048 }
3049 
3050 void MacroAssembler::ldmxcsr(AddressLiteral src) {
3051   if (reachable(src)) {
3052     Assembler::ldmxcsr(as_Address(src));
3053   } else {
3054     lea(rscratch1, src);
3055     Assembler::ldmxcsr(Address(rscratch1, 0));
3056   }
3057 }
3058 
3059 int MacroAssembler::load_signed_byte(Register dst, Address src) {
3060   int off;
3061   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3062     off = offset();
3063     movsbl(dst, src); // movsxb
3064   } else {
3065     off = load_unsigned_byte(dst, src);
3066     shll(dst, 24);
3067     sarl(dst, 24);
3068   }
3069   return off;
3070 }
3071 
3072 // Note: load_signed_short used to be called load_signed_word.
3073 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
3074 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
3075 // The term "word" in HotSpot means a 32- or 64-bit machine word.
3076 int MacroAssembler::load_signed_short(Register dst, Address src) {
3077   int off;
3078   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3079     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
3080     // version but this is what 64bit has always done. This seems to imply
3081     // that users are only using 32bits worth.
3082     off = offset();
3083     movswl(dst, src); // movsxw
3084   } else {
3085     off = load_unsigned_short(dst, src);
3086     shll(dst, 16);
3087     sarl(dst, 16);
3088   }
3089   return off;
3090 }
3091 
3092 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
3093   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3094   // and "3.9 Partial Register Penalties", p. 22).
3095   int off;
3096   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
3097     off = offset();
3098     movzbl(dst, src); // movzxb
3099   } else {
3100     xorl(dst, dst);
3101     off = offset();
3102     movb(dst, src);
3103   }
3104   return off;
3105 }
3106 
3107 // Note: load_unsigned_short used to be called load_unsigned_word.
3108 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
3109   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3110   // and "3.9 Partial Register Penalties", p. 22).
3111   int off;
3112   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
3113     off = offset();
3114     movzwl(dst, src); // movzxw
3115   } else {
3116     xorl(dst, dst);
3117     off = offset();
3118     movw(dst, src);
3119   }
3120   return off;
3121 }
3122 
3123 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
3124   switch (size_in_bytes) {
3125 #ifndef _LP64
3126   case  8:
3127     assert(dst2 != noreg, "second dest register required");
3128     movl(dst,  src);
3129     movl(dst2, src.plus_disp(BytesPerInt));
3130     break;
3131 #else
3132   case  8:  movq(dst, src); break;
3133 #endif
3134   case  4:  movl(dst, src); break;
3135   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
3136   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
3137   default:  ShouldNotReachHere();
3138   }
3139 }
3140 
3141 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
3142   switch (size_in_bytes) {
3143 #ifndef _LP64
3144   case  8:
3145     assert(src2 != noreg, "second source register required");
3146     movl(dst,                        src);
3147     movl(dst.plus_disp(BytesPerInt), src2);
3148     break;
3149 #else
3150   case  8:  movq(dst, src); break;
3151 #endif
3152   case  4:  movl(dst, src); break;
3153   case  2:  movw(dst, src); break;
3154   case  1:  movb(dst, src); break;
3155   default:  ShouldNotReachHere();
3156   }
3157 }
3158 
3159 void MacroAssembler::mov32(AddressLiteral dst, Register src) {
3160   if (reachable(dst)) {
3161     movl(as_Address(dst), src);
3162   } else {
3163     lea(rscratch1, dst);
3164     movl(Address(rscratch1, 0), src);
3165   }
3166 }
3167 
3168 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
3169   if (reachable(src)) {
3170     movl(dst, as_Address(src));
3171   } else {
3172     lea(rscratch1, src);
3173     movl(dst, Address(rscratch1, 0));
3174   }
3175 }
3176 
3177 // C++ bool manipulation
3178 
3179 void MacroAssembler::movbool(Register dst, Address src) {
3180   if(sizeof(bool) == 1)
3181     movb(dst, src);
3182   else if(sizeof(bool) == 2)
3183     movw(dst, src);
3184   else if(sizeof(bool) == 4)
3185     movl(dst, src);
3186   else
3187     // unsupported
3188     ShouldNotReachHere();
3189 }
3190 
3191 void MacroAssembler::movbool(Address dst, bool boolconst) {
3192   if(sizeof(bool) == 1)
3193     movb(dst, (int) boolconst);
3194   else if(sizeof(bool) == 2)
3195     movw(dst, (int) boolconst);
3196   else if(sizeof(bool) == 4)
3197     movl(dst, (int) boolconst);
3198   else
3199     // unsupported
3200     ShouldNotReachHere();
3201 }
3202 
3203 void MacroAssembler::movbool(Address dst, Register src) {
3204   if(sizeof(bool) == 1)
3205     movb(dst, src);
3206   else if(sizeof(bool) == 2)
3207     movw(dst, src);
3208   else if(sizeof(bool) == 4)
3209     movl(dst, src);
3210   else
3211     // unsupported
3212     ShouldNotReachHere();
3213 }
3214 
3215 void MacroAssembler::movbyte(ArrayAddress dst, int src) {
3216   movb(as_Address(dst), src);
3217 }
3218 
3219 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src) {
3220   if (reachable(src)) {
3221     movdl(dst, as_Address(src));
3222   } else {
3223     lea(rscratch1, src);
3224     movdl(dst, Address(rscratch1, 0));
3225   }
3226 }
3227 
3228 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src) {
3229   if (reachable(src)) {
3230     movq(dst, as_Address(src));
3231   } else {
3232     lea(rscratch1, src);
3233     movq(dst, Address(rscratch1, 0));
3234   }
3235 }
3236 
3237 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
3238   if (reachable(src)) {
3239     if (UseXmmLoadAndClearUpper) {
3240       movsd (dst, as_Address(src));
3241     } else {
3242       movlpd(dst, as_Address(src));
3243     }
3244   } else {
3245     lea(rscratch1, src);
3246     if (UseXmmLoadAndClearUpper) {
3247       movsd (dst, Address(rscratch1, 0));
3248     } else {
3249       movlpd(dst, Address(rscratch1, 0));
3250     }
3251   }
3252 }
3253 
3254 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
3255   if (reachable(src)) {
3256     movss(dst, as_Address(src));
3257   } else {
3258     lea(rscratch1, src);
3259     movss(dst, Address(rscratch1, 0));
3260   }
3261 }
3262 
3263 void MacroAssembler::movptr(Register dst, Register src) {
3264   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3265 }
3266 
3267 void MacroAssembler::movptr(Register dst, Address src) {
3268   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3269 }
3270 
3271 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
3272 void MacroAssembler::movptr(Register dst, intptr_t src) {
3273   LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src));
3274 }
3275 
3276 void MacroAssembler::movptr(Address dst, Register src) {
3277   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3278 }
3279 
3280 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src) {
3281   if (reachable(src)) {
3282     Assembler::movdqu(dst, as_Address(src));
3283   } else {
3284     lea(rscratch1, src);
3285     Assembler::movdqu(dst, Address(rscratch1, 0));
3286   }
3287 }
3288 
3289 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) {
3290   if (reachable(src)) {
3291     Assembler::movdqa(dst, as_Address(src));
3292   } else {
3293     lea(rscratch1, src);
3294     Assembler::movdqa(dst, Address(rscratch1, 0));
3295   }
3296 }
3297 
3298 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
3299   if (reachable(src)) {
3300     Assembler::movsd(dst, as_Address(src));
3301   } else {
3302     lea(rscratch1, src);
3303     Assembler::movsd(dst, Address(rscratch1, 0));
3304   }
3305 }
3306 
3307 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
3308   if (reachable(src)) {
3309     Assembler::movss(dst, as_Address(src));
3310   } else {
3311     lea(rscratch1, src);
3312     Assembler::movss(dst, Address(rscratch1, 0));
3313   }
3314 }
3315 
3316 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) {
3317   if (reachable(src)) {
3318     Assembler::mulsd(dst, as_Address(src));
3319   } else {
3320     lea(rscratch1, src);
3321     Assembler::mulsd(dst, Address(rscratch1, 0));
3322   }
3323 }
3324 
3325 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) {
3326   if (reachable(src)) {
3327     Assembler::mulss(dst, as_Address(src));
3328   } else {
3329     lea(rscratch1, src);
3330     Assembler::mulss(dst, Address(rscratch1, 0));
3331   }
3332 }
3333 
3334 void MacroAssembler::null_check(Register reg, int offset) {
3335   if (needs_explicit_null_check(offset)) {
3336     // provoke OS NULL exception if reg = NULL by
3337     // accessing M[reg] w/o changing any (non-CC) registers
3338     // NOTE: cmpl is plenty here to provoke a segv
3339     cmpptr(rax, Address(reg, 0));
3340     // Note: should probably use testl(rax, Address(reg, 0));
3341     //       may be shorter code (however, this version of
3342     //       testl needs to be implemented first)
3343   } else {
3344     // nothing to do, (later) access of M[reg + offset]
3345     // will provoke OS NULL exception if reg = NULL
3346   }
3347 }
3348 
3349 void MacroAssembler::os_breakpoint() {
3350   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
3351   // (e.g., MSVC can't call ps() otherwise)
3352   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
3353 }
3354 
3355 void MacroAssembler::pop_CPU_state() {
3356   pop_FPU_state();
3357   pop_IU_state();
3358 }
3359 
3360 void MacroAssembler::pop_FPU_state() {
3361   NOT_LP64(frstor(Address(rsp, 0));)
3362   LP64_ONLY(fxrstor(Address(rsp, 0));)
3363   addptr(rsp, FPUStateSizeInWords * wordSize);
3364 }
3365 
3366 void MacroAssembler::pop_IU_state() {
3367   popa();
3368   LP64_ONLY(addq(rsp, 8));
3369   popf();
3370 }
3371 
3372 // Save Integer and Float state
3373 // Warning: Stack must be 16 byte aligned (64bit)
3374 void MacroAssembler::push_CPU_state() {
3375   push_IU_state();
3376   push_FPU_state();
3377 }
3378 
3379 void MacroAssembler::push_FPU_state() {
3380   subptr(rsp, FPUStateSizeInWords * wordSize);
3381 #ifndef _LP64
3382   fnsave(Address(rsp, 0));
3383   fwait();
3384 #else
3385   fxsave(Address(rsp, 0));
3386 #endif // LP64
3387 }
3388 
3389 void MacroAssembler::push_IU_state() {
3390   // Push flags first because pusha kills them
3391   pushf();
3392   // Make sure rsp stays 16-byte aligned
3393   LP64_ONLY(subq(rsp, 8));
3394   pusha();
3395 }
3396 
3397 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp, bool clear_pc) {
3398   // determine java_thread register
3399   if (!java_thread->is_valid()) {
3400     java_thread = rdi;
3401     get_thread(java_thread);
3402   }
3403   // we must set sp to zero to clear frame
3404   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
3405   if (clear_fp) {
3406     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
3407   }
3408 
3409   if (clear_pc)
3410     movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
3411 
3412 }
3413 
3414 void MacroAssembler::restore_rax(Register tmp) {
3415   if (tmp == noreg) pop(rax);
3416   else if (tmp != rax) mov(rax, tmp);
3417 }
3418 
3419 void MacroAssembler::round_to(Register reg, int modulus) {
3420   addptr(reg, modulus - 1);
3421   andptr(reg, -modulus);
3422 }
3423 
3424 void MacroAssembler::save_rax(Register tmp) {
3425   if (tmp == noreg) push(rax);
3426   else if (tmp != rax) mov(tmp, rax);
3427 }
3428 
3429 // Write serialization page so VM thread can do a pseudo remote membar.
3430 // We use the current thread pointer to calculate a thread specific
3431 // offset to write to within the page. This minimizes bus traffic
3432 // due to cache line collision.
3433 void MacroAssembler::serialize_memory(Register thread, Register tmp) {
3434   movl(tmp, thread);
3435   shrl(tmp, os::get_serialize_page_shift_count());
3436   andl(tmp, (os::vm_page_size() - sizeof(int)));
3437 
3438   Address index(noreg, tmp, Address::times_1);
3439   ExternalAddress page(os::get_memory_serialize_page());
3440 
3441   // Size of store must match masking code above
3442   movl(as_Address(ArrayAddress(page, index)), tmp);
3443 }
3444 
3445 // Calls to C land
3446 //
3447 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
3448 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
3449 // has to be reset to 0. This is required to allow proper stack traversal.
3450 void MacroAssembler::set_last_Java_frame(Register java_thread,
3451                                          Register last_java_sp,
3452                                          Register last_java_fp,
3453                                          address  last_java_pc) {
3454   // determine java_thread register
3455   if (!java_thread->is_valid()) {
3456     java_thread = rdi;
3457     get_thread(java_thread);
3458   }
3459   // determine last_java_sp register
3460   if (!last_java_sp->is_valid()) {
3461     last_java_sp = rsp;
3462   }
3463 
3464   // last_java_fp is optional
3465 
3466   if (last_java_fp->is_valid()) {
3467     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
3468   }
3469 
3470   // last_java_pc is optional
3471 
3472   if (last_java_pc != NULL) {
3473     lea(Address(java_thread,
3474                  JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()),
3475         InternalAddress(last_java_pc));
3476 
3477   }
3478   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
3479 }
3480 
3481 void MacroAssembler::shlptr(Register dst, int imm8) {
3482   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
3483 }
3484 
3485 void MacroAssembler::shrptr(Register dst, int imm8) {
3486   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
3487 }
3488 
3489 void MacroAssembler::sign_extend_byte(Register reg) {
3490   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
3491     movsbl(reg, reg); // movsxb
3492   } else {
3493     shll(reg, 24);
3494     sarl(reg, 24);
3495   }
3496 }
3497 
3498 void MacroAssembler::sign_extend_short(Register reg) {
3499   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3500     movswl(reg, reg); // movsxw
3501   } else {
3502     shll(reg, 16);
3503     sarl(reg, 16);
3504   }
3505 }
3506 
3507 void MacroAssembler::testl(Register dst, AddressLiteral src) {
3508   assert(reachable(src), "Address should be reachable");
3509   testl(dst, as_Address(src));
3510 }
3511 
3512 void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) {
3513   if (reachable(src)) {
3514     Assembler::sqrtsd(dst, as_Address(src));
3515   } else {
3516     lea(rscratch1, src);
3517     Assembler::sqrtsd(dst, Address(rscratch1, 0));
3518   }
3519 }
3520 
3521 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) {
3522   if (reachable(src)) {
3523     Assembler::sqrtss(dst, as_Address(src));
3524   } else {
3525     lea(rscratch1, src);
3526     Assembler::sqrtss(dst, Address(rscratch1, 0));
3527   }
3528 }
3529 
3530 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) {
3531   if (reachable(src)) {
3532     Assembler::subsd(dst, as_Address(src));
3533   } else {
3534     lea(rscratch1, src);
3535     Assembler::subsd(dst, Address(rscratch1, 0));
3536   }
3537 }
3538 
3539 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) {
3540   if (reachable(src)) {
3541     Assembler::subss(dst, as_Address(src));
3542   } else {
3543     lea(rscratch1, src);
3544     Assembler::subss(dst, Address(rscratch1, 0));
3545   }
3546 }
3547 
3548 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
3549   if (reachable(src)) {
3550     Assembler::ucomisd(dst, as_Address(src));
3551   } else {
3552     lea(rscratch1, src);
3553     Assembler::ucomisd(dst, Address(rscratch1, 0));
3554   }
3555 }
3556 
3557 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
3558   if (reachable(src)) {
3559     Assembler::ucomiss(dst, as_Address(src));
3560   } else {
3561     lea(rscratch1, src);
3562     Assembler::ucomiss(dst, Address(rscratch1, 0));
3563   }
3564 }
3565 
3566 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
3567   // Used in sign-bit flipping with aligned address.
3568   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3569   if (reachable(src)) {
3570     Assembler::xorpd(dst, as_Address(src));
3571   } else {
3572     lea(rscratch1, src);
3573     Assembler::xorpd(dst, Address(rscratch1, 0));
3574   }
3575 }
3576 
3577 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
3578   // Used in sign-bit flipping with aligned address.
3579   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3580   if (reachable(src)) {
3581     Assembler::xorps(dst, as_Address(src));
3582   } else {
3583     lea(rscratch1, src);
3584     Assembler::xorps(dst, Address(rscratch1, 0));
3585   }
3586 }
3587 
3588 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) {
3589   // Used in sign-bit flipping with aligned address.
3590   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
3591   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
3592   if (reachable(src)) {
3593     Assembler::pshufb(dst, as_Address(src));
3594   } else {
3595     lea(rscratch1, src);
3596     Assembler::pshufb(dst, Address(rscratch1, 0));
3597   }
3598 }
3599 
3600 // AVX 3-operands instructions
3601 
3602 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3603   if (reachable(src)) {
3604     vaddsd(dst, nds, as_Address(src));
3605   } else {
3606     lea(rscratch1, src);
3607     vaddsd(dst, nds, Address(rscratch1, 0));
3608   }
3609 }
3610 
3611 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3612   if (reachable(src)) {
3613     vaddss(dst, nds, as_Address(src));
3614   } else {
3615     lea(rscratch1, src);
3616     vaddss(dst, nds, Address(rscratch1, 0));
3617   }
3618 }
3619 
3620 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) {
3621   if (reachable(src)) {
3622     vandpd(dst, nds, as_Address(src), vector256);
3623   } else {
3624     lea(rscratch1, src);
3625     vandpd(dst, nds, Address(rscratch1, 0), vector256);
3626   }
3627 }
3628 
3629 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) {
3630   if (reachable(src)) {
3631     vandps(dst, nds, as_Address(src), vector256);
3632   } else {
3633     lea(rscratch1, src);
3634     vandps(dst, nds, Address(rscratch1, 0), vector256);
3635   }
3636 }
3637 
3638 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3639   if (reachable(src)) {
3640     vdivsd(dst, nds, as_Address(src));
3641   } else {
3642     lea(rscratch1, src);
3643     vdivsd(dst, nds, Address(rscratch1, 0));
3644   }
3645 }
3646 
3647 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3648   if (reachable(src)) {
3649     vdivss(dst, nds, as_Address(src));
3650   } else {
3651     lea(rscratch1, src);
3652     vdivss(dst, nds, Address(rscratch1, 0));
3653   }
3654 }
3655 
3656 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3657   if (reachable(src)) {
3658     vmulsd(dst, nds, as_Address(src));
3659   } else {
3660     lea(rscratch1, src);
3661     vmulsd(dst, nds, Address(rscratch1, 0));
3662   }
3663 }
3664 
3665 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3666   if (reachable(src)) {
3667     vmulss(dst, nds, as_Address(src));
3668   } else {
3669     lea(rscratch1, src);
3670     vmulss(dst, nds, Address(rscratch1, 0));
3671   }
3672 }
3673 
3674 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3675   if (reachable(src)) {
3676     vsubsd(dst, nds, as_Address(src));
3677   } else {
3678     lea(rscratch1, src);
3679     vsubsd(dst, nds, Address(rscratch1, 0));
3680   }
3681 }
3682 
3683 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3684   if (reachable(src)) {
3685     vsubss(dst, nds, as_Address(src));
3686   } else {
3687     lea(rscratch1, src);
3688     vsubss(dst, nds, Address(rscratch1, 0));
3689   }
3690 }
3691 
3692 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) {
3693   if (reachable(src)) {
3694     vxorpd(dst, nds, as_Address(src), vector256);
3695   } else {
3696     lea(rscratch1, src);
3697     vxorpd(dst, nds, Address(rscratch1, 0), vector256);
3698   }
3699 }
3700 
3701 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) {
3702   if (reachable(src)) {
3703     vxorps(dst, nds, as_Address(src), vector256);
3704   } else {
3705     lea(rscratch1, src);
3706     vxorps(dst, nds, Address(rscratch1, 0), vector256);
3707   }
3708 }
3709 
3710 
3711 //////////////////////////////////////////////////////////////////////////////////
3712 #if INCLUDE_ALL_GCS
3713 
3714 void MacroAssembler::g1_write_barrier_pre(Register obj,
3715                                           Register pre_val,
3716                                           Register thread,
3717                                           Register tmp,
3718                                           bool tosca_live,
3719                                           bool expand_call) {
3720 
3721   // If expand_call is true then we expand the call_VM_leaf macro
3722   // directly to skip generating the check by
3723   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
3724 
3725 #ifdef _LP64
3726   assert(thread == r15_thread, "must be");
3727 #endif // _LP64
3728 
3729   Label done;
3730   Label runtime;
3731 
3732   assert(pre_val != noreg, "check this code");
3733 
3734   if (obj != noreg) {
3735     assert_different_registers(obj, pre_val, tmp);
3736     assert(pre_val != rax, "check this code");
3737   }
3738 
3739   Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
3740                                        PtrQueue::byte_offset_of_active()));
3741   Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
3742                                        PtrQueue::byte_offset_of_index()));
3743   Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
3744                                        PtrQueue::byte_offset_of_buf()));
3745 
3746 
3747   // Is marking active?
3748   if (in_bytes(PtrQueue::byte_width_of_active()) == 4) {
3749     cmpl(in_progress, 0);
3750   } else {
3751     assert(in_bytes(PtrQueue::byte_width_of_active()) == 1, "Assumption");
3752     cmpb(in_progress, 0);
3753   }
3754   jcc(Assembler::equal, done);
3755 
3756   // Do we need to load the previous value?
3757   if (obj != noreg) {
3758     load_heap_oop(pre_val, Address(obj, 0));
3759   }
3760 
3761   // Is the previous value null?
3762   cmpptr(pre_val, (int32_t) NULL_WORD);
3763   jcc(Assembler::equal, done);
3764 
3765   // Can we store original value in the thread's buffer?
3766   // Is index == 0?
3767   // (The index field is typed as size_t.)
3768 
3769   movptr(tmp, index);                   // tmp := *index_adr
3770   cmpptr(tmp, 0);                       // tmp == 0?
3771   jcc(Assembler::equal, runtime);       // If yes, goto runtime
3772 
3773   subptr(tmp, wordSize);                // tmp := tmp - wordSize
3774   movptr(index, tmp);                   // *index_adr := tmp
3775   addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
3776 
3777   // Record the previous value
3778   movptr(Address(tmp, 0), pre_val);
3779   jmp(done);
3780 
3781   bind(runtime);
3782   // save the live input values
3783   if(tosca_live) push(rax);
3784 
3785   if (obj != noreg && obj != rax)
3786     push(obj);
3787 
3788   if (pre_val != rax)
3789     push(pre_val);
3790 
3791   // Calling the runtime using the regular call_VM_leaf mechanism generates
3792   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
3793   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
3794   //
3795   // If we care generating the pre-barrier without a frame (e.g. in the
3796   // intrinsified Reference.get() routine) then ebp might be pointing to
3797   // the caller frame and so this check will most likely fail at runtime.
3798   //
3799   // Expanding the call directly bypasses the generation of the check.
3800   // So when we do not have have a full interpreter frame on the stack
3801   // expand_call should be passed true.
3802 
3803   NOT_LP64( push(thread); )
3804 
3805   if (expand_call) {
3806     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
3807     pass_arg1(this, thread);
3808     pass_arg0(this, pre_val);
3809     MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), 2);
3810   } else {
3811     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
3812   }
3813 
3814   NOT_LP64( pop(thread); )
3815 
3816   // save the live input values
3817   if (pre_val != rax)
3818     pop(pre_val);
3819 
3820   if (obj != noreg && obj != rax)
3821     pop(obj);
3822 
3823   if(tosca_live) pop(rax);
3824 
3825   bind(done);
3826 }
3827 
3828 void MacroAssembler::g1_write_barrier_post(Register store_addr,
3829                                            Register new_val,
3830                                            Register thread,
3831                                            Register tmp,
3832                                            Register tmp2) {
3833 #ifdef _LP64
3834   assert(thread == r15_thread, "must be");
3835 #endif // _LP64
3836 
3837   Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
3838                                        PtrQueue::byte_offset_of_index()));
3839   Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
3840                                        PtrQueue::byte_offset_of_buf()));
3841 
3842   BarrierSet* bs = Universe::heap()->barrier_set();
3843   CardTableModRefBS* ct = (CardTableModRefBS*)bs;
3844   assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
3845 
3846   Label done;
3847   Label runtime;
3848 
3849   // Does store cross heap regions?
3850 
3851   movptr(tmp, store_addr);
3852   xorptr(tmp, new_val);
3853   shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
3854   jcc(Assembler::equal, done);
3855 
3856   // crosses regions, storing NULL?
3857 
3858   cmpptr(new_val, (int32_t) NULL_WORD);
3859   jcc(Assembler::equal, done);
3860 
3861   // storing region crossing non-NULL, is card already dirty?
3862 
3863   const Register card_addr = tmp;
3864   const Register cardtable = tmp2;
3865 
3866   movptr(card_addr, store_addr);
3867   shrptr(card_addr, CardTableModRefBS::card_shift);
3868   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
3869   // a valid address and therefore is not properly handled by the relocation code.
3870   movptr(cardtable, (intptr_t)ct->byte_map_base);
3871   addptr(card_addr, cardtable);
3872 
3873   cmpb(Address(card_addr, 0), (int)G1SATBCardTableModRefBS::g1_young_card_val());
3874   jcc(Assembler::equal, done);
3875 
3876   membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3877   cmpb(Address(card_addr, 0), (int)CardTableModRefBS::dirty_card_val());
3878   jcc(Assembler::equal, done);
3879 
3880 
3881   // storing a region crossing, non-NULL oop, card is clean.
3882   // dirty card and log.
3883 
3884   movb(Address(card_addr, 0), (int)CardTableModRefBS::dirty_card_val());
3885 
3886   cmpl(queue_index, 0);
3887   jcc(Assembler::equal, runtime);
3888   subl(queue_index, wordSize);
3889   movptr(tmp2, buffer);
3890 #ifdef _LP64
3891   movslq(rscratch1, queue_index);
3892   addq(tmp2, rscratch1);
3893   movq(Address(tmp2, 0), card_addr);
3894 #else
3895   addl(tmp2, queue_index);
3896   movl(Address(tmp2, 0), card_addr);
3897 #endif
3898   jmp(done);
3899 
3900   bind(runtime);
3901   // save the live input values
3902   push(store_addr);
3903   push(new_val);
3904 #ifdef _LP64
3905   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread);
3906 #else
3907   push(thread);
3908   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
3909   pop(thread);
3910 #endif
3911   pop(new_val);
3912   pop(store_addr);
3913 
3914   bind(done);
3915 }
3916 
3917 #endif // INCLUDE_ALL_GCS
3918 //////////////////////////////////////////////////////////////////////////////////
3919 
3920 
3921 void MacroAssembler::store_check(Register obj) {
3922   // Does a store check for the oop in register obj. The content of
3923   // register obj is destroyed afterwards.
3924   store_check_part_1(obj);
3925   store_check_part_2(obj);
3926 }
3927 
3928 void MacroAssembler::store_check(Register obj, Address dst) {
3929   store_check(obj);
3930 }
3931 
3932 
3933 // split the store check operation so that other instructions can be scheduled inbetween
3934 void MacroAssembler::store_check_part_1(Register obj) {
3935   BarrierSet* bs = Universe::heap()->barrier_set();
3936   assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
3937   shrptr(obj, CardTableModRefBS::card_shift);
3938 }
3939 
3940 void MacroAssembler::store_check_part_2(Register obj) {
3941   BarrierSet* bs = Universe::heap()->barrier_set();
3942   assert(bs->kind() == BarrierSet::CardTableModRef, "Wrong barrier set kind");
3943   CardTableModRefBS* ct = (CardTableModRefBS*)bs;
3944   assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
3945 
3946   // The calculation for byte_map_base is as follows:
3947   // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
3948   // So this essentially converts an address to a displacement and it will
3949   // never need to be relocated. On 64bit however the value may be too
3950   // large for a 32bit displacement.
3951   intptr_t disp = (intptr_t) ct->byte_map_base;
3952   if (is_simm32(disp)) {
3953     Address cardtable(noreg, obj, Address::times_1, disp);
3954     movb(cardtable, 0);
3955   } else {
3956     // By doing it as an ExternalAddress 'disp' could be converted to a rip-relative
3957     // displacement and done in a single instruction given favorable mapping and a
3958     // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
3959     // entry and that entry is not properly handled by the relocation code.
3960     AddressLiteral cardtable((address)ct->byte_map_base, relocInfo::none);
3961     Address index(noreg, obj, Address::times_1);
3962     movb(as_Address(ArrayAddress(cardtable, index)), 0);
3963   }
3964 }
3965 
3966 void MacroAssembler::subptr(Register dst, int32_t imm32) {
3967   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
3968 }
3969 
3970 // Force generation of a 4 byte immediate value even if it fits into 8bit
3971 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
3972   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
3973 }
3974 
3975 void MacroAssembler::subptr(Register dst, Register src) {
3976   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
3977 }
3978 
3979 // C++ bool manipulation
3980 void MacroAssembler::testbool(Register dst) {
3981   if(sizeof(bool) == 1)
3982     testb(dst, 0xff);
3983   else if(sizeof(bool) == 2) {
3984     // testw implementation needed for two byte bools
3985     ShouldNotReachHere();
3986   } else if(sizeof(bool) == 4)
3987     testl(dst, dst);
3988   else
3989     // unsupported
3990     ShouldNotReachHere();
3991 }
3992 
3993 void MacroAssembler::testptr(Register dst, Register src) {
3994   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
3995 }
3996 
3997 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
3998 void MacroAssembler::tlab_allocate(Register obj,
3999                                    Register var_size_in_bytes,
4000                                    int con_size_in_bytes,
4001                                    Register t1,
4002                                    Register t2,
4003                                    Label& slow_case) {
4004   assert_different_registers(obj, t1, t2);
4005   assert_different_registers(obj, var_size_in_bytes, t1);
4006   Register end = t2;
4007   Register thread = NOT_LP64(t1) LP64_ONLY(r15_thread);
4008 
4009   verify_tlab();
4010 
4011   NOT_LP64(get_thread(thread));
4012 
4013   movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
4014   if (var_size_in_bytes == noreg) {
4015     lea(end, Address(obj, con_size_in_bytes));
4016   } else {
4017     lea(end, Address(obj, var_size_in_bytes, Address::times_1));
4018   }
4019   cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
4020   jcc(Assembler::above, slow_case);
4021 
4022   // update the tlab top pointer
4023   movptr(Address(thread, JavaThread::tlab_top_offset()), end);
4024 
4025   // recover var_size_in_bytes if necessary
4026   if (var_size_in_bytes == end) {
4027     subptr(var_size_in_bytes, obj);
4028   }
4029   verify_tlab();
4030 }
4031 
4032 // Preserves rbx, and rdx.
4033 Register MacroAssembler::tlab_refill(Label& retry,
4034                                      Label& try_eden,
4035                                      Label& slow_case) {
4036   Register top = rax;
4037   Register t1  = rcx;
4038   Register t2  = rsi;
4039   Register thread_reg = NOT_LP64(rdi) LP64_ONLY(r15_thread);
4040   assert_different_registers(top, thread_reg, t1, t2, /* preserve: */ rbx, rdx);
4041   Label do_refill, discard_tlab;
4042 
4043   if (CMSIncrementalMode || !Universe::heap()->supports_inline_contig_alloc()) {
4044     // No allocation in the shared eden.
4045     jmp(slow_case);
4046   }
4047 
4048   NOT_LP64(get_thread(thread_reg));
4049 
4050   movptr(top, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
4051   movptr(t1,  Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
4052 
4053   // calculate amount of free space
4054   subptr(t1, top);
4055   shrptr(t1, LogHeapWordSize);
4056 
4057   // Retain tlab and allocate object in shared space if
4058   // the amount free in the tlab is too large to discard.
4059   cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())));
4060   jcc(Assembler::lessEqual, discard_tlab);
4061 
4062   // Retain
4063   // %%% yuck as movptr...
4064   movptr(t2, (int32_t) ThreadLocalAllocBuffer::refill_waste_limit_increment());
4065   addptr(Address(thread_reg, in_bytes(JavaThread::tlab_refill_waste_limit_offset())), t2);
4066   if (TLABStats) {
4067     // increment number of slow_allocations
4068     addl(Address(thread_reg, in_bytes(JavaThread::tlab_slow_allocations_offset())), 1);
4069   }
4070   jmp(try_eden);
4071 
4072   bind(discard_tlab);
4073   if (TLABStats) {
4074     // increment number of refills
4075     addl(Address(thread_reg, in_bytes(JavaThread::tlab_number_of_refills_offset())), 1);
4076     // accumulate wastage -- t1 is amount free in tlab
4077     addl(Address(thread_reg, in_bytes(JavaThread::tlab_fast_refill_waste_offset())), t1);
4078   }
4079 
4080   // if tlab is currently allocated (top or end != null) then
4081   // fill [top, end + alignment_reserve) with array object
4082   testptr(top, top);
4083   jcc(Assembler::zero, do_refill);
4084 
4085   // set up the mark word
4086   movptr(Address(top, oopDesc::mark_offset_in_bytes()), (intptr_t)markOopDesc::prototype()->copy_set_hash(0x2));
4087   // set the length to the remaining space
4088   subptr(t1, typeArrayOopDesc::header_size(T_INT));
4089   addptr(t1, (int32_t)ThreadLocalAllocBuffer::alignment_reserve());
4090   shlptr(t1, log2_intptr(HeapWordSize/sizeof(jint)));
4091   movl(Address(top, arrayOopDesc::length_offset_in_bytes()), t1);
4092   // set klass to intArrayKlass
4093   // dubious reloc why not an oop reloc?
4094   movptr(t1, ExternalAddress((address)Universe::intArrayKlassObj_addr()));
4095   // store klass last.  concurrent gcs assumes klass length is valid if
4096   // klass field is not null.
4097   store_klass(top, t1);
4098 
4099   movptr(t1, top);
4100   subptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
4101   incr_allocated_bytes(thread_reg, t1, 0);
4102 
4103   // refill the tlab with an eden allocation
4104   bind(do_refill);
4105   movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
4106   shlptr(t1, LogHeapWordSize);
4107   // allocate new tlab, address returned in top
4108   eden_allocate(top, t1, 0, t2, slow_case);
4109 
4110   // Check that t1 was preserved in eden_allocate.
4111 #ifdef ASSERT
4112   if (UseTLAB) {
4113     Label ok;
4114     Register tsize = rsi;
4115     assert_different_registers(tsize, thread_reg, t1);
4116     push(tsize);
4117     movptr(tsize, Address(thread_reg, in_bytes(JavaThread::tlab_size_offset())));
4118     shlptr(tsize, LogHeapWordSize);
4119     cmpptr(t1, tsize);
4120     jcc(Assembler::equal, ok);
4121     STOP("assert(t1 != tlab size)");
4122     should_not_reach_here();
4123 
4124     bind(ok);
4125     pop(tsize);
4126   }
4127 #endif
4128   movptr(Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())), top);
4129   movptr(Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())), top);
4130   addptr(top, t1);
4131   subptr(top, (int32_t)ThreadLocalAllocBuffer::alignment_reserve_in_bytes());
4132   movptr(Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())), top);
4133   verify_tlab();
4134   jmp(retry);
4135 
4136   return thread_reg; // for use by caller
4137 }
4138 
4139 void MacroAssembler::incr_allocated_bytes(Register thread,
4140                                           Register var_size_in_bytes,
4141                                           int con_size_in_bytes,
4142                                           Register t1) {
4143   if (!thread->is_valid()) {
4144 #ifdef _LP64
4145     thread = r15_thread;
4146 #else
4147     assert(t1->is_valid(), "need temp reg");
4148     thread = t1;
4149     get_thread(thread);
4150 #endif
4151   }
4152 
4153 #ifdef _LP64
4154   if (var_size_in_bytes->is_valid()) {
4155     addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
4156   } else {
4157     addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
4158   }
4159 #else
4160   if (var_size_in_bytes->is_valid()) {
4161     addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
4162   } else {
4163     addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
4164   }
4165   adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0);
4166 #endif
4167 }
4168 
4169 void MacroAssembler::fp_runtime_fallback(address runtime_entry, int nb_args, int num_fpu_regs_in_use) {
4170   pusha();
4171 
4172   // if we are coming from c1, xmm registers may be live
4173   int off = 0;
4174   if (UseSSE == 1)  {
4175     subptr(rsp, sizeof(jdouble)*8);
4176     movflt(Address(rsp,off++*sizeof(jdouble)),xmm0);
4177     movflt(Address(rsp,off++*sizeof(jdouble)),xmm1);
4178     movflt(Address(rsp,off++*sizeof(jdouble)),xmm2);
4179     movflt(Address(rsp,off++*sizeof(jdouble)),xmm3);
4180     movflt(Address(rsp,off++*sizeof(jdouble)),xmm4);
4181     movflt(Address(rsp,off++*sizeof(jdouble)),xmm5);
4182     movflt(Address(rsp,off++*sizeof(jdouble)),xmm6);
4183     movflt(Address(rsp,off++*sizeof(jdouble)),xmm7);
4184   } else if (UseSSE >= 2)  {
4185 #ifdef COMPILER2
4186     if (MaxVectorSize > 16) {
4187       assert(UseAVX > 0, "256bit vectors are supported only with AVX");
4188       // Save upper half of YMM registes
4189       subptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8));
4190       vextractf128h(Address(rsp,  0),xmm0);
4191       vextractf128h(Address(rsp, 16),xmm1);
4192       vextractf128h(Address(rsp, 32),xmm2);
4193       vextractf128h(Address(rsp, 48),xmm3);
4194       vextractf128h(Address(rsp, 64),xmm4);
4195       vextractf128h(Address(rsp, 80),xmm5);
4196       vextractf128h(Address(rsp, 96),xmm6);
4197       vextractf128h(Address(rsp,112),xmm7);
4198 #ifdef _LP64
4199       vextractf128h(Address(rsp,128),xmm8);
4200       vextractf128h(Address(rsp,144),xmm9);
4201       vextractf128h(Address(rsp,160),xmm10);
4202       vextractf128h(Address(rsp,176),xmm11);
4203       vextractf128h(Address(rsp,192),xmm12);
4204       vextractf128h(Address(rsp,208),xmm13);
4205       vextractf128h(Address(rsp,224),xmm14);
4206       vextractf128h(Address(rsp,240),xmm15);
4207 #endif
4208     }
4209 #endif
4210     // Save whole 128bit (16 bytes) XMM regiters
4211     subptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8));
4212     movdqu(Address(rsp,off++*16),xmm0);
4213     movdqu(Address(rsp,off++*16),xmm1);
4214     movdqu(Address(rsp,off++*16),xmm2);
4215     movdqu(Address(rsp,off++*16),xmm3);
4216     movdqu(Address(rsp,off++*16),xmm4);
4217     movdqu(Address(rsp,off++*16),xmm5);
4218     movdqu(Address(rsp,off++*16),xmm6);
4219     movdqu(Address(rsp,off++*16),xmm7);
4220 #ifdef _LP64
4221     movdqu(Address(rsp,off++*16),xmm8);
4222     movdqu(Address(rsp,off++*16),xmm9);
4223     movdqu(Address(rsp,off++*16),xmm10);
4224     movdqu(Address(rsp,off++*16),xmm11);
4225     movdqu(Address(rsp,off++*16),xmm12);
4226     movdqu(Address(rsp,off++*16),xmm13);
4227     movdqu(Address(rsp,off++*16),xmm14);
4228     movdqu(Address(rsp,off++*16),xmm15);
4229 #endif
4230   }
4231 
4232   // Preserve registers across runtime call
4233   int incoming_argument_and_return_value_offset = -1;
4234   if (num_fpu_regs_in_use > 1) {
4235     // Must preserve all other FPU regs (could alternatively convert
4236     // SharedRuntime::dsin, dcos etc. into assembly routines known not to trash
4237     // FPU state, but can not trust C compiler)
4238     NEEDS_CLEANUP;
4239     // NOTE that in this case we also push the incoming argument(s) to
4240     // the stack and restore it later; we also use this stack slot to
4241     // hold the return value from dsin, dcos etc.
4242     for (int i = 0; i < num_fpu_regs_in_use; i++) {
4243       subptr(rsp, sizeof(jdouble));
4244       fstp_d(Address(rsp, 0));
4245     }
4246     incoming_argument_and_return_value_offset = sizeof(jdouble)*(num_fpu_regs_in_use-1);
4247     for (int i = nb_args-1; i >= 0; i--) {
4248       fld_d(Address(rsp, incoming_argument_and_return_value_offset-i*sizeof(jdouble)));
4249     }
4250   }
4251 
4252   subptr(rsp, nb_args*sizeof(jdouble));
4253   for (int i = 0; i < nb_args; i++) {
4254     fstp_d(Address(rsp, i*sizeof(jdouble)));
4255   }
4256 
4257 #ifdef _LP64
4258   if (nb_args > 0) {
4259     movdbl(xmm0, Address(rsp, 0));
4260   }
4261   if (nb_args > 1) {
4262     movdbl(xmm1, Address(rsp, sizeof(jdouble)));
4263   }
4264   assert(nb_args <= 2, "unsupported number of args");
4265 #endif // _LP64
4266 
4267   // NOTE: we must not use call_VM_leaf here because that requires a
4268   // complete interpreter frame in debug mode -- same bug as 4387334
4269   // MacroAssembler::call_VM_leaf_base is perfectly safe and will
4270   // do proper 64bit abi
4271 
4272   NEEDS_CLEANUP;
4273   // Need to add stack banging before this runtime call if it needs to
4274   // be taken; however, there is no generic stack banging routine at
4275   // the MacroAssembler level
4276 
4277   MacroAssembler::call_VM_leaf_base(runtime_entry, 0);
4278 
4279 #ifdef _LP64
4280   movsd(Address(rsp, 0), xmm0);
4281   fld_d(Address(rsp, 0));
4282 #endif // _LP64
4283   addptr(rsp, sizeof(jdouble) * nb_args);
4284   if (num_fpu_regs_in_use > 1) {
4285     // Must save return value to stack and then restore entire FPU
4286     // stack except incoming arguments
4287     fstp_d(Address(rsp, incoming_argument_and_return_value_offset));
4288     for (int i = 0; i < num_fpu_regs_in_use - nb_args; i++) {
4289       fld_d(Address(rsp, 0));
4290       addptr(rsp, sizeof(jdouble));
4291     }
4292     fld_d(Address(rsp, (nb_args-1)*sizeof(jdouble)));
4293     addptr(rsp, sizeof(jdouble) * nb_args);
4294   }
4295 
4296   off = 0;
4297   if (UseSSE == 1)  {
4298     movflt(xmm0, Address(rsp,off++*sizeof(jdouble)));
4299     movflt(xmm1, Address(rsp,off++*sizeof(jdouble)));
4300     movflt(xmm2, Address(rsp,off++*sizeof(jdouble)));
4301     movflt(xmm3, Address(rsp,off++*sizeof(jdouble)));
4302     movflt(xmm4, Address(rsp,off++*sizeof(jdouble)));
4303     movflt(xmm5, Address(rsp,off++*sizeof(jdouble)));
4304     movflt(xmm6, Address(rsp,off++*sizeof(jdouble)));
4305     movflt(xmm7, Address(rsp,off++*sizeof(jdouble)));
4306     addptr(rsp, sizeof(jdouble)*8);
4307   } else if (UseSSE >= 2)  {
4308     // Restore whole 128bit (16 bytes) XMM regiters
4309     movdqu(xmm0, Address(rsp,off++*16));
4310     movdqu(xmm1, Address(rsp,off++*16));
4311     movdqu(xmm2, Address(rsp,off++*16));
4312     movdqu(xmm3, Address(rsp,off++*16));
4313     movdqu(xmm4, Address(rsp,off++*16));
4314     movdqu(xmm5, Address(rsp,off++*16));
4315     movdqu(xmm6, Address(rsp,off++*16));
4316     movdqu(xmm7, Address(rsp,off++*16));
4317 #ifdef _LP64
4318     movdqu(xmm8, Address(rsp,off++*16));
4319     movdqu(xmm9, Address(rsp,off++*16));
4320     movdqu(xmm10, Address(rsp,off++*16));
4321     movdqu(xmm11, Address(rsp,off++*16));
4322     movdqu(xmm12, Address(rsp,off++*16));
4323     movdqu(xmm13, Address(rsp,off++*16));
4324     movdqu(xmm14, Address(rsp,off++*16));
4325     movdqu(xmm15, Address(rsp,off++*16));
4326 #endif
4327     addptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8));
4328 #ifdef COMPILER2
4329     if (MaxVectorSize > 16) {
4330       // Restore upper half of YMM registes.
4331       vinsertf128h(xmm0, Address(rsp,  0));
4332       vinsertf128h(xmm1, Address(rsp, 16));
4333       vinsertf128h(xmm2, Address(rsp, 32));
4334       vinsertf128h(xmm3, Address(rsp, 48));
4335       vinsertf128h(xmm4, Address(rsp, 64));
4336       vinsertf128h(xmm5, Address(rsp, 80));
4337       vinsertf128h(xmm6, Address(rsp, 96));
4338       vinsertf128h(xmm7, Address(rsp,112));
4339 #ifdef _LP64
4340       vinsertf128h(xmm8, Address(rsp,128));
4341       vinsertf128h(xmm9, Address(rsp,144));
4342       vinsertf128h(xmm10, Address(rsp,160));
4343       vinsertf128h(xmm11, Address(rsp,176));
4344       vinsertf128h(xmm12, Address(rsp,192));
4345       vinsertf128h(xmm13, Address(rsp,208));
4346       vinsertf128h(xmm14, Address(rsp,224));
4347       vinsertf128h(xmm15, Address(rsp,240));
4348 #endif
4349       addptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8));
4350     }
4351 #endif
4352   }
4353   popa();
4354 }
4355 
4356 static const double     pi_4 =  0.7853981633974483;
4357 
4358 void MacroAssembler::trigfunc(char trig, int num_fpu_regs_in_use) {
4359   // A hand-coded argument reduction for values in fabs(pi/4, pi/2)
4360   // was attempted in this code; unfortunately it appears that the
4361   // switch to 80-bit precision and back causes this to be
4362   // unprofitable compared with simply performing a runtime call if
4363   // the argument is out of the (-pi/4, pi/4) range.
4364 
4365   Register tmp = noreg;
4366   if (!VM_Version::supports_cmov()) {
4367     // fcmp needs a temporary so preserve rbx,
4368     tmp = rbx;
4369     push(tmp);
4370   }
4371 
4372   Label slow_case, done;
4373 
4374   ExternalAddress pi4_adr = (address)&pi_4;
4375   if (reachable(pi4_adr)) {
4376     // x ?<= pi/4
4377     fld_d(pi4_adr);
4378     fld_s(1);                // Stack:  X  PI/4  X
4379     fabs();                  // Stack: |X| PI/4  X
4380     fcmp(tmp);
4381     jcc(Assembler::above, slow_case);
4382 
4383     // fastest case: -pi/4 <= x <= pi/4
4384     switch(trig) {
4385     case 's':
4386       fsin();
4387       break;
4388     case 'c':
4389       fcos();
4390       break;
4391     case 't':
4392       ftan();
4393       break;
4394     default:
4395       assert(false, "bad intrinsic");
4396       break;
4397     }
4398     jmp(done);
4399   }
4400 
4401   // slow case: runtime call
4402   bind(slow_case);
4403 
4404   switch(trig) {
4405   case 's':
4406     {
4407       fp_runtime_fallback(CAST_FROM_FN_PTR(address, SharedRuntime::dsin), 1, num_fpu_regs_in_use);
4408     }
4409     break;
4410   case 'c':
4411     {
4412       fp_runtime_fallback(CAST_FROM_FN_PTR(address, SharedRuntime::dcos), 1, num_fpu_regs_in_use);
4413     }
4414     break;
4415   case 't':
4416     {
4417       fp_runtime_fallback(CAST_FROM_FN_PTR(address, SharedRuntime::dtan), 1, num_fpu_regs_in_use);
4418     }
4419     break;
4420   default:
4421     assert(false, "bad intrinsic");
4422     break;
4423   }
4424 
4425   // Come here with result in F-TOS
4426   bind(done);
4427 
4428   if (tmp != noreg) {
4429     pop(tmp);
4430   }
4431 }
4432 
4433 
4434 // Look up the method for a megamorphic invokeinterface call.
4435 // The target method is determined by <intf_klass, itable_index>.
4436 // The receiver klass is in recv_klass.
4437 // On success, the result will be in method_result, and execution falls through.
4438 // On failure, execution transfers to the given label.
4439 void MacroAssembler::lookup_interface_method(Register recv_klass,
4440                                              Register intf_klass,
4441                                              RegisterOrConstant itable_index,
4442                                              Register method_result,
4443                                              Register scan_temp,
4444                                              Label& L_no_such_interface) {
4445   assert_different_registers(recv_klass, intf_klass, method_result, scan_temp);
4446   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
4447          "caller must use same register for non-constant itable index as for method");
4448 
4449   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
4450   int vtable_base = InstanceKlass::vtable_start_offset() * wordSize;
4451   int itentry_off = itableMethodEntry::method_offset_in_bytes();
4452   int scan_step   = itableOffsetEntry::size() * wordSize;
4453   int vte_size    = vtableEntry::size() * wordSize;
4454   Address::ScaleFactor times_vte_scale = Address::times_ptr;
4455   assert(vte_size == wordSize, "else adjust times_vte_scale");
4456 
4457   movl(scan_temp, Address(recv_klass, InstanceKlass::vtable_length_offset() * wordSize));
4458 
4459   // %%% Could store the aligned, prescaled offset in the klassoop.
4460   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
4461   if (HeapWordsPerLong > 1) {
4462     // Round up to align_object_offset boundary
4463     // see code for InstanceKlass::start_of_itable!
4464     round_to(scan_temp, BytesPerLong);
4465   }
4466 
4467   // Adjust recv_klass by scaled itable_index, so we can free itable_index.
4468   assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
4469   lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
4470 
4471   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
4472   //   if (scan->interface() == intf) {
4473   //     result = (klass + scan->offset() + itable_index);
4474   //   }
4475   // }
4476   Label search, found_method;
4477 
4478   for (int peel = 1; peel >= 0; peel--) {
4479     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
4480     cmpptr(intf_klass, method_result);
4481 
4482     if (peel) {
4483       jccb(Assembler::equal, found_method);
4484     } else {
4485       jccb(Assembler::notEqual, search);
4486       // (invert the test to fall through to found_method...)
4487     }
4488 
4489     if (!peel)  break;
4490 
4491     bind(search);
4492 
4493     // Check that the previous entry is non-null.  A null entry means that
4494     // the receiver class doesn't implement the interface, and wasn't the
4495     // same as when the caller was compiled.
4496     testptr(method_result, method_result);
4497     jcc(Assembler::zero, L_no_such_interface);
4498     addptr(scan_temp, scan_step);
4499   }
4500 
4501   bind(found_method);
4502 
4503   // Got a hit.
4504   movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
4505   movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
4506 }
4507 
4508 
4509 // virtual method calling
4510 void MacroAssembler::lookup_virtual_method(Register recv_klass,
4511                                            RegisterOrConstant vtable_index,
4512                                            Register method_result) {
4513   const int base = InstanceKlass::vtable_start_offset() * wordSize;
4514   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
4515   Address vtable_entry_addr(recv_klass,
4516                             vtable_index, Address::times_ptr,
4517                             base + vtableEntry::method_offset_in_bytes());
4518   movptr(method_result, vtable_entry_addr);
4519 }
4520 
4521 
4522 void MacroAssembler::check_klass_subtype(Register sub_klass,
4523                            Register super_klass,
4524                            Register temp_reg,
4525                            Label& L_success) {
4526   Label L_failure;
4527   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, NULL);
4528   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
4529   bind(L_failure);
4530 }
4531 
4532 
4533 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
4534                                                    Register super_klass,
4535                                                    Register temp_reg,
4536                                                    Label* L_success,
4537                                                    Label* L_failure,
4538                                                    Label* L_slow_path,
4539                                         RegisterOrConstant super_check_offset) {
4540   assert_different_registers(sub_klass, super_klass, temp_reg);
4541   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
4542   if (super_check_offset.is_register()) {
4543     assert_different_registers(sub_klass, super_klass,
4544                                super_check_offset.as_register());
4545   } else if (must_load_sco) {
4546     assert(temp_reg != noreg, "supply either a temp or a register offset");
4547   }
4548 
4549   Label L_fallthrough;
4550   int label_nulls = 0;
4551   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
4552   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
4553   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
4554   assert(label_nulls <= 1, "at most one NULL in the batch");
4555 
4556   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
4557   int sco_offset = in_bytes(Klass::super_check_offset_offset());
4558   Address super_check_offset_addr(super_klass, sco_offset);
4559 
4560   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
4561   // range of a jccb.  If this routine grows larger, reconsider at
4562   // least some of these.
4563 #define local_jcc(assembler_cond, label)                                \
4564   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
4565   else                             jcc( assembler_cond, label) /*omit semi*/
4566 
4567   // Hacked jmp, which may only be used just before L_fallthrough.
4568 #define final_jmp(label)                                                \
4569   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
4570   else                            jmp(label)                /*omit semi*/
4571 
4572   // If the pointers are equal, we are done (e.g., String[] elements).
4573   // This self-check enables sharing of secondary supertype arrays among
4574   // non-primary types such as array-of-interface.  Otherwise, each such
4575   // type would need its own customized SSA.
4576   // We move this check to the front of the fast path because many
4577   // type checks are in fact trivially successful in this manner,
4578   // so we get a nicely predicted branch right at the start of the check.
4579   cmpptr(sub_klass, super_klass);
4580   local_jcc(Assembler::equal, *L_success);
4581 
4582   // Check the supertype display:
4583   if (must_load_sco) {
4584     // Positive movl does right thing on LP64.
4585     movl(temp_reg, super_check_offset_addr);
4586     super_check_offset = RegisterOrConstant(temp_reg);
4587   }
4588   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
4589   cmpptr(super_klass, super_check_addr); // load displayed supertype
4590 
4591   // This check has worked decisively for primary supers.
4592   // Secondary supers are sought in the super_cache ('super_cache_addr').
4593   // (Secondary supers are interfaces and very deeply nested subtypes.)
4594   // This works in the same check above because of a tricky aliasing
4595   // between the super_cache and the primary super display elements.
4596   // (The 'super_check_addr' can address either, as the case requires.)
4597   // Note that the cache is updated below if it does not help us find
4598   // what we need immediately.
4599   // So if it was a primary super, we can just fail immediately.
4600   // Otherwise, it's the slow path for us (no success at this point).
4601 
4602   if (super_check_offset.is_register()) {
4603     local_jcc(Assembler::equal, *L_success);
4604     cmpl(super_check_offset.as_register(), sc_offset);
4605     if (L_failure == &L_fallthrough) {
4606       local_jcc(Assembler::equal, *L_slow_path);
4607     } else {
4608       local_jcc(Assembler::notEqual, *L_failure);
4609       final_jmp(*L_slow_path);
4610     }
4611   } else if (super_check_offset.as_constant() == sc_offset) {
4612     // Need a slow path; fast failure is impossible.
4613     if (L_slow_path == &L_fallthrough) {
4614       local_jcc(Assembler::equal, *L_success);
4615     } else {
4616       local_jcc(Assembler::notEqual, *L_slow_path);
4617       final_jmp(*L_success);
4618     }
4619   } else {
4620     // No slow path; it's a fast decision.
4621     if (L_failure == &L_fallthrough) {
4622       local_jcc(Assembler::equal, *L_success);
4623     } else {
4624       local_jcc(Assembler::notEqual, *L_failure);
4625       final_jmp(*L_success);
4626     }
4627   }
4628 
4629   bind(L_fallthrough);
4630 
4631 #undef local_jcc
4632 #undef final_jmp
4633 }
4634 
4635 
4636 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
4637                                                    Register super_klass,
4638                                                    Register temp_reg,
4639                                                    Register temp2_reg,
4640                                                    Label* L_success,
4641                                                    Label* L_failure,
4642                                                    bool set_cond_codes) {
4643   assert_different_registers(sub_klass, super_klass, temp_reg);
4644   if (temp2_reg != noreg)
4645     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
4646 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
4647 
4648   Label L_fallthrough;
4649   int label_nulls = 0;
4650   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
4651   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
4652   assert(label_nulls <= 1, "at most one NULL in the batch");
4653 
4654   // a couple of useful fields in sub_klass:
4655   int ss_offset = in_bytes(Klass::secondary_supers_offset());
4656   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
4657   Address secondary_supers_addr(sub_klass, ss_offset);
4658   Address super_cache_addr(     sub_klass, sc_offset);
4659 
4660   // Do a linear scan of the secondary super-klass chain.
4661   // This code is rarely used, so simplicity is a virtue here.
4662   // The repne_scan instruction uses fixed registers, which we must spill.
4663   // Don't worry too much about pre-existing connections with the input regs.
4664 
4665   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
4666   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
4667 
4668   // Get super_klass value into rax (even if it was in rdi or rcx).
4669   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
4670   if (super_klass != rax || UseCompressedOops) {
4671     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
4672     mov(rax, super_klass);
4673   }
4674   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
4675   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
4676 
4677 #ifndef PRODUCT
4678   int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
4679   ExternalAddress pst_counter_addr((address) pst_counter);
4680   NOT_LP64(  incrementl(pst_counter_addr) );
4681   LP64_ONLY( lea(rcx, pst_counter_addr) );
4682   LP64_ONLY( incrementl(Address(rcx, 0)) );
4683 #endif //PRODUCT
4684 
4685   // We will consult the secondary-super array.
4686   movptr(rdi, secondary_supers_addr);
4687   // Load the array length.  (Positive movl does right thing on LP64.)
4688   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
4689   // Skip to start of data.
4690   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
4691 
4692   // Scan RCX words at [RDI] for an occurrence of RAX.
4693   // Set NZ/Z based on last compare.
4694   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
4695   // not change flags (only scas instruction which is repeated sets flags).
4696   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
4697 
4698     testptr(rax,rax); // Set Z = 0
4699     repne_scan();
4700 
4701   // Unspill the temp. registers:
4702   if (pushed_rdi)  pop(rdi);
4703   if (pushed_rcx)  pop(rcx);
4704   if (pushed_rax)  pop(rax);
4705 
4706   if (set_cond_codes) {
4707     // Special hack for the AD files:  rdi is guaranteed non-zero.
4708     assert(!pushed_rdi, "rdi must be left non-NULL");
4709     // Also, the condition codes are properly set Z/NZ on succeed/failure.
4710   }
4711 
4712   if (L_failure == &L_fallthrough)
4713         jccb(Assembler::notEqual, *L_failure);
4714   else  jcc(Assembler::notEqual, *L_failure);
4715 
4716   // Success.  Cache the super we found and proceed in triumph.
4717   movptr(super_cache_addr, super_klass);
4718 
4719   if (L_success != &L_fallthrough) {
4720     jmp(*L_success);
4721   }
4722 
4723 #undef IS_A_TEMP
4724 
4725   bind(L_fallthrough);
4726 }
4727 
4728 
4729 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
4730   if (VM_Version::supports_cmov()) {
4731     cmovl(cc, dst, src);
4732   } else {
4733     Label L;
4734     jccb(negate_condition(cc), L);
4735     movl(dst, src);
4736     bind(L);
4737   }
4738 }
4739 
4740 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
4741   if (VM_Version::supports_cmov()) {
4742     cmovl(cc, dst, src);
4743   } else {
4744     Label L;
4745     jccb(negate_condition(cc), L);
4746     movl(dst, src);
4747     bind(L);
4748   }
4749 }
4750 
4751 void MacroAssembler::verify_oop(Register reg, const char* s) {
4752   if (!VerifyOops) return;
4753 
4754   // Pass register number to verify_oop_subroutine
4755   const char* b = NULL;
4756   {
4757     ResourceMark rm;
4758     stringStream ss;
4759     ss.print("verify_oop: %s: %s", reg->name(), s);
4760     b = code_string(ss.as_string());
4761   }
4762   BLOCK_COMMENT("verify_oop {");
4763 #ifdef _LP64
4764   push(rscratch1);                    // save r10, trashed by movptr()
4765 #endif
4766   push(rax);                          // save rax,
4767   push(reg);                          // pass register argument
4768   ExternalAddress buffer((address) b);
4769   // avoid using pushptr, as it modifies scratch registers
4770   // and our contract is not to modify anything
4771   movptr(rax, buffer.addr());
4772   push(rax);
4773   // call indirectly to solve generation ordering problem
4774   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
4775   call(rax);
4776   // Caller pops the arguments (oop, message) and restores rax, r10
4777   BLOCK_COMMENT("} verify_oop");
4778 }
4779 
4780 
4781 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
4782                                                       Register tmp,
4783                                                       int offset) {
4784   intptr_t value = *delayed_value_addr;
4785   if (value != 0)
4786     return RegisterOrConstant(value + offset);
4787 
4788   // load indirectly to solve generation ordering problem
4789   movptr(tmp, ExternalAddress((address) delayed_value_addr));
4790 
4791 #ifdef ASSERT
4792   { Label L;
4793     testptr(tmp, tmp);
4794     if (WizardMode) {
4795       const char* buf = NULL;
4796       {
4797         ResourceMark rm;
4798         stringStream ss;
4799         ss.print("DelayedValue="INTPTR_FORMAT, delayed_value_addr[1]);
4800         buf = code_string(ss.as_string());
4801       }
4802       jcc(Assembler::notZero, L);
4803       STOP(buf);
4804     } else {
4805       jccb(Assembler::notZero, L);
4806       hlt();
4807     }
4808     bind(L);
4809   }
4810 #endif
4811 
4812   if (offset != 0)
4813     addptr(tmp, offset);
4814 
4815   return RegisterOrConstant(tmp);
4816 }
4817 
4818 
4819 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
4820                                          int extra_slot_offset) {
4821   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
4822   int stackElementSize = Interpreter::stackElementSize;
4823   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
4824 #ifdef ASSERT
4825   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
4826   assert(offset1 - offset == stackElementSize, "correct arithmetic");
4827 #endif
4828   Register             scale_reg    = noreg;
4829   Address::ScaleFactor scale_factor = Address::no_scale;
4830   if (arg_slot.is_constant()) {
4831     offset += arg_slot.as_constant() * stackElementSize;
4832   } else {
4833     scale_reg    = arg_slot.as_register();
4834     scale_factor = Address::times(stackElementSize);
4835   }
4836   offset += wordSize;           // return PC is on stack
4837   return Address(rsp, scale_reg, scale_factor, offset);
4838 }
4839 
4840 
4841 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
4842   if (!VerifyOops) return;
4843 
4844   // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
4845   // Pass register number to verify_oop_subroutine
4846   const char* b = NULL;
4847   {
4848     ResourceMark rm;
4849     stringStream ss;
4850     ss.print("verify_oop_addr: %s", s);
4851     b = code_string(ss.as_string());
4852   }
4853 #ifdef _LP64
4854   push(rscratch1);                    // save r10, trashed by movptr()
4855 #endif
4856   push(rax);                          // save rax,
4857   // addr may contain rsp so we will have to adjust it based on the push
4858   // we just did (and on 64 bit we do two pushes)
4859   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
4860   // stores rax into addr which is backwards of what was intended.
4861   if (addr.uses(rsp)) {
4862     lea(rax, addr);
4863     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
4864   } else {
4865     pushptr(addr);
4866   }
4867 
4868   ExternalAddress buffer((address) b);
4869   // pass msg argument
4870   // avoid using pushptr, as it modifies scratch registers
4871   // and our contract is not to modify anything
4872   movptr(rax, buffer.addr());
4873   push(rax);
4874 
4875   // call indirectly to solve generation ordering problem
4876   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
4877   call(rax);
4878   // Caller pops the arguments (addr, message) and restores rax, r10.
4879 }
4880 
4881 void MacroAssembler::verify_tlab() {
4882 #ifdef ASSERT
4883   if (UseTLAB && VerifyOops) {
4884     Label next, ok;
4885     Register t1 = rsi;
4886     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
4887 
4888     push(t1);
4889     NOT_LP64(push(thread_reg));
4890     NOT_LP64(get_thread(thread_reg));
4891 
4892     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
4893     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
4894     jcc(Assembler::aboveEqual, next);
4895     STOP("assert(top >= start)");
4896     should_not_reach_here();
4897 
4898     bind(next);
4899     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
4900     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
4901     jcc(Assembler::aboveEqual, ok);
4902     STOP("assert(top <= end)");
4903     should_not_reach_here();
4904 
4905     bind(ok);
4906     NOT_LP64(pop(thread_reg));
4907     pop(t1);
4908   }
4909 #endif
4910 }
4911 
4912 class ControlWord {
4913  public:
4914   int32_t _value;
4915 
4916   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
4917   int  precision_control() const       { return  (_value >>  8) & 3      ; }
4918   bool precision() const               { return ((_value >>  5) & 1) != 0; }
4919   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
4920   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
4921   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
4922   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
4923   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
4924 
4925   void print() const {
4926     // rounding control
4927     const char* rc;
4928     switch (rounding_control()) {
4929       case 0: rc = "round near"; break;
4930       case 1: rc = "round down"; break;
4931       case 2: rc = "round up  "; break;
4932       case 3: rc = "chop      "; break;
4933     };
4934     // precision control
4935     const char* pc;
4936     switch (precision_control()) {
4937       case 0: pc = "24 bits "; break;
4938       case 1: pc = "reserved"; break;
4939       case 2: pc = "53 bits "; break;
4940       case 3: pc = "64 bits "; break;
4941     };
4942     // flags
4943     char f[9];
4944     f[0] = ' ';
4945     f[1] = ' ';
4946     f[2] = (precision   ()) ? 'P' : 'p';
4947     f[3] = (underflow   ()) ? 'U' : 'u';
4948     f[4] = (overflow    ()) ? 'O' : 'o';
4949     f[5] = (zero_divide ()) ? 'Z' : 'z';
4950     f[6] = (denormalized()) ? 'D' : 'd';
4951     f[7] = (invalid     ()) ? 'I' : 'i';
4952     f[8] = '\x0';
4953     // output
4954     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
4955   }
4956 
4957 };
4958 
4959 class StatusWord {
4960  public:
4961   int32_t _value;
4962 
4963   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
4964   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
4965   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
4966   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
4967   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
4968   int  top() const                     { return  (_value >> 11) & 7      ; }
4969   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
4970   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
4971   bool precision() const               { return ((_value >>  5) & 1) != 0; }
4972   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
4973   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
4974   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
4975   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
4976   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
4977 
4978   void print() const {
4979     // condition codes
4980     char c[5];
4981     c[0] = (C3()) ? '3' : '-';
4982     c[1] = (C2()) ? '2' : '-';
4983     c[2] = (C1()) ? '1' : '-';
4984     c[3] = (C0()) ? '0' : '-';
4985     c[4] = '\x0';
4986     // flags
4987     char f[9];
4988     f[0] = (error_status()) ? 'E' : '-';
4989     f[1] = (stack_fault ()) ? 'S' : '-';
4990     f[2] = (precision   ()) ? 'P' : '-';
4991     f[3] = (underflow   ()) ? 'U' : '-';
4992     f[4] = (overflow    ()) ? 'O' : '-';
4993     f[5] = (zero_divide ()) ? 'Z' : '-';
4994     f[6] = (denormalized()) ? 'D' : '-';
4995     f[7] = (invalid     ()) ? 'I' : '-';
4996     f[8] = '\x0';
4997     // output
4998     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
4999   }
5000 
5001 };
5002 
5003 class TagWord {
5004  public:
5005   int32_t _value;
5006 
5007   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
5008 
5009   void print() const {
5010     printf("%04x", _value & 0xFFFF);
5011   }
5012 
5013 };
5014 
5015 class FPU_Register {
5016  public:
5017   int32_t _m0;
5018   int32_t _m1;
5019   int16_t _ex;
5020 
5021   bool is_indefinite() const           {
5022     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
5023   }
5024 
5025   void print() const {
5026     char  sign = (_ex < 0) ? '-' : '+';
5027     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
5028     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
5029   };
5030 
5031 };
5032 
5033 class FPU_State {
5034  public:
5035   enum {
5036     register_size       = 10,
5037     number_of_registers =  8,
5038     register_mask       =  7
5039   };
5040 
5041   ControlWord  _control_word;
5042   StatusWord   _status_word;
5043   TagWord      _tag_word;
5044   int32_t      _error_offset;
5045   int32_t      _error_selector;
5046   int32_t      _data_offset;
5047   int32_t      _data_selector;
5048   int8_t       _register[register_size * number_of_registers];
5049 
5050   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
5051   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
5052 
5053   const char* tag_as_string(int tag) const {
5054     switch (tag) {
5055       case 0: return "valid";
5056       case 1: return "zero";
5057       case 2: return "special";
5058       case 3: return "empty";
5059     }
5060     ShouldNotReachHere();
5061     return NULL;
5062   }
5063 
5064   void print() const {
5065     // print computation registers
5066     { int t = _status_word.top();
5067       for (int i = 0; i < number_of_registers; i++) {
5068         int j = (i - t) & register_mask;
5069         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
5070         st(j)->print();
5071         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
5072       }
5073     }
5074     printf("\n");
5075     // print control registers
5076     printf("ctrl = "); _control_word.print(); printf("\n");
5077     printf("stat = "); _status_word .print(); printf("\n");
5078     printf("tags = "); _tag_word    .print(); printf("\n");
5079   }
5080 
5081 };
5082 
5083 class Flag_Register {
5084  public:
5085   int32_t _value;
5086 
5087   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
5088   bool direction() const               { return ((_value >> 10) & 1) != 0; }
5089   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
5090   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
5091   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
5092   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
5093   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
5094 
5095   void print() const {
5096     // flags
5097     char f[8];
5098     f[0] = (overflow       ()) ? 'O' : '-';
5099     f[1] = (direction      ()) ? 'D' : '-';
5100     f[2] = (sign           ()) ? 'S' : '-';
5101     f[3] = (zero           ()) ? 'Z' : '-';
5102     f[4] = (auxiliary_carry()) ? 'A' : '-';
5103     f[5] = (parity         ()) ? 'P' : '-';
5104     f[6] = (carry          ()) ? 'C' : '-';
5105     f[7] = '\x0';
5106     // output
5107     printf("%08x  flags = %s", _value, f);
5108   }
5109 
5110 };
5111 
5112 class IU_Register {
5113  public:
5114   int32_t _value;
5115 
5116   void print() const {
5117     printf("%08x  %11d", _value, _value);
5118   }
5119 
5120 };
5121 
5122 class IU_State {
5123  public:
5124   Flag_Register _eflags;
5125   IU_Register   _rdi;
5126   IU_Register   _rsi;
5127   IU_Register   _rbp;
5128   IU_Register   _rsp;
5129   IU_Register   _rbx;
5130   IU_Register   _rdx;
5131   IU_Register   _rcx;
5132   IU_Register   _rax;
5133 
5134   void print() const {
5135     // computation registers
5136     printf("rax,  = "); _rax.print(); printf("\n");
5137     printf("rbx,  = "); _rbx.print(); printf("\n");
5138     printf("rcx  = "); _rcx.print(); printf("\n");
5139     printf("rdx  = "); _rdx.print(); printf("\n");
5140     printf("rdi  = "); _rdi.print(); printf("\n");
5141     printf("rsi  = "); _rsi.print(); printf("\n");
5142     printf("rbp,  = "); _rbp.print(); printf("\n");
5143     printf("rsp  = "); _rsp.print(); printf("\n");
5144     printf("\n");
5145     // control registers
5146     printf("flgs = "); _eflags.print(); printf("\n");
5147   }
5148 };
5149 
5150 
5151 class CPU_State {
5152  public:
5153   FPU_State _fpu_state;
5154   IU_State  _iu_state;
5155 
5156   void print() const {
5157     printf("--------------------------------------------------\n");
5158     _iu_state .print();
5159     printf("\n");
5160     _fpu_state.print();
5161     printf("--------------------------------------------------\n");
5162   }
5163 
5164 };
5165 
5166 
5167 static void _print_CPU_state(CPU_State* state) {
5168   state->print();
5169 };
5170 
5171 
5172 void MacroAssembler::print_CPU_state() {
5173   push_CPU_state();
5174   push(rsp);                // pass CPU state
5175   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
5176   addptr(rsp, wordSize);       // discard argument
5177   pop_CPU_state();
5178 }
5179 
5180 
5181 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
5182   static int counter = 0;
5183   FPU_State* fs = &state->_fpu_state;
5184   counter++;
5185   // For leaf calls, only verify that the top few elements remain empty.
5186   // We only need 1 empty at the top for C2 code.
5187   if( stack_depth < 0 ) {
5188     if( fs->tag_for_st(7) != 3 ) {
5189       printf("FPR7 not empty\n");
5190       state->print();
5191       assert(false, "error");
5192       return false;
5193     }
5194     return true;                // All other stack states do not matter
5195   }
5196 
5197   assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std,
5198          "bad FPU control word");
5199 
5200   // compute stack depth
5201   int i = 0;
5202   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
5203   int d = i;
5204   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
5205   // verify findings
5206   if (i != FPU_State::number_of_registers) {
5207     // stack not contiguous
5208     printf("%s: stack not contiguous at ST%d\n", s, i);
5209     state->print();
5210     assert(false, "error");
5211     return false;
5212   }
5213   // check if computed stack depth corresponds to expected stack depth
5214   if (stack_depth < 0) {
5215     // expected stack depth is -stack_depth or less
5216     if (d > -stack_depth) {
5217       // too many elements on the stack
5218       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
5219       state->print();
5220       assert(false, "error");
5221       return false;
5222     }
5223   } else {
5224     // expected stack depth is stack_depth
5225     if (d != stack_depth) {
5226       // wrong stack depth
5227       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
5228       state->print();
5229       assert(false, "error");
5230       return false;
5231     }
5232   }
5233   // everything is cool
5234   return true;
5235 }
5236 
5237 
5238 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
5239   if (!VerifyFPU) return;
5240   push_CPU_state();
5241   push(rsp);                // pass CPU state
5242   ExternalAddress msg((address) s);
5243   // pass message string s
5244   pushptr(msg.addr());
5245   push(stack_depth);        // pass stack depth
5246   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
5247   addptr(rsp, 3 * wordSize);   // discard arguments
5248   // check for error
5249   { Label L;
5250     testl(rax, rax);
5251     jcc(Assembler::notZero, L);
5252     int3();                  // break if error condition
5253     bind(L);
5254   }
5255   pop_CPU_state();
5256 }
5257 
5258 void MacroAssembler::restore_cpu_control_state_after_jni() {
5259   // Either restore the MXCSR register after returning from the JNI Call
5260   // or verify that it wasn't changed (with -Xcheck:jni flag).
5261   if (VM_Version::supports_sse()) {
5262     if (RestoreMXCSROnJNICalls) {
5263       ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std()));
5264     } else if (CheckJNICalls) {
5265       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
5266     }
5267   }
5268   if (VM_Version::supports_avx()) {
5269     // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
5270     vzeroupper();
5271   }
5272 
5273 #ifndef _LP64
5274   // Either restore the x87 floating pointer control word after returning
5275   // from the JNI call or verify that it wasn't changed.
5276   if (CheckJNICalls) {
5277     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
5278   }
5279 #endif // _LP64
5280 }
5281 
5282 
5283 void MacroAssembler::load_klass(Register dst, Register src) {
5284 #ifdef _LP64
5285   if (UseCompressedClassPointers) {
5286     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5287     decode_klass_not_null(dst);
5288   } else
5289 #endif
5290     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5291 }
5292 
5293 void MacroAssembler::load_prototype_header(Register dst, Register src) {
5294   load_klass(dst, src);
5295   movptr(dst, Address(dst, Klass::prototype_header_offset()));
5296 }
5297 
5298 void MacroAssembler::store_klass(Register dst, Register src) {
5299 #ifdef _LP64
5300   if (UseCompressedClassPointers) {
5301     encode_klass_not_null(src);
5302     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5303   } else
5304 #endif
5305     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5306 }
5307 
5308 void MacroAssembler::load_heap_oop(Register dst, Address src) {
5309 #ifdef _LP64
5310   // FIXME: Must change all places where we try to load the klass.
5311   if (UseCompressedOops) {
5312     movl(dst, src);
5313     decode_heap_oop(dst);
5314   } else
5315 #endif
5316     movptr(dst, src);
5317 }
5318 
5319 // Doesn't do verfication, generates fixed size code
5320 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src) {
5321 #ifdef _LP64
5322   if (UseCompressedOops) {
5323     movl(dst, src);
5324     decode_heap_oop_not_null(dst);
5325   } else
5326 #endif
5327     movptr(dst, src);
5328 }
5329 
5330 void MacroAssembler::store_heap_oop(Address dst, Register src) {
5331 #ifdef _LP64
5332   if (UseCompressedOops) {
5333     assert(!dst.uses(src), "not enough registers");
5334     encode_heap_oop(src);
5335     movl(dst, src);
5336   } else
5337 #endif
5338     movptr(dst, src);
5339 }
5340 
5341 void MacroAssembler::cmp_heap_oop(Register src1, Address src2, Register tmp) {
5342   assert_different_registers(src1, tmp);
5343 #ifdef _LP64
5344   if (UseCompressedOops) {
5345     bool did_push = false;
5346     if (tmp == noreg) {
5347       tmp = rax;
5348       push(tmp);
5349       did_push = true;
5350       assert(!src2.uses(rsp), "can't push");
5351     }
5352     load_heap_oop(tmp, src2);
5353     cmpptr(src1, tmp);
5354     if (did_push)  pop(tmp);
5355   } else
5356 #endif
5357     cmpptr(src1, src2);
5358 }
5359 
5360 // Used for storing NULLs.
5361 void MacroAssembler::store_heap_oop_null(Address dst) {
5362 #ifdef _LP64
5363   if (UseCompressedOops) {
5364     movl(dst, (int32_t)NULL_WORD);
5365   } else {
5366     movslq(dst, (int32_t)NULL_WORD);
5367   }
5368 #else
5369   movl(dst, (int32_t)NULL_WORD);
5370 #endif
5371 }
5372 
5373 #ifdef _LP64
5374 void MacroAssembler::store_klass_gap(Register dst, Register src) {
5375   if (UseCompressedClassPointers) {
5376     // Store to klass gap in destination
5377     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
5378   }
5379 }
5380 
5381 #ifdef ASSERT
5382 void MacroAssembler::verify_heapbase(const char* msg) {
5383   assert (UseCompressedOops, "should be compressed");
5384   assert (Universe::heap() != NULL, "java heap should be initialized");
5385   if (CheckCompressedOops) {
5386     Label ok;
5387     push(rscratch1); // cmpptr trashes rscratch1
5388     cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
5389     jcc(Assembler::equal, ok);
5390     STOP(msg);
5391     bind(ok);
5392     pop(rscratch1);
5393   }
5394 }
5395 #endif
5396 
5397 // Algorithm must match oop.inline.hpp encode_heap_oop.
5398 void MacroAssembler::encode_heap_oop(Register r) {
5399 #ifdef ASSERT
5400   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
5401 #endif
5402   verify_oop(r, "broken oop in encode_heap_oop");
5403   if (Universe::narrow_oop_base() == NULL) {
5404     if (Universe::narrow_oop_shift() != 0) {
5405       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
5406       shrq(r, LogMinObjAlignmentInBytes);
5407     }
5408     return;
5409   }
5410   testq(r, r);
5411   cmovq(Assembler::equal, r, r12_heapbase);
5412   subq(r, r12_heapbase);
5413   shrq(r, LogMinObjAlignmentInBytes);
5414 }
5415 
5416 void MacroAssembler::encode_heap_oop_not_null(Register r) {
5417 #ifdef ASSERT
5418   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
5419   if (CheckCompressedOops) {
5420     Label ok;
5421     testq(r, r);
5422     jcc(Assembler::notEqual, ok);
5423     STOP("null oop passed to encode_heap_oop_not_null");
5424     bind(ok);
5425   }
5426 #endif
5427   verify_oop(r, "broken oop in encode_heap_oop_not_null");
5428   if (Universe::narrow_oop_base() != NULL) {
5429     subq(r, r12_heapbase);
5430   }
5431   if (Universe::narrow_oop_shift() != 0) {
5432     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
5433     shrq(r, LogMinObjAlignmentInBytes);
5434   }
5435 }
5436 
5437 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
5438 #ifdef ASSERT
5439   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
5440   if (CheckCompressedOops) {
5441     Label ok;
5442     testq(src, src);
5443     jcc(Assembler::notEqual, ok);
5444     STOP("null oop passed to encode_heap_oop_not_null2");
5445     bind(ok);
5446   }
5447 #endif
5448   verify_oop(src, "broken oop in encode_heap_oop_not_null2");
5449   if (dst != src) {
5450     movq(dst, src);
5451   }
5452   if (Universe::narrow_oop_base() != NULL) {
5453     subq(dst, r12_heapbase);
5454   }
5455   if (Universe::narrow_oop_shift() != 0) {
5456     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
5457     shrq(dst, LogMinObjAlignmentInBytes);
5458   }
5459 }
5460 
5461 void  MacroAssembler::decode_heap_oop(Register r) {
5462 #ifdef ASSERT
5463   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
5464 #endif
5465   if (Universe::narrow_oop_base() == NULL) {
5466     if (Universe::narrow_oop_shift() != 0) {
5467       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
5468       shlq(r, LogMinObjAlignmentInBytes);
5469     }
5470   } else {
5471     Label done;
5472     shlq(r, LogMinObjAlignmentInBytes);
5473     jccb(Assembler::equal, done);
5474     addq(r, r12_heapbase);
5475     bind(done);
5476   }
5477   verify_oop(r, "broken oop in decode_heap_oop");
5478 }
5479 
5480 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
5481   // Note: it will change flags
5482   assert (UseCompressedOops, "should only be used for compressed headers");
5483   assert (Universe::heap() != NULL, "java heap should be initialized");
5484   // Cannot assert, unverified entry point counts instructions (see .ad file)
5485   // vtableStubs also counts instructions in pd_code_size_limit.
5486   // Also do not verify_oop as this is called by verify_oop.
5487   if (Universe::narrow_oop_shift() != 0) {
5488     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
5489     shlq(r, LogMinObjAlignmentInBytes);
5490     if (Universe::narrow_oop_base() != NULL) {
5491       addq(r, r12_heapbase);
5492     }
5493   } else {
5494     assert (Universe::narrow_oop_base() == NULL, "sanity");
5495   }
5496 }
5497 
5498 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
5499   // Note: it will change flags
5500   assert (UseCompressedOops, "should only be used for compressed headers");
5501   assert (Universe::heap() != NULL, "java heap should be initialized");
5502   // Cannot assert, unverified entry point counts instructions (see .ad file)
5503   // vtableStubs also counts instructions in pd_code_size_limit.
5504   // Also do not verify_oop as this is called by verify_oop.
5505   if (Universe::narrow_oop_shift() != 0) {
5506     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
5507     if (LogMinObjAlignmentInBytes == Address::times_8) {
5508       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
5509     } else {
5510       if (dst != src) {
5511         movq(dst, src);
5512       }
5513       shlq(dst, LogMinObjAlignmentInBytes);
5514       if (Universe::narrow_oop_base() != NULL) {
5515         addq(dst, r12_heapbase);
5516       }
5517     }
5518   } else {
5519     assert (Universe::narrow_oop_base() == NULL, "sanity");
5520     if (dst != src) {
5521       movq(dst, src);
5522     }
5523   }
5524 }
5525 
5526 void MacroAssembler::encode_klass_not_null(Register r) {
5527   if (Universe::narrow_klass_base() != NULL) {
5528     // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
5529     assert(r != r12_heapbase, "Encoding a klass in r12");
5530     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
5531     subq(r, r12_heapbase);
5532   }
5533   if (Universe::narrow_klass_shift() != 0) {
5534     assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
5535     shrq(r, LogKlassAlignmentInBytes);
5536   }
5537   if (Universe::narrow_klass_base() != NULL) {
5538     reinit_heapbase();
5539   }
5540 }
5541 
5542 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
5543   if (dst == src) {
5544     encode_klass_not_null(src);
5545   } else {
5546     if (Universe::narrow_klass_base() != NULL) {
5547       mov64(dst, (int64_t)Universe::narrow_klass_base());
5548       negq(dst);
5549       addq(dst, src);
5550     } else {
5551       movptr(dst, src);
5552     }
5553     if (Universe::narrow_klass_shift() != 0) {
5554       assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
5555       shrq(dst, LogKlassAlignmentInBytes);
5556     }
5557   }
5558 }
5559 
5560 // Function instr_size_for_decode_klass_not_null() counts the instructions
5561 // generated by decode_klass_not_null(register r) and reinit_heapbase(),
5562 // when (Universe::heap() != NULL).  Hence, if the instructions they
5563 // generate change, then this method needs to be updated.
5564 int MacroAssembler::instr_size_for_decode_klass_not_null() {
5565   assert (UseCompressedClassPointers, "only for compressed klass ptrs");
5566   if (Universe::narrow_klass_base() != NULL) {
5567     // mov64 + addq + shlq? + mov64  (for reinit_heapbase()).
5568     return (Universe::narrow_klass_shift() == 0 ? 20 : 24);
5569   } else {
5570     // longest load decode klass function, mov64, leaq
5571     return 16;
5572   }
5573 }
5574 
5575 // !!! If the instructions that get generated here change then function
5576 // instr_size_for_decode_klass_not_null() needs to get updated.
5577 void  MacroAssembler::decode_klass_not_null(Register r) {
5578   // Note: it will change flags
5579   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5580   assert(r != r12_heapbase, "Decoding a klass in r12");
5581   // Cannot assert, unverified entry point counts instructions (see .ad file)
5582   // vtableStubs also counts instructions in pd_code_size_limit.
5583   // Also do not verify_oop as this is called by verify_oop.
5584   if (Universe::narrow_klass_shift() != 0) {
5585     assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
5586     shlq(r, LogKlassAlignmentInBytes);
5587   }
5588   // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
5589   if (Universe::narrow_klass_base() != NULL) {
5590     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
5591     addq(r, r12_heapbase);
5592     reinit_heapbase();
5593   }
5594 }
5595 
5596 void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
5597   // Note: it will change flags
5598   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5599   if (dst == src) {
5600     decode_klass_not_null(dst);
5601   } else {
5602     // Cannot assert, unverified entry point counts instructions (see .ad file)
5603     // vtableStubs also counts instructions in pd_code_size_limit.
5604     // Also do not verify_oop as this is called by verify_oop.
5605     mov64(dst, (int64_t)Universe::narrow_klass_base());
5606     if (Universe::narrow_klass_shift() != 0) {
5607       assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
5608       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
5609       leaq(dst, Address(dst, src, Address::times_8, 0));
5610     } else {
5611       addq(dst, src);
5612     }
5613   }
5614 }
5615 
5616 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
5617   assert (UseCompressedOops, "should only be used for compressed headers");
5618   assert (Universe::heap() != NULL, "java heap should be initialized");
5619   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
5620   int oop_index = oop_recorder()->find_index(obj);
5621   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5622   mov_narrow_oop(dst, oop_index, rspec);
5623 }
5624 
5625 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
5626   assert (UseCompressedOops, "should only be used for compressed headers");
5627   assert (Universe::heap() != NULL, "java heap should be initialized");
5628   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
5629   int oop_index = oop_recorder()->find_index(obj);
5630   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5631   mov_narrow_oop(dst, oop_index, rspec);
5632 }
5633 
5634 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
5635   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5636   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
5637   int klass_index = oop_recorder()->find_index(k);
5638   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5639   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
5640 }
5641 
5642 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
5643   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5644   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
5645   int klass_index = oop_recorder()->find_index(k);
5646   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5647   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
5648 }
5649 
5650 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
5651   assert (UseCompressedOops, "should only be used for compressed headers");
5652   assert (Universe::heap() != NULL, "java heap should be initialized");
5653   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
5654   int oop_index = oop_recorder()->find_index(obj);
5655   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5656   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
5657 }
5658 
5659 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
5660   assert (UseCompressedOops, "should only be used for compressed headers");
5661   assert (Universe::heap() != NULL, "java heap should be initialized");
5662   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
5663   int oop_index = oop_recorder()->find_index(obj);
5664   RelocationHolder rspec = oop_Relocation::spec(oop_index);
5665   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
5666 }
5667 
5668 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
5669   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5670   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
5671   int klass_index = oop_recorder()->find_index(k);
5672   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5673   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
5674 }
5675 
5676 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
5677   assert (UseCompressedClassPointers, "should only be used for compressed headers");
5678   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
5679   int klass_index = oop_recorder()->find_index(k);
5680   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5681   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
5682 }
5683 
5684 void MacroAssembler::reinit_heapbase() {
5685   if (UseCompressedOops || UseCompressedClassPointers) {
5686     if (Universe::heap() != NULL) {
5687       if (Universe::narrow_oop_base() == NULL) {
5688         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
5689       } else {
5690         mov64(r12_heapbase, (int64_t)Universe::narrow_ptrs_base());
5691       }
5692     } else {
5693       movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
5694     }
5695   }
5696 }
5697 
5698 #endif // _LP64
5699 
5700 
5701 // C2 compiled method's prolog code.
5702 void MacroAssembler::verified_entry(int framesize, bool stack_bang, bool fp_mode_24b) {
5703 
5704   // WARNING: Initial instruction MUST be 5 bytes or longer so that
5705   // NativeJump::patch_verified_entry will be able to patch out the entry
5706   // code safely. The push to verify stack depth is ok at 5 bytes,
5707   // the frame allocation can be either 3 or 6 bytes. So if we don't do
5708   // stack bang then we must use the 6 byte frame allocation even if
5709   // we have no frame. :-(
5710 
5711   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
5712   // Remove word for return addr
5713   framesize -= wordSize;
5714 
5715   // Calls to C2R adapters often do not accept exceptional returns.
5716   // We require that their callers must bang for them.  But be careful, because
5717   // some VM calls (such as call site linkage) can use several kilobytes of
5718   // stack.  But the stack safety zone should account for that.
5719   // See bugs 4446381, 4468289, 4497237.
5720   if (stack_bang) {
5721     generate_stack_overflow_check(framesize);
5722 
5723     // We always push rbp, so that on return to interpreter rbp, will be
5724     // restored correctly and we can correct the stack.
5725     push(rbp);
5726     // Remove word for ebp
5727     framesize -= wordSize;
5728 
5729     // Create frame
5730     if (framesize) {
5731       subptr(rsp, framesize);
5732     }
5733   } else {
5734     // Create frame (force generation of a 4 byte immediate value)
5735     subptr_imm32(rsp, framesize);
5736 
5737     // Save RBP register now.
5738     framesize -= wordSize;
5739     movptr(Address(rsp, framesize), rbp);
5740   }
5741 
5742   if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
5743     framesize -= wordSize;
5744     movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
5745   }
5746 
5747 #ifndef _LP64
5748   // If method sets FPU control word do it now
5749   if (fp_mode_24b) {
5750     fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
5751   }
5752   if (UseSSE >= 2 && VerifyFPU) {
5753     verify_FPU(0, "FPU stack must be clean on entry");
5754   }
5755 #endif
5756 
5757 #ifdef ASSERT
5758   if (VerifyStackAtCalls) {
5759     Label L;
5760     push(rax);
5761     mov(rax, rsp);
5762     andptr(rax, StackAlignmentInBytes-1);
5763     cmpptr(rax, StackAlignmentInBytes-wordSize);
5764     pop(rax);
5765     jcc(Assembler::equal, L);
5766     STOP("Stack is not properly aligned!");
5767     bind(L);
5768   }
5769 #endif
5770 
5771 }
5772 
5773 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp) {
5774   // cnt - number of qwords (8-byte words).
5775   // base - start address, qword aligned.
5776   assert(base==rdi, "base register must be edi for rep stos");
5777   assert(tmp==rax,   "tmp register must be eax for rep stos");
5778   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
5779 
5780   xorptr(tmp, tmp);
5781   if (UseFastStosb) {
5782     shlptr(cnt,3); // convert to number of bytes
5783     rep_stosb();
5784   } else {
5785     NOT_LP64(shlptr(cnt,1);) // convert to number of dwords for 32-bit VM
5786     rep_stos();
5787   }
5788 }
5789 
5790 // IndexOf for constant substrings with size >= 8 chars
5791 // which don't need to be loaded through stack.
5792 void MacroAssembler::string_indexofC8(Register str1, Register str2,
5793                                       Register cnt1, Register cnt2,
5794                                       int int_cnt2,  Register result,
5795                                       XMMRegister vec, Register tmp) {
5796   ShortBranchVerifier sbv(this);
5797   assert(UseSSE42Intrinsics, "SSE4.2 is required");
5798 
5799   // This method uses pcmpestri inxtruction with bound registers
5800   //   inputs:
5801   //     xmm - substring
5802   //     rax - substring length (elements count)
5803   //     mem - scanned string
5804   //     rdx - string length (elements count)
5805   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
5806   //   outputs:
5807   //     rcx - matched index in string
5808   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
5809 
5810   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR,
5811         RET_FOUND, RET_NOT_FOUND, EXIT, FOUND_SUBSTR,
5812         MATCH_SUBSTR_HEAD, RELOAD_STR, FOUND_CANDIDATE;
5813 
5814   // Note, inline_string_indexOf() generates checks:
5815   // if (substr.count > string.count) return -1;
5816   // if (substr.count == 0) return 0;
5817   assert(int_cnt2 >= 8, "this code isused only for cnt2 >= 8 chars");
5818 
5819   // Load substring.
5820   movdqu(vec, Address(str2, 0));
5821   movl(cnt2, int_cnt2);
5822   movptr(result, str1); // string addr
5823 
5824   if (int_cnt2 > 8) {
5825     jmpb(SCAN_TO_SUBSTR);
5826 
5827     // Reload substr for rescan, this code
5828     // is executed only for large substrings (> 8 chars)
5829     bind(RELOAD_SUBSTR);
5830     movdqu(vec, Address(str2, 0));
5831     negptr(cnt2); // Jumped here with negative cnt2, convert to positive
5832 
5833     bind(RELOAD_STR);
5834     // We came here after the beginning of the substring was
5835     // matched but the rest of it was not so we need to search
5836     // again. Start from the next element after the previous match.
5837 
5838     // cnt2 is number of substring reminding elements and
5839     // cnt1 is number of string reminding elements when cmp failed.
5840     // Restored cnt1 = cnt1 - cnt2 + int_cnt2
5841     subl(cnt1, cnt2);
5842     addl(cnt1, int_cnt2);
5843     movl(cnt2, int_cnt2); // Now restore cnt2
5844 
5845     decrementl(cnt1);     // Shift to next element
5846     cmpl(cnt1, cnt2);
5847     jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
5848 
5849     addptr(result, 2);
5850 
5851   } // (int_cnt2 > 8)
5852 
5853   // Scan string for start of substr in 16-byte vectors
5854   bind(SCAN_TO_SUBSTR);
5855   pcmpestri(vec, Address(result, 0), 0x0d);
5856   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
5857   subl(cnt1, 8);
5858   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
5859   cmpl(cnt1, cnt2);
5860   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
5861   addptr(result, 16);
5862   jmpb(SCAN_TO_SUBSTR);
5863 
5864   // Found a potential substr
5865   bind(FOUND_CANDIDATE);
5866   // Matched whole vector if first element matched (tmp(rcx) == 0).
5867   if (int_cnt2 == 8) {
5868     jccb(Assembler::overflow, RET_FOUND);    // OF == 1
5869   } else { // int_cnt2 > 8
5870     jccb(Assembler::overflow, FOUND_SUBSTR);
5871   }
5872   // After pcmpestri tmp(rcx) contains matched element index
5873   // Compute start addr of substr
5874   lea(result, Address(result, tmp, Address::times_2));
5875 
5876   // Make sure string is still long enough
5877   subl(cnt1, tmp);
5878   cmpl(cnt1, cnt2);
5879   if (int_cnt2 == 8) {
5880     jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
5881   } else { // int_cnt2 > 8
5882     jccb(Assembler::greaterEqual, MATCH_SUBSTR_HEAD);
5883   }
5884   // Left less then substring.
5885 
5886   bind(RET_NOT_FOUND);
5887   movl(result, -1);
5888   jmpb(EXIT);
5889 
5890   if (int_cnt2 > 8) {
5891     // This code is optimized for the case when whole substring
5892     // is matched if its head is matched.
5893     bind(MATCH_SUBSTR_HEAD);
5894     pcmpestri(vec, Address(result, 0), 0x0d);
5895     // Reload only string if does not match
5896     jccb(Assembler::noOverflow, RELOAD_STR); // OF == 0
5897 
5898     Label CONT_SCAN_SUBSTR;
5899     // Compare the rest of substring (> 8 chars).
5900     bind(FOUND_SUBSTR);
5901     // First 8 chars are already matched.
5902     negptr(cnt2);
5903     addptr(cnt2, 8);
5904 
5905     bind(SCAN_SUBSTR);
5906     subl(cnt1, 8);
5907     cmpl(cnt2, -8); // Do not read beyond substring
5908     jccb(Assembler::lessEqual, CONT_SCAN_SUBSTR);
5909     // Back-up strings to avoid reading beyond substring:
5910     // cnt1 = cnt1 - cnt2 + 8
5911     addl(cnt1, cnt2); // cnt2 is negative
5912     addl(cnt1, 8);
5913     movl(cnt2, 8); negptr(cnt2);
5914     bind(CONT_SCAN_SUBSTR);
5915     if (int_cnt2 < (int)G) {
5916       movdqu(vec, Address(str2, cnt2, Address::times_2, int_cnt2*2));
5917       pcmpestri(vec, Address(result, cnt2, Address::times_2, int_cnt2*2), 0x0d);
5918     } else {
5919       // calculate index in register to avoid integer overflow (int_cnt2*2)
5920       movl(tmp, int_cnt2);
5921       addptr(tmp, cnt2);
5922       movdqu(vec, Address(str2, tmp, Address::times_2, 0));
5923       pcmpestri(vec, Address(result, tmp, Address::times_2, 0), 0x0d);
5924     }
5925     // Need to reload strings pointers if not matched whole vector
5926     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
5927     addptr(cnt2, 8);
5928     jcc(Assembler::negative, SCAN_SUBSTR);
5929     // Fall through if found full substring
5930 
5931   } // (int_cnt2 > 8)
5932 
5933   bind(RET_FOUND);
5934   // Found result if we matched full small substring.
5935   // Compute substr offset
5936   subptr(result, str1);
5937   shrl(result, 1); // index
5938   bind(EXIT);
5939 
5940 } // string_indexofC8
5941 
5942 // Small strings are loaded through stack if they cross page boundary.
5943 void MacroAssembler::string_indexof(Register str1, Register str2,
5944                                     Register cnt1, Register cnt2,
5945                                     int int_cnt2,  Register result,
5946                                     XMMRegister vec, Register tmp) {
5947   ShortBranchVerifier sbv(this);
5948   assert(UseSSE42Intrinsics, "SSE4.2 is required");
5949   //
5950   // int_cnt2 is length of small (< 8 chars) constant substring
5951   // or (-1) for non constant substring in which case its length
5952   // is in cnt2 register.
5953   //
5954   // Note, inline_string_indexOf() generates checks:
5955   // if (substr.count > string.count) return -1;
5956   // if (substr.count == 0) return 0;
5957   //
5958   assert(int_cnt2 == -1 || (0 < int_cnt2 && int_cnt2 < 8), "should be != 0");
5959 
5960   // This method uses pcmpestri inxtruction with bound registers
5961   //   inputs:
5962   //     xmm - substring
5963   //     rax - substring length (elements count)
5964   //     mem - scanned string
5965   //     rdx - string length (elements count)
5966   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
5967   //   outputs:
5968   //     rcx - matched index in string
5969   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
5970 
5971   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, ADJUST_STR,
5972         RET_FOUND, RET_NOT_FOUND, CLEANUP, FOUND_SUBSTR,
5973         FOUND_CANDIDATE;
5974 
5975   { //========================================================
5976     // We don't know where these strings are located
5977     // and we can't read beyond them. Load them through stack.
5978     Label BIG_STRINGS, CHECK_STR, COPY_SUBSTR, COPY_STR;
5979 
5980     movptr(tmp, rsp); // save old SP
5981 
5982     if (int_cnt2 > 0) {     // small (< 8 chars) constant substring
5983       if (int_cnt2 == 1) {  // One char
5984         load_unsigned_short(result, Address(str2, 0));
5985         movdl(vec, result); // move 32 bits
5986       } else if (int_cnt2 == 2) { // Two chars
5987         movdl(vec, Address(str2, 0)); // move 32 bits
5988       } else if (int_cnt2 == 4) { // Four chars
5989         movq(vec, Address(str2, 0));  // move 64 bits
5990       } else { // cnt2 = { 3, 5, 6, 7 }
5991         // Array header size is 12 bytes in 32-bit VM
5992         // + 6 bytes for 3 chars == 18 bytes,
5993         // enough space to load vec and shift.
5994         assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity");
5995         movdqu(vec, Address(str2, (int_cnt2*2)-16));
5996         psrldq(vec, 16-(int_cnt2*2));
5997       }
5998     } else { // not constant substring
5999       cmpl(cnt2, 8);
6000       jccb(Assembler::aboveEqual, BIG_STRINGS); // Both strings are big enough
6001 
6002       // We can read beyond string if srt+16 does not cross page boundary
6003       // since heaps are aligned and mapped by pages.
6004       assert(os::vm_page_size() < (int)G, "default page should be small");
6005       movl(result, str2); // We need only low 32 bits
6006       andl(result, (os::vm_page_size()-1));
6007       cmpl(result, (os::vm_page_size()-16));
6008       jccb(Assembler::belowEqual, CHECK_STR);
6009 
6010       // Move small strings to stack to allow load 16 bytes into vec.
6011       subptr(rsp, 16);
6012       int stk_offset = wordSize-2;
6013       push(cnt2);
6014 
6015       bind(COPY_SUBSTR);
6016       load_unsigned_short(result, Address(str2, cnt2, Address::times_2, -2));
6017       movw(Address(rsp, cnt2, Address::times_2, stk_offset), result);
6018       decrement(cnt2);
6019       jccb(Assembler::notZero, COPY_SUBSTR);
6020 
6021       pop(cnt2);
6022       movptr(str2, rsp);  // New substring address
6023     } // non constant
6024 
6025     bind(CHECK_STR);
6026     cmpl(cnt1, 8);
6027     jccb(Assembler::aboveEqual, BIG_STRINGS);
6028 
6029     // Check cross page boundary.
6030     movl(result, str1); // We need only low 32 bits
6031     andl(result, (os::vm_page_size()-1));
6032     cmpl(result, (os::vm_page_size()-16));
6033     jccb(Assembler::belowEqual, BIG_STRINGS);
6034 
6035     subptr(rsp, 16);
6036     int stk_offset = -2;
6037     if (int_cnt2 < 0) { // not constant
6038       push(cnt2);
6039       stk_offset += wordSize;
6040     }
6041     movl(cnt2, cnt1);
6042 
6043     bind(COPY_STR);
6044     load_unsigned_short(result, Address(str1, cnt2, Address::times_2, -2));
6045     movw(Address(rsp, cnt2, Address::times_2, stk_offset), result);
6046     decrement(cnt2);
6047     jccb(Assembler::notZero, COPY_STR);
6048 
6049     if (int_cnt2 < 0) { // not constant
6050       pop(cnt2);
6051     }
6052     movptr(str1, rsp);  // New string address
6053 
6054     bind(BIG_STRINGS);
6055     // Load substring.
6056     if (int_cnt2 < 0) { // -1
6057       movdqu(vec, Address(str2, 0));
6058       push(cnt2);       // substr count
6059       push(str2);       // substr addr
6060       push(str1);       // string addr
6061     } else {
6062       // Small (< 8 chars) constant substrings are loaded already.
6063       movl(cnt2, int_cnt2);
6064     }
6065     push(tmp);  // original SP
6066 
6067   } // Finished loading
6068 
6069   //========================================================
6070   // Start search
6071   //
6072 
6073   movptr(result, str1); // string addr
6074 
6075   if (int_cnt2  < 0) {  // Only for non constant substring
6076     jmpb(SCAN_TO_SUBSTR);
6077 
6078     // SP saved at sp+0
6079     // String saved at sp+1*wordSize
6080     // Substr saved at sp+2*wordSize
6081     // Substr count saved at sp+3*wordSize
6082 
6083     // Reload substr for rescan, this code
6084     // is executed only for large substrings (> 8 chars)
6085     bind(RELOAD_SUBSTR);
6086     movptr(str2, Address(rsp, 2*wordSize));
6087     movl(cnt2, Address(rsp, 3*wordSize));
6088     movdqu(vec, Address(str2, 0));
6089     // We came here after the beginning of the substring was
6090     // matched but the rest of it was not so we need to search
6091     // again. Start from the next element after the previous match.
6092     subptr(str1, result); // Restore counter
6093     shrl(str1, 1);
6094     addl(cnt1, str1);
6095     decrementl(cnt1);   // Shift to next element
6096     cmpl(cnt1, cnt2);
6097     jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6098 
6099     addptr(result, 2);
6100   } // non constant
6101 
6102   // Scan string for start of substr in 16-byte vectors
6103   bind(SCAN_TO_SUBSTR);
6104   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
6105   pcmpestri(vec, Address(result, 0), 0x0d);
6106   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
6107   subl(cnt1, 8);
6108   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
6109   cmpl(cnt1, cnt2);
6110   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6111   addptr(result, 16);
6112 
6113   bind(ADJUST_STR);
6114   cmpl(cnt1, 8); // Do not read beyond string
6115   jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
6116   // Back-up string to avoid reading beyond string.
6117   lea(result, Address(result, cnt1, Address::times_2, -16));
6118   movl(cnt1, 8);
6119   jmpb(SCAN_TO_SUBSTR);
6120 
6121   // Found a potential substr
6122   bind(FOUND_CANDIDATE);
6123   // After pcmpestri tmp(rcx) contains matched element index
6124 
6125   // Make sure string is still long enough
6126   subl(cnt1, tmp);
6127   cmpl(cnt1, cnt2);
6128   jccb(Assembler::greaterEqual, FOUND_SUBSTR);
6129   // Left less then substring.
6130 
6131   bind(RET_NOT_FOUND);
6132   movl(result, -1);
6133   jmpb(CLEANUP);
6134 
6135   bind(FOUND_SUBSTR);
6136   // Compute start addr of substr
6137   lea(result, Address(result, tmp, Address::times_2));
6138 
6139   if (int_cnt2 > 0) { // Constant substring
6140     // Repeat search for small substring (< 8 chars)
6141     // from new point without reloading substring.
6142     // Have to check that we don't read beyond string.
6143     cmpl(tmp, 8-int_cnt2);
6144     jccb(Assembler::greater, ADJUST_STR);
6145     // Fall through if matched whole substring.
6146   } else { // non constant
6147     assert(int_cnt2 == -1, "should be != 0");
6148 
6149     addl(tmp, cnt2);
6150     // Found result if we matched whole substring.
6151     cmpl(tmp, 8);
6152     jccb(Assembler::lessEqual, RET_FOUND);
6153 
6154     // Repeat search for small substring (<= 8 chars)
6155     // from new point 'str1' without reloading substring.
6156     cmpl(cnt2, 8);
6157     // Have to check that we don't read beyond string.
6158     jccb(Assembler::lessEqual, ADJUST_STR);
6159 
6160     Label CHECK_NEXT, CONT_SCAN_SUBSTR, RET_FOUND_LONG;
6161     // Compare the rest of substring (> 8 chars).
6162     movptr(str1, result);
6163 
6164     cmpl(tmp, cnt2);
6165     // First 8 chars are already matched.
6166     jccb(Assembler::equal, CHECK_NEXT);
6167 
6168     bind(SCAN_SUBSTR);
6169     pcmpestri(vec, Address(str1, 0), 0x0d);
6170     // Need to reload strings pointers if not matched whole vector
6171     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
6172 
6173     bind(CHECK_NEXT);
6174     subl(cnt2, 8);
6175     jccb(Assembler::lessEqual, RET_FOUND_LONG); // Found full substring
6176     addptr(str1, 16);
6177     addptr(str2, 16);
6178     subl(cnt1, 8);
6179     cmpl(cnt2, 8); // Do not read beyond substring
6180     jccb(Assembler::greaterEqual, CONT_SCAN_SUBSTR);
6181     // Back-up strings to avoid reading beyond substring.
6182     lea(str2, Address(str2, cnt2, Address::times_2, -16));
6183     lea(str1, Address(str1, cnt2, Address::times_2, -16));
6184     subl(cnt1, cnt2);
6185     movl(cnt2, 8);
6186     addl(cnt1, 8);
6187     bind(CONT_SCAN_SUBSTR);
6188     movdqu(vec, Address(str2, 0));
6189     jmpb(SCAN_SUBSTR);
6190 
6191     bind(RET_FOUND_LONG);
6192     movptr(str1, Address(rsp, wordSize));
6193   } // non constant
6194 
6195   bind(RET_FOUND);
6196   // Compute substr offset
6197   subptr(result, str1);
6198   shrl(result, 1); // index
6199 
6200   bind(CLEANUP);
6201   pop(rsp); // restore SP
6202 
6203 } // string_indexof
6204 
6205 // Compare strings.
6206 void MacroAssembler::string_compare(Register str1, Register str2,
6207                                     Register cnt1, Register cnt2, Register result,
6208                                     XMMRegister vec1) {
6209   ShortBranchVerifier sbv(this);
6210   Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
6211 
6212   // Compute the minimum of the string lengths and the
6213   // difference of the string lengths (stack).
6214   // Do the conditional move stuff
6215   movl(result, cnt1);
6216   subl(cnt1, cnt2);
6217   push(cnt1);
6218   cmov32(Assembler::lessEqual, cnt2, result);
6219 
6220   // Is the minimum length zero?
6221   testl(cnt2, cnt2);
6222   jcc(Assembler::zero, LENGTH_DIFF_LABEL);
6223 
6224   // Compare first characters
6225   load_unsigned_short(result, Address(str1, 0));
6226   load_unsigned_short(cnt1, Address(str2, 0));
6227   subl(result, cnt1);
6228   jcc(Assembler::notZero,  POP_LABEL);
6229   cmpl(cnt2, 1);
6230   jcc(Assembler::equal, LENGTH_DIFF_LABEL);
6231 
6232   // Check if the strings start at the same location.
6233   cmpptr(str1, str2);
6234   jcc(Assembler::equal, LENGTH_DIFF_LABEL);
6235 
6236   Address::ScaleFactor scale = Address::times_2;
6237   int stride = 8;
6238 
6239   if (UseAVX >= 2 && UseSSE42Intrinsics) {
6240     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_WIDE_TAIL, COMPARE_SMALL_STR;
6241     Label COMPARE_WIDE_VECTORS_LOOP, COMPARE_16_CHARS, COMPARE_INDEX_CHAR;
6242     Label COMPARE_TAIL_LONG;
6243     int pcmpmask = 0x19;
6244 
6245     // Setup to compare 16-chars (32-bytes) vectors,
6246     // start from first character again because it has aligned address.
6247     int stride2 = 16;
6248     int adr_stride  = stride  << scale;
6249     int adr_stride2 = stride2 << scale;
6250 
6251     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
6252     // rax and rdx are used by pcmpestri as elements counters
6253     movl(result, cnt2);
6254     andl(cnt2, ~(stride2-1));   // cnt2 holds the vector count
6255     jcc(Assembler::zero, COMPARE_TAIL_LONG);
6256 
6257     // fast path : compare first 2 8-char vectors.
6258     bind(COMPARE_16_CHARS);
6259     movdqu(vec1, Address(str1, 0));
6260     pcmpestri(vec1, Address(str2, 0), pcmpmask);
6261     jccb(Assembler::below, COMPARE_INDEX_CHAR);
6262 
6263     movdqu(vec1, Address(str1, adr_stride));
6264     pcmpestri(vec1, Address(str2, adr_stride), pcmpmask);
6265     jccb(Assembler::aboveEqual, COMPARE_WIDE_VECTORS);
6266     addl(cnt1, stride);
6267 
6268     // Compare the characters at index in cnt1
6269     bind(COMPARE_INDEX_CHAR); //cnt1 has the offset of the mismatching character
6270     load_unsigned_short(result, Address(str1, cnt1, scale));
6271     load_unsigned_short(cnt2, Address(str2, cnt1, scale));
6272     subl(result, cnt2);
6273     jmp(POP_LABEL);
6274 
6275     // Setup the registers to start vector comparison loop
6276     bind(COMPARE_WIDE_VECTORS);
6277     lea(str1, Address(str1, result, scale));
6278     lea(str2, Address(str2, result, scale));
6279     subl(result, stride2);
6280     subl(cnt2, stride2);
6281     jccb(Assembler::zero, COMPARE_WIDE_TAIL);
6282     negptr(result);
6283 
6284     //  In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest)
6285     bind(COMPARE_WIDE_VECTORS_LOOP);
6286     vmovdqu(vec1, Address(str1, result, scale));
6287     vpxor(vec1, Address(str2, result, scale));
6288     vptest(vec1, vec1);
6289     jccb(Assembler::notZero, VECTOR_NOT_EQUAL);
6290     addptr(result, stride2);
6291     subl(cnt2, stride2);
6292     jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP);
6293     // clean upper bits of YMM registers
6294     vzeroupper();
6295 
6296     // compare wide vectors tail
6297     bind(COMPARE_WIDE_TAIL);
6298     testptr(result, result);
6299     jccb(Assembler::zero, LENGTH_DIFF_LABEL);
6300 
6301     movl(result, stride2);
6302     movl(cnt2, result);
6303     negptr(result);
6304     jmpb(COMPARE_WIDE_VECTORS_LOOP);
6305 
6306     // Identifies the mismatching (higher or lower)16-bytes in the 32-byte vectors.
6307     bind(VECTOR_NOT_EQUAL);
6308     // clean upper bits of YMM registers
6309     vzeroupper();
6310     lea(str1, Address(str1, result, scale));
6311     lea(str2, Address(str2, result, scale));
6312     jmp(COMPARE_16_CHARS);
6313 
6314     // Compare tail chars, length between 1 to 15 chars
6315     bind(COMPARE_TAIL_LONG);
6316     movl(cnt2, result);
6317     cmpl(cnt2, stride);
6318     jccb(Assembler::less, COMPARE_SMALL_STR);
6319 
6320     movdqu(vec1, Address(str1, 0));
6321     pcmpestri(vec1, Address(str2, 0), pcmpmask);
6322     jcc(Assembler::below, COMPARE_INDEX_CHAR);
6323     subptr(cnt2, stride);
6324     jccb(Assembler::zero, LENGTH_DIFF_LABEL);
6325     lea(str1, Address(str1, result, scale));
6326     lea(str2, Address(str2, result, scale));
6327     negptr(cnt2);
6328     jmpb(WHILE_HEAD_LABEL);
6329 
6330     bind(COMPARE_SMALL_STR);
6331   } else if (UseSSE42Intrinsics) {
6332     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
6333     int pcmpmask = 0x19;
6334     // Setup to compare 8-char (16-byte) vectors,
6335     // start from first character again because it has aligned address.
6336     movl(result, cnt2);
6337     andl(cnt2, ~(stride - 1));   // cnt2 holds the vector count
6338     jccb(Assembler::zero, COMPARE_TAIL);
6339 
6340     lea(str1, Address(str1, result, scale));
6341     lea(str2, Address(str2, result, scale));
6342     negptr(result);
6343 
6344     // pcmpestri
6345     //   inputs:
6346     //     vec1- substring
6347     //     rax - negative string length (elements count)
6348     //     mem - scaned string
6349     //     rdx - string length (elements count)
6350     //     pcmpmask - cmp mode: 11000 (string compare with negated result)
6351     //               + 00 (unsigned bytes) or  + 01 (unsigned shorts)
6352     //   outputs:
6353     //     rcx - first mismatched element index
6354     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
6355 
6356     bind(COMPARE_WIDE_VECTORS);
6357     movdqu(vec1, Address(str1, result, scale));
6358     pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
6359     // After pcmpestri cnt1(rcx) contains mismatched element index
6360 
6361     jccb(Assembler::below, VECTOR_NOT_EQUAL);  // CF==1
6362     addptr(result, stride);
6363     subptr(cnt2, stride);
6364     jccb(Assembler::notZero, COMPARE_WIDE_VECTORS);
6365 
6366     // compare wide vectors tail
6367     testptr(result, result);
6368     jccb(Assembler::zero, LENGTH_DIFF_LABEL);
6369 
6370     movl(cnt2, stride);
6371     movl(result, stride);
6372     negptr(result);
6373     movdqu(vec1, Address(str1, result, scale));
6374     pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
6375     jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL);
6376 
6377     // Mismatched characters in the vectors
6378     bind(VECTOR_NOT_EQUAL);
6379     addptr(cnt1, result);
6380     load_unsigned_short(result, Address(str1, cnt1, scale));
6381     load_unsigned_short(cnt2, Address(str2, cnt1, scale));
6382     subl(result, cnt2);
6383     jmpb(POP_LABEL);
6384 
6385     bind(COMPARE_TAIL); // limit is zero
6386     movl(cnt2, result);
6387     // Fallthru to tail compare
6388   }
6389   // Shift str2 and str1 to the end of the arrays, negate min
6390   lea(str1, Address(str1, cnt2, scale));
6391   lea(str2, Address(str2, cnt2, scale));
6392   decrementl(cnt2);  // first character was compared already
6393   negptr(cnt2);
6394 
6395   // Compare the rest of the elements
6396   bind(WHILE_HEAD_LABEL);
6397   load_unsigned_short(result, Address(str1, cnt2, scale, 0));
6398   load_unsigned_short(cnt1, Address(str2, cnt2, scale, 0));
6399   subl(result, cnt1);
6400   jccb(Assembler::notZero, POP_LABEL);
6401   increment(cnt2);
6402   jccb(Assembler::notZero, WHILE_HEAD_LABEL);
6403 
6404   // Strings are equal up to min length.  Return the length difference.
6405   bind(LENGTH_DIFF_LABEL);
6406   pop(result);
6407   jmpb(DONE_LABEL);
6408 
6409   // Discard the stored length difference
6410   bind(POP_LABEL);
6411   pop(cnt1);
6412 
6413   // That's it
6414   bind(DONE_LABEL);
6415 }
6416 
6417 // Compare char[] arrays aligned to 4 bytes or substrings.
6418 void MacroAssembler::char_arrays_equals(bool is_array_equ, Register ary1, Register ary2,
6419                                         Register limit, Register result, Register chr,
6420                                         XMMRegister vec1, XMMRegister vec2) {
6421   ShortBranchVerifier sbv(this);
6422   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR;
6423 
6424   int length_offset  = arrayOopDesc::length_offset_in_bytes();
6425   int base_offset    = arrayOopDesc::base_offset_in_bytes(T_CHAR);
6426 
6427   // Check the input args
6428   cmpptr(ary1, ary2);
6429   jcc(Assembler::equal, TRUE_LABEL);
6430 
6431   if (is_array_equ) {
6432     // Need additional checks for arrays_equals.
6433     testptr(ary1, ary1);
6434     jcc(Assembler::zero, FALSE_LABEL);
6435     testptr(ary2, ary2);
6436     jcc(Assembler::zero, FALSE_LABEL);
6437 
6438     // Check the lengths
6439     movl(limit, Address(ary1, length_offset));
6440     cmpl(limit, Address(ary2, length_offset));
6441     jcc(Assembler::notEqual, FALSE_LABEL);
6442   }
6443 
6444   // count == 0
6445   testl(limit, limit);
6446   jcc(Assembler::zero, TRUE_LABEL);
6447 
6448   if (is_array_equ) {
6449     // Load array address
6450     lea(ary1, Address(ary1, base_offset));
6451     lea(ary2, Address(ary2, base_offset));
6452   }
6453 
6454   shll(limit, 1);      // byte count != 0
6455   movl(result, limit); // copy
6456 
6457   if (UseAVX >= 2) {
6458     // With AVX2, use 32-byte vector compare
6459     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
6460 
6461     // Compare 32-byte vectors
6462     andl(result, 0x0000001e);  //   tail count (in bytes)
6463     andl(limit, 0xffffffe0);   // vector count (in bytes)
6464     jccb(Assembler::zero, COMPARE_TAIL);
6465 
6466     lea(ary1, Address(ary1, limit, Address::times_1));
6467     lea(ary2, Address(ary2, limit, Address::times_1));
6468     negptr(limit);
6469 
6470     bind(COMPARE_WIDE_VECTORS);
6471     vmovdqu(vec1, Address(ary1, limit, Address::times_1));
6472     vmovdqu(vec2, Address(ary2, limit, Address::times_1));
6473     vpxor(vec1, vec2);
6474 
6475     vptest(vec1, vec1);
6476     jccb(Assembler::notZero, FALSE_LABEL);
6477     addptr(limit, 32);
6478     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
6479 
6480     testl(result, result);
6481     jccb(Assembler::zero, TRUE_LABEL);
6482 
6483     vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
6484     vmovdqu(vec2, Address(ary2, result, Address::times_1, -32));
6485     vpxor(vec1, vec2);
6486 
6487     vptest(vec1, vec1);
6488     jccb(Assembler::notZero, FALSE_LABEL);
6489     jmpb(TRUE_LABEL);
6490 
6491     bind(COMPARE_TAIL); // limit is zero
6492     movl(limit, result);
6493     // Fallthru to tail compare
6494   } else if (UseSSE42Intrinsics) {
6495     // With SSE4.2, use double quad vector compare
6496     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
6497 
6498     // Compare 16-byte vectors
6499     andl(result, 0x0000000e);  //   tail count (in bytes)
6500     andl(limit, 0xfffffff0);   // vector count (in bytes)
6501     jccb(Assembler::zero, COMPARE_TAIL);
6502 
6503     lea(ary1, Address(ary1, limit, Address::times_1));
6504     lea(ary2, Address(ary2, limit, Address::times_1));
6505     negptr(limit);
6506 
6507     bind(COMPARE_WIDE_VECTORS);
6508     movdqu(vec1, Address(ary1, limit, Address::times_1));
6509     movdqu(vec2, Address(ary2, limit, Address::times_1));
6510     pxor(vec1, vec2);
6511 
6512     ptest(vec1, vec1);
6513     jccb(Assembler::notZero, FALSE_LABEL);
6514     addptr(limit, 16);
6515     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
6516 
6517     testl(result, result);
6518     jccb(Assembler::zero, TRUE_LABEL);
6519 
6520     movdqu(vec1, Address(ary1, result, Address::times_1, -16));
6521     movdqu(vec2, Address(ary2, result, Address::times_1, -16));
6522     pxor(vec1, vec2);
6523 
6524     ptest(vec1, vec1);
6525     jccb(Assembler::notZero, FALSE_LABEL);
6526     jmpb(TRUE_LABEL);
6527 
6528     bind(COMPARE_TAIL); // limit is zero
6529     movl(limit, result);
6530     // Fallthru to tail compare
6531   }
6532 
6533   // Compare 4-byte vectors
6534   andl(limit, 0xfffffffc); // vector count (in bytes)
6535   jccb(Assembler::zero, COMPARE_CHAR);
6536 
6537   lea(ary1, Address(ary1, limit, Address::times_1));
6538   lea(ary2, Address(ary2, limit, Address::times_1));
6539   negptr(limit);
6540 
6541   bind(COMPARE_VECTORS);
6542   movl(chr, Address(ary1, limit, Address::times_1));
6543   cmpl(chr, Address(ary2, limit, Address::times_1));
6544   jccb(Assembler::notEqual, FALSE_LABEL);
6545   addptr(limit, 4);
6546   jcc(Assembler::notZero, COMPARE_VECTORS);
6547 
6548   // Compare trailing char (final 2 bytes), if any
6549   bind(COMPARE_CHAR);
6550   testl(result, 0x2);   // tail  char
6551   jccb(Assembler::zero, TRUE_LABEL);
6552   load_unsigned_short(chr, Address(ary1, 0));
6553   load_unsigned_short(limit, Address(ary2, 0));
6554   cmpl(chr, limit);
6555   jccb(Assembler::notEqual, FALSE_LABEL);
6556 
6557   bind(TRUE_LABEL);
6558   movl(result, 1);   // return true
6559   jmpb(DONE);
6560 
6561   bind(FALSE_LABEL);
6562   xorl(result, result); // return false
6563 
6564   // That's it
6565   bind(DONE);
6566   if (UseAVX >= 2) {
6567     // clean upper bits of YMM registers
6568     vzeroupper();
6569   }
6570 }
6571 
6572 void MacroAssembler::generate_fill(BasicType t, bool aligned,
6573                                    Register to, Register value, Register count,
6574                                    Register rtmp, XMMRegister xtmp) {
6575   ShortBranchVerifier sbv(this);
6576   assert_different_registers(to, value, count, rtmp);
6577   Label L_exit, L_skip_align1, L_skip_align2, L_fill_byte;
6578   Label L_fill_2_bytes, L_fill_4_bytes;
6579 
6580   int shift = -1;
6581   switch (t) {
6582     case T_BYTE:
6583       shift = 2;
6584       break;
6585     case T_SHORT:
6586       shift = 1;
6587       break;
6588     case T_INT:
6589       shift = 0;
6590       break;
6591     default: ShouldNotReachHere();
6592   }
6593 
6594   if (t == T_BYTE) {
6595     andl(value, 0xff);
6596     movl(rtmp, value);
6597     shll(rtmp, 8);
6598     orl(value, rtmp);
6599   }
6600   if (t == T_SHORT) {
6601     andl(value, 0xffff);
6602   }
6603   if (t == T_BYTE || t == T_SHORT) {
6604     movl(rtmp, value);
6605     shll(rtmp, 16);
6606     orl(value, rtmp);
6607   }
6608 
6609   cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
6610   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
6611   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
6612     // align source address at 4 bytes address boundary
6613     if (t == T_BYTE) {
6614       // One byte misalignment happens only for byte arrays
6615       testptr(to, 1);
6616       jccb(Assembler::zero, L_skip_align1);
6617       movb(Address(to, 0), value);
6618       increment(to);
6619       decrement(count);
6620       BIND(L_skip_align1);
6621     }
6622     // Two bytes misalignment happens only for byte and short (char) arrays
6623     testptr(to, 2);
6624     jccb(Assembler::zero, L_skip_align2);
6625     movw(Address(to, 0), value);
6626     addptr(to, 2);
6627     subl(count, 1<<(shift-1));
6628     BIND(L_skip_align2);
6629   }
6630   if (UseSSE < 2) {
6631     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
6632     // Fill 32-byte chunks
6633     subl(count, 8 << shift);
6634     jcc(Assembler::less, L_check_fill_8_bytes);
6635     align(16);
6636 
6637     BIND(L_fill_32_bytes_loop);
6638 
6639     for (int i = 0; i < 32; i += 4) {
6640       movl(Address(to, i), value);
6641     }
6642 
6643     addptr(to, 32);
6644     subl(count, 8 << shift);
6645     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
6646     BIND(L_check_fill_8_bytes);
6647     addl(count, 8 << shift);
6648     jccb(Assembler::zero, L_exit);
6649     jmpb(L_fill_8_bytes);
6650 
6651     //
6652     // length is too short, just fill qwords
6653     //
6654     BIND(L_fill_8_bytes_loop);
6655     movl(Address(to, 0), value);
6656     movl(Address(to, 4), value);
6657     addptr(to, 8);
6658     BIND(L_fill_8_bytes);
6659     subl(count, 1 << (shift + 1));
6660     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
6661     // fall through to fill 4 bytes
6662   } else {
6663     Label L_fill_32_bytes;
6664     if (!UseUnalignedLoadStores) {
6665       // align to 8 bytes, we know we are 4 byte aligned to start
6666       testptr(to, 4);
6667       jccb(Assembler::zero, L_fill_32_bytes);
6668       movl(Address(to, 0), value);
6669       addptr(to, 4);
6670       subl(count, 1<<shift);
6671     }
6672     BIND(L_fill_32_bytes);
6673     {
6674       assert( UseSSE >= 2, "supported cpu only" );
6675       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
6676       movdl(xtmp, value);
6677       if (UseAVX >= 2 && UseUnalignedLoadStores) {
6678         // Fill 64-byte chunks
6679         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
6680         vpbroadcastd(xtmp, xtmp);
6681 
6682         subl(count, 16 << shift);
6683         jcc(Assembler::less, L_check_fill_32_bytes);
6684         align(16);
6685 
6686         BIND(L_fill_64_bytes_loop);
6687         vmovdqu(Address(to, 0), xtmp);
6688         vmovdqu(Address(to, 32), xtmp);
6689         addptr(to, 64);
6690         subl(count, 16 << shift);
6691         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
6692 
6693         BIND(L_check_fill_32_bytes);
6694         addl(count, 8 << shift);
6695         jccb(Assembler::less, L_check_fill_8_bytes);
6696         vmovdqu(Address(to, 0), xtmp);
6697         addptr(to, 32);
6698         subl(count, 8 << shift);
6699 
6700         BIND(L_check_fill_8_bytes);
6701         // clean upper bits of YMM registers
6702         vzeroupper();
6703       } else {
6704         // Fill 32-byte chunks
6705         pshufd(xtmp, xtmp, 0);
6706 
6707         subl(count, 8 << shift);
6708         jcc(Assembler::less, L_check_fill_8_bytes);
6709         align(16);
6710 
6711         BIND(L_fill_32_bytes_loop);
6712 
6713         if (UseUnalignedLoadStores) {
6714           movdqu(Address(to, 0), xtmp);
6715           movdqu(Address(to, 16), xtmp);
6716         } else {
6717           movq(Address(to, 0), xtmp);
6718           movq(Address(to, 8), xtmp);
6719           movq(Address(to, 16), xtmp);
6720           movq(Address(to, 24), xtmp);
6721         }
6722 
6723         addptr(to, 32);
6724         subl(count, 8 << shift);
6725         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
6726 
6727         BIND(L_check_fill_8_bytes);
6728       }
6729       addl(count, 8 << shift);
6730       jccb(Assembler::zero, L_exit);
6731       jmpb(L_fill_8_bytes);
6732 
6733       //
6734       // length is too short, just fill qwords
6735       //
6736       BIND(L_fill_8_bytes_loop);
6737       movq(Address(to, 0), xtmp);
6738       addptr(to, 8);
6739       BIND(L_fill_8_bytes);
6740       subl(count, 1 << (shift + 1));
6741       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
6742     }
6743   }
6744   // fill trailing 4 bytes
6745   BIND(L_fill_4_bytes);
6746   testl(count, 1<<shift);
6747   jccb(Assembler::zero, L_fill_2_bytes);
6748   movl(Address(to, 0), value);
6749   if (t == T_BYTE || t == T_SHORT) {
6750     addptr(to, 4);
6751     BIND(L_fill_2_bytes);
6752     // fill trailing 2 bytes
6753     testl(count, 1<<(shift-1));
6754     jccb(Assembler::zero, L_fill_byte);
6755     movw(Address(to, 0), value);
6756     if (t == T_BYTE) {
6757       addptr(to, 2);
6758       BIND(L_fill_byte);
6759       // fill trailing byte
6760       testl(count, 1);
6761       jccb(Assembler::zero, L_exit);
6762       movb(Address(to, 0), value);
6763     } else {
6764       BIND(L_fill_byte);
6765     }
6766   } else {
6767     BIND(L_fill_2_bytes);
6768   }
6769   BIND(L_exit);
6770 }
6771 
6772 // encode char[] to byte[] in ISO_8859_1
6773 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
6774                                       XMMRegister tmp1Reg, XMMRegister tmp2Reg,
6775                                       XMMRegister tmp3Reg, XMMRegister tmp4Reg,
6776                                       Register tmp5, Register result) {
6777   // rsi: src
6778   // rdi: dst
6779   // rdx: len
6780   // rcx: tmp5
6781   // rax: result
6782   ShortBranchVerifier sbv(this);
6783   assert_different_registers(src, dst, len, tmp5, result);
6784   Label L_done, L_copy_1_char, L_copy_1_char_exit;
6785 
6786   // set result
6787   xorl(result, result);
6788   // check for zero length
6789   testl(len, len);
6790   jcc(Assembler::zero, L_done);
6791   movl(result, len);
6792 
6793   // Setup pointers
6794   lea(src, Address(src, len, Address::times_2)); // char[]
6795   lea(dst, Address(dst, len, Address::times_1)); // byte[]
6796   negptr(len);
6797 
6798   if (UseSSE42Intrinsics || UseAVX >= 2) {
6799     Label L_chars_8_check, L_copy_8_chars, L_copy_8_chars_exit;
6800     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
6801 
6802     if (UseAVX >= 2) {
6803       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
6804       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
6805       movdl(tmp1Reg, tmp5);
6806       vpbroadcastd(tmp1Reg, tmp1Reg);
6807       jmpb(L_chars_32_check);
6808 
6809       bind(L_copy_32_chars);
6810       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
6811       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
6812       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector256 */ true);
6813       vptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
6814       jccb(Assembler::notZero, L_copy_32_chars_exit);
6815       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector256 */ true);
6816       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector256 */ true);
6817       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
6818 
6819       bind(L_chars_32_check);
6820       addptr(len, 32);
6821       jccb(Assembler::lessEqual, L_copy_32_chars);
6822 
6823       bind(L_copy_32_chars_exit);
6824       subptr(len, 16);
6825       jccb(Assembler::greater, L_copy_16_chars_exit);
6826 
6827     } else if (UseSSE42Intrinsics) {
6828       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
6829       movdl(tmp1Reg, tmp5);
6830       pshufd(tmp1Reg, tmp1Reg, 0);
6831       jmpb(L_chars_16_check);
6832     }
6833 
6834     bind(L_copy_16_chars);
6835     if (UseAVX >= 2) {
6836       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
6837       vptest(tmp2Reg, tmp1Reg);
6838       jccb(Assembler::notZero, L_copy_16_chars_exit);
6839       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector256 */ true);
6840       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector256 */ true);
6841     } else {
6842       if (UseAVX > 0) {
6843         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
6844         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
6845         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector256 */ false);
6846       } else {
6847         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
6848         por(tmp2Reg, tmp3Reg);
6849         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
6850         por(tmp2Reg, tmp4Reg);
6851       }
6852       ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
6853       jccb(Assembler::notZero, L_copy_16_chars_exit);
6854       packuswb(tmp3Reg, tmp4Reg);
6855     }
6856     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
6857 
6858     bind(L_chars_16_check);
6859     addptr(len, 16);
6860     jccb(Assembler::lessEqual, L_copy_16_chars);
6861 
6862     bind(L_copy_16_chars_exit);
6863     if (UseAVX >= 2) {
6864       // clean upper bits of YMM registers
6865       vzeroupper();
6866     }
6867     subptr(len, 8);
6868     jccb(Assembler::greater, L_copy_8_chars_exit);
6869 
6870     bind(L_copy_8_chars);
6871     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
6872     ptest(tmp3Reg, tmp1Reg);
6873     jccb(Assembler::notZero, L_copy_8_chars_exit);
6874     packuswb(tmp3Reg, tmp1Reg);
6875     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
6876     addptr(len, 8);
6877     jccb(Assembler::lessEqual, L_copy_8_chars);
6878 
6879     bind(L_copy_8_chars_exit);
6880     subptr(len, 8);
6881     jccb(Assembler::zero, L_done);
6882   }
6883 
6884   bind(L_copy_1_char);
6885   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
6886   testl(tmp5, 0xff00);      // check if Unicode char
6887   jccb(Assembler::notZero, L_copy_1_char_exit);
6888   movb(Address(dst, len, Address::times_1, 0), tmp5);
6889   addptr(len, 1);
6890   jccb(Assembler::less, L_copy_1_char);
6891 
6892   bind(L_copy_1_char_exit);
6893   addptr(result, len); // len is negative count of not processed elements
6894   bind(L_done);
6895 }
6896 
6897 /**
6898  * Emits code to update CRC-32 with a byte value according to constants in table
6899  *
6900  * @param [in,out]crc   Register containing the crc.
6901  * @param [in]val       Register containing the byte to fold into the CRC.
6902  * @param [in]table     Register containing the table of crc constants.
6903  *
6904  * uint32_t crc;
6905  * val = crc_table[(val ^ crc) & 0xFF];
6906  * crc = val ^ (crc >> 8);
6907  *
6908  */
6909 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
6910   xorl(val, crc);
6911   andl(val, 0xFF);
6912   shrl(crc, 8); // unsigned shift
6913   xorl(crc, Address(table, val, Address::times_4, 0));
6914 }
6915 
6916 /**
6917  * Fold 128-bit data chunk
6918  */
6919 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
6920   vpclmulhdq(xtmp, xK, xcrc); // [123:64]
6921   vpclmulldq(xcrc, xK, xcrc); // [63:0]
6922   vpxor(xcrc, xcrc, Address(buf, offset), false /* vector256 */);
6923   pxor(xcrc, xtmp);
6924 }
6925 
6926 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
6927   vpclmulhdq(xtmp, xK, xcrc);
6928   vpclmulldq(xcrc, xK, xcrc);
6929   pxor(xcrc, xbuf);
6930   pxor(xcrc, xtmp);
6931 }
6932 
6933 /**
6934  * 8-bit folds to compute 32-bit CRC
6935  *
6936  * uint64_t xcrc;
6937  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
6938  */
6939 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
6940   movdl(tmp, xcrc);
6941   andl(tmp, 0xFF);
6942   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
6943   psrldq(xcrc, 1); // unsigned shift one byte
6944   pxor(xcrc, xtmp);
6945 }
6946 
6947 /**
6948  * uint32_t crc;
6949  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
6950  */
6951 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
6952   movl(tmp, crc);
6953   andl(tmp, 0xFF);
6954   shrl(crc, 8);
6955   xorl(crc, Address(table, tmp, Address::times_4, 0));
6956 }
6957 
6958 /**
6959  * @param crc   register containing existing CRC (32-bit)
6960  * @param buf   register pointing to input byte buffer (byte*)
6961  * @param len   register containing number of bytes
6962  * @param table register that will contain address of CRC table
6963  * @param tmp   scratch register
6964  */
6965 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
6966   assert_different_registers(crc, buf, len, table, tmp, rax);
6967 
6968   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
6969   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
6970 
6971   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
6972   notl(crc); // ~crc
6973   cmpl(len, 16);
6974   jcc(Assembler::less, L_tail);
6975 
6976   // Align buffer to 16 bytes
6977   movl(tmp, buf);
6978   andl(tmp, 0xF);
6979   jccb(Assembler::zero, L_aligned);
6980   subl(tmp,  16);
6981   addl(len, tmp);
6982 
6983   align(4);
6984   BIND(L_align_loop);
6985   movsbl(rax, Address(buf, 0)); // load byte with sign extension
6986   update_byte_crc32(crc, rax, table);
6987   increment(buf);
6988   incrementl(tmp);
6989   jccb(Assembler::less, L_align_loop);
6990 
6991   BIND(L_aligned);
6992   movl(tmp, len); // save
6993   shrl(len, 4);
6994   jcc(Assembler::zero, L_tail_restore);
6995 
6996   // Fold crc into first bytes of vector
6997   movdqa(xmm1, Address(buf, 0));
6998   movdl(rax, xmm1);
6999   xorl(crc, rax);
7000   pinsrd(xmm1, crc, 0);
7001   addptr(buf, 16);
7002   subl(len, 4); // len > 0
7003   jcc(Assembler::less, L_fold_tail);
7004 
7005   movdqa(xmm2, Address(buf,  0));
7006   movdqa(xmm3, Address(buf, 16));
7007   movdqa(xmm4, Address(buf, 32));
7008   addptr(buf, 48);
7009   subl(len, 3);
7010   jcc(Assembler::lessEqual, L_fold_512b);
7011 
7012   // Fold total 512 bits of polynomial on each iteration,
7013   // 128 bits per each of 4 parallel streams.
7014   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
7015 
7016   align(32);
7017   BIND(L_fold_512b_loop);
7018   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
7019   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
7020   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
7021   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
7022   addptr(buf, 64);
7023   subl(len, 4);
7024   jcc(Assembler::greater, L_fold_512b_loop);
7025 
7026   // Fold 512 bits to 128 bits.
7027   BIND(L_fold_512b);
7028   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
7029   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
7030   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
7031   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
7032 
7033   // Fold the rest of 128 bits data chunks
7034   BIND(L_fold_tail);
7035   addl(len, 3);
7036   jccb(Assembler::lessEqual, L_fold_128b);
7037   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
7038 
7039   BIND(L_fold_tail_loop);
7040   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
7041   addptr(buf, 16);
7042   decrementl(len);
7043   jccb(Assembler::greater, L_fold_tail_loop);
7044 
7045   // Fold 128 bits in xmm1 down into 32 bits in crc register.
7046   BIND(L_fold_128b);
7047   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()));
7048   vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
7049   vpand(xmm3, xmm0, xmm2, false /* vector256 */);
7050   vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
7051   psrldq(xmm1, 8);
7052   psrldq(xmm2, 4);
7053   pxor(xmm0, xmm1);
7054   pxor(xmm0, xmm2);
7055 
7056   // 8 8-bit folds to compute 32-bit CRC.
7057   for (int j = 0; j < 4; j++) {
7058     fold_8bit_crc32(xmm0, table, xmm1, rax);
7059   }
7060   movdl(crc, xmm0); // mov 32 bits to general register
7061   for (int j = 0; j < 4; j++) {
7062     fold_8bit_crc32(crc, table, rax);
7063   }
7064 
7065   BIND(L_tail_restore);
7066   movl(len, tmp); // restore
7067   BIND(L_tail);
7068   andl(len, 0xf);
7069   jccb(Assembler::zero, L_exit);
7070 
7071   // Fold the rest of bytes
7072   align(4);
7073   BIND(L_tail_loop);
7074   movsbl(rax, Address(buf, 0)); // load byte with sign extension
7075   update_byte_crc32(crc, rax, table);
7076   increment(buf);
7077   decrementl(len);
7078   jccb(Assembler::greater, L_tail_loop);
7079 
7080   BIND(L_exit);
7081   notl(crc); // ~c
7082 }
7083 
7084 #undef BIND
7085 #undef BLOCK_COMMENT
7086 
7087 
7088 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
7089   switch (cond) {
7090     // Note some conditions are synonyms for others
7091     case Assembler::zero:         return Assembler::notZero;
7092     case Assembler::notZero:      return Assembler::zero;
7093     case Assembler::less:         return Assembler::greaterEqual;
7094     case Assembler::lessEqual:    return Assembler::greater;
7095     case Assembler::greater:      return Assembler::lessEqual;
7096     case Assembler::greaterEqual: return Assembler::less;
7097     case Assembler::below:        return Assembler::aboveEqual;
7098     case Assembler::belowEqual:   return Assembler::above;
7099     case Assembler::above:        return Assembler::belowEqual;
7100     case Assembler::aboveEqual:   return Assembler::below;
7101     case Assembler::overflow:     return Assembler::noOverflow;
7102     case Assembler::noOverflow:   return Assembler::overflow;
7103     case Assembler::negative:     return Assembler::positive;
7104     case Assembler::positive:     return Assembler::negative;
7105     case Assembler::parity:       return Assembler::noParity;
7106     case Assembler::noParity:     return Assembler::parity;
7107   }
7108   ShouldNotReachHere(); return Assembler::overflow;
7109 }
7110 
7111 SkipIfEqual::SkipIfEqual(
7112     MacroAssembler* masm, const bool* flag_addr, bool value) {
7113   _masm = masm;
7114   _masm->cmp8(ExternalAddress((address)flag_addr), value);
7115   _masm->jcc(Assembler::equal, _label);
7116 }
7117 
7118 SkipIfEqual::~SkipIfEqual() {
7119   _masm->bind(_label);
7120 }