1 /*
   2  * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "asm/assembler.hpp"
  28 #include "asm/assembler.inline.hpp"
  29 #include "compiler/disassembler.hpp"
  30 #include "gc/shared/cardTable.hpp"
  31 #include "gc/shared/cardTableModRefBS.hpp"
  32 #include "gc/shared/collectedHeap.inline.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "memory/universe.hpp"
  36 #include "oops/klass.inline.hpp"
  37 #include "prims/methodHandles.hpp"
  38 #include "runtime/biasedLocking.hpp"
  39 #include "runtime/interfaceSupport.hpp"
  40 #include "runtime/objectMonitor.hpp"
  41 #include "runtime/os.hpp"
  42 #include "runtime/safepoint.hpp"
  43 #include "runtime/safepointMechanism.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 #include "runtime/thread.hpp"
  47 #include "utilities/macros.hpp"
  48 #if INCLUDE_ALL_GCS
  49 #include "gc/g1/g1CardTable.hpp"
  50 #include "gc/g1/g1CollectedHeap.inline.hpp"
  51 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
  52 #include "gc/g1/heapRegion.hpp"
  53 #endif // INCLUDE_ALL_GCS
  54 #include "crc32c.h"
  55 #ifdef COMPILER2
  56 #include "opto/intrinsicnode.hpp"
  57 #endif
  58 
  59 #ifdef PRODUCT
  60 #define BLOCK_COMMENT(str) /* nothing */
  61 #define STOP(error) stop(error)
  62 #else
  63 #define BLOCK_COMMENT(str) block_comment(str)
  64 #define STOP(error) block_comment(error); stop(error)
  65 #endif
  66 
  67 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  68 
  69 #ifdef ASSERT
  70 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  71 #endif
  72 
  73 static Assembler::Condition reverse[] = {
  74     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  75     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  76     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  77     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  78     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
  79     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
  80     Assembler::above          /* belowEqual    = 0x6 */ ,
  81     Assembler::belowEqual     /* above         = 0x7 */ ,
  82     Assembler::positive       /* negative      = 0x8 */ ,
  83     Assembler::negative       /* positive      = 0x9 */ ,
  84     Assembler::noParity       /* parity        = 0xa */ ,
  85     Assembler::parity         /* noParity      = 0xb */ ,
  86     Assembler::greaterEqual   /* less          = 0xc */ ,
  87     Assembler::less           /* greaterEqual  = 0xd */ ,
  88     Assembler::greater        /* lessEqual     = 0xe */ ,
  89     Assembler::lessEqual      /* greater       = 0xf, */
  90 
  91 };
  92 
  93 
  94 // Implementation of MacroAssembler
  95 
  96 // First all the versions that have distinct versions depending on 32/64 bit
  97 // Unless the difference is trivial (1 line or so).
  98 
  99 #ifndef _LP64
 100 
 101 // 32bit versions
 102 
 103 Address MacroAssembler::as_Address(AddressLiteral adr) {
 104   return Address(adr.target(), adr.rspec());
 105 }
 106 
 107 Address MacroAssembler::as_Address(ArrayAddress adr) {
 108   return Address::make_array(adr);
 109 }
 110 
 111 void MacroAssembler::call_VM_leaf_base(address entry_point,
 112                                        int number_of_arguments) {
 113   call(RuntimeAddress(entry_point));
 114   increment(rsp, number_of_arguments * wordSize);
 115 }
 116 
 117 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
 118   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 119 }
 120 
 121 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
 122   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 123 }
 124 
 125 void MacroAssembler::cmpoop(Address src1, jobject obj) {
 126   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 127 }
 128 
 129 void MacroAssembler::cmpoop(Register src1, jobject obj) {
 130   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 131 }
 132 
 133 void MacroAssembler::extend_sign(Register hi, Register lo) {
 134   // According to Intel Doc. AP-526, "Integer Divide", p.18.
 135   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
 136     cdql();
 137   } else {
 138     movl(hi, lo);
 139     sarl(hi, 31);
 140   }
 141 }
 142 
 143 void MacroAssembler::jC2(Register tmp, Label& L) {
 144   // set parity bit if FPU flag C2 is set (via rax)
 145   save_rax(tmp);
 146   fwait(); fnstsw_ax();
 147   sahf();
 148   restore_rax(tmp);
 149   // branch
 150   jcc(Assembler::parity, L);
 151 }
 152 
 153 void MacroAssembler::jnC2(Register tmp, Label& L) {
 154   // set parity bit if FPU flag C2 is set (via rax)
 155   save_rax(tmp);
 156   fwait(); fnstsw_ax();
 157   sahf();
 158   restore_rax(tmp);
 159   // branch
 160   jcc(Assembler::noParity, L);
 161 }
 162 
 163 // 32bit can do a case table jump in one instruction but we no longer allow the base
 164 // to be installed in the Address class
 165 void MacroAssembler::jump(ArrayAddress entry) {
 166   jmp(as_Address(entry));
 167 }
 168 
 169 // Note: y_lo will be destroyed
 170 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 171   // Long compare for Java (semantics as described in JVM spec.)
 172   Label high, low, done;
 173 
 174   cmpl(x_hi, y_hi);
 175   jcc(Assembler::less, low);
 176   jcc(Assembler::greater, high);
 177   // x_hi is the return register
 178   xorl(x_hi, x_hi);
 179   cmpl(x_lo, y_lo);
 180   jcc(Assembler::below, low);
 181   jcc(Assembler::equal, done);
 182 
 183   bind(high);
 184   xorl(x_hi, x_hi);
 185   increment(x_hi);
 186   jmp(done);
 187 
 188   bind(low);
 189   xorl(x_hi, x_hi);
 190   decrementl(x_hi);
 191 
 192   bind(done);
 193 }
 194 
 195 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 196     mov_literal32(dst, (int32_t)src.target(), src.rspec());
 197 }
 198 
 199 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 200   // leal(dst, as_Address(adr));
 201   // see note in movl as to why we must use a move
 202   mov_literal32(dst, (int32_t) adr.target(), adr.rspec());
 203 }
 204 
 205 void MacroAssembler::leave() {
 206   mov(rsp, rbp);
 207   pop(rbp);
 208 }
 209 
 210 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
 211   // Multiplication of two Java long values stored on the stack
 212   // as illustrated below. Result is in rdx:rax.
 213   //
 214   // rsp ---> [  ??  ] \               \
 215   //            ....    | y_rsp_offset  |
 216   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
 217   //          [ y_hi ]                  | (in bytes)
 218   //            ....                    |
 219   //          [ x_lo ]                 /
 220   //          [ x_hi ]
 221   //            ....
 222   //
 223   // Basic idea: lo(result) = lo(x_lo * y_lo)
 224   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
 225   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
 226   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
 227   Label quick;
 228   // load x_hi, y_hi and check if quick
 229   // multiplication is possible
 230   movl(rbx, x_hi);
 231   movl(rcx, y_hi);
 232   movl(rax, rbx);
 233   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
 234   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
 235   // do full multiplication
 236   // 1st step
 237   mull(y_lo);                                    // x_hi * y_lo
 238   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
 239   // 2nd step
 240   movl(rax, x_lo);
 241   mull(rcx);                                     // x_lo * y_hi
 242   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
 243   // 3rd step
 244   bind(quick);                                   // note: rbx, = 0 if quick multiply!
 245   movl(rax, x_lo);
 246   mull(y_lo);                                    // x_lo * y_lo
 247   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
 248 }
 249 
 250 void MacroAssembler::lneg(Register hi, Register lo) {
 251   negl(lo);
 252   adcl(hi, 0);
 253   negl(hi);
 254 }
 255 
 256 void MacroAssembler::lshl(Register hi, Register lo) {
 257   // Java shift left long support (semantics as described in JVM spec., p.305)
 258   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
 259   // shift value is in rcx !
 260   assert(hi != rcx, "must not use rcx");
 261   assert(lo != rcx, "must not use rcx");
 262   const Register s = rcx;                        // shift count
 263   const int      n = BitsPerWord;
 264   Label L;
 265   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 266   cmpl(s, n);                                    // if (s < n)
 267   jcc(Assembler::less, L);                       // else (s >= n)
 268   movl(hi, lo);                                  // x := x << n
 269   xorl(lo, lo);
 270   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 271   bind(L);                                       // s (mod n) < n
 272   shldl(hi, lo);                                 // x := x << s
 273   shll(lo);
 274 }
 275 
 276 
 277 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
 278   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
 279   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
 280   assert(hi != rcx, "must not use rcx");
 281   assert(lo != rcx, "must not use rcx");
 282   const Register s = rcx;                        // shift count
 283   const int      n = BitsPerWord;
 284   Label L;
 285   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 286   cmpl(s, n);                                    // if (s < n)
 287   jcc(Assembler::less, L);                       // else (s >= n)
 288   movl(lo, hi);                                  // x := x >> n
 289   if (sign_extension) sarl(hi, 31);
 290   else                xorl(hi, hi);
 291   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 292   bind(L);                                       // s (mod n) < n
 293   shrdl(lo, hi);                                 // x := x >> s
 294   if (sign_extension) sarl(hi);
 295   else                shrl(hi);
 296 }
 297 
 298 void MacroAssembler::movoop(Register dst, jobject obj) {
 299   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 300 }
 301 
 302 void MacroAssembler::movoop(Address dst, jobject obj) {
 303   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 304 }
 305 
 306 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 307   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 308 }
 309 
 310 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 311   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 312 }
 313 
 314 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 315   // scratch register is not used,
 316   // it is defined to match parameters of 64-bit version of this method.
 317   if (src.is_lval()) {
 318     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
 319   } else {
 320     movl(dst, as_Address(src));
 321   }
 322 }
 323 
 324 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 325   movl(as_Address(dst), src);
 326 }
 327 
 328 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 329   movl(dst, as_Address(src));
 330 }
 331 
 332 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 333 void MacroAssembler::movptr(Address dst, intptr_t src) {
 334   movl(dst, src);
 335 }
 336 
 337 
 338 void MacroAssembler::pop_callee_saved_registers() {
 339   pop(rcx);
 340   pop(rdx);
 341   pop(rdi);
 342   pop(rsi);
 343 }
 344 
 345 void MacroAssembler::pop_fTOS() {
 346   fld_d(Address(rsp, 0));
 347   addl(rsp, 2 * wordSize);
 348 }
 349 
 350 void MacroAssembler::push_callee_saved_registers() {
 351   push(rsi);
 352   push(rdi);
 353   push(rdx);
 354   push(rcx);
 355 }
 356 
 357 void MacroAssembler::push_fTOS() {
 358   subl(rsp, 2 * wordSize);
 359   fstp_d(Address(rsp, 0));
 360 }
 361 
 362 
 363 void MacroAssembler::pushoop(jobject obj) {
 364   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
 365 }
 366 
 367 void MacroAssembler::pushklass(Metadata* obj) {
 368   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
 369 }
 370 
 371 void MacroAssembler::pushptr(AddressLiteral src) {
 372   if (src.is_lval()) {
 373     push_literal32((int32_t)src.target(), src.rspec());
 374   } else {
 375     pushl(as_Address(src));
 376   }
 377 }
 378 
 379 void MacroAssembler::set_word_if_not_zero(Register dst) {
 380   xorl(dst, dst);
 381   set_byte_if_not_zero(dst);
 382 }
 383 
 384 static void pass_arg0(MacroAssembler* masm, Register arg) {
 385   masm->push(arg);
 386 }
 387 
 388 static void pass_arg1(MacroAssembler* masm, Register arg) {
 389   masm->push(arg);
 390 }
 391 
 392 static void pass_arg2(MacroAssembler* masm, Register arg) {
 393   masm->push(arg);
 394 }
 395 
 396 static void pass_arg3(MacroAssembler* masm, Register arg) {
 397   masm->push(arg);
 398 }
 399 
 400 #ifndef PRODUCT
 401 extern "C" void findpc(intptr_t x);
 402 #endif
 403 
 404 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
 405   // In order to get locks to work, we need to fake a in_VM state
 406   JavaThread* thread = JavaThread::current();
 407   JavaThreadState saved_state = thread->thread_state();
 408   thread->set_thread_state(_thread_in_vm);
 409   if (ShowMessageBoxOnError) {
 410     JavaThread* thread = JavaThread::current();
 411     JavaThreadState saved_state = thread->thread_state();
 412     thread->set_thread_state(_thread_in_vm);
 413     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 414       ttyLocker ttyl;
 415       BytecodeCounter::print();
 416     }
 417     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 418     // This is the value of eip which points to where verify_oop will return.
 419     if (os::message_box(msg, "Execution stopped, print registers?")) {
 420       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
 421       BREAKPOINT;
 422     }
 423   } else {
 424     ttyLocker ttyl;
 425     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
 426   }
 427   // Don't assert holding the ttyLock
 428     assert(false, "DEBUG MESSAGE: %s", msg);
 429   ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 430 }
 431 
 432 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
 433   ttyLocker ttyl;
 434   FlagSetting fs(Debugging, true);
 435   tty->print_cr("eip = 0x%08x", eip);
 436 #ifndef PRODUCT
 437   if ((WizardMode || Verbose) && PrintMiscellaneous) {
 438     tty->cr();
 439     findpc(eip);
 440     tty->cr();
 441   }
 442 #endif
 443 #define PRINT_REG(rax) \
 444   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
 445   PRINT_REG(rax);
 446   PRINT_REG(rbx);
 447   PRINT_REG(rcx);
 448   PRINT_REG(rdx);
 449   PRINT_REG(rdi);
 450   PRINT_REG(rsi);
 451   PRINT_REG(rbp);
 452   PRINT_REG(rsp);
 453 #undef PRINT_REG
 454   // Print some words near top of staack.
 455   int* dump_sp = (int*) rsp;
 456   for (int col1 = 0; col1 < 8; col1++) {
 457     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 458     os::print_location(tty, *dump_sp++);
 459   }
 460   for (int row = 0; row < 16; row++) {
 461     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 462     for (int col = 0; col < 8; col++) {
 463       tty->print(" 0x%08x", *dump_sp++);
 464     }
 465     tty->cr();
 466   }
 467   // Print some instructions around pc:
 468   Disassembler::decode((address)eip-64, (address)eip);
 469   tty->print_cr("--------");
 470   Disassembler::decode((address)eip, (address)eip+32);
 471 }
 472 
 473 void MacroAssembler::stop(const char* msg) {
 474   ExternalAddress message((address)msg);
 475   // push address of message
 476   pushptr(message.addr());
 477   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 478   pusha();                                            // push registers
 479   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 480   hlt();
 481 }
 482 
 483 void MacroAssembler::warn(const char* msg) {
 484   push_CPU_state();
 485 
 486   ExternalAddress message((address) msg);
 487   // push address of message
 488   pushptr(message.addr());
 489 
 490   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
 491   addl(rsp, wordSize);       // discard argument
 492   pop_CPU_state();
 493 }
 494 
 495 void MacroAssembler::print_state() {
 496   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 497   pusha();                                            // push registers
 498 
 499   push_CPU_state();
 500   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
 501   pop_CPU_state();
 502 
 503   popa();
 504   addl(rsp, wordSize);
 505 }
 506 
 507 #else // _LP64
 508 
 509 // 64 bit versions
 510 
 511 Address MacroAssembler::as_Address(AddressLiteral adr) {
 512   // amd64 always does this as a pc-rel
 513   // we can be absolute or disp based on the instruction type
 514   // jmp/call are displacements others are absolute
 515   assert(!adr.is_lval(), "must be rval");
 516   assert(reachable(adr), "must be");
 517   return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
 518 
 519 }
 520 
 521 Address MacroAssembler::as_Address(ArrayAddress adr) {
 522   AddressLiteral base = adr.base();
 523   lea(rscratch1, base);
 524   Address index = adr.index();
 525   assert(index._disp == 0, "must not have disp"); // maybe it can?
 526   Address array(rscratch1, index._index, index._scale, index._disp);
 527   return array;
 528 }
 529 
 530 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
 531   Label L, E;
 532 
 533 #ifdef _WIN64
 534   // Windows always allocates space for it's register args
 535   assert(num_args <= 4, "only register arguments supported");
 536   subq(rsp,  frame::arg_reg_save_area_bytes);
 537 #endif
 538 
 539   // Align stack if necessary
 540   testl(rsp, 15);
 541   jcc(Assembler::zero, L);
 542 
 543   subq(rsp, 8);
 544   {
 545     call(RuntimeAddress(entry_point));
 546   }
 547   addq(rsp, 8);
 548   jmp(E);
 549 
 550   bind(L);
 551   {
 552     call(RuntimeAddress(entry_point));
 553   }
 554 
 555   bind(E);
 556 
 557 #ifdef _WIN64
 558   // restore stack pointer
 559   addq(rsp, frame::arg_reg_save_area_bytes);
 560 #endif
 561 
 562 }
 563 
 564 void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
 565   assert(!src2.is_lval(), "should use cmpptr");
 566 
 567   if (reachable(src2)) {
 568     cmpq(src1, as_Address(src2));
 569   } else {
 570     lea(rscratch1, src2);
 571     Assembler::cmpq(src1, Address(rscratch1, 0));
 572   }
 573 }
 574 
 575 int MacroAssembler::corrected_idivq(Register reg) {
 576   // Full implementation of Java ldiv and lrem; checks for special
 577   // case as described in JVM spec., p.243 & p.271.  The function
 578   // returns the (pc) offset of the idivl instruction - may be needed
 579   // for implicit exceptions.
 580   //
 581   //         normal case                           special case
 582   //
 583   // input : rax: dividend                         min_long
 584   //         reg: divisor   (may not be eax/edx)   -1
 585   //
 586   // output: rax: quotient  (= rax idiv reg)       min_long
 587   //         rdx: remainder (= rax irem reg)       0
 588   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
 589   static const int64_t min_long = 0x8000000000000000;
 590   Label normal_case, special_case;
 591 
 592   // check for special case
 593   cmp64(rax, ExternalAddress((address) &min_long));
 594   jcc(Assembler::notEqual, normal_case);
 595   xorl(rdx, rdx); // prepare rdx for possible special case (where
 596                   // remainder = 0)
 597   cmpq(reg, -1);
 598   jcc(Assembler::equal, special_case);
 599 
 600   // handle normal case
 601   bind(normal_case);
 602   cdqq();
 603   int idivq_offset = offset();
 604   idivq(reg);
 605 
 606   // normal and special case exit
 607   bind(special_case);
 608 
 609   return idivq_offset;
 610 }
 611 
 612 void MacroAssembler::decrementq(Register reg, int value) {
 613   if (value == min_jint) { subq(reg, value); return; }
 614   if (value <  0) { incrementq(reg, -value); return; }
 615   if (value == 0) {                        ; return; }
 616   if (value == 1 && UseIncDec) { decq(reg) ; return; }
 617   /* else */      { subq(reg, value)       ; return; }
 618 }
 619 
 620 void MacroAssembler::decrementq(Address dst, int value) {
 621   if (value == min_jint) { subq(dst, value); return; }
 622   if (value <  0) { incrementq(dst, -value); return; }
 623   if (value == 0) {                        ; return; }
 624   if (value == 1 && UseIncDec) { decq(dst) ; return; }
 625   /* else */      { subq(dst, value)       ; return; }
 626 }
 627 
 628 void MacroAssembler::incrementq(AddressLiteral dst) {
 629   if (reachable(dst)) {
 630     incrementq(as_Address(dst));
 631   } else {
 632     lea(rscratch1, dst);
 633     incrementq(Address(rscratch1, 0));
 634   }
 635 }
 636 
 637 void MacroAssembler::incrementq(Register reg, int value) {
 638   if (value == min_jint) { addq(reg, value); return; }
 639   if (value <  0) { decrementq(reg, -value); return; }
 640   if (value == 0) {                        ; return; }
 641   if (value == 1 && UseIncDec) { incq(reg) ; return; }
 642   /* else */      { addq(reg, value)       ; return; }
 643 }
 644 
 645 void MacroAssembler::incrementq(Address dst, int value) {
 646   if (value == min_jint) { addq(dst, value); return; }
 647   if (value <  0) { decrementq(dst, -value); return; }
 648   if (value == 0) {                        ; return; }
 649   if (value == 1 && UseIncDec) { incq(dst) ; return; }
 650   /* else */      { addq(dst, value)       ; return; }
 651 }
 652 
 653 // 32bit can do a case table jump in one instruction but we no longer allow the base
 654 // to be installed in the Address class
 655 void MacroAssembler::jump(ArrayAddress entry) {
 656   lea(rscratch1, entry.base());
 657   Address dispatch = entry.index();
 658   assert(dispatch._base == noreg, "must be");
 659   dispatch._base = rscratch1;
 660   jmp(dispatch);
 661 }
 662 
 663 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 664   ShouldNotReachHere(); // 64bit doesn't use two regs
 665   cmpq(x_lo, y_lo);
 666 }
 667 
 668 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 669     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 670 }
 671 
 672 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 673   mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());
 674   movptr(dst, rscratch1);
 675 }
 676 
 677 void MacroAssembler::leave() {
 678   // %%% is this really better? Why not on 32bit too?
 679   emit_int8((unsigned char)0xC9); // LEAVE
 680 }
 681 
 682 void MacroAssembler::lneg(Register hi, Register lo) {
 683   ShouldNotReachHere(); // 64bit doesn't use two regs
 684   negq(lo);
 685 }
 686 
 687 void MacroAssembler::movoop(Register dst, jobject obj) {
 688   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 689 }
 690 
 691 void MacroAssembler::movoop(Address dst, jobject obj) {
 692   mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 693   movq(dst, rscratch1);
 694 }
 695 
 696 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 697   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 698 }
 699 
 700 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 701   mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 702   movq(dst, rscratch1);
 703 }
 704 
 705 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 706   if (src.is_lval()) {
 707     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 708   } else {
 709     if (reachable(src)) {
 710       movq(dst, as_Address(src));
 711     } else {
 712       lea(scratch, src);
 713       movq(dst, Address(scratch, 0));
 714     }
 715   }
 716 }
 717 
 718 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 719   movq(as_Address(dst), src);
 720 }
 721 
 722 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 723   movq(dst, as_Address(src));
 724 }
 725 
 726 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 727 void MacroAssembler::movptr(Address dst, intptr_t src) {
 728   mov64(rscratch1, src);
 729   movq(dst, rscratch1);
 730 }
 731 
 732 // These are mostly for initializing NULL
 733 void MacroAssembler::movptr(Address dst, int32_t src) {
 734   movslq(dst, src);
 735 }
 736 
 737 void MacroAssembler::movptr(Register dst, int32_t src) {
 738   mov64(dst, (intptr_t)src);
 739 }
 740 
 741 void MacroAssembler::pushoop(jobject obj) {
 742   movoop(rscratch1, obj);
 743   push(rscratch1);
 744 }
 745 
 746 void MacroAssembler::pushklass(Metadata* obj) {
 747   mov_metadata(rscratch1, obj);
 748   push(rscratch1);
 749 }
 750 
 751 void MacroAssembler::pushptr(AddressLiteral src) {
 752   lea(rscratch1, src);
 753   if (src.is_lval()) {
 754     push(rscratch1);
 755   } else {
 756     pushq(Address(rscratch1, 0));
 757   }
 758 }
 759 
 760 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
 761   // we must set sp to zero to clear frame
 762   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 763   // must clear fp, so that compiled frames are not confused; it is
 764   // possible that we need it only for debugging
 765   if (clear_fp) {
 766     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 767   }
 768 
 769   // Always clear the pc because it could have been set by make_walkable()
 770   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 771   vzeroupper();
 772 }
 773 
 774 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 775                                          Register last_java_fp,
 776                                          address  last_java_pc) {
 777   vzeroupper();
 778   // determine last_java_sp register
 779   if (!last_java_sp->is_valid()) {
 780     last_java_sp = rsp;
 781   }
 782 
 783   // last_java_fp is optional
 784   if (last_java_fp->is_valid()) {
 785     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),
 786            last_java_fp);
 787   }
 788 
 789   // last_java_pc is optional
 790   if (last_java_pc != NULL) {
 791     Address java_pc(r15_thread,
 792                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 793     lea(rscratch1, InternalAddress(last_java_pc));
 794     movptr(java_pc, rscratch1);
 795   }
 796 
 797   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 798 }
 799 
 800 static void pass_arg0(MacroAssembler* masm, Register arg) {
 801   if (c_rarg0 != arg ) {
 802     masm->mov(c_rarg0, arg);
 803   }
 804 }
 805 
 806 static void pass_arg1(MacroAssembler* masm, Register arg) {
 807   if (c_rarg1 != arg ) {
 808     masm->mov(c_rarg1, arg);
 809   }
 810 }
 811 
 812 static void pass_arg2(MacroAssembler* masm, Register arg) {
 813   if (c_rarg2 != arg ) {
 814     masm->mov(c_rarg2, arg);
 815   }
 816 }
 817 
 818 static void pass_arg3(MacroAssembler* masm, Register arg) {
 819   if (c_rarg3 != arg ) {
 820     masm->mov(c_rarg3, arg);
 821   }
 822 }
 823 
 824 void MacroAssembler::stop(const char* msg) {
 825   address rip = pc();
 826   pusha(); // get regs on stack
 827   lea(c_rarg0, ExternalAddress((address) msg));
 828   lea(c_rarg1, InternalAddress(rip));
 829   movq(c_rarg2, rsp); // pass pointer to regs array
 830   andq(rsp, -16); // align stack as required by ABI
 831   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
 832   hlt();
 833 }
 834 
 835 void MacroAssembler::warn(const char* msg) {
 836   push(rbp);
 837   movq(rbp, rsp);
 838   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 839   push_CPU_state();   // keeps alignment at 16 bytes
 840   lea(c_rarg0, ExternalAddress((address) msg));
 841   lea(rax, ExternalAddress(CAST_FROM_FN_PTR(address, warning)));
 842   call(rax);
 843   pop_CPU_state();
 844   mov(rsp, rbp);
 845   pop(rbp);
 846 }
 847 
 848 void MacroAssembler::print_state() {
 849   address rip = pc();
 850   pusha();            // get regs on stack
 851   push(rbp);
 852   movq(rbp, rsp);
 853   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 854   push_CPU_state();   // keeps alignment at 16 bytes
 855 
 856   lea(c_rarg0, InternalAddress(rip));
 857   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
 858   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
 859 
 860   pop_CPU_state();
 861   mov(rsp, rbp);
 862   pop(rbp);
 863   popa();
 864 }
 865 
 866 #ifndef PRODUCT
 867 extern "C" void findpc(intptr_t x);
 868 #endif
 869 
 870 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
 871   // In order to get locks to work, we need to fake a in_VM state
 872   if (ShowMessageBoxOnError) {
 873     JavaThread* thread = JavaThread::current();
 874     JavaThreadState saved_state = thread->thread_state();
 875     thread->set_thread_state(_thread_in_vm);
 876 #ifndef PRODUCT
 877     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 878       ttyLocker ttyl;
 879       BytecodeCounter::print();
 880     }
 881 #endif
 882     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 883     // XXX correct this offset for amd64
 884     // This is the value of eip which points to where verify_oop will return.
 885     if (os::message_box(msg, "Execution stopped, print registers?")) {
 886       print_state64(pc, regs);
 887       BREAKPOINT;
 888       assert(false, "start up GDB");
 889     }
 890     ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 891   } else {
 892     ttyLocker ttyl;
 893     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
 894                     msg);
 895     assert(false, "DEBUG MESSAGE: %s", msg);
 896   }
 897 }
 898 
 899 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
 900   ttyLocker ttyl;
 901   FlagSetting fs(Debugging, true);
 902   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
 903 #ifndef PRODUCT
 904   tty->cr();
 905   findpc(pc);
 906   tty->cr();
 907 #endif
 908 #define PRINT_REG(rax, value) \
 909   { tty->print("%s = ", #rax); os::print_location(tty, value); }
 910   PRINT_REG(rax, regs[15]);
 911   PRINT_REG(rbx, regs[12]);
 912   PRINT_REG(rcx, regs[14]);
 913   PRINT_REG(rdx, regs[13]);
 914   PRINT_REG(rdi, regs[8]);
 915   PRINT_REG(rsi, regs[9]);
 916   PRINT_REG(rbp, regs[10]);
 917   PRINT_REG(rsp, regs[11]);
 918   PRINT_REG(r8 , regs[7]);
 919   PRINT_REG(r9 , regs[6]);
 920   PRINT_REG(r10, regs[5]);
 921   PRINT_REG(r11, regs[4]);
 922   PRINT_REG(r12, regs[3]);
 923   PRINT_REG(r13, regs[2]);
 924   PRINT_REG(r14, regs[1]);
 925   PRINT_REG(r15, regs[0]);
 926 #undef PRINT_REG
 927   // Print some words near top of staack.
 928   int64_t* rsp = (int64_t*) regs[11];
 929   int64_t* dump_sp = rsp;
 930   for (int col1 = 0; col1 < 8; col1++) {
 931     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 932     os::print_location(tty, *dump_sp++);
 933   }
 934   for (int row = 0; row < 25; row++) {
 935     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 936     for (int col = 0; col < 4; col++) {
 937       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
 938     }
 939     tty->cr();
 940   }
 941   // Print some instructions around pc:
 942   Disassembler::decode((address)pc-64, (address)pc);
 943   tty->print_cr("--------");
 944   Disassembler::decode((address)pc, (address)pc+32);
 945 }
 946 
 947 #endif // _LP64
 948 
 949 // Now versions that are common to 32/64 bit
 950 
 951 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 952   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 953 }
 954 
 955 void MacroAssembler::addptr(Register dst, Register src) {
 956   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 957 }
 958 
 959 void MacroAssembler::addptr(Address dst, Register src) {
 960   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 961 }
 962 
 963 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {
 964   if (reachable(src)) {
 965     Assembler::addsd(dst, as_Address(src));
 966   } else {
 967     lea(rscratch1, src);
 968     Assembler::addsd(dst, Address(rscratch1, 0));
 969   }
 970 }
 971 
 972 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {
 973   if (reachable(src)) {
 974     addss(dst, as_Address(src));
 975   } else {
 976     lea(rscratch1, src);
 977     addss(dst, Address(rscratch1, 0));
 978   }
 979 }
 980 
 981 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) {
 982   if (reachable(src)) {
 983     Assembler::addpd(dst, as_Address(src));
 984   } else {
 985     lea(rscratch1, src);
 986     Assembler::addpd(dst, Address(rscratch1, 0));
 987   }
 988 }
 989 
 990 void MacroAssembler::align(int modulus) {
 991   align(modulus, offset());
 992 }
 993 
 994 void MacroAssembler::align(int modulus, int target) {
 995   if (target % modulus != 0) {
 996     nop(modulus - (target % modulus));
 997   }
 998 }
 999 
1000 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {
1001   // Used in sign-masking with aligned address.
1002   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1003   if (reachable(src)) {
1004     Assembler::andpd(dst, as_Address(src));
1005   } else {
1006     lea(rscratch1, src);
1007     Assembler::andpd(dst, Address(rscratch1, 0));
1008   }
1009 }
1010 
1011 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {
1012   // Used in sign-masking with aligned address.
1013   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1014   if (reachable(src)) {
1015     Assembler::andps(dst, as_Address(src));
1016   } else {
1017     lea(rscratch1, src);
1018     Assembler::andps(dst, Address(rscratch1, 0));
1019   }
1020 }
1021 
1022 void MacroAssembler::andptr(Register dst, int32_t imm32) {
1023   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
1024 }
1025 
1026 void MacroAssembler::atomic_incl(Address counter_addr) {
1027   if (os::is_MP())
1028     lock();
1029   incrementl(counter_addr);
1030 }
1031 
1032 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
1033   if (reachable(counter_addr)) {
1034     atomic_incl(as_Address(counter_addr));
1035   } else {
1036     lea(scr, counter_addr);
1037     atomic_incl(Address(scr, 0));
1038   }
1039 }
1040 
1041 #ifdef _LP64
1042 void MacroAssembler::atomic_incq(Address counter_addr) {
1043   if (os::is_MP())
1044     lock();
1045   incrementq(counter_addr);
1046 }
1047 
1048 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) {
1049   if (reachable(counter_addr)) {
1050     atomic_incq(as_Address(counter_addr));
1051   } else {
1052     lea(scr, counter_addr);
1053     atomic_incq(Address(scr, 0));
1054   }
1055 }
1056 #endif
1057 
1058 // Writes to stack successive pages until offset reached to check for
1059 // stack overflow + shadow pages.  This clobbers tmp.
1060 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
1061   movptr(tmp, rsp);
1062   // Bang stack for total size given plus shadow page size.
1063   // Bang one page at a time because large size can bang beyond yellow and
1064   // red zones.
1065   Label loop;
1066   bind(loop);
1067   movl(Address(tmp, (-os::vm_page_size())), size );
1068   subptr(tmp, os::vm_page_size());
1069   subl(size, os::vm_page_size());
1070   jcc(Assembler::greater, loop);
1071 
1072   // Bang down shadow pages too.
1073   // At this point, (tmp-0) is the last address touched, so don't
1074   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
1075   // was post-decremented.)  Skip this address by starting at i=1, and
1076   // touch a few more pages below.  N.B.  It is important to touch all
1077   // the way down including all pages in the shadow zone.
1078   for (int i = 1; i < ((int)JavaThread::stack_shadow_zone_size() / os::vm_page_size()); i++) {
1079     // this could be any sized move but this is can be a debugging crumb
1080     // so the bigger the better.
1081     movptr(Address(tmp, (-i*os::vm_page_size())), size );
1082   }
1083 }
1084 
1085 void MacroAssembler::reserved_stack_check() {
1086     // testing if reserved zone needs to be enabled
1087     Label no_reserved_zone_enabling;
1088     Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
1089     NOT_LP64(get_thread(rsi);)
1090 
1091     cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
1092     jcc(Assembler::below, no_reserved_zone_enabling);
1093 
1094     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
1095     jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
1096     should_not_reach_here();
1097 
1098     bind(no_reserved_zone_enabling);
1099 }
1100 
1101 int MacroAssembler::biased_locking_enter(Register lock_reg,
1102                                          Register obj_reg,
1103                                          Register swap_reg,
1104                                          Register tmp_reg,
1105                                          bool swap_reg_contains_mark,
1106                                          Label& done,
1107                                          Label* slow_case,
1108                                          BiasedLockingCounters* counters) {
1109   assert(UseBiasedLocking, "why call this otherwise?");
1110   assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
1111   assert(tmp_reg != noreg, "tmp_reg must be supplied");
1112   assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
1113   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
1114   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
1115   NOT_LP64( Address saved_mark_addr(lock_reg, 0); )
1116 
1117   if (PrintBiasedLockingStatistics && counters == NULL) {
1118     counters = BiasedLocking::counters();
1119   }
1120   // Biased locking
1121   // See whether the lock is currently biased toward our thread and
1122   // whether the epoch is still valid
1123   // Note that the runtime guarantees sufficient alignment of JavaThread
1124   // pointers to allow age to be placed into low bits
1125   // First check to see whether biasing is even enabled for this object
1126   Label cas_label;
1127   int null_check_offset = -1;
1128   if (!swap_reg_contains_mark) {
1129     null_check_offset = offset();
1130     movptr(swap_reg, mark_addr);
1131   }
1132   movptr(tmp_reg, swap_reg);
1133   andptr(tmp_reg, markOopDesc::biased_lock_mask_in_place);
1134   cmpptr(tmp_reg, markOopDesc::biased_lock_pattern);
1135   jcc(Assembler::notEqual, cas_label);
1136   // The bias pattern is present in the object's header. Need to check
1137   // whether the bias owner and the epoch are both still current.
1138 #ifndef _LP64
1139   // Note that because there is no current thread register on x86_32 we
1140   // need to store off the mark word we read out of the object to
1141   // avoid reloading it and needing to recheck invariants below. This
1142   // store is unfortunate but it makes the overall code shorter and
1143   // simpler.
1144   movptr(saved_mark_addr, swap_reg);
1145 #endif
1146   if (swap_reg_contains_mark) {
1147     null_check_offset = offset();
1148   }
1149   load_prototype_header(tmp_reg, obj_reg);
1150 #ifdef _LP64
1151   orptr(tmp_reg, r15_thread);
1152   xorptr(tmp_reg, swap_reg);
1153   Register header_reg = tmp_reg;
1154 #else
1155   xorptr(tmp_reg, swap_reg);
1156   get_thread(swap_reg);
1157   xorptr(swap_reg, tmp_reg);
1158   Register header_reg = swap_reg;
1159 #endif
1160   andptr(header_reg, ~((int) markOopDesc::age_mask_in_place));
1161   if (counters != NULL) {
1162     cond_inc32(Assembler::zero,
1163                ExternalAddress((address) counters->biased_lock_entry_count_addr()));
1164   }
1165   jcc(Assembler::equal, done);
1166 
1167   Label try_revoke_bias;
1168   Label try_rebias;
1169 
1170   // At this point we know that the header has the bias pattern and
1171   // that we are not the bias owner in the current epoch. We need to
1172   // figure out more details about the state of the header in order to
1173   // know what operations can be legally performed on the object's
1174   // header.
1175 
1176   // If the low three bits in the xor result aren't clear, that means
1177   // the prototype header is no longer biased and we have to revoke
1178   // the bias on this object.
1179   testptr(header_reg, markOopDesc::biased_lock_mask_in_place);
1180   jccb(Assembler::notZero, try_revoke_bias);
1181 
1182   // Biasing is still enabled for this data type. See whether the
1183   // epoch of the current bias is still valid, meaning that the epoch
1184   // bits of the mark word are equal to the epoch bits of the
1185   // prototype header. (Note that the prototype header's epoch bits
1186   // only change at a safepoint.) If not, attempt to rebias the object
1187   // toward the current thread. Note that we must be absolutely sure
1188   // that the current epoch is invalid in order to do this because
1189   // otherwise the manipulations it performs on the mark word are
1190   // illegal.
1191   testptr(header_reg, markOopDesc::epoch_mask_in_place);
1192   jccb(Assembler::notZero, try_rebias);
1193 
1194   // The epoch of the current bias is still valid but we know nothing
1195   // about the owner; it might be set or it might be clear. Try to
1196   // acquire the bias of the object using an atomic operation. If this
1197   // fails we will go in to the runtime to revoke the object's bias.
1198   // Note that we first construct the presumed unbiased header so we
1199   // don't accidentally blow away another thread's valid bias.
1200   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1201   andptr(swap_reg,
1202          markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
1203 #ifdef _LP64
1204   movptr(tmp_reg, swap_reg);
1205   orptr(tmp_reg, r15_thread);
1206 #else
1207   get_thread(tmp_reg);
1208   orptr(tmp_reg, swap_reg);
1209 #endif
1210   if (os::is_MP()) {
1211     lock();
1212   }
1213   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1214   // If the biasing toward our thread failed, this means that
1215   // another thread succeeded in biasing it toward itself and we
1216   // need to revoke that bias. The revocation will occur in the
1217   // interpreter runtime in the slow case.
1218   if (counters != NULL) {
1219     cond_inc32(Assembler::zero,
1220                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
1221   }
1222   if (slow_case != NULL) {
1223     jcc(Assembler::notZero, *slow_case);
1224   }
1225   jmp(done);
1226 
1227   bind(try_rebias);
1228   // At this point we know the epoch has expired, meaning that the
1229   // current "bias owner", if any, is actually invalid. Under these
1230   // circumstances _only_, we are allowed to use the current header's
1231   // value as the comparison value when doing the cas to acquire the
1232   // bias in the current epoch. In other words, we allow transfer of
1233   // the bias from one thread to another directly in this situation.
1234   //
1235   // FIXME: due to a lack of registers we currently blow away the age
1236   // bits in this situation. Should attempt to preserve them.
1237   load_prototype_header(tmp_reg, obj_reg);
1238 #ifdef _LP64
1239   orptr(tmp_reg, r15_thread);
1240 #else
1241   get_thread(swap_reg);
1242   orptr(tmp_reg, swap_reg);
1243   movptr(swap_reg, saved_mark_addr);
1244 #endif
1245   if (os::is_MP()) {
1246     lock();
1247   }
1248   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1249   // If the biasing toward our thread failed, then another thread
1250   // succeeded in biasing it toward itself and we need to revoke that
1251   // bias. The revocation will occur in the runtime in the slow case.
1252   if (counters != NULL) {
1253     cond_inc32(Assembler::zero,
1254                ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
1255   }
1256   if (slow_case != NULL) {
1257     jcc(Assembler::notZero, *slow_case);
1258   }
1259   jmp(done);
1260 
1261   bind(try_revoke_bias);
1262   // The prototype mark in the klass doesn't have the bias bit set any
1263   // more, indicating that objects of this data type are not supposed
1264   // to be biased any more. We are going to try to reset the mark of
1265   // this object to the prototype value and fall through to the
1266   // CAS-based locking scheme. Note that if our CAS fails, it means
1267   // that another thread raced us for the privilege of revoking the
1268   // bias of this particular object, so it's okay to continue in the
1269   // normal locking code.
1270   //
1271   // FIXME: due to a lack of registers we currently blow away the age
1272   // bits in this situation. Should attempt to preserve them.
1273   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1274   load_prototype_header(tmp_reg, obj_reg);
1275   if (os::is_MP()) {
1276     lock();
1277   }
1278   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1279   // Fall through to the normal CAS-based lock, because no matter what
1280   // the result of the above CAS, some thread must have succeeded in
1281   // removing the bias bit from the object's header.
1282   if (counters != NULL) {
1283     cond_inc32(Assembler::zero,
1284                ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
1285   }
1286 
1287   bind(cas_label);
1288 
1289   return null_check_offset;
1290 }
1291 
1292 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
1293   assert(UseBiasedLocking, "why call this otherwise?");
1294 
1295   // Check for biased locking unlock case, which is a no-op
1296   // Note: we do not have to check the thread ID for two reasons.
1297   // First, the interpreter checks for IllegalMonitorStateException at
1298   // a higher level. Second, if the bias was revoked while we held the
1299   // lock, the object could not be rebiased toward another thread, so
1300   // the bias bit would be clear.
1301   movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1302   andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);
1303   cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
1304   jcc(Assembler::equal, done);
1305 }
1306 
1307 #ifdef COMPILER2
1308 
1309 #if INCLUDE_RTM_OPT
1310 
1311 // Update rtm_counters based on abort status
1312 // input: abort_status
1313 //        rtm_counters (RTMLockingCounters*)
1314 // flags are killed
1315 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) {
1316 
1317   atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset()));
1318   if (PrintPreciseRTMLockingStatistics) {
1319     for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
1320       Label check_abort;
1321       testl(abort_status, (1<<i));
1322       jccb(Assembler::equal, check_abort);
1323       atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx))));
1324       bind(check_abort);
1325     }
1326   }
1327 }
1328 
1329 // Branch if (random & (count-1) != 0), count is 2^n
1330 // tmp, scr and flags are killed
1331 void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) {
1332   assert(tmp == rax, "");
1333   assert(scr == rdx, "");
1334   rdtsc(); // modifies EDX:EAX
1335   andptr(tmp, count-1);
1336   jccb(Assembler::notZero, brLabel);
1337 }
1338 
1339 // Perform abort ratio calculation, set no_rtm bit if high ratio
1340 // input:  rtm_counters_Reg (RTMLockingCounters* address)
1341 // tmpReg, rtm_counters_Reg and flags are killed
1342 void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg,
1343                                                  Register rtm_counters_Reg,
1344                                                  RTMLockingCounters* rtm_counters,
1345                                                  Metadata* method_data) {
1346   Label L_done, L_check_always_rtm1, L_check_always_rtm2;
1347 
1348   if (RTMLockingCalculationDelay > 0) {
1349     // Delay calculation
1350     movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg);
1351     testptr(tmpReg, tmpReg);
1352     jccb(Assembler::equal, L_done);
1353   }
1354   // Abort ratio calculation only if abort_count > RTMAbortThreshold
1355   //   Aborted transactions = abort_count * 100
1356   //   All transactions = total_count *  RTMTotalCountIncrRate
1357   //   Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio)
1358 
1359   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset()));
1360   cmpptr(tmpReg, RTMAbortThreshold);
1361   jccb(Assembler::below, L_check_always_rtm2);
1362   imulptr(tmpReg, tmpReg, 100);
1363 
1364   Register scrReg = rtm_counters_Reg;
1365   movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1366   imulptr(scrReg, scrReg, RTMTotalCountIncrRate);
1367   imulptr(scrReg, scrReg, RTMAbortRatio);
1368   cmpptr(tmpReg, scrReg);
1369   jccb(Assembler::below, L_check_always_rtm1);
1370   if (method_data != NULL) {
1371     // set rtm_state to "no rtm" in MDO
1372     mov_metadata(tmpReg, method_data);
1373     if (os::is_MP()) {
1374       lock();
1375     }
1376     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM);
1377   }
1378   jmpb(L_done);
1379   bind(L_check_always_rtm1);
1380   // Reload RTMLockingCounters* address
1381   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1382   bind(L_check_always_rtm2);
1383   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1384   cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate);
1385   jccb(Assembler::below, L_done);
1386   if (method_data != NULL) {
1387     // set rtm_state to "always rtm" in MDO
1388     mov_metadata(tmpReg, method_data);
1389     if (os::is_MP()) {
1390       lock();
1391     }
1392     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM);
1393   }
1394   bind(L_done);
1395 }
1396 
1397 // Update counters and perform abort ratio calculation
1398 // input:  abort_status_Reg
1399 // rtm_counters_Reg, flags are killed
1400 void MacroAssembler::rtm_profiling(Register abort_status_Reg,
1401                                    Register rtm_counters_Reg,
1402                                    RTMLockingCounters* rtm_counters,
1403                                    Metadata* method_data,
1404                                    bool profile_rtm) {
1405 
1406   assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1407   // update rtm counters based on rax value at abort
1408   // reads abort_status_Reg, updates flags
1409   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1410   rtm_counters_update(abort_status_Reg, rtm_counters_Reg);
1411   if (profile_rtm) {
1412     // Save abort status because abort_status_Reg is used by following code.
1413     if (RTMRetryCount > 0) {
1414       push(abort_status_Reg);
1415     }
1416     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1417     rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data);
1418     // restore abort status
1419     if (RTMRetryCount > 0) {
1420       pop(abort_status_Reg);
1421     }
1422   }
1423 }
1424 
1425 // Retry on abort if abort's status is 0x6: can retry (0x2) | memory conflict (0x4)
1426 // inputs: retry_count_Reg
1427 //       : abort_status_Reg
1428 // output: retry_count_Reg decremented by 1
1429 // flags are killed
1430 void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) {
1431   Label doneRetry;
1432   assert(abort_status_Reg == rax, "");
1433   // The abort reason bits are in eax (see all states in rtmLocking.hpp)
1434   // 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4)
1435   // if reason is in 0x6 and retry count != 0 then retry
1436   andptr(abort_status_Reg, 0x6);
1437   jccb(Assembler::zero, doneRetry);
1438   testl(retry_count_Reg, retry_count_Reg);
1439   jccb(Assembler::zero, doneRetry);
1440   pause();
1441   decrementl(retry_count_Reg);
1442   jmp(retryLabel);
1443   bind(doneRetry);
1444 }
1445 
1446 // Spin and retry if lock is busy,
1447 // inputs: box_Reg (monitor address)
1448 //       : retry_count_Reg
1449 // output: retry_count_Reg decremented by 1
1450 //       : clear z flag if retry count exceeded
1451 // tmp_Reg, scr_Reg, flags are killed
1452 void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg,
1453                                             Register tmp_Reg, Register scr_Reg, Label& retryLabel) {
1454   Label SpinLoop, SpinExit, doneRetry;
1455   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1456 
1457   testl(retry_count_Reg, retry_count_Reg);
1458   jccb(Assembler::zero, doneRetry);
1459   decrementl(retry_count_Reg);
1460   movptr(scr_Reg, RTMSpinLoopCount);
1461 
1462   bind(SpinLoop);
1463   pause();
1464   decrementl(scr_Reg);
1465   jccb(Assembler::lessEqual, SpinExit);
1466   movptr(tmp_Reg, Address(box_Reg, owner_offset));
1467   testptr(tmp_Reg, tmp_Reg);
1468   jccb(Assembler::notZero, SpinLoop);
1469 
1470   bind(SpinExit);
1471   jmp(retryLabel);
1472   bind(doneRetry);
1473   incrementl(retry_count_Reg); // clear z flag
1474 }
1475 
1476 // Use RTM for normal stack locks
1477 // Input: objReg (object to lock)
1478 void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg,
1479                                        Register retry_on_abort_count_Reg,
1480                                        RTMLockingCounters* stack_rtm_counters,
1481                                        Metadata* method_data, bool profile_rtm,
1482                                        Label& DONE_LABEL, Label& IsInflated) {
1483   assert(UseRTMForStackLocks, "why call this otherwise?");
1484   assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1485   assert(tmpReg == rax, "");
1486   assert(scrReg == rdx, "");
1487   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1488 
1489   if (RTMRetryCount > 0) {
1490     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1491     bind(L_rtm_retry);
1492   }
1493   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1494   testptr(tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased
1495   jcc(Assembler::notZero, IsInflated);
1496 
1497   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1498     Label L_noincrement;
1499     if (RTMTotalCountIncrRate > 1) {
1500       // tmpReg, scrReg and flags are killed
1501       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1502     }
1503     assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM");
1504     atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg);
1505     bind(L_noincrement);
1506   }
1507   xbegin(L_on_abort);
1508   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));       // fetch markword
1509   andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
1510   cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
1511   jcc(Assembler::equal, DONE_LABEL);        // all done if unlocked
1512 
1513   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1514   if (UseRTMXendForLockBusy) {
1515     xend();
1516     movptr(abort_status_Reg, 0x2);   // Set the abort status to 2 (so we can retry)
1517     jmp(L_decrement_retry);
1518   }
1519   else {
1520     xabort(0);
1521   }
1522   bind(L_on_abort);
1523   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1524     rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm);
1525   }
1526   bind(L_decrement_retry);
1527   if (RTMRetryCount > 0) {
1528     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1529     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1530   }
1531 }
1532 
1533 // Use RTM for inflating locks
1534 // inputs: objReg (object to lock)
1535 //         boxReg (on-stack box address (displaced header location) - KILLED)
1536 //         tmpReg (ObjectMonitor address + markOopDesc::monitor_value)
1537 void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg,
1538                                           Register scrReg, Register retry_on_busy_count_Reg,
1539                                           Register retry_on_abort_count_Reg,
1540                                           RTMLockingCounters* rtm_counters,
1541                                           Metadata* method_data, bool profile_rtm,
1542                                           Label& DONE_LABEL) {
1543   assert(UseRTMLocking, "why call this otherwise?");
1544   assert(tmpReg == rax, "");
1545   assert(scrReg == rdx, "");
1546   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1547   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1548 
1549   // Without cast to int32_t a movptr will destroy r10 which is typically obj
1550   movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1551   movptr(boxReg, tmpReg); // Save ObjectMonitor address
1552 
1553   if (RTMRetryCount > 0) {
1554     movl(retry_on_busy_count_Reg, RTMRetryCount);  // Retry on lock busy
1555     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1556     bind(L_rtm_retry);
1557   }
1558   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1559     Label L_noincrement;
1560     if (RTMTotalCountIncrRate > 1) {
1561       // tmpReg, scrReg and flags are killed
1562       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1563     }
1564     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1565     atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);
1566     bind(L_noincrement);
1567   }
1568   xbegin(L_on_abort);
1569   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1570   movptr(tmpReg, Address(tmpReg, owner_offset));
1571   testptr(tmpReg, tmpReg);
1572   jcc(Assembler::zero, DONE_LABEL);
1573   if (UseRTMXendForLockBusy) {
1574     xend();
1575     jmp(L_decrement_retry);
1576   }
1577   else {
1578     xabort(0);
1579   }
1580   bind(L_on_abort);
1581   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1582   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1583     rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm);
1584   }
1585   if (RTMRetryCount > 0) {
1586     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1587     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1588   }
1589 
1590   movptr(tmpReg, Address(boxReg, owner_offset)) ;
1591   testptr(tmpReg, tmpReg) ;
1592   jccb(Assembler::notZero, L_decrement_retry) ;
1593 
1594   // Appears unlocked - try to swing _owner from null to non-null.
1595   // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1596 #ifdef _LP64
1597   Register threadReg = r15_thread;
1598 #else
1599   get_thread(scrReg);
1600   Register threadReg = scrReg;
1601 #endif
1602   if (os::is_MP()) {
1603     lock();
1604   }
1605   cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg
1606 
1607   if (RTMRetryCount > 0) {
1608     // success done else retry
1609     jccb(Assembler::equal, DONE_LABEL) ;
1610     bind(L_decrement_retry);
1611     // Spin and retry if lock is busy.
1612     rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
1613   }
1614   else {
1615     bind(L_decrement_retry);
1616   }
1617 }
1618 
1619 #endif //  INCLUDE_RTM_OPT
1620 
1621 // Fast_Lock and Fast_Unlock used by C2
1622 
1623 // Because the transitions from emitted code to the runtime
1624 // monitorenter/exit helper stubs are so slow it's critical that
1625 // we inline both the stack-locking fast-path and the inflated fast path.
1626 //
1627 // See also: cmpFastLock and cmpFastUnlock.
1628 //
1629 // What follows is a specialized inline transliteration of the code
1630 // in slow_enter() and slow_exit().  If we're concerned about I$ bloat
1631 // another option would be to emit TrySlowEnter and TrySlowExit methods
1632 // at startup-time.  These methods would accept arguments as
1633 // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure
1634 // indications in the icc.ZFlag.  Fast_Lock and Fast_Unlock would simply
1635 // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.
1636 // In practice, however, the # of lock sites is bounded and is usually small.
1637 // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer
1638 // if the processor uses simple bimodal branch predictors keyed by EIP
1639 // Since the helper routines would be called from multiple synchronization
1640 // sites.
1641 //
1642 // An even better approach would be write "MonitorEnter()" and "MonitorExit()"
1643 // in java - using j.u.c and unsafe - and just bind the lock and unlock sites
1644 // to those specialized methods.  That'd give us a mostly platform-independent
1645 // implementation that the JITs could optimize and inline at their pleasure.
1646 // Done correctly, the only time we'd need to cross to native could would be
1647 // to park() or unpark() threads.  We'd also need a few more unsafe operators
1648 // to (a) prevent compiler-JIT reordering of non-volatile accesses, and
1649 // (b) explicit barriers or fence operations.
1650 //
1651 // TODO:
1652 //
1653 // *  Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr).
1654 //    This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals.
1655 //    Given TLAB allocation, Self is usually manifested in a register, so passing it into
1656 //    the lock operators would typically be faster than reifying Self.
1657 //
1658 // *  Ideally I'd define the primitives as:
1659 //       fast_lock   (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.
1660 //       fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED
1661 //    Unfortunately ADLC bugs prevent us from expressing the ideal form.
1662 //    Instead, we're stuck with a rather awkward and brittle register assignments below.
1663 //    Furthermore the register assignments are overconstrained, possibly resulting in
1664 //    sub-optimal code near the synchronization site.
1665 //
1666 // *  Eliminate the sp-proximity tests and just use "== Self" tests instead.
1667 //    Alternately, use a better sp-proximity test.
1668 //
1669 // *  Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.
1670 //    Either one is sufficient to uniquely identify a thread.
1671 //    TODO: eliminate use of sp in _owner and use get_thread(tr) instead.
1672 //
1673 // *  Intrinsify notify() and notifyAll() for the common cases where the
1674 //    object is locked by the calling thread but the waitlist is empty.
1675 //    avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().
1676 //
1677 // *  use jccb and jmpb instead of jcc and jmp to improve code density.
1678 //    But beware of excessive branch density on AMD Opterons.
1679 //
1680 // *  Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success
1681 //    or failure of the fast-path.  If the fast-path fails then we pass
1682 //    control to the slow-path, typically in C.  In Fast_Lock and
1683 //    Fast_Unlock we often branch to DONE_LABEL, just to find that C2
1684 //    will emit a conditional branch immediately after the node.
1685 //    So we have branches to branches and lots of ICC.ZF games.
1686 //    Instead, it might be better to have C2 pass a "FailureLabel"
1687 //    into Fast_Lock and Fast_Unlock.  In the case of success, control
1688 //    will drop through the node.  ICC.ZF is undefined at exit.
1689 //    In the case of failure, the node will branch directly to the
1690 //    FailureLabel
1691 
1692 
1693 // obj: object to lock
1694 // box: on-stack box address (displaced header location) - KILLED
1695 // rax,: tmp -- KILLED
1696 // scr: tmp -- KILLED
1697 void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
1698                                Register scrReg, Register cx1Reg, Register cx2Reg,
1699                                BiasedLockingCounters* counters,
1700                                RTMLockingCounters* rtm_counters,
1701                                RTMLockingCounters* stack_rtm_counters,
1702                                Metadata* method_data,
1703                                bool use_rtm, bool profile_rtm) {
1704   // Ensure the register assignments are disjoint
1705   assert(tmpReg == rax, "");
1706 
1707   if (use_rtm) {
1708     assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg);
1709   } else {
1710     assert(cx1Reg == noreg, "");
1711     assert(cx2Reg == noreg, "");
1712     assert_different_registers(objReg, boxReg, tmpReg, scrReg);
1713   }
1714 
1715   if (counters != NULL) {
1716     atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);
1717   }
1718   if (EmitSync & 1) {
1719       // set box->dhw = markOopDesc::unused_mark()
1720       // Force all sync thru slow-path: slow_enter() and slow_exit()
1721       movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1722       cmpptr (rsp, (int32_t)NULL_WORD);
1723   } else {
1724     // Possible cases that we'll encounter in fast_lock
1725     // ------------------------------------------------
1726     // * Inflated
1727     //    -- unlocked
1728     //    -- Locked
1729     //       = by self
1730     //       = by other
1731     // * biased
1732     //    -- by Self
1733     //    -- by other
1734     // * neutral
1735     // * stack-locked
1736     //    -- by self
1737     //       = sp-proximity test hits
1738     //       = sp-proximity test generates false-negative
1739     //    -- by other
1740     //
1741 
1742     Label IsInflated, DONE_LABEL;
1743 
1744     // it's stack-locked, biased or neutral
1745     // TODO: optimize away redundant LDs of obj->mark and improve the markword triage
1746     // order to reduce the number of conditional branches in the most common cases.
1747     // Beware -- there's a subtle invariant that fetch of the markword
1748     // at [FETCH], below, will never observe a biased encoding (*101b).
1749     // If this invariant is not held we risk exclusion (safety) failure.
1750     if (UseBiasedLocking && !UseOptoBiasInlining) {
1751       biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters);
1752     }
1753 
1754 #if INCLUDE_RTM_OPT
1755     if (UseRTMForStackLocks && use_rtm) {
1756       rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
1757                         stack_rtm_counters, method_data, profile_rtm,
1758                         DONE_LABEL, IsInflated);
1759     }
1760 #endif // INCLUDE_RTM_OPT
1761 
1762     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));          // [FETCH]
1763     testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
1764     jccb(Assembler::notZero, IsInflated);
1765 
1766     // Attempt stack-locking ...
1767     orptr (tmpReg, markOopDesc::unlocked_value);
1768     movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
1769     if (os::is_MP()) {
1770       lock();
1771     }
1772     cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes()));      // Updates tmpReg
1773     if (counters != NULL) {
1774       cond_inc32(Assembler::equal,
1775                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
1776     }
1777     jcc(Assembler::equal, DONE_LABEL);           // Success
1778 
1779     // Recursive locking.
1780     // The object is stack-locked: markword contains stack pointer to BasicLock.
1781     // Locked by current thread if difference with current SP is less than one page.
1782     subptr(tmpReg, rsp);
1783     // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
1784     andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
1785     movptr(Address(boxReg, 0), tmpReg);
1786     if (counters != NULL) {
1787       cond_inc32(Assembler::equal,
1788                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
1789     }
1790     jmp(DONE_LABEL);
1791 
1792     bind(IsInflated);
1793     // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markOopDesc::monitor_value
1794 
1795 #if INCLUDE_RTM_OPT
1796     // Use the same RTM locking code in 32- and 64-bit VM.
1797     if (use_rtm) {
1798       rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg,
1799                            rtm_counters, method_data, profile_rtm, DONE_LABEL);
1800     } else {
1801 #endif // INCLUDE_RTM_OPT
1802 
1803 #ifndef _LP64
1804     // The object is inflated.
1805 
1806     // boxReg refers to the on-stack BasicLock in the current frame.
1807     // We'd like to write:
1808     //   set box->_displaced_header = markOopDesc::unused_mark().  Any non-0 value suffices.
1809     // This is convenient but results a ST-before-CAS penalty.  The following CAS suffers
1810     // additional latency as we have another ST in the store buffer that must drain.
1811 
1812     if (EmitSync & 8192) {
1813        movptr(Address(boxReg, 0), 3);            // results in ST-before-CAS penalty
1814        get_thread (scrReg);
1815        movptr(boxReg, tmpReg);                    // consider: LEA box, [tmp-2]
1816        movptr(tmpReg, NULL_WORD);                 // consider: xor vs mov
1817        if (os::is_MP()) {
1818          lock();
1819        }
1820        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1821     } else
1822     if ((EmitSync & 128) == 0) {                      // avoid ST-before-CAS
1823        // register juggle because we need tmpReg for cmpxchgptr below
1824        movptr(scrReg, boxReg);
1825        movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2]
1826 
1827        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
1828        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
1829           // prefetchw [eax + Offset(_owner)-2]
1830           prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1831        }
1832 
1833        if ((EmitSync & 64) == 0) {
1834          // Optimistic form: consider XORL tmpReg,tmpReg
1835          movptr(tmpReg, NULL_WORD);
1836        } else {
1837          // Can suffer RTS->RTO upgrades on shared or cold $ lines
1838          // Test-And-CAS instead of CAS
1839          movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));   // rax, = m->_owner
1840          testptr(tmpReg, tmpReg);                   // Locked ?
1841          jccb  (Assembler::notZero, DONE_LABEL);
1842        }
1843 
1844        // Appears unlocked - try to swing _owner from null to non-null.
1845        // Ideally, I'd manifest "Self" with get_thread and then attempt
1846        // to CAS the register containing Self into m->Owner.
1847        // But we don't have enough registers, so instead we can either try to CAS
1848        // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
1849        // we later store "Self" into m->Owner.  Transiently storing a stack address
1850        // (rsp or the address of the box) into  m->owner is harmless.
1851        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1852        if (os::is_MP()) {
1853          lock();
1854        }
1855        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1856        movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3
1857        // If we weren't able to swing _owner from NULL to the BasicLock
1858        // then take the slow path.
1859        jccb  (Assembler::notZero, DONE_LABEL);
1860        // update _owner from BasicLock to thread
1861        get_thread (scrReg);                    // beware: clobbers ICCs
1862        movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
1863        xorptr(boxReg, boxReg);                 // set icc.ZFlag = 1 to indicate success
1864 
1865        // If the CAS fails we can either retry or pass control to the slow-path.
1866        // We use the latter tactic.
1867        // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1868        // If the CAS was successful ...
1869        //   Self has acquired the lock
1870        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1871        // Intentional fall-through into DONE_LABEL ...
1872     } else {
1873        movptr(Address(boxReg, 0), intptr_t(markOopDesc::unused_mark()));  // results in ST-before-CAS penalty
1874        movptr(boxReg, tmpReg);
1875 
1876        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
1877        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
1878           // prefetchw [eax + Offset(_owner)-2]
1879           prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1880        }
1881 
1882        if ((EmitSync & 64) == 0) {
1883          // Optimistic form
1884          xorptr  (tmpReg, tmpReg);
1885        } else {
1886          // Can suffer RTS->RTO upgrades on shared or cold $ lines
1887          movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));   // rax, = m->_owner
1888          testptr(tmpReg, tmpReg);                   // Locked ?
1889          jccb  (Assembler::notZero, DONE_LABEL);
1890        }
1891 
1892        // Appears unlocked - try to swing _owner from null to non-null.
1893        // Use either "Self" (in scr) or rsp as thread identity in _owner.
1894        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1895        get_thread (scrReg);
1896        if (os::is_MP()) {
1897          lock();
1898        }
1899        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1900 
1901        // If the CAS fails we can either retry or pass control to the slow-path.
1902        // We use the latter tactic.
1903        // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1904        // If the CAS was successful ...
1905        //   Self has acquired the lock
1906        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1907        // Intentional fall-through into DONE_LABEL ...
1908     }
1909 #else // _LP64
1910     // It's inflated
1911     movq(scrReg, tmpReg);
1912     xorq(tmpReg, tmpReg);
1913 
1914     if (os::is_MP()) {
1915       lock();
1916     }
1917     cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1918     // Unconditionally set box->_displaced_header = markOopDesc::unused_mark().
1919     // Without cast to int32_t movptr will destroy r10 which is typically obj.
1920     movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1921     // Intentional fall-through into DONE_LABEL ...
1922     // Propagate ICC.ZF from CAS above into DONE_LABEL.
1923 #endif // _LP64
1924 #if INCLUDE_RTM_OPT
1925     } // use_rtm()
1926 #endif
1927     // DONE_LABEL is a hot target - we'd really like to place it at the
1928     // start of cache line by padding with NOPs.
1929     // See the AMD and Intel software optimization manuals for the
1930     // most efficient "long" NOP encodings.
1931     // Unfortunately none of our alignment mechanisms suffice.
1932     bind(DONE_LABEL);
1933 
1934     // At DONE_LABEL the icc ZFlag is set as follows ...
1935     // Fast_Unlock uses the same protocol.
1936     // ZFlag == 1 -> Success
1937     // ZFlag == 0 -> Failure - force control through the slow-path
1938   }
1939 }
1940 
1941 // obj: object to unlock
1942 // box: box address (displaced header location), killed.  Must be EAX.
1943 // tmp: killed, cannot be obj nor box.
1944 //
1945 // Some commentary on balanced locking:
1946 //
1947 // Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites.
1948 // Methods that don't have provably balanced locking are forced to run in the
1949 // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
1950 // The interpreter provides two properties:
1951 // I1:  At return-time the interpreter automatically and quietly unlocks any
1952 //      objects acquired the current activation (frame).  Recall that the
1953 //      interpreter maintains an on-stack list of locks currently held by
1954 //      a frame.
1955 // I2:  If a method attempts to unlock an object that is not held by the
1956 //      the frame the interpreter throws IMSX.
1957 //
1958 // Lets say A(), which has provably balanced locking, acquires O and then calls B().
1959 // B() doesn't have provably balanced locking so it runs in the interpreter.
1960 // Control returns to A() and A() unlocks O.  By I1 and I2, above, we know that O
1961 // is still locked by A().
1962 //
1963 // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:
1964 // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
1965 // should not be unlocked by "normal" java-level locking and vice-versa.  The specification
1966 // doesn't specify what will occur if a program engages in such mixed-mode locking, however.
1967 // Arguably given that the spec legislates the JNI case as undefined our implementation
1968 // could reasonably *avoid* checking owner in Fast_Unlock().
1969 // In the interest of performance we elide m->Owner==Self check in unlock.
1970 // A perfectly viable alternative is to elide the owner check except when
1971 // Xcheck:jni is enabled.
1972 
1973 void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
1974   assert(boxReg == rax, "");
1975   assert_different_registers(objReg, boxReg, tmpReg);
1976 
1977   if (EmitSync & 4) {
1978     // Disable - inhibit all inlining.  Force control through the slow-path
1979     cmpptr (rsp, 0);
1980   } else {
1981     Label DONE_LABEL, Stacked, CheckSucc;
1982 
1983     // Critically, the biased locking test must have precedence over
1984     // and appear before the (box->dhw == 0) recursive stack-lock test.
1985     if (UseBiasedLocking && !UseOptoBiasInlining) {
1986        biased_locking_exit(objReg, tmpReg, DONE_LABEL);
1987     }
1988 
1989 #if INCLUDE_RTM_OPT
1990     if (UseRTMForStackLocks && use_rtm) {
1991       assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1992       Label L_regular_unlock;
1993       movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));           // fetch markword
1994       andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
1995       cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
1996       jccb(Assembler::notEqual, L_regular_unlock);  // if !HLE RegularLock
1997       xend();                                       // otherwise end...
1998       jmp(DONE_LABEL);                              // ... and we're done
1999       bind(L_regular_unlock);
2000     }
2001 #endif
2002 
2003     cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
2004     jcc   (Assembler::zero, DONE_LABEL);            // 0 indicates recursive stack-lock
2005     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));             // Examine the object's markword
2006     testptr(tmpReg, markOopDesc::monitor_value);    // Inflated?
2007     jccb  (Assembler::zero, Stacked);
2008 
2009     // It's inflated.
2010 #if INCLUDE_RTM_OPT
2011     if (use_rtm) {
2012       Label L_regular_inflated_unlock;
2013       int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
2014       movptr(boxReg, Address(tmpReg, owner_offset));
2015       testptr(boxReg, boxReg);
2016       jccb(Assembler::notZero, L_regular_inflated_unlock);
2017       xend();
2018       jmpb(DONE_LABEL);
2019       bind(L_regular_inflated_unlock);
2020     }
2021 #endif
2022 
2023     // Despite our balanced locking property we still check that m->_owner == Self
2024     // as java routines or native JNI code called by this thread might
2025     // have released the lock.
2026     // Refer to the comments in synchronizer.cpp for how we might encode extra
2027     // state in _succ so we can avoid fetching EntryList|cxq.
2028     //
2029     // I'd like to add more cases in fast_lock() and fast_unlock() --
2030     // such as recursive enter and exit -- but we have to be wary of
2031     // I$ bloat, T$ effects and BP$ effects.
2032     //
2033     // If there's no contention try a 1-0 exit.  That is, exit without
2034     // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how
2035     // we detect and recover from the race that the 1-0 exit admits.
2036     //
2037     // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier
2038     // before it STs null into _owner, releasing the lock.  Updates
2039     // to data protected by the critical section must be visible before
2040     // we drop the lock (and thus before any other thread could acquire
2041     // the lock and observe the fields protected by the lock).
2042     // IA32's memory-model is SPO, so STs are ordered with respect to
2043     // each other and there's no need for an explicit barrier (fence).
2044     // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
2045 #ifndef _LP64
2046     get_thread (boxReg);
2047     if ((EmitSync & 4096) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
2048       // prefetchw [ebx + Offset(_owner)-2]
2049       prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2050     }
2051 
2052     // Note that we could employ various encoding schemes to reduce
2053     // the number of loads below (currently 4) to just 2 or 3.
2054     // Refer to the comments in synchronizer.cpp.
2055     // In practice the chain of fetches doesn't seem to impact performance, however.
2056     xorptr(boxReg, boxReg);
2057     if ((EmitSync & 65536) == 0 && (EmitSync & 256)) {
2058        // Attempt to reduce branch density - AMD's branch predictor.
2059        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2060        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2061        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2062        jccb  (Assembler::notZero, DONE_LABEL);
2063        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2064        jmpb  (DONE_LABEL);
2065     } else {
2066        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2067        jccb  (Assembler::notZero, DONE_LABEL);
2068        movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2069        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2070        jccb  (Assembler::notZero, CheckSucc);
2071        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2072        jmpb  (DONE_LABEL);
2073     }
2074 
2075     // The Following code fragment (EmitSync & 65536) improves the performance of
2076     // contended applications and contended synchronization microbenchmarks.
2077     // Unfortunately the emission of the code - even though not executed - causes regressions
2078     // in scimark and jetstream, evidently because of $ effects.  Replacing the code
2079     // with an equal number of never-executed NOPs results in the same regression.
2080     // We leave it off by default.
2081 
2082     if ((EmitSync & 65536) != 0) {
2083        Label LSuccess, LGoSlowPath ;
2084 
2085        bind  (CheckSucc);
2086 
2087        // Optional pre-test ... it's safe to elide this
2088        cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2089        jccb(Assembler::zero, LGoSlowPath);
2090 
2091        // We have a classic Dekker-style idiom:
2092        //    ST m->_owner = 0 ; MEMBAR; LD m->_succ
2093        // There are a number of ways to implement the barrier:
2094        // (1) lock:andl &m->_owner, 0
2095        //     is fast, but mask doesn't currently support the "ANDL M,IMM32" form.
2096        //     LOCK: ANDL [ebx+Offset(_Owner)-2], 0
2097        //     Encodes as 81 31 OFF32 IMM32 or 83 63 OFF8 IMM8
2098        // (2) If supported, an explicit MFENCE is appealing.
2099        //     In older IA32 processors MFENCE is slower than lock:add or xchg
2100        //     particularly if the write-buffer is full as might be the case if
2101        //     if stores closely precede the fence or fence-equivalent instruction.
2102        //     See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences
2103        //     as the situation has changed with Nehalem and Shanghai.
2104        // (3) In lieu of an explicit fence, use lock:addl to the top-of-stack
2105        //     The $lines underlying the top-of-stack should be in M-state.
2106        //     The locked add instruction is serializing, of course.
2107        // (4) Use xchg, which is serializing
2108        //     mov boxReg, 0; xchgl boxReg, [tmpReg + Offset(_owner)-2] also works
2109        // (5) ST m->_owner = 0 and then execute lock:orl &m->_succ, 0.
2110        //     The integer condition codes will tell us if succ was 0.
2111        //     Since _succ and _owner should reside in the same $line and
2112        //     we just stored into _owner, it's likely that the $line
2113        //     remains in M-state for the lock:orl.
2114        //
2115        // We currently use (3), although it's likely that switching to (2)
2116        // is correct for the future.
2117 
2118        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2119        if (os::is_MP()) {
2120          lock(); addptr(Address(rsp, 0), 0);
2121        }
2122        // Ratify _succ remains non-null
2123        cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), 0);
2124        jccb  (Assembler::notZero, LSuccess);
2125 
2126        xorptr(boxReg, boxReg);                  // box is really EAX
2127        if (os::is_MP()) { lock(); }
2128        cmpxchgptr(rsp, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2129        // There's no successor so we tried to regrab the lock with the
2130        // placeholder value. If that didn't work, then another thread
2131        // grabbed the lock so we're done (and exit was a success).
2132        jccb  (Assembler::notEqual, LSuccess);
2133        // Since we're low on registers we installed rsp as a placeholding in _owner.
2134        // Now install Self over rsp.  This is safe as we're transitioning from
2135        // non-null to non=null
2136        get_thread (boxReg);
2137        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), boxReg);
2138        // Intentional fall-through into LGoSlowPath ...
2139 
2140        bind  (LGoSlowPath);
2141        orptr(boxReg, 1);                      // set ICC.ZF=0 to indicate failure
2142        jmpb  (DONE_LABEL);
2143 
2144        bind  (LSuccess);
2145        xorptr(boxReg, boxReg);                 // set ICC.ZF=1 to indicate success
2146        jmpb  (DONE_LABEL);
2147     }
2148 
2149     bind (Stacked);
2150     // It's not inflated and it's not recursively stack-locked and it's not biased.
2151     // It must be stack-locked.
2152     // Try to reset the header to displaced header.
2153     // The "box" value on the stack is stable, so we can reload
2154     // and be assured we observe the same value as above.
2155     movptr(tmpReg, Address(boxReg, 0));
2156     if (os::is_MP()) {
2157       lock();
2158     }
2159     cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
2160     // Intention fall-thru into DONE_LABEL
2161 
2162     // DONE_LABEL is a hot target - we'd really like to place it at the
2163     // start of cache line by padding with NOPs.
2164     // See the AMD and Intel software optimization manuals for the
2165     // most efficient "long" NOP encodings.
2166     // Unfortunately none of our alignment mechanisms suffice.
2167     if ((EmitSync & 65536) == 0) {
2168        bind (CheckSucc);
2169     }
2170 #else // _LP64
2171     // It's inflated
2172     if (EmitSync & 1024) {
2173       // Emit code to check that _owner == Self
2174       // We could fold the _owner test into subsequent code more efficiently
2175       // than using a stand-alone check, but since _owner checking is off by
2176       // default we don't bother. We also might consider predicating the
2177       // _owner==Self check on Xcheck:jni or running on a debug build.
2178       movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2179       xorptr(boxReg, r15_thread);
2180     } else {
2181       xorptr(boxReg, boxReg);
2182     }
2183     orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2184     jccb  (Assembler::notZero, DONE_LABEL);
2185     movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2186     orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2187     jccb  (Assembler::notZero, CheckSucc);
2188     movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2189     jmpb  (DONE_LABEL);
2190 
2191     if ((EmitSync & 65536) == 0) {
2192       // Try to avoid passing control into the slow_path ...
2193       Label LSuccess, LGoSlowPath ;
2194       bind  (CheckSucc);
2195 
2196       // The following optional optimization can be elided if necessary
2197       // Effectively: if (succ == null) goto SlowPath
2198       // The code reduces the window for a race, however,
2199       // and thus benefits performance.
2200       cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2201       jccb  (Assembler::zero, LGoSlowPath);
2202 
2203       xorptr(boxReg, boxReg);
2204       if ((EmitSync & 16) && os::is_MP()) {
2205         xchgptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2206       } else {
2207         movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2208         if (os::is_MP()) {
2209           // Memory barrier/fence
2210           // Dekker pivot point -- fulcrum : ST Owner; MEMBAR; LD Succ
2211           // Instead of MFENCE we use a dummy locked add of 0 to the top-of-stack.
2212           // This is faster on Nehalem and AMD Shanghai/Barcelona.
2213           // See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences
2214           // We might also restructure (ST Owner=0;barrier;LD _Succ) to
2215           // (mov box,0; xchgq box, &m->Owner; LD _succ) .
2216           lock(); addl(Address(rsp, 0), 0);
2217         }
2218       }
2219       cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2220       jccb  (Assembler::notZero, LSuccess);
2221 
2222       // Rare inopportune interleaving - race.
2223       // The successor vanished in the small window above.
2224       // The lock is contended -- (cxq|EntryList) != null -- and there's no apparent successor.
2225       // We need to ensure progress and succession.
2226       // Try to reacquire the lock.
2227       // If that fails then the new owner is responsible for succession and this
2228       // thread needs to take no further action and can exit via the fast path (success).
2229       // If the re-acquire succeeds then pass control into the slow path.
2230       // As implemented, this latter mode is horrible because we generated more
2231       // coherence traffic on the lock *and* artifically extended the critical section
2232       // length while by virtue of passing control into the slow path.
2233 
2234       // box is really RAX -- the following CMPXCHG depends on that binding
2235       // cmpxchg R,[M] is equivalent to rax = CAS(M,rax,R)
2236       if (os::is_MP()) { lock(); }
2237       cmpxchgptr(r15_thread, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2238       // There's no successor so we tried to regrab the lock.
2239       // If that didn't work, then another thread grabbed the
2240       // lock so we're done (and exit was a success).
2241       jccb  (Assembler::notEqual, LSuccess);
2242       // Intentional fall-through into slow-path
2243 
2244       bind  (LGoSlowPath);
2245       orl   (boxReg, 1);                      // set ICC.ZF=0 to indicate failure
2246       jmpb  (DONE_LABEL);
2247 
2248       bind  (LSuccess);
2249       testl (boxReg, 0);                      // set ICC.ZF=1 to indicate success
2250       jmpb  (DONE_LABEL);
2251     }
2252 
2253     bind  (Stacked);
2254     movptr(tmpReg, Address (boxReg, 0));      // re-fetch
2255     if (os::is_MP()) { lock(); }
2256     cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
2257 
2258     if (EmitSync & 65536) {
2259        bind (CheckSucc);
2260     }
2261 #endif
2262     bind(DONE_LABEL);
2263   }
2264 }
2265 #endif // COMPILER2
2266 
2267 void MacroAssembler::c2bool(Register x) {
2268   // implements x == 0 ? 0 : 1
2269   // note: must only look at least-significant byte of x
2270   //       since C-style booleans are stored in one byte
2271   //       only! (was bug)
2272   andl(x, 0xFF);
2273   setb(Assembler::notZero, x);
2274 }
2275 
2276 // Wouldn't need if AddressLiteral version had new name
2277 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
2278   Assembler::call(L, rtype);
2279 }
2280 
2281 void MacroAssembler::call(Register entry) {
2282   Assembler::call(entry);
2283 }
2284 
2285 void MacroAssembler::call(AddressLiteral entry) {
2286   if (reachable(entry)) {
2287     Assembler::call_literal(entry.target(), entry.rspec());
2288   } else {
2289     lea(rscratch1, entry);
2290     Assembler::call(rscratch1);
2291   }
2292 }
2293 
2294 void MacroAssembler::ic_call(address entry, jint method_index) {
2295   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
2296   movptr(rax, (intptr_t)Universe::non_oop_word());
2297   call(AddressLiteral(entry, rh));
2298 }
2299 
2300 // Implementation of call_VM versions
2301 
2302 void MacroAssembler::call_VM(Register oop_result,
2303                              address entry_point,
2304                              bool check_exceptions) {
2305   Label C, E;
2306   call(C, relocInfo::none);
2307   jmp(E);
2308 
2309   bind(C);
2310   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
2311   ret(0);
2312 
2313   bind(E);
2314 }
2315 
2316 void MacroAssembler::call_VM(Register oop_result,
2317                              address entry_point,
2318                              Register arg_1,
2319                              bool check_exceptions) {
2320   Label C, E;
2321   call(C, relocInfo::none);
2322   jmp(E);
2323 
2324   bind(C);
2325   pass_arg1(this, arg_1);
2326   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
2327   ret(0);
2328 
2329   bind(E);
2330 }
2331 
2332 void MacroAssembler::call_VM(Register oop_result,
2333                              address entry_point,
2334                              Register arg_1,
2335                              Register arg_2,
2336                              bool check_exceptions) {
2337   Label C, E;
2338   call(C, relocInfo::none);
2339   jmp(E);
2340 
2341   bind(C);
2342 
2343   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2344 
2345   pass_arg2(this, arg_2);
2346   pass_arg1(this, arg_1);
2347   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
2348   ret(0);
2349 
2350   bind(E);
2351 }
2352 
2353 void MacroAssembler::call_VM(Register oop_result,
2354                              address entry_point,
2355                              Register arg_1,
2356                              Register arg_2,
2357                              Register arg_3,
2358                              bool check_exceptions) {
2359   Label C, E;
2360   call(C, relocInfo::none);
2361   jmp(E);
2362 
2363   bind(C);
2364 
2365   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2366   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2367   pass_arg3(this, arg_3);
2368 
2369   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2370   pass_arg2(this, arg_2);
2371 
2372   pass_arg1(this, arg_1);
2373   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
2374   ret(0);
2375 
2376   bind(E);
2377 }
2378 
2379 void MacroAssembler::call_VM(Register oop_result,
2380                              Register last_java_sp,
2381                              address entry_point,
2382                              int number_of_arguments,
2383                              bool check_exceptions) {
2384   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2385   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2386 }
2387 
2388 void MacroAssembler::call_VM(Register oop_result,
2389                              Register last_java_sp,
2390                              address entry_point,
2391                              Register arg_1,
2392                              bool check_exceptions) {
2393   pass_arg1(this, arg_1);
2394   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2395 }
2396 
2397 void MacroAssembler::call_VM(Register oop_result,
2398                              Register last_java_sp,
2399                              address entry_point,
2400                              Register arg_1,
2401                              Register arg_2,
2402                              bool check_exceptions) {
2403 
2404   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2405   pass_arg2(this, arg_2);
2406   pass_arg1(this, arg_1);
2407   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2408 }
2409 
2410 void MacroAssembler::call_VM(Register oop_result,
2411                              Register last_java_sp,
2412                              address entry_point,
2413                              Register arg_1,
2414                              Register arg_2,
2415                              Register arg_3,
2416                              bool check_exceptions) {
2417   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2418   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2419   pass_arg3(this, arg_3);
2420   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2421   pass_arg2(this, arg_2);
2422   pass_arg1(this, arg_1);
2423   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2424 }
2425 
2426 void MacroAssembler::super_call_VM(Register oop_result,
2427                                    Register last_java_sp,
2428                                    address entry_point,
2429                                    int number_of_arguments,
2430                                    bool check_exceptions) {
2431   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2432   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2433 }
2434 
2435 void MacroAssembler::super_call_VM(Register oop_result,
2436                                    Register last_java_sp,
2437                                    address entry_point,
2438                                    Register arg_1,
2439                                    bool check_exceptions) {
2440   pass_arg1(this, arg_1);
2441   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2442 }
2443 
2444 void MacroAssembler::super_call_VM(Register oop_result,
2445                                    Register last_java_sp,
2446                                    address entry_point,
2447                                    Register arg_1,
2448                                    Register arg_2,
2449                                    bool check_exceptions) {
2450 
2451   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2452   pass_arg2(this, arg_2);
2453   pass_arg1(this, arg_1);
2454   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2455 }
2456 
2457 void MacroAssembler::super_call_VM(Register oop_result,
2458                                    Register last_java_sp,
2459                                    address entry_point,
2460                                    Register arg_1,
2461                                    Register arg_2,
2462                                    Register arg_3,
2463                                    bool check_exceptions) {
2464   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2465   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2466   pass_arg3(this, arg_3);
2467   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2468   pass_arg2(this, arg_2);
2469   pass_arg1(this, arg_1);
2470   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2471 }
2472 
2473 void MacroAssembler::call_VM_base(Register oop_result,
2474                                   Register java_thread,
2475                                   Register last_java_sp,
2476                                   address  entry_point,
2477                                   int      number_of_arguments,
2478                                   bool     check_exceptions) {
2479   // determine java_thread register
2480   if (!java_thread->is_valid()) {
2481 #ifdef _LP64
2482     java_thread = r15_thread;
2483 #else
2484     java_thread = rdi;
2485     get_thread(java_thread);
2486 #endif // LP64
2487   }
2488   // determine last_java_sp register
2489   if (!last_java_sp->is_valid()) {
2490     last_java_sp = rsp;
2491   }
2492   // debugging support
2493   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
2494   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
2495 #ifdef ASSERT
2496   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
2497   // r12 is the heapbase.
2498   LP64_ONLY(if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
2499 #endif // ASSERT
2500 
2501   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
2502   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
2503 
2504   // push java thread (becomes first argument of C function)
2505 
2506   NOT_LP64(push(java_thread); number_of_arguments++);
2507   LP64_ONLY(mov(c_rarg0, r15_thread));
2508 
2509   // set last Java frame before call
2510   assert(last_java_sp != rbp, "can't use ebp/rbp");
2511 
2512   // Only interpreter should have to set fp
2513   set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);
2514 
2515   // do the call, remove parameters
2516   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
2517 
2518   // restore the thread (cannot use the pushed argument since arguments
2519   // may be overwritten by C code generated by an optimizing compiler);
2520   // however can use the register value directly if it is callee saved.
2521   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
2522     // rdi & rsi (also r15) are callee saved -> nothing to do
2523 #ifdef ASSERT
2524     guarantee(java_thread != rax, "change this code");
2525     push(rax);
2526     { Label L;
2527       get_thread(rax);
2528       cmpptr(java_thread, rax);
2529       jcc(Assembler::equal, L);
2530       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
2531       bind(L);
2532     }
2533     pop(rax);
2534 #endif
2535   } else {
2536     get_thread(java_thread);
2537   }
2538   // reset last Java frame
2539   // Only interpreter should have to clear fp
2540   reset_last_Java_frame(java_thread, true);
2541 
2542    // C++ interp handles this in the interpreter
2543   check_and_handle_popframe(java_thread);
2544   check_and_handle_earlyret(java_thread);
2545 
2546   if (check_exceptions) {
2547     // check for pending exceptions (java_thread is set upon return)
2548     cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
2549 #ifndef _LP64
2550     jump_cc(Assembler::notEqual,
2551             RuntimeAddress(StubRoutines::forward_exception_entry()));
2552 #else
2553     // This used to conditionally jump to forward_exception however it is
2554     // possible if we relocate that the branch will not reach. So we must jump
2555     // around so we can always reach
2556 
2557     Label ok;
2558     jcc(Assembler::equal, ok);
2559     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2560     bind(ok);
2561 #endif // LP64
2562   }
2563 
2564   // get oop result if there is one and reset the value in the thread
2565   if (oop_result->is_valid()) {
2566     get_vm_result(oop_result, java_thread);
2567   }
2568 }
2569 
2570 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
2571 
2572   // Calculate the value for last_Java_sp
2573   // somewhat subtle. call_VM does an intermediate call
2574   // which places a return address on the stack just under the
2575   // stack pointer as the user finsihed with it. This allows
2576   // use to retrieve last_Java_pc from last_Java_sp[-1].
2577   // On 32bit we then have to push additional args on the stack to accomplish
2578   // the actual requested call. On 64bit call_VM only can use register args
2579   // so the only extra space is the return address that call_VM created.
2580   // This hopefully explains the calculations here.
2581 
2582 #ifdef _LP64
2583   // We've pushed one address, correct last_Java_sp
2584   lea(rax, Address(rsp, wordSize));
2585 #else
2586   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
2587 #endif // LP64
2588 
2589   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
2590 
2591 }
2592 
2593 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
2594 void MacroAssembler::call_VM_leaf0(address entry_point) {
2595   MacroAssembler::call_VM_leaf_base(entry_point, 0);
2596 }
2597 
2598 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
2599   call_VM_leaf_base(entry_point, number_of_arguments);
2600 }
2601 
2602 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
2603   pass_arg0(this, arg_0);
2604   call_VM_leaf(entry_point, 1);
2605 }
2606 
2607 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2608 
2609   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2610   pass_arg1(this, arg_1);
2611   pass_arg0(this, arg_0);
2612   call_VM_leaf(entry_point, 2);
2613 }
2614 
2615 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2616   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2617   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2618   pass_arg2(this, arg_2);
2619   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2620   pass_arg1(this, arg_1);
2621   pass_arg0(this, arg_0);
2622   call_VM_leaf(entry_point, 3);
2623 }
2624 
2625 void MacroAssembler::super_call_VM_leaf(address entry_point) {
2626   MacroAssembler::call_VM_leaf_base(entry_point, 1);
2627 }
2628 
2629 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
2630   pass_arg0(this, arg_0);
2631   MacroAssembler::call_VM_leaf_base(entry_point, 1);
2632 }
2633 
2634 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2635 
2636   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2637   pass_arg1(this, arg_1);
2638   pass_arg0(this, arg_0);
2639   MacroAssembler::call_VM_leaf_base(entry_point, 2);
2640 }
2641 
2642 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2643   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2644   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2645   pass_arg2(this, arg_2);
2646   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2647   pass_arg1(this, arg_1);
2648   pass_arg0(this, arg_0);
2649   MacroAssembler::call_VM_leaf_base(entry_point, 3);
2650 }
2651 
2652 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
2653   LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg"));
2654   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2655   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2656   pass_arg3(this, arg_3);
2657   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2658   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2659   pass_arg2(this, arg_2);
2660   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2661   pass_arg1(this, arg_1);
2662   pass_arg0(this, arg_0);
2663   MacroAssembler::call_VM_leaf_base(entry_point, 4);
2664 }
2665 
2666 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
2667   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
2668   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
2669   verify_oop(oop_result, "broken oop in call_VM_base");
2670 }
2671 
2672 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
2673   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
2674   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
2675 }
2676 
2677 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
2678 }
2679 
2680 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
2681 }
2682 
2683 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) {
2684   if (reachable(src1)) {
2685     cmpl(as_Address(src1), imm);
2686   } else {
2687     lea(rscratch1, src1);
2688     cmpl(Address(rscratch1, 0), imm);
2689   }
2690 }
2691 
2692 void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
2693   assert(!src2.is_lval(), "use cmpptr");
2694   if (reachable(src2)) {
2695     cmpl(src1, as_Address(src2));
2696   } else {
2697     lea(rscratch1, src2);
2698     cmpl(src1, Address(rscratch1, 0));
2699   }
2700 }
2701 
2702 void MacroAssembler::cmp32(Register src1, int32_t imm) {
2703   Assembler::cmpl(src1, imm);
2704 }
2705 
2706 void MacroAssembler::cmp32(Register src1, Address src2) {
2707   Assembler::cmpl(src1, src2);
2708 }
2709 
2710 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2711   ucomisd(opr1, opr2);
2712 
2713   Label L;
2714   if (unordered_is_less) {
2715     movl(dst, -1);
2716     jcc(Assembler::parity, L);
2717     jcc(Assembler::below , L);
2718     movl(dst, 0);
2719     jcc(Assembler::equal , L);
2720     increment(dst);
2721   } else { // unordered is greater
2722     movl(dst, 1);
2723     jcc(Assembler::parity, L);
2724     jcc(Assembler::above , L);
2725     movl(dst, 0);
2726     jcc(Assembler::equal , L);
2727     decrementl(dst);
2728   }
2729   bind(L);
2730 }
2731 
2732 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2733   ucomiss(opr1, opr2);
2734 
2735   Label L;
2736   if (unordered_is_less) {
2737     movl(dst, -1);
2738     jcc(Assembler::parity, L);
2739     jcc(Assembler::below , L);
2740     movl(dst, 0);
2741     jcc(Assembler::equal , L);
2742     increment(dst);
2743   } else { // unordered is greater
2744     movl(dst, 1);
2745     jcc(Assembler::parity, L);
2746     jcc(Assembler::above , L);
2747     movl(dst, 0);
2748     jcc(Assembler::equal , L);
2749     decrementl(dst);
2750   }
2751   bind(L);
2752 }
2753 
2754 
2755 void MacroAssembler::cmp8(AddressLiteral src1, int imm) {
2756   if (reachable(src1)) {
2757     cmpb(as_Address(src1), imm);
2758   } else {
2759     lea(rscratch1, src1);
2760     cmpb(Address(rscratch1, 0), imm);
2761   }
2762 }
2763 
2764 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
2765 #ifdef _LP64
2766   if (src2.is_lval()) {
2767     movptr(rscratch1, src2);
2768     Assembler::cmpq(src1, rscratch1);
2769   } else if (reachable(src2)) {
2770     cmpq(src1, as_Address(src2));
2771   } else {
2772     lea(rscratch1, src2);
2773     Assembler::cmpq(src1, Address(rscratch1, 0));
2774   }
2775 #else
2776   if (src2.is_lval()) {
2777     cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2778   } else {
2779     cmpl(src1, as_Address(src2));
2780   }
2781 #endif // _LP64
2782 }
2783 
2784 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
2785   assert(src2.is_lval(), "not a mem-mem compare");
2786 #ifdef _LP64
2787   // moves src2's literal address
2788   movptr(rscratch1, src2);
2789   Assembler::cmpq(src1, rscratch1);
2790 #else
2791   cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2792 #endif // _LP64
2793 }
2794 
2795 void MacroAssembler::cmpoop(Register src1, Register src2) {
2796   cmpptr(src1, src2);
2797 }
2798 
2799 void MacroAssembler::cmpoop(Register src1, Address src2) {
2800   cmpptr(src1, src2);
2801 }
2802 
2803 #ifdef _LP64
2804 void MacroAssembler::cmpoop(Register src1, jobject src2) {
2805   movoop(rscratch1, src2);
2806   cmpptr(src1, rscratch1);
2807 }
2808 #endif
2809 
2810 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) {
2811   if (reachable(adr)) {
2812     if (os::is_MP())
2813       lock();
2814     cmpxchgptr(reg, as_Address(adr));
2815   } else {
2816     lea(rscratch1, adr);
2817     if (os::is_MP())
2818       lock();
2819     cmpxchgptr(reg, Address(rscratch1, 0));
2820   }
2821 }
2822 
2823 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
2824   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
2825 }
2826 
2827 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {
2828   if (reachable(src)) {
2829     Assembler::comisd(dst, as_Address(src));
2830   } else {
2831     lea(rscratch1, src);
2832     Assembler::comisd(dst, Address(rscratch1, 0));
2833   }
2834 }
2835 
2836 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {
2837   if (reachable(src)) {
2838     Assembler::comiss(dst, as_Address(src));
2839   } else {
2840     lea(rscratch1, src);
2841     Assembler::comiss(dst, Address(rscratch1, 0));
2842   }
2843 }
2844 
2845 
2846 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
2847   Condition negated_cond = negate_condition(cond);
2848   Label L;
2849   jcc(negated_cond, L);
2850   pushf(); // Preserve flags
2851   atomic_incl(counter_addr);
2852   popf();
2853   bind(L);
2854 }
2855 
2856 int MacroAssembler::corrected_idivl(Register reg) {
2857   // Full implementation of Java idiv and irem; checks for
2858   // special case as described in JVM spec., p.243 & p.271.
2859   // The function returns the (pc) offset of the idivl
2860   // instruction - may be needed for implicit exceptions.
2861   //
2862   //         normal case                           special case
2863   //
2864   // input : rax,: dividend                         min_int
2865   //         reg: divisor   (may not be rax,/rdx)   -1
2866   //
2867   // output: rax,: quotient  (= rax, idiv reg)       min_int
2868   //         rdx: remainder (= rax, irem reg)       0
2869   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
2870   const int min_int = 0x80000000;
2871   Label normal_case, special_case;
2872 
2873   // check for special case
2874   cmpl(rax, min_int);
2875   jcc(Assembler::notEqual, normal_case);
2876   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
2877   cmpl(reg, -1);
2878   jcc(Assembler::equal, special_case);
2879 
2880   // handle normal case
2881   bind(normal_case);
2882   cdql();
2883   int idivl_offset = offset();
2884   idivl(reg);
2885 
2886   // normal and special case exit
2887   bind(special_case);
2888 
2889   return idivl_offset;
2890 }
2891 
2892 
2893 
2894 void MacroAssembler::decrementl(Register reg, int value) {
2895   if (value == min_jint) {subl(reg, value) ; return; }
2896   if (value <  0) { incrementl(reg, -value); return; }
2897   if (value == 0) {                        ; return; }
2898   if (value == 1 && UseIncDec) { decl(reg) ; return; }
2899   /* else */      { subl(reg, value)       ; return; }
2900 }
2901 
2902 void MacroAssembler::decrementl(Address dst, int value) {
2903   if (value == min_jint) {subl(dst, value) ; return; }
2904   if (value <  0) { incrementl(dst, -value); return; }
2905   if (value == 0) {                        ; return; }
2906   if (value == 1 && UseIncDec) { decl(dst) ; return; }
2907   /* else */      { subl(dst, value)       ; return; }
2908 }
2909 
2910 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
2911   assert (shift_value > 0, "illegal shift value");
2912   Label _is_positive;
2913   testl (reg, reg);
2914   jcc (Assembler::positive, _is_positive);
2915   int offset = (1 << shift_value) - 1 ;
2916 
2917   if (offset == 1) {
2918     incrementl(reg);
2919   } else {
2920     addl(reg, offset);
2921   }
2922 
2923   bind (_is_positive);
2924   sarl(reg, shift_value);
2925 }
2926 
2927 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
2928   if (reachable(src)) {
2929     Assembler::divsd(dst, as_Address(src));
2930   } else {
2931     lea(rscratch1, src);
2932     Assembler::divsd(dst, Address(rscratch1, 0));
2933   }
2934 }
2935 
2936 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
2937   if (reachable(src)) {
2938     Assembler::divss(dst, as_Address(src));
2939   } else {
2940     lea(rscratch1, src);
2941     Assembler::divss(dst, Address(rscratch1, 0));
2942   }
2943 }
2944 
2945 // !defined(COMPILER2) is because of stupid core builds
2946 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) || INCLUDE_JVMCI
2947 void MacroAssembler::empty_FPU_stack() {
2948   if (VM_Version::supports_mmx()) {
2949     emms();
2950   } else {
2951     for (int i = 8; i-- > 0; ) ffree(i);
2952   }
2953 }
2954 #endif // !LP64 || C1 || !C2 || INCLUDE_JVMCI
2955 
2956 
2957 // Defines obj, preserves var_size_in_bytes
2958 void MacroAssembler::eden_allocate(Register obj,
2959                                    Register var_size_in_bytes,
2960                                    int con_size_in_bytes,
2961                                    Register t1,
2962                                    Label& slow_case) {
2963   assert(obj == rax, "obj must be in rax, for cmpxchg");
2964   assert_different_registers(obj, var_size_in_bytes, t1);
2965   if (!Universe::heap()->supports_inline_contig_alloc()) {
2966     jmp(slow_case);
2967   } else {
2968     Register end = t1;
2969     Label retry;
2970     bind(retry);
2971     ExternalAddress heap_top((address) Universe::heap()->top_addr());
2972     movptr(obj, heap_top);
2973     if (var_size_in_bytes == noreg) {
2974       lea(end, Address(obj, con_size_in_bytes));
2975     } else {
2976       lea(end, Address(obj, var_size_in_bytes, Address::times_1));
2977     }
2978     // if end < obj then we wrapped around => object too long => slow case
2979     cmpptr(end, obj);
2980     jcc(Assembler::below, slow_case);
2981     cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
2982     jcc(Assembler::above, slow_case);
2983     // Compare obj with the top addr, and if still equal, store the new top addr in
2984     // end at the address of the top addr pointer. Sets ZF if was equal, and clears
2985     // it otherwise. Use lock prefix for atomicity on MPs.
2986     locked_cmpxchgptr(end, heap_top);
2987     jcc(Assembler::notEqual, retry);
2988   }
2989 }
2990 
2991 void MacroAssembler::enter() {
2992   push(rbp);
2993   mov(rbp, rsp);
2994 }
2995 
2996 // A 5 byte nop that is safe for patching (see patch_verified_entry)
2997 void MacroAssembler::fat_nop() {
2998   if (UseAddressNop) {
2999     addr_nop_5();
3000   } else {
3001     emit_int8(0x26); // es:
3002     emit_int8(0x2e); // cs:
3003     emit_int8(0x64); // fs:
3004     emit_int8(0x65); // gs:
3005     emit_int8((unsigned char)0x90);
3006   }
3007 }
3008 
3009 void MacroAssembler::fcmp(Register tmp) {
3010   fcmp(tmp, 1, true, true);
3011 }
3012 
3013 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
3014   assert(!pop_right || pop_left, "usage error");
3015   if (VM_Version::supports_cmov()) {
3016     assert(tmp == noreg, "unneeded temp");
3017     if (pop_left) {
3018       fucomip(index);
3019     } else {
3020       fucomi(index);
3021     }
3022     if (pop_right) {
3023       fpop();
3024     }
3025   } else {
3026     assert(tmp != noreg, "need temp");
3027     if (pop_left) {
3028       if (pop_right) {
3029         fcompp();
3030       } else {
3031         fcomp(index);
3032       }
3033     } else {
3034       fcom(index);
3035     }
3036     // convert FPU condition into eflags condition via rax,
3037     save_rax(tmp);
3038     fwait(); fnstsw_ax();
3039     sahf();
3040     restore_rax(tmp);
3041   }
3042   // condition codes set as follows:
3043   //
3044   // CF (corresponds to C0) if x < y
3045   // PF (corresponds to C2) if unordered
3046   // ZF (corresponds to C3) if x = y
3047 }
3048 
3049 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
3050   fcmp2int(dst, unordered_is_less, 1, true, true);
3051 }
3052 
3053 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
3054   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
3055   Label L;
3056   if (unordered_is_less) {
3057     movl(dst, -1);
3058     jcc(Assembler::parity, L);
3059     jcc(Assembler::below , L);
3060     movl(dst, 0);
3061     jcc(Assembler::equal , L);
3062     increment(dst);
3063   } else { // unordered is greater
3064     movl(dst, 1);
3065     jcc(Assembler::parity, L);
3066     jcc(Assembler::above , L);
3067     movl(dst, 0);
3068     jcc(Assembler::equal , L);
3069     decrementl(dst);
3070   }
3071   bind(L);
3072 }
3073 
3074 void MacroAssembler::fld_d(AddressLiteral src) {
3075   fld_d(as_Address(src));
3076 }
3077 
3078 void MacroAssembler::fld_s(AddressLiteral src) {
3079   fld_s(as_Address(src));
3080 }
3081 
3082 void MacroAssembler::fld_x(AddressLiteral src) {
3083   Assembler::fld_x(as_Address(src));
3084 }
3085 
3086 void MacroAssembler::fldcw(AddressLiteral src) {
3087   Assembler::fldcw(as_Address(src));
3088 }
3089 
3090 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src) {
3091   if (reachable(src)) {
3092     Assembler::mulpd(dst, as_Address(src));
3093   } else {
3094     lea(rscratch1, src);
3095     Assembler::mulpd(dst, Address(rscratch1, 0));
3096   }
3097 }
3098 
3099 void MacroAssembler::increase_precision() {
3100   subptr(rsp, BytesPerWord);
3101   fnstcw(Address(rsp, 0));
3102   movl(rax, Address(rsp, 0));
3103   orl(rax, 0x300);
3104   push(rax);
3105   fldcw(Address(rsp, 0));
3106   pop(rax);
3107 }
3108 
3109 void MacroAssembler::restore_precision() {
3110   fldcw(Address(rsp, 0));
3111   addptr(rsp, BytesPerWord);
3112 }
3113 
3114 void MacroAssembler::fpop() {
3115   ffree();
3116   fincstp();
3117 }
3118 
3119 void MacroAssembler::load_float(Address src) {
3120   if (UseSSE >= 1) {
3121     movflt(xmm0, src);
3122   } else {
3123     LP64_ONLY(ShouldNotReachHere());
3124     NOT_LP64(fld_s(src));
3125   }
3126 }
3127 
3128 void MacroAssembler::store_float(Address dst) {
3129   if (UseSSE >= 1) {
3130     movflt(dst, xmm0);
3131   } else {
3132     LP64_ONLY(ShouldNotReachHere());
3133     NOT_LP64(fstp_s(dst));
3134   }
3135 }
3136 
3137 void MacroAssembler::load_double(Address src) {
3138   if (UseSSE >= 2) {
3139     movdbl(xmm0, src);
3140   } else {
3141     LP64_ONLY(ShouldNotReachHere());
3142     NOT_LP64(fld_d(src));
3143   }
3144 }
3145 
3146 void MacroAssembler::store_double(Address dst) {
3147   if (UseSSE >= 2) {
3148     movdbl(dst, xmm0);
3149   } else {
3150     LP64_ONLY(ShouldNotReachHere());
3151     NOT_LP64(fstp_d(dst));
3152   }
3153 }
3154 
3155 void MacroAssembler::fremr(Register tmp) {
3156   save_rax(tmp);
3157   { Label L;
3158     bind(L);
3159     fprem();
3160     fwait(); fnstsw_ax();
3161 #ifdef _LP64
3162     testl(rax, 0x400);
3163     jcc(Assembler::notEqual, L);
3164 #else
3165     sahf();
3166     jcc(Assembler::parity, L);
3167 #endif // _LP64
3168   }
3169   restore_rax(tmp);
3170   // Result is in ST0.
3171   // Note: fxch & fpop to get rid of ST1
3172   // (otherwise FPU stack could overflow eventually)
3173   fxch(1);
3174   fpop();
3175 }
3176 
3177 // dst = c = a * b + c
3178 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
3179   Assembler::vfmadd231sd(c, a, b);
3180   if (dst != c) {
3181     movdbl(dst, c);
3182   }
3183 }
3184 
3185 // dst = c = a * b + c
3186 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
3187   Assembler::vfmadd231ss(c, a, b);
3188   if (dst != c) {
3189     movflt(dst, c);
3190   }
3191 }
3192 
3193 // dst = c = a * b + c
3194 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
3195   Assembler::vfmadd231pd(c, a, b, vector_len);
3196   if (dst != c) {
3197     vmovdqu(dst, c);
3198   }
3199 }
3200 
3201 // dst = c = a * b + c
3202 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
3203   Assembler::vfmadd231ps(c, a, b, vector_len);
3204   if (dst != c) {
3205     vmovdqu(dst, c);
3206   }
3207 }
3208 
3209 // dst = c = a * b + c
3210 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
3211   Assembler::vfmadd231pd(c, a, b, vector_len);
3212   if (dst != c) {
3213     vmovdqu(dst, c);
3214   }
3215 }
3216 
3217 // dst = c = a * b + c
3218 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
3219   Assembler::vfmadd231ps(c, a, b, vector_len);
3220   if (dst != c) {
3221     vmovdqu(dst, c);
3222   }
3223 }
3224 
3225 void MacroAssembler::incrementl(AddressLiteral dst) {
3226   if (reachable(dst)) {
3227     incrementl(as_Address(dst));
3228   } else {
3229     lea(rscratch1, dst);
3230     incrementl(Address(rscratch1, 0));
3231   }
3232 }
3233 
3234 void MacroAssembler::incrementl(ArrayAddress dst) {
3235   incrementl(as_Address(dst));
3236 }
3237 
3238 void MacroAssembler::incrementl(Register reg, int value) {
3239   if (value == min_jint) {addl(reg, value) ; return; }
3240   if (value <  0) { decrementl(reg, -value); return; }
3241   if (value == 0) {                        ; return; }
3242   if (value == 1 && UseIncDec) { incl(reg) ; return; }
3243   /* else */      { addl(reg, value)       ; return; }
3244 }
3245 
3246 void MacroAssembler::incrementl(Address dst, int value) {
3247   if (value == min_jint) {addl(dst, value) ; return; }
3248   if (value <  0) { decrementl(dst, -value); return; }
3249   if (value == 0) {                        ; return; }
3250   if (value == 1 && UseIncDec) { incl(dst) ; return; }
3251   /* else */      { addl(dst, value)       ; return; }
3252 }
3253 
3254 void MacroAssembler::jump(AddressLiteral dst) {
3255   if (reachable(dst)) {
3256     jmp_literal(dst.target(), dst.rspec());
3257   } else {
3258     lea(rscratch1, dst);
3259     jmp(rscratch1);
3260   }
3261 }
3262 
3263 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
3264   if (reachable(dst)) {
3265     InstructionMark im(this);
3266     relocate(dst.reloc());
3267     const int short_size = 2;
3268     const int long_size = 6;
3269     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
3270     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
3271       // 0111 tttn #8-bit disp
3272       emit_int8(0x70 | cc);
3273       emit_int8((offs - short_size) & 0xFF);
3274     } else {
3275       // 0000 1111 1000 tttn #32-bit disp
3276       emit_int8(0x0F);
3277       emit_int8((unsigned char)(0x80 | cc));
3278       emit_int32(offs - long_size);
3279     }
3280   } else {
3281 #ifdef ASSERT
3282     warning("reversing conditional branch");
3283 #endif /* ASSERT */
3284     Label skip;
3285     jccb(reverse[cc], skip);
3286     lea(rscratch1, dst);
3287     Assembler::jmp(rscratch1);
3288     bind(skip);
3289   }
3290 }
3291 
3292 void MacroAssembler::ldmxcsr(AddressLiteral src) {
3293   if (reachable(src)) {
3294     Assembler::ldmxcsr(as_Address(src));
3295   } else {
3296     lea(rscratch1, src);
3297     Assembler::ldmxcsr(Address(rscratch1, 0));
3298   }
3299 }
3300 
3301 int MacroAssembler::load_signed_byte(Register dst, Address src) {
3302   int off;
3303   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3304     off = offset();
3305     movsbl(dst, src); // movsxb
3306   } else {
3307     off = load_unsigned_byte(dst, src);
3308     shll(dst, 24);
3309     sarl(dst, 24);
3310   }
3311   return off;
3312 }
3313 
3314 // Note: load_signed_short used to be called load_signed_word.
3315 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
3316 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
3317 // The term "word" in HotSpot means a 32- or 64-bit machine word.
3318 int MacroAssembler::load_signed_short(Register dst, Address src) {
3319   int off;
3320   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3321     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
3322     // version but this is what 64bit has always done. This seems to imply
3323     // that users are only using 32bits worth.
3324     off = offset();
3325     movswl(dst, src); // movsxw
3326   } else {
3327     off = load_unsigned_short(dst, src);
3328     shll(dst, 16);
3329     sarl(dst, 16);
3330   }
3331   return off;
3332 }
3333 
3334 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
3335   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3336   // and "3.9 Partial Register Penalties", p. 22).
3337   int off;
3338   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
3339     off = offset();
3340     movzbl(dst, src); // movzxb
3341   } else {
3342     xorl(dst, dst);
3343     off = offset();
3344     movb(dst, src);
3345   }
3346   return off;
3347 }
3348 
3349 // Note: load_unsigned_short used to be called load_unsigned_word.
3350 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
3351   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3352   // and "3.9 Partial Register Penalties", p. 22).
3353   int off;
3354   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
3355     off = offset();
3356     movzwl(dst, src); // movzxw
3357   } else {
3358     xorl(dst, dst);
3359     off = offset();
3360     movw(dst, src);
3361   }
3362   return off;
3363 }
3364 
3365 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
3366   switch (size_in_bytes) {
3367 #ifndef _LP64
3368   case  8:
3369     assert(dst2 != noreg, "second dest register required");
3370     movl(dst,  src);
3371     movl(dst2, src.plus_disp(BytesPerInt));
3372     break;
3373 #else
3374   case  8:  movq(dst, src); break;
3375 #endif
3376   case  4:  movl(dst, src); break;
3377   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
3378   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
3379   default:  ShouldNotReachHere();
3380   }
3381 }
3382 
3383 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
3384   switch (size_in_bytes) {
3385 #ifndef _LP64
3386   case  8:
3387     assert(src2 != noreg, "second source register required");
3388     movl(dst,                        src);
3389     movl(dst.plus_disp(BytesPerInt), src2);
3390     break;
3391 #else
3392   case  8:  movq(dst, src); break;
3393 #endif
3394   case  4:  movl(dst, src); break;
3395   case  2:  movw(dst, src); break;
3396   case  1:  movb(dst, src); break;
3397   default:  ShouldNotReachHere();
3398   }
3399 }
3400 
3401 void MacroAssembler::mov32(AddressLiteral dst, Register src) {
3402   if (reachable(dst)) {
3403     movl(as_Address(dst), src);
3404   } else {
3405     lea(rscratch1, dst);
3406     movl(Address(rscratch1, 0), src);
3407   }
3408 }
3409 
3410 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
3411   if (reachable(src)) {
3412     movl(dst, as_Address(src));
3413   } else {
3414     lea(rscratch1, src);
3415     movl(dst, Address(rscratch1, 0));
3416   }
3417 }
3418 
3419 // C++ bool manipulation
3420 
3421 void MacroAssembler::movbool(Register dst, Address src) {
3422   if(sizeof(bool) == 1)
3423     movb(dst, src);
3424   else if(sizeof(bool) == 2)
3425     movw(dst, src);
3426   else if(sizeof(bool) == 4)
3427     movl(dst, src);
3428   else
3429     // unsupported
3430     ShouldNotReachHere();
3431 }
3432 
3433 void MacroAssembler::movbool(Address dst, bool boolconst) {
3434   if(sizeof(bool) == 1)
3435     movb(dst, (int) boolconst);
3436   else if(sizeof(bool) == 2)
3437     movw(dst, (int) boolconst);
3438   else if(sizeof(bool) == 4)
3439     movl(dst, (int) boolconst);
3440   else
3441     // unsupported
3442     ShouldNotReachHere();
3443 }
3444 
3445 void MacroAssembler::movbool(Address dst, Register src) {
3446   if(sizeof(bool) == 1)
3447     movb(dst, src);
3448   else if(sizeof(bool) == 2)
3449     movw(dst, src);
3450   else if(sizeof(bool) == 4)
3451     movl(dst, src);
3452   else
3453     // unsupported
3454     ShouldNotReachHere();
3455 }
3456 
3457 void MacroAssembler::movbyte(ArrayAddress dst, int src) {
3458   movb(as_Address(dst), src);
3459 }
3460 
3461 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src) {
3462   if (reachable(src)) {
3463     movdl(dst, as_Address(src));
3464   } else {
3465     lea(rscratch1, src);
3466     movdl(dst, Address(rscratch1, 0));
3467   }
3468 }
3469 
3470 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src) {
3471   if (reachable(src)) {
3472     movq(dst, as_Address(src));
3473   } else {
3474     lea(rscratch1, src);
3475     movq(dst, Address(rscratch1, 0));
3476   }
3477 }
3478 
3479 void MacroAssembler::setvectmask(Register dst, Register src) {
3480   Assembler::movl(dst, 1);
3481   Assembler::shlxl(dst, dst, src);
3482   Assembler::decl(dst);
3483   Assembler::kmovdl(k1, dst);
3484   Assembler::movl(dst, src);
3485 }
3486 
3487 void MacroAssembler::restorevectmask() {
3488   Assembler::knotwl(k1, k0);
3489 }
3490 
3491 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
3492   if (reachable(src)) {
3493     if (UseXmmLoadAndClearUpper) {
3494       movsd (dst, as_Address(src));
3495     } else {
3496       movlpd(dst, as_Address(src));
3497     }
3498   } else {
3499     lea(rscratch1, src);
3500     if (UseXmmLoadAndClearUpper) {
3501       movsd (dst, Address(rscratch1, 0));
3502     } else {
3503       movlpd(dst, Address(rscratch1, 0));
3504     }
3505   }
3506 }
3507 
3508 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
3509   if (reachable(src)) {
3510     movss(dst, as_Address(src));
3511   } else {
3512     lea(rscratch1, src);
3513     movss(dst, Address(rscratch1, 0));
3514   }
3515 }
3516 
3517 void MacroAssembler::movptr(Register dst, Register src) {
3518   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3519 }
3520 
3521 void MacroAssembler::movptr(Register dst, Address src) {
3522   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3523 }
3524 
3525 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
3526 void MacroAssembler::movptr(Register dst, intptr_t src) {
3527   LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src));
3528 }
3529 
3530 void MacroAssembler::movptr(Address dst, Register src) {
3531   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3532 }
3533 
3534 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
3535   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3536     Assembler::vextractf32x4(dst, src, 0);
3537   } else {
3538     Assembler::movdqu(dst, src);
3539   }
3540 }
3541 
3542 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
3543   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3544     Assembler::vinsertf32x4(dst, dst, src, 0);
3545   } else {
3546     Assembler::movdqu(dst, src);
3547   }
3548 }
3549 
3550 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
3551   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3552     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3553   } else {
3554     Assembler::movdqu(dst, src);
3555   }
3556 }
3557 
3558 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register scratchReg) {
3559   if (reachable(src)) {
3560     movdqu(dst, as_Address(src));
3561   } else {
3562     lea(scratchReg, src);
3563     movdqu(dst, Address(scratchReg, 0));
3564   }
3565 }
3566 
3567 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
3568   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3569     vextractf64x4_low(dst, src);
3570   } else {
3571     Assembler::vmovdqu(dst, src);
3572   }
3573 }
3574 
3575 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
3576   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3577     vinsertf64x4_low(dst, src);
3578   } else {
3579     Assembler::vmovdqu(dst, src);
3580   }
3581 }
3582 
3583 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3584   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3585     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3586   }
3587   else {
3588     Assembler::vmovdqu(dst, src);
3589   }
3590 }
3591 
3592 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src) {
3593   if (reachable(src)) {
3594     vmovdqu(dst, as_Address(src));
3595   }
3596   else {
3597     lea(rscratch1, src);
3598     vmovdqu(dst, Address(rscratch1, 0));
3599   }
3600 }
3601 
3602 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) {
3603   if (reachable(src)) {
3604     Assembler::movdqa(dst, as_Address(src));
3605   } else {
3606     lea(rscratch1, src);
3607     Assembler::movdqa(dst, Address(rscratch1, 0));
3608   }
3609 }
3610 
3611 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
3612   if (reachable(src)) {
3613     Assembler::movsd(dst, as_Address(src));
3614   } else {
3615     lea(rscratch1, src);
3616     Assembler::movsd(dst, Address(rscratch1, 0));
3617   }
3618 }
3619 
3620 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
3621   if (reachable(src)) {
3622     Assembler::movss(dst, as_Address(src));
3623   } else {
3624     lea(rscratch1, src);
3625     Assembler::movss(dst, Address(rscratch1, 0));
3626   }
3627 }
3628 
3629 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) {
3630   if (reachable(src)) {
3631     Assembler::mulsd(dst, as_Address(src));
3632   } else {
3633     lea(rscratch1, src);
3634     Assembler::mulsd(dst, Address(rscratch1, 0));
3635   }
3636 }
3637 
3638 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) {
3639   if (reachable(src)) {
3640     Assembler::mulss(dst, as_Address(src));
3641   } else {
3642     lea(rscratch1, src);
3643     Assembler::mulss(dst, Address(rscratch1, 0));
3644   }
3645 }
3646 
3647 void MacroAssembler::null_check(Register reg, int offset) {
3648   if (needs_explicit_null_check(offset)) {
3649     // provoke OS NULL exception if reg = NULL by
3650     // accessing M[reg] w/o changing any (non-CC) registers
3651     // NOTE: cmpl is plenty here to provoke a segv
3652     cmpptr(rax, Address(reg, 0));
3653     // Note: should probably use testl(rax, Address(reg, 0));
3654     //       may be shorter code (however, this version of
3655     //       testl needs to be implemented first)
3656   } else {
3657     // nothing to do, (later) access of M[reg + offset]
3658     // will provoke OS NULL exception if reg = NULL
3659   }
3660 }
3661 
3662 void MacroAssembler::test_klass_is_value(Register klass, Register temp_reg, Label& is_value) {
3663   movl(temp_reg, Address(klass, Klass::access_flags_offset()));
3664   testl(temp_reg, JVM_ACC_VALUE);
3665   jcc(Assembler::notZero, is_value);
3666 }
3667 
3668 void MacroAssembler::test_field_is_flattenable(Register flags, Register temp_reg, Label& is_flattenable) {
3669   movl(temp_reg, flags);
3670   shrl(temp_reg, ConstantPoolCacheEntry::is_flattenable_field_shift);
3671   andl(temp_reg, 0x1);
3672   testl(temp_reg, temp_reg);
3673   jcc(Assembler::notZero, is_flattenable);
3674 }
3675 
3676 void MacroAssembler::test_field_is_not_flattenable(Register flags, Register temp_reg, Label& notFlattenable) {
3677   movl(temp_reg, flags);
3678   shrl(temp_reg, ConstantPoolCacheEntry::is_flattenable_field_shift);
3679   andl(temp_reg, 0x1);
3680   testl(temp_reg, temp_reg);
3681   jcc(Assembler::zero, notFlattenable);
3682 }
3683 
3684 void MacroAssembler::test_field_is_flattened(Register flags, Register temp_reg, Label& is_flattened) {
3685   movl(temp_reg, flags);
3686   shrl(temp_reg, ConstantPoolCacheEntry::is_flattened_field_shift);
3687   andl(temp_reg, 0x1);
3688   testl(temp_reg, temp_reg);
3689   jcc(Assembler::notZero, is_flattened);
3690 }
3691 
3692 void MacroAssembler::test_flat_array_klass(Register klass, Register temp_reg,
3693                                            Label& is_flat_array) {
3694   movl(temp_reg, Address(klass, Klass::layout_helper_offset()));
3695   sarl(temp_reg, Klass::_lh_array_tag_shift);
3696   cmpl(temp_reg, Klass::_lh_array_tag_vt_value);
3697   jcc(Assembler::equal, is_flat_array);
3698 }
3699 
3700 
3701 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg,
3702                                          Label& is_flat_array) {
3703   load_klass(temp_reg, oop);
3704   test_flat_array_klass(temp_reg, temp_reg, is_flat_array);
3705 }
3706 
3707 void MacroAssembler::test_value_is_not_buffered(Register value, Register temp_reg, Label& not_buffered) {
3708   ExternalAddress VTBuffer_top(VTBuffer::top_addr());
3709   ExternalAddress VTBuffer_end(VTBuffer::end_addr());
3710 
3711   // Test below is ordered based on the relative positions of
3712   // the Java heap and the VTBuffer to execute a single test for heap-allocated values
3713 
3714   if (VTBuffer::base() < Universe::heap()->base()) {
3715     lea(temp_reg, VTBuffer_end);
3716     cmpptr(value, temp_reg);
3717     jcc(Assembler::greaterEqual, not_buffered);
3718     lea(temp_reg, VTBuffer_top);
3719     cmpptr(value, temp_reg);
3720     jcc(Assembler::less, not_buffered);
3721   } else {
3722     lea(temp_reg, VTBuffer_top);
3723     cmpptr(value, temp_reg);
3724     jcc(Assembler::less, not_buffered);
3725     lea(temp_reg, VTBuffer_end);
3726     cmpptr(value, temp_reg);
3727     jcc(Assembler::greaterEqual, not_buffered);
3728   }
3729 }
3730 
3731 void MacroAssembler::test_oop_is_not_value(Register oop, Register temp, Label& is_not_value) {
3732   const int mask = Universe::oop_metadata_valuetype_mask();
3733 #ifdef _LP64
3734   if (UseCompressedClassPointers) {
3735     movl(temp, Address(oop, oopDesc::klass_offset_in_bytes()));
3736   } else
3737 #endif
3738   movptr(temp, Address(oop, oopDesc::klass_offset_in_bytes()));
3739 
3740   andl(temp, mask);
3741   testl(temp, temp);
3742   jcc(Assembler::zero, is_not_value);
3743 }
3744 
3745 void MacroAssembler::os_breakpoint() {
3746   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
3747   // (e.g., MSVC can't call ps() otherwise)
3748   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
3749 }
3750 
3751 void MacroAssembler::unimplemented(const char* what) {
3752   const char* buf = NULL;
3753   {
3754     ResourceMark rm;
3755     stringStream ss;
3756     ss.print("unimplemented: %s", what);
3757     buf = code_string(ss.as_string());
3758   }
3759   stop(buf);
3760 }
3761 
3762 #ifdef _LP64
3763 #define XSTATE_BV 0x200
3764 #endif
3765 
3766 void MacroAssembler::pop_CPU_state() {
3767   pop_FPU_state();
3768   pop_IU_state();
3769 }
3770 
3771 void MacroAssembler::pop_FPU_state() {
3772 #ifndef _LP64
3773   frstor(Address(rsp, 0));
3774 #else
3775   fxrstor(Address(rsp, 0));
3776 #endif
3777   addptr(rsp, FPUStateSizeInWords * wordSize);
3778 }
3779 
3780 void MacroAssembler::pop_IU_state() {
3781   popa();
3782   LP64_ONLY(addq(rsp, 8));
3783   popf();
3784 }
3785 
3786 // Save Integer and Float state
3787 // Warning: Stack must be 16 byte aligned (64bit)
3788 void MacroAssembler::push_CPU_state() {
3789   push_IU_state();
3790   push_FPU_state();
3791 }
3792 
3793 void MacroAssembler::push_FPU_state() {
3794   subptr(rsp, FPUStateSizeInWords * wordSize);
3795 #ifndef _LP64
3796   fnsave(Address(rsp, 0));
3797   fwait();
3798 #else
3799   fxsave(Address(rsp, 0));
3800 #endif // LP64
3801 }
3802 
3803 void MacroAssembler::push_IU_state() {
3804   // Push flags first because pusha kills them
3805   pushf();
3806   // Make sure rsp stays 16-byte aligned
3807   LP64_ONLY(subq(rsp, 8));
3808   pusha();
3809 }
3810 
3811 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
3812   if (!java_thread->is_valid()) {
3813     java_thread = rdi;
3814     get_thread(java_thread);
3815   }
3816   // we must set sp to zero to clear frame
3817   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
3818   if (clear_fp) {
3819     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
3820   }
3821 
3822   // Always clear the pc because it could have been set by make_walkable()
3823   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
3824 
3825   vzeroupper();
3826 }
3827 
3828 void MacroAssembler::restore_rax(Register tmp) {
3829   if (tmp == noreg) pop(rax);
3830   else if (tmp != rax) mov(rax, tmp);
3831 }
3832 
3833 void MacroAssembler::round_to(Register reg, int modulus) {
3834   addptr(reg, modulus - 1);
3835   andptr(reg, -modulus);
3836 }
3837 
3838 void MacroAssembler::save_rax(Register tmp) {
3839   if (tmp == noreg) push(rax);
3840   else if (tmp != rax) mov(tmp, rax);
3841 }
3842 
3843 // Write serialization page so VM thread can do a pseudo remote membar.
3844 // We use the current thread pointer to calculate a thread specific
3845 // offset to write to within the page. This minimizes bus traffic
3846 // due to cache line collision.
3847 void MacroAssembler::serialize_memory(Register thread, Register tmp) {
3848   movl(tmp, thread);
3849   shrl(tmp, os::get_serialize_page_shift_count());
3850   andl(tmp, (os::vm_page_size() - sizeof(int)));
3851 
3852   Address index(noreg, tmp, Address::times_1);
3853   ExternalAddress page(os::get_memory_serialize_page());
3854 
3855   // Size of store must match masking code above
3856   movl(as_Address(ArrayAddress(page, index)), tmp);
3857 }
3858 
3859 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, Register temp_reg) {
3860   if (SafepointMechanism::uses_thread_local_poll()) {
3861 #ifdef _LP64
3862     assert(thread_reg == r15_thread, "should be");
3863 #else
3864     if (thread_reg == noreg) {
3865       thread_reg = temp_reg;
3866       get_thread(thread_reg);
3867     }
3868 #endif
3869     testb(Address(thread_reg, Thread::polling_page_offset()), SafepointMechanism::poll_bit());
3870     jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
3871   } else {
3872     cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
3873         SafepointSynchronize::_not_synchronized);
3874     jcc(Assembler::notEqual, slow_path);
3875   }
3876 }
3877 
3878 // Calls to C land
3879 //
3880 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
3881 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
3882 // has to be reset to 0. This is required to allow proper stack traversal.
3883 void MacroAssembler::set_last_Java_frame(Register java_thread,
3884                                          Register last_java_sp,
3885                                          Register last_java_fp,
3886                                          address  last_java_pc) {
3887   vzeroupper();
3888   // determine java_thread register
3889   if (!java_thread->is_valid()) {
3890     java_thread = rdi;
3891     get_thread(java_thread);
3892   }
3893   // determine last_java_sp register
3894   if (!last_java_sp->is_valid()) {
3895     last_java_sp = rsp;
3896   }
3897 
3898   // last_java_fp is optional
3899 
3900   if (last_java_fp->is_valid()) {
3901     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
3902   }
3903 
3904   // last_java_pc is optional
3905 
3906   if (last_java_pc != NULL) {
3907     lea(Address(java_thread,
3908                  JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()),
3909         InternalAddress(last_java_pc));
3910 
3911   }
3912   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
3913 }
3914 
3915 void MacroAssembler::shlptr(Register dst, int imm8) {
3916   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
3917 }
3918 
3919 void MacroAssembler::shrptr(Register dst, int imm8) {
3920   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
3921 }
3922 
3923 void MacroAssembler::sign_extend_byte(Register reg) {
3924   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
3925     movsbl(reg, reg); // movsxb
3926   } else {
3927     shll(reg, 24);
3928     sarl(reg, 24);
3929   }
3930 }
3931 
3932 void MacroAssembler::sign_extend_short(Register reg) {
3933   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3934     movswl(reg, reg); // movsxw
3935   } else {
3936     shll(reg, 16);
3937     sarl(reg, 16);
3938   }
3939 }
3940 
3941 void MacroAssembler::testl(Register dst, AddressLiteral src) {
3942   assert(reachable(src), "Address should be reachable");
3943   testl(dst, as_Address(src));
3944 }
3945 
3946 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3947   int dst_enc = dst->encoding();
3948   int src_enc = src->encoding();
3949   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3950     Assembler::pcmpeqb(dst, src);
3951   } else if ((dst_enc < 16) && (src_enc < 16)) {
3952     Assembler::pcmpeqb(dst, src);
3953   } else if (src_enc < 16) {
3954     subptr(rsp, 64);
3955     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3956     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3957     Assembler::pcmpeqb(xmm0, src);
3958     movdqu(dst, xmm0);
3959     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3960     addptr(rsp, 64);
3961   } else if (dst_enc < 16) {
3962     subptr(rsp, 64);
3963     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3964     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3965     Assembler::pcmpeqb(dst, xmm0);
3966     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3967     addptr(rsp, 64);
3968   } else {
3969     subptr(rsp, 64);
3970     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3971     subptr(rsp, 64);
3972     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
3973     movdqu(xmm0, src);
3974     movdqu(xmm1, dst);
3975     Assembler::pcmpeqb(xmm1, xmm0);
3976     movdqu(dst, xmm1);
3977     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
3978     addptr(rsp, 64);
3979     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3980     addptr(rsp, 64);
3981   }
3982 }
3983 
3984 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3985   int dst_enc = dst->encoding();
3986   int src_enc = src->encoding();
3987   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3988     Assembler::pcmpeqw(dst, src);
3989   } else if ((dst_enc < 16) && (src_enc < 16)) {
3990     Assembler::pcmpeqw(dst, src);
3991   } else if (src_enc < 16) {
3992     subptr(rsp, 64);
3993     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3994     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3995     Assembler::pcmpeqw(xmm0, src);
3996     movdqu(dst, xmm0);
3997     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3998     addptr(rsp, 64);
3999   } else if (dst_enc < 16) {
4000     subptr(rsp, 64);
4001     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4002     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4003     Assembler::pcmpeqw(dst, xmm0);
4004     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4005     addptr(rsp, 64);
4006   } else {
4007     subptr(rsp, 64);
4008     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4009     subptr(rsp, 64);
4010     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4011     movdqu(xmm0, src);
4012     movdqu(xmm1, dst);
4013     Assembler::pcmpeqw(xmm1, xmm0);
4014     movdqu(dst, xmm1);
4015     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4016     addptr(rsp, 64);
4017     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4018     addptr(rsp, 64);
4019   }
4020 }
4021 
4022 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
4023   int dst_enc = dst->encoding();
4024   if (dst_enc < 16) {
4025     Assembler::pcmpestri(dst, src, imm8);
4026   } else {
4027     subptr(rsp, 64);
4028     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4029     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4030     Assembler::pcmpestri(xmm0, src, imm8);
4031     movdqu(dst, xmm0);
4032     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4033     addptr(rsp, 64);
4034   }
4035 }
4036 
4037 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
4038   int dst_enc = dst->encoding();
4039   int src_enc = src->encoding();
4040   if ((dst_enc < 16) && (src_enc < 16)) {
4041     Assembler::pcmpestri(dst, src, imm8);
4042   } else if (src_enc < 16) {
4043     subptr(rsp, 64);
4044     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4045     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4046     Assembler::pcmpestri(xmm0, src, imm8);
4047     movdqu(dst, xmm0);
4048     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4049     addptr(rsp, 64);
4050   } else if (dst_enc < 16) {
4051     subptr(rsp, 64);
4052     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4053     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4054     Assembler::pcmpestri(dst, xmm0, imm8);
4055     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4056     addptr(rsp, 64);
4057   } else {
4058     subptr(rsp, 64);
4059     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4060     subptr(rsp, 64);
4061     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4062     movdqu(xmm0, src);
4063     movdqu(xmm1, dst);
4064     Assembler::pcmpestri(xmm1, xmm0, imm8);
4065     movdqu(dst, xmm1);
4066     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4067     addptr(rsp, 64);
4068     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4069     addptr(rsp, 64);
4070   }
4071 }
4072 
4073 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
4074   int dst_enc = dst->encoding();
4075   int src_enc = src->encoding();
4076   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4077     Assembler::pmovzxbw(dst, src);
4078   } else if ((dst_enc < 16) && (src_enc < 16)) {
4079     Assembler::pmovzxbw(dst, src);
4080   } else if (src_enc < 16) {
4081     subptr(rsp, 64);
4082     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4083     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4084     Assembler::pmovzxbw(xmm0, src);
4085     movdqu(dst, xmm0);
4086     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4087     addptr(rsp, 64);
4088   } else if (dst_enc < 16) {
4089     subptr(rsp, 64);
4090     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4091     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4092     Assembler::pmovzxbw(dst, xmm0);
4093     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4094     addptr(rsp, 64);
4095   } else {
4096     subptr(rsp, 64);
4097     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4098     subptr(rsp, 64);
4099     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4100     movdqu(xmm0, src);
4101     movdqu(xmm1, dst);
4102     Assembler::pmovzxbw(xmm1, xmm0);
4103     movdqu(dst, xmm1);
4104     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4105     addptr(rsp, 64);
4106     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4107     addptr(rsp, 64);
4108   }
4109 }
4110 
4111 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
4112   int dst_enc = dst->encoding();
4113   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4114     Assembler::pmovzxbw(dst, src);
4115   } else if (dst_enc < 16) {
4116     Assembler::pmovzxbw(dst, src);
4117   } else {
4118     subptr(rsp, 64);
4119     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4120     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4121     Assembler::pmovzxbw(xmm0, src);
4122     movdqu(dst, xmm0);
4123     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4124     addptr(rsp, 64);
4125   }
4126 }
4127 
4128 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
4129   int src_enc = src->encoding();
4130   if (src_enc < 16) {
4131     Assembler::pmovmskb(dst, src);
4132   } else {
4133     subptr(rsp, 64);
4134     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4135     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4136     Assembler::pmovmskb(dst, xmm0);
4137     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4138     addptr(rsp, 64);
4139   }
4140 }
4141 
4142 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
4143   int dst_enc = dst->encoding();
4144   int src_enc = src->encoding();
4145   if ((dst_enc < 16) && (src_enc < 16)) {
4146     Assembler::ptest(dst, src);
4147   } else if (src_enc < 16) {
4148     subptr(rsp, 64);
4149     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4150     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4151     Assembler::ptest(xmm0, src);
4152     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4153     addptr(rsp, 64);
4154   } else if (dst_enc < 16) {
4155     subptr(rsp, 64);
4156     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4157     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4158     Assembler::ptest(dst, xmm0);
4159     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4160     addptr(rsp, 64);
4161   } else {
4162     subptr(rsp, 64);
4163     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4164     subptr(rsp, 64);
4165     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4166     movdqu(xmm0, src);
4167     movdqu(xmm1, dst);
4168     Assembler::ptest(xmm1, xmm0);
4169     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4170     addptr(rsp, 64);
4171     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4172     addptr(rsp, 64);
4173   }
4174 }
4175 
4176 void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) {
4177   if (reachable(src)) {
4178     Assembler::sqrtsd(dst, as_Address(src));
4179   } else {
4180     lea(rscratch1, src);
4181     Assembler::sqrtsd(dst, Address(rscratch1, 0));
4182   }
4183 }
4184 
4185 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) {
4186   if (reachable(src)) {
4187     Assembler::sqrtss(dst, as_Address(src));
4188   } else {
4189     lea(rscratch1, src);
4190     Assembler::sqrtss(dst, Address(rscratch1, 0));
4191   }
4192 }
4193 
4194 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) {
4195   if (reachable(src)) {
4196     Assembler::subsd(dst, as_Address(src));
4197   } else {
4198     lea(rscratch1, src);
4199     Assembler::subsd(dst, Address(rscratch1, 0));
4200   }
4201 }
4202 
4203 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) {
4204   if (reachable(src)) {
4205     Assembler::subss(dst, as_Address(src));
4206   } else {
4207     lea(rscratch1, src);
4208     Assembler::subss(dst, Address(rscratch1, 0));
4209   }
4210 }
4211 
4212 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
4213   if (reachable(src)) {
4214     Assembler::ucomisd(dst, as_Address(src));
4215   } else {
4216     lea(rscratch1, src);
4217     Assembler::ucomisd(dst, Address(rscratch1, 0));
4218   }
4219 }
4220 
4221 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
4222   if (reachable(src)) {
4223     Assembler::ucomiss(dst, as_Address(src));
4224   } else {
4225     lea(rscratch1, src);
4226     Assembler::ucomiss(dst, Address(rscratch1, 0));
4227   }
4228 }
4229 
4230 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
4231   // Used in sign-bit flipping with aligned address.
4232   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
4233   if (reachable(src)) {
4234     Assembler::xorpd(dst, as_Address(src));
4235   } else {
4236     lea(rscratch1, src);
4237     Assembler::xorpd(dst, Address(rscratch1, 0));
4238   }
4239 }
4240 
4241 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
4242   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
4243     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
4244   }
4245   else {
4246     Assembler::xorpd(dst, src);
4247   }
4248 }
4249 
4250 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
4251   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
4252     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
4253   } else {
4254     Assembler::xorps(dst, src);
4255   }
4256 }
4257 
4258 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
4259   // Used in sign-bit flipping with aligned address.
4260   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
4261   if (reachable(src)) {
4262     Assembler::xorps(dst, as_Address(src));
4263   } else {
4264     lea(rscratch1, src);
4265     Assembler::xorps(dst, Address(rscratch1, 0));
4266   }
4267 }
4268 
4269 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) {
4270   // Used in sign-bit flipping with aligned address.
4271   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
4272   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
4273   if (reachable(src)) {
4274     Assembler::pshufb(dst, as_Address(src));
4275   } else {
4276     lea(rscratch1, src);
4277     Assembler::pshufb(dst, Address(rscratch1, 0));
4278   }
4279 }
4280 
4281 // AVX 3-operands instructions
4282 
4283 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4284   if (reachable(src)) {
4285     vaddsd(dst, nds, as_Address(src));
4286   } else {
4287     lea(rscratch1, src);
4288     vaddsd(dst, nds, Address(rscratch1, 0));
4289   }
4290 }
4291 
4292 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4293   if (reachable(src)) {
4294     vaddss(dst, nds, as_Address(src));
4295   } else {
4296     lea(rscratch1, src);
4297     vaddss(dst, nds, Address(rscratch1, 0));
4298   }
4299 }
4300 
4301 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
4302   int dst_enc = dst->encoding();
4303   int nds_enc = nds->encoding();
4304   int src_enc = src->encoding();
4305   if ((dst_enc < 16) && (nds_enc < 16)) {
4306     vandps(dst, nds, negate_field, vector_len);
4307   } else if ((src_enc < 16) && (dst_enc < 16)) {
4308     evmovdqul(src, nds, Assembler::AVX_512bit);
4309     vandps(dst, src, negate_field, vector_len);
4310   } else if (src_enc < 16) {
4311     evmovdqul(src, nds, Assembler::AVX_512bit);
4312     vandps(src, src, negate_field, vector_len);
4313     evmovdqul(dst, src, Assembler::AVX_512bit);
4314   } else if (dst_enc < 16) {
4315     evmovdqul(src, xmm0, Assembler::AVX_512bit);
4316     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4317     vandps(dst, xmm0, negate_field, vector_len);
4318     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4319   } else {
4320     if (src_enc != dst_enc) {
4321       evmovdqul(src, xmm0, Assembler::AVX_512bit);
4322       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4323       vandps(xmm0, xmm0, negate_field, vector_len);
4324       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4325       evmovdqul(xmm0, src, Assembler::AVX_512bit);
4326     } else {
4327       subptr(rsp, 64);
4328       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4329       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4330       vandps(xmm0, xmm0, negate_field, vector_len);
4331       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4332       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4333       addptr(rsp, 64);
4334     }
4335   }
4336 }
4337 
4338 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
4339   int dst_enc = dst->encoding();
4340   int nds_enc = nds->encoding();
4341   int src_enc = src->encoding();
4342   if ((dst_enc < 16) && (nds_enc < 16)) {
4343     vandpd(dst, nds, negate_field, vector_len);
4344   } else if ((src_enc < 16) && (dst_enc < 16)) {
4345     evmovdqul(src, nds, Assembler::AVX_512bit);
4346     vandpd(dst, src, negate_field, vector_len);
4347   } else if (src_enc < 16) {
4348     evmovdqul(src, nds, Assembler::AVX_512bit);
4349     vandpd(src, src, negate_field, vector_len);
4350     evmovdqul(dst, src, Assembler::AVX_512bit);
4351   } else if (dst_enc < 16) {
4352     evmovdqul(src, xmm0, Assembler::AVX_512bit);
4353     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4354     vandpd(dst, xmm0, negate_field, vector_len);
4355     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4356   } else {
4357     if (src_enc != dst_enc) {
4358       evmovdqul(src, xmm0, Assembler::AVX_512bit);
4359       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4360       vandpd(xmm0, xmm0, negate_field, vector_len);
4361       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4362       evmovdqul(xmm0, src, Assembler::AVX_512bit);
4363     } else {
4364       subptr(rsp, 64);
4365       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4366       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4367       vandpd(xmm0, xmm0, negate_field, vector_len);
4368       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4369       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4370       addptr(rsp, 64);
4371     }
4372   }
4373 }
4374 
4375 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4376   int dst_enc = dst->encoding();
4377   int nds_enc = nds->encoding();
4378   int src_enc = src->encoding();
4379   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4380     Assembler::vpaddb(dst, nds, src, vector_len);
4381   } else if ((dst_enc < 16) && (src_enc < 16)) {
4382     Assembler::vpaddb(dst, dst, src, vector_len);
4383   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4384     // use nds as scratch for src
4385     evmovdqul(nds, src, Assembler::AVX_512bit);
4386     Assembler::vpaddb(dst, dst, nds, vector_len);
4387   } else if ((src_enc < 16) && (nds_enc < 16)) {
4388     // use nds as scratch for dst
4389     evmovdqul(nds, dst, Assembler::AVX_512bit);
4390     Assembler::vpaddb(nds, nds, src, vector_len);
4391     evmovdqul(dst, nds, Assembler::AVX_512bit);
4392   } else if (dst_enc < 16) {
4393     // use nds as scatch for xmm0 to hold src
4394     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4395     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4396     Assembler::vpaddb(dst, dst, xmm0, vector_len);
4397     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4398   } else {
4399     // worse case scenario, all regs are in the upper bank
4400     subptr(rsp, 64);
4401     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4402     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4403     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4404     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4405     Assembler::vpaddb(xmm0, xmm0, xmm1, vector_len);
4406     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4407     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4408     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4409     addptr(rsp, 64);
4410   }
4411 }
4412 
4413 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4414   int dst_enc = dst->encoding();
4415   int nds_enc = nds->encoding();
4416   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4417     Assembler::vpaddb(dst, nds, src, vector_len);
4418   } else if (dst_enc < 16) {
4419     Assembler::vpaddb(dst, dst, src, vector_len);
4420   } else if (nds_enc < 16) {
4421     // implies dst_enc in upper bank with src as scratch
4422     evmovdqul(nds, dst, Assembler::AVX_512bit);
4423     Assembler::vpaddb(nds, nds, src, vector_len);
4424     evmovdqul(dst, nds, Assembler::AVX_512bit);
4425   } else {
4426     // worse case scenario, all regs in upper bank
4427     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4428     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4429     Assembler::vpaddb(xmm0, xmm0, src, vector_len);
4430     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4431   }
4432 }
4433 
4434 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4435   int dst_enc = dst->encoding();
4436   int nds_enc = nds->encoding();
4437   int src_enc = src->encoding();
4438   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4439     Assembler::vpaddw(dst, nds, src, vector_len);
4440   } else if ((dst_enc < 16) && (src_enc < 16)) {
4441     Assembler::vpaddw(dst, dst, src, vector_len);
4442   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4443     // use nds as scratch for src
4444     evmovdqul(nds, src, Assembler::AVX_512bit);
4445     Assembler::vpaddw(dst, dst, nds, vector_len);
4446   } else if ((src_enc < 16) && (nds_enc < 16)) {
4447     // use nds as scratch for dst
4448     evmovdqul(nds, dst, Assembler::AVX_512bit);
4449     Assembler::vpaddw(nds, nds, src, vector_len);
4450     evmovdqul(dst, nds, Assembler::AVX_512bit);
4451   } else if (dst_enc < 16) {
4452     // use nds as scatch for xmm0 to hold src
4453     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4454     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4455     Assembler::vpaddw(dst, dst, xmm0, vector_len);
4456     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4457   } else {
4458     // worse case scenario, all regs are in the upper bank
4459     subptr(rsp, 64);
4460     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4461     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4462     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4463     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4464     Assembler::vpaddw(xmm0, xmm0, xmm1, vector_len);
4465     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4466     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4467     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4468     addptr(rsp, 64);
4469   }
4470 }
4471 
4472 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4473   int dst_enc = dst->encoding();
4474   int nds_enc = nds->encoding();
4475   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4476     Assembler::vpaddw(dst, nds, src, vector_len);
4477   } else if (dst_enc < 16) {
4478     Assembler::vpaddw(dst, dst, src, vector_len);
4479   } else if (nds_enc < 16) {
4480     // implies dst_enc in upper bank with src as scratch
4481     evmovdqul(nds, dst, Assembler::AVX_512bit);
4482     Assembler::vpaddw(nds, nds, src, vector_len);
4483     evmovdqul(dst, nds, Assembler::AVX_512bit);
4484   } else {
4485     // worse case scenario, all regs in upper bank
4486     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4487     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4488     Assembler::vpaddw(xmm0, xmm0, src, vector_len);
4489     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4490   }
4491 }
4492 
4493 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4494   if (reachable(src)) {
4495     Assembler::vpand(dst, nds, as_Address(src), vector_len);
4496   } else {
4497     lea(rscratch1, src);
4498     Assembler::vpand(dst, nds, Address(rscratch1, 0), vector_len);
4499   }
4500 }
4501 
4502 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
4503   int dst_enc = dst->encoding();
4504   int src_enc = src->encoding();
4505   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4506     Assembler::vpbroadcastw(dst, src);
4507   } else if ((dst_enc < 16) && (src_enc < 16)) {
4508     Assembler::vpbroadcastw(dst, src);
4509   } else if (src_enc < 16) {
4510     subptr(rsp, 64);
4511     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4512     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4513     Assembler::vpbroadcastw(xmm0, src);
4514     movdqu(dst, xmm0);
4515     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4516     addptr(rsp, 64);
4517   } else if (dst_enc < 16) {
4518     subptr(rsp, 64);
4519     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4520     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4521     Assembler::vpbroadcastw(dst, xmm0);
4522     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4523     addptr(rsp, 64);
4524   } else {
4525     subptr(rsp, 64);
4526     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4527     subptr(rsp, 64);
4528     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4529     movdqu(xmm0, src);
4530     movdqu(xmm1, dst);
4531     Assembler::vpbroadcastw(xmm1, xmm0);
4532     movdqu(dst, xmm1);
4533     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4534     addptr(rsp, 64);
4535     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4536     addptr(rsp, 64);
4537   }
4538 }
4539 
4540 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4541   int dst_enc = dst->encoding();
4542   int nds_enc = nds->encoding();
4543   int src_enc = src->encoding();
4544   assert(dst_enc == nds_enc, "");
4545   if ((dst_enc < 16) && (src_enc < 16)) {
4546     Assembler::vpcmpeqb(dst, nds, src, vector_len);
4547   } else if (src_enc < 16) {
4548     subptr(rsp, 64);
4549     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4550     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4551     Assembler::vpcmpeqb(xmm0, xmm0, src, vector_len);
4552     movdqu(dst, xmm0);
4553     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4554     addptr(rsp, 64);
4555   } else if (dst_enc < 16) {
4556     subptr(rsp, 64);
4557     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4558     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4559     Assembler::vpcmpeqb(dst, dst, xmm0, vector_len);
4560     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4561     addptr(rsp, 64);
4562   } else {
4563     subptr(rsp, 64);
4564     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4565     subptr(rsp, 64);
4566     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4567     movdqu(xmm0, src);
4568     movdqu(xmm1, dst);
4569     Assembler::vpcmpeqb(xmm1, xmm1, xmm0, vector_len);
4570     movdqu(dst, xmm1);
4571     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4572     addptr(rsp, 64);
4573     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4574     addptr(rsp, 64);
4575   }
4576 }
4577 
4578 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4579   int dst_enc = dst->encoding();
4580   int nds_enc = nds->encoding();
4581   int src_enc = src->encoding();
4582   assert(dst_enc == nds_enc, "");
4583   if ((dst_enc < 16) && (src_enc < 16)) {
4584     Assembler::vpcmpeqw(dst, nds, src, vector_len);
4585   } else if (src_enc < 16) {
4586     subptr(rsp, 64);
4587     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4588     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4589     Assembler::vpcmpeqw(xmm0, xmm0, src, vector_len);
4590     movdqu(dst, xmm0);
4591     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4592     addptr(rsp, 64);
4593   } else if (dst_enc < 16) {
4594     subptr(rsp, 64);
4595     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4596     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4597     Assembler::vpcmpeqw(dst, dst, xmm0, vector_len);
4598     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4599     addptr(rsp, 64);
4600   } else {
4601     subptr(rsp, 64);
4602     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4603     subptr(rsp, 64);
4604     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4605     movdqu(xmm0, src);
4606     movdqu(xmm1, dst);
4607     Assembler::vpcmpeqw(xmm1, xmm1, xmm0, vector_len);
4608     movdqu(dst, xmm1);
4609     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4610     addptr(rsp, 64);
4611     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4612     addptr(rsp, 64);
4613   }
4614 }
4615 
4616 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
4617   int dst_enc = dst->encoding();
4618   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4619     Assembler::vpmovzxbw(dst, src, vector_len);
4620   } else if (dst_enc < 16) {
4621     Assembler::vpmovzxbw(dst, src, vector_len);
4622   } else {
4623     subptr(rsp, 64);
4624     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4625     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4626     Assembler::vpmovzxbw(xmm0, src, vector_len);
4627     movdqu(dst, xmm0);
4628     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4629     addptr(rsp, 64);
4630   }
4631 }
4632 
4633 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src) {
4634   int src_enc = src->encoding();
4635   if (src_enc < 16) {
4636     Assembler::vpmovmskb(dst, src);
4637   } else {
4638     subptr(rsp, 64);
4639     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4640     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4641     Assembler::vpmovmskb(dst, xmm0);
4642     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4643     addptr(rsp, 64);
4644   }
4645 }
4646 
4647 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4648   int dst_enc = dst->encoding();
4649   int nds_enc = nds->encoding();
4650   int src_enc = src->encoding();
4651   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4652     Assembler::vpmullw(dst, nds, src, vector_len);
4653   } else if ((dst_enc < 16) && (src_enc < 16)) {
4654     Assembler::vpmullw(dst, dst, src, vector_len);
4655   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4656     // use nds as scratch for src
4657     evmovdqul(nds, src, Assembler::AVX_512bit);
4658     Assembler::vpmullw(dst, dst, nds, vector_len);
4659   } else if ((src_enc < 16) && (nds_enc < 16)) {
4660     // use nds as scratch for dst
4661     evmovdqul(nds, dst, Assembler::AVX_512bit);
4662     Assembler::vpmullw(nds, nds, src, vector_len);
4663     evmovdqul(dst, nds, Assembler::AVX_512bit);
4664   } else if (dst_enc < 16) {
4665     // use nds as scatch for xmm0 to hold src
4666     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4667     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4668     Assembler::vpmullw(dst, dst, xmm0, vector_len);
4669     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4670   } else {
4671     // worse case scenario, all regs are in the upper bank
4672     subptr(rsp, 64);
4673     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4674     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4675     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4676     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4677     Assembler::vpmullw(xmm0, xmm0, xmm1, vector_len);
4678     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4679     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4680     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4681     addptr(rsp, 64);
4682   }
4683 }
4684 
4685 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4686   int dst_enc = dst->encoding();
4687   int nds_enc = nds->encoding();
4688   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4689     Assembler::vpmullw(dst, nds, src, vector_len);
4690   } else if (dst_enc < 16) {
4691     Assembler::vpmullw(dst, dst, src, vector_len);
4692   } else if (nds_enc < 16) {
4693     // implies dst_enc in upper bank with src as scratch
4694     evmovdqul(nds, dst, Assembler::AVX_512bit);
4695     Assembler::vpmullw(nds, nds, src, vector_len);
4696     evmovdqul(dst, nds, Assembler::AVX_512bit);
4697   } else {
4698     // worse case scenario, all regs in upper bank
4699     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4700     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4701     Assembler::vpmullw(xmm0, xmm0, src, vector_len);
4702     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4703   }
4704 }
4705 
4706 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4707   int dst_enc = dst->encoding();
4708   int nds_enc = nds->encoding();
4709   int src_enc = src->encoding();
4710   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4711     Assembler::vpsubb(dst, nds, src, vector_len);
4712   } else if ((dst_enc < 16) && (src_enc < 16)) {
4713     Assembler::vpsubb(dst, dst, src, vector_len);
4714   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4715     // use nds as scratch for src
4716     evmovdqul(nds, src, Assembler::AVX_512bit);
4717     Assembler::vpsubb(dst, dst, nds, vector_len);
4718   } else if ((src_enc < 16) && (nds_enc < 16)) {
4719     // use nds as scratch for dst
4720     evmovdqul(nds, dst, Assembler::AVX_512bit);
4721     Assembler::vpsubb(nds, nds, src, vector_len);
4722     evmovdqul(dst, nds, Assembler::AVX_512bit);
4723   } else if (dst_enc < 16) {
4724     // use nds as scatch for xmm0 to hold src
4725     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4726     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4727     Assembler::vpsubb(dst, dst, xmm0, vector_len);
4728     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4729   } else {
4730     // worse case scenario, all regs are in the upper bank
4731     subptr(rsp, 64);
4732     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4733     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4734     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4735     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4736     Assembler::vpsubb(xmm0, xmm0, xmm1, vector_len);
4737     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4738     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4739     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4740     addptr(rsp, 64);
4741   }
4742 }
4743 
4744 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4745   int dst_enc = dst->encoding();
4746   int nds_enc = nds->encoding();
4747   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4748     Assembler::vpsubb(dst, nds, src, vector_len);
4749   } else if (dst_enc < 16) {
4750     Assembler::vpsubb(dst, dst, src, vector_len);
4751   } else if (nds_enc < 16) {
4752     // implies dst_enc in upper bank with src as scratch
4753     evmovdqul(nds, dst, Assembler::AVX_512bit);
4754     Assembler::vpsubb(nds, nds, src, vector_len);
4755     evmovdqul(dst, nds, Assembler::AVX_512bit);
4756   } else {
4757     // worse case scenario, all regs in upper bank
4758     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4759     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4760     Assembler::vpsubw(xmm0, xmm0, src, vector_len);
4761     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4762   }
4763 }
4764 
4765 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4766   int dst_enc = dst->encoding();
4767   int nds_enc = nds->encoding();
4768   int src_enc = src->encoding();
4769   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4770     Assembler::vpsubw(dst, nds, src, vector_len);
4771   } else if ((dst_enc < 16) && (src_enc < 16)) {
4772     Assembler::vpsubw(dst, dst, src, vector_len);
4773   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4774     // use nds as scratch for src
4775     evmovdqul(nds, src, Assembler::AVX_512bit);
4776     Assembler::vpsubw(dst, dst, nds, vector_len);
4777   } else if ((src_enc < 16) && (nds_enc < 16)) {
4778     // use nds as scratch for dst
4779     evmovdqul(nds, dst, Assembler::AVX_512bit);
4780     Assembler::vpsubw(nds, nds, src, vector_len);
4781     evmovdqul(dst, nds, Assembler::AVX_512bit);
4782   } else if (dst_enc < 16) {
4783     // use nds as scatch for xmm0 to hold src
4784     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4785     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4786     Assembler::vpsubw(dst, dst, xmm0, vector_len);
4787     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4788   } else {
4789     // worse case scenario, all regs are in the upper bank
4790     subptr(rsp, 64);
4791     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4792     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4793     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4794     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4795     Assembler::vpsubw(xmm0, xmm0, xmm1, vector_len);
4796     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4797     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4798     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4799     addptr(rsp, 64);
4800   }
4801 }
4802 
4803 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4804   int dst_enc = dst->encoding();
4805   int nds_enc = nds->encoding();
4806   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4807     Assembler::vpsubw(dst, nds, src, vector_len);
4808   } else if (dst_enc < 16) {
4809     Assembler::vpsubw(dst, dst, src, vector_len);
4810   } else if (nds_enc < 16) {
4811     // implies dst_enc in upper bank with src as scratch
4812     evmovdqul(nds, dst, Assembler::AVX_512bit);
4813     Assembler::vpsubw(nds, nds, src, vector_len);
4814     evmovdqul(dst, nds, Assembler::AVX_512bit);
4815   } else {
4816     // worse case scenario, all regs in upper bank
4817     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4818     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4819     Assembler::vpsubw(xmm0, xmm0, src, vector_len);
4820     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4821   }
4822 }
4823 
4824 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4825   int dst_enc = dst->encoding();
4826   int nds_enc = nds->encoding();
4827   int shift_enc = shift->encoding();
4828   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4829     Assembler::vpsraw(dst, nds, shift, vector_len);
4830   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4831     Assembler::vpsraw(dst, dst, shift, vector_len);
4832   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4833     // use nds_enc as scratch with shift
4834     evmovdqul(nds, shift, Assembler::AVX_512bit);
4835     Assembler::vpsraw(dst, dst, nds, vector_len);
4836   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4837     // use nds as scratch with dst
4838     evmovdqul(nds, dst, Assembler::AVX_512bit);
4839     Assembler::vpsraw(nds, nds, shift, vector_len);
4840     evmovdqul(dst, nds, Assembler::AVX_512bit);
4841   } else if (dst_enc < 16) {
4842     // use nds to save a copy of xmm0 and hold shift
4843     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4844     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4845     Assembler::vpsraw(dst, dst, xmm0, vector_len);
4846     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4847   } else if (nds_enc < 16) {
4848     // use nds as dest as temps
4849     evmovdqul(nds, dst, Assembler::AVX_512bit);
4850     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4851     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4852     Assembler::vpsraw(nds, nds, xmm0, vector_len);
4853     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4854     evmovdqul(dst, nds, Assembler::AVX_512bit);
4855   } else {
4856     // worse case scenario, all regs are in the upper bank
4857     subptr(rsp, 64);
4858     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4859     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4860     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4861     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4862     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4863     evmovdqul(xmm1, dst, Assembler::AVX_512bit);
4864     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4865     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4866     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4867     addptr(rsp, 64);
4868   }
4869 }
4870 
4871 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4872   int dst_enc = dst->encoding();
4873   int nds_enc = nds->encoding();
4874   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4875     Assembler::vpsraw(dst, nds, shift, vector_len);
4876   } else if (dst_enc < 16) {
4877     Assembler::vpsraw(dst, dst, shift, vector_len);
4878   } else if (nds_enc < 16) {
4879     // use nds as scratch
4880     evmovdqul(nds, dst, Assembler::AVX_512bit);
4881     Assembler::vpsraw(nds, nds, shift, vector_len);
4882     evmovdqul(dst, nds, Assembler::AVX_512bit);
4883   } else {
4884     // use nds as scratch for xmm0
4885     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4886     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4887     Assembler::vpsraw(xmm0, xmm0, shift, vector_len);
4888     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4889   }
4890 }
4891 
4892 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4893   int dst_enc = dst->encoding();
4894   int nds_enc = nds->encoding();
4895   int shift_enc = shift->encoding();
4896   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4897     Assembler::vpsrlw(dst, nds, shift, vector_len);
4898   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4899     Assembler::vpsrlw(dst, dst, shift, vector_len);
4900   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4901     // use nds_enc as scratch with shift
4902     evmovdqul(nds, shift, Assembler::AVX_512bit);
4903     Assembler::vpsrlw(dst, dst, nds, vector_len);
4904   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4905     // use nds as scratch with dst
4906     evmovdqul(nds, dst, Assembler::AVX_512bit);
4907     Assembler::vpsrlw(nds, nds, shift, vector_len);
4908     evmovdqul(dst, nds, Assembler::AVX_512bit);
4909   } else if (dst_enc < 16) {
4910     // use nds to save a copy of xmm0 and hold shift
4911     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4912     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4913     Assembler::vpsrlw(dst, dst, xmm0, vector_len);
4914     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4915   } else if (nds_enc < 16) {
4916     // use nds as dest as temps
4917     evmovdqul(nds, dst, Assembler::AVX_512bit);
4918     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4919     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4920     Assembler::vpsrlw(nds, nds, xmm0, vector_len);
4921     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4922     evmovdqul(dst, nds, Assembler::AVX_512bit);
4923   } else {
4924     // worse case scenario, all regs are in the upper bank
4925     subptr(rsp, 64);
4926     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4927     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4928     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4929     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4930     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4931     evmovdqul(xmm1, dst, Assembler::AVX_512bit);
4932     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4933     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4934     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4935     addptr(rsp, 64);
4936   }
4937 }
4938 
4939 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4940   int dst_enc = dst->encoding();
4941   int nds_enc = nds->encoding();
4942   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4943     Assembler::vpsrlw(dst, nds, shift, vector_len);
4944   } else if (dst_enc < 16) {
4945     Assembler::vpsrlw(dst, dst, shift, vector_len);
4946   } else if (nds_enc < 16) {
4947     // use nds as scratch
4948     evmovdqul(nds, dst, Assembler::AVX_512bit);
4949     Assembler::vpsrlw(nds, nds, shift, vector_len);
4950     evmovdqul(dst, nds, Assembler::AVX_512bit);
4951   } else {
4952     // use nds as scratch for xmm0
4953     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4954     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4955     Assembler::vpsrlw(xmm0, xmm0, shift, vector_len);
4956     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4957   }
4958 }
4959 
4960 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4961   int dst_enc = dst->encoding();
4962   int nds_enc = nds->encoding();
4963   int shift_enc = shift->encoding();
4964   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4965     Assembler::vpsllw(dst, nds, shift, vector_len);
4966   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4967     Assembler::vpsllw(dst, dst, shift, vector_len);
4968   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4969     // use nds_enc as scratch with shift
4970     evmovdqul(nds, shift, Assembler::AVX_512bit);
4971     Assembler::vpsllw(dst, dst, nds, vector_len);
4972   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4973     // use nds as scratch with dst
4974     evmovdqul(nds, dst, Assembler::AVX_512bit);
4975     Assembler::vpsllw(nds, nds, shift, vector_len);
4976     evmovdqul(dst, nds, Assembler::AVX_512bit);
4977   } else if (dst_enc < 16) {
4978     // use nds to save a copy of xmm0 and hold shift
4979     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4980     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4981     Assembler::vpsllw(dst, dst, xmm0, vector_len);
4982     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4983   } else if (nds_enc < 16) {
4984     // use nds as dest as temps
4985     evmovdqul(nds, dst, Assembler::AVX_512bit);
4986     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4987     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4988     Assembler::vpsllw(nds, nds, xmm0, vector_len);
4989     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4990     evmovdqul(dst, nds, Assembler::AVX_512bit);
4991   } else {
4992     // worse case scenario, all regs are in the upper bank
4993     subptr(rsp, 64);
4994     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4995     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4996     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4997     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4998     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4999     evmovdqul(xmm1, dst, Assembler::AVX_512bit);
5000     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5001     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
5002     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
5003     addptr(rsp, 64);
5004   }
5005 }
5006 
5007 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
5008   int dst_enc = dst->encoding();
5009   int nds_enc = nds->encoding();
5010   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
5011     Assembler::vpsllw(dst, nds, shift, vector_len);
5012   } else if (dst_enc < 16) {
5013     Assembler::vpsllw(dst, dst, shift, vector_len);
5014   } else if (nds_enc < 16) {
5015     // use nds as scratch
5016     evmovdqul(nds, dst, Assembler::AVX_512bit);
5017     Assembler::vpsllw(nds, nds, shift, vector_len);
5018     evmovdqul(dst, nds, Assembler::AVX_512bit);
5019   } else {
5020     // use nds as scratch for xmm0
5021     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
5022     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5023     Assembler::vpsllw(xmm0, xmm0, shift, vector_len);
5024     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
5025   }
5026 }
5027 
5028 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
5029   int dst_enc = dst->encoding();
5030   int src_enc = src->encoding();
5031   if ((dst_enc < 16) && (src_enc < 16)) {
5032     Assembler::vptest(dst, src);
5033   } else if (src_enc < 16) {
5034     subptr(rsp, 64);
5035     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5036     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5037     Assembler::vptest(xmm0, src);
5038     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5039     addptr(rsp, 64);
5040   } else if (dst_enc < 16) {
5041     subptr(rsp, 64);
5042     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5043     evmovdqul(xmm0, src, Assembler::AVX_512bit);
5044     Assembler::vptest(dst, xmm0);
5045     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5046     addptr(rsp, 64);
5047   } else {
5048     subptr(rsp, 64);
5049     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5050     subptr(rsp, 64);
5051     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
5052     movdqu(xmm0, src);
5053     movdqu(xmm1, dst);
5054     Assembler::vptest(xmm1, xmm0);
5055     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
5056     addptr(rsp, 64);
5057     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5058     addptr(rsp, 64);
5059   }
5060 }
5061 
5062 // This instruction exists within macros, ergo we cannot control its input
5063 // when emitted through those patterns.
5064 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
5065   if (VM_Version::supports_avx512nobw()) {
5066     int dst_enc = dst->encoding();
5067     int src_enc = src->encoding();
5068     if (dst_enc == src_enc) {
5069       if (dst_enc < 16) {
5070         Assembler::punpcklbw(dst, src);
5071       } else {
5072         subptr(rsp, 64);
5073         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5074         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5075         Assembler::punpcklbw(xmm0, xmm0);
5076         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5077         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5078         addptr(rsp, 64);
5079       }
5080     } else {
5081       if ((src_enc < 16) && (dst_enc < 16)) {
5082         Assembler::punpcklbw(dst, src);
5083       } else if (src_enc < 16) {
5084         subptr(rsp, 64);
5085         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5086         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5087         Assembler::punpcklbw(xmm0, src);
5088         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5089         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5090         addptr(rsp, 64);
5091       } else if (dst_enc < 16) {
5092         subptr(rsp, 64);
5093         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5094         evmovdqul(xmm0, src, Assembler::AVX_512bit);
5095         Assembler::punpcklbw(dst, xmm0);
5096         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5097         addptr(rsp, 64);
5098       } else {
5099         subptr(rsp, 64);
5100         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5101         subptr(rsp, 64);
5102         evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
5103         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5104         evmovdqul(xmm1, src, Assembler::AVX_512bit);
5105         Assembler::punpcklbw(xmm0, xmm1);
5106         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5107         evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
5108         addptr(rsp, 64);
5109         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5110         addptr(rsp, 64);
5111       }
5112     }
5113   } else {
5114     Assembler::punpcklbw(dst, src);
5115   }
5116 }
5117 
5118 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
5119   if (VM_Version::supports_avx512vl()) {
5120     Assembler::pshufd(dst, src, mode);
5121   } else {
5122     int dst_enc = dst->encoding();
5123     if (dst_enc < 16) {
5124       Assembler::pshufd(dst, src, mode);
5125     } else {
5126       subptr(rsp, 64);
5127       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5128       Assembler::pshufd(xmm0, src, mode);
5129       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5130       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5131       addptr(rsp, 64);
5132     }
5133   }
5134 }
5135 
5136 // This instruction exists within macros, ergo we cannot control its input
5137 // when emitted through those patterns.
5138 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
5139   if (VM_Version::supports_avx512nobw()) {
5140     int dst_enc = dst->encoding();
5141     int src_enc = src->encoding();
5142     if (dst_enc == src_enc) {
5143       if (dst_enc < 16) {
5144         Assembler::pshuflw(dst, src, mode);
5145       } else {
5146         subptr(rsp, 64);
5147         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5148         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5149         Assembler::pshuflw(xmm0, xmm0, mode);
5150         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5151         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5152         addptr(rsp, 64);
5153       }
5154     } else {
5155       if ((src_enc < 16) && (dst_enc < 16)) {
5156         Assembler::pshuflw(dst, src, mode);
5157       } else if (src_enc < 16) {
5158         subptr(rsp, 64);
5159         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5160         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5161         Assembler::pshuflw(xmm0, src, mode);
5162         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5163         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5164         addptr(rsp, 64);
5165       } else if (dst_enc < 16) {
5166         subptr(rsp, 64);
5167         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5168         evmovdqul(xmm0, src, Assembler::AVX_512bit);
5169         Assembler::pshuflw(dst, xmm0, mode);
5170         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5171         addptr(rsp, 64);
5172       } else {
5173         subptr(rsp, 64);
5174         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5175         subptr(rsp, 64);
5176         evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
5177         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5178         evmovdqul(xmm1, src, Assembler::AVX_512bit);
5179         Assembler::pshuflw(xmm0, xmm1, mode);
5180         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5181         evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
5182         addptr(rsp, 64);
5183         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5184         addptr(rsp, 64);
5185       }
5186     }
5187   } else {
5188     Assembler::pshuflw(dst, src, mode);
5189   }
5190 }
5191 
5192 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5193   if (reachable(src)) {
5194     vandpd(dst, nds, as_Address(src), vector_len);
5195   } else {
5196     lea(rscratch1, src);
5197     vandpd(dst, nds, Address(rscratch1, 0), vector_len);
5198   }
5199 }
5200 
5201 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5202   if (reachable(src)) {
5203     vandps(dst, nds, as_Address(src), vector_len);
5204   } else {
5205     lea(rscratch1, src);
5206     vandps(dst, nds, Address(rscratch1, 0), vector_len);
5207   }
5208 }
5209 
5210 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5211   if (reachable(src)) {
5212     vdivsd(dst, nds, as_Address(src));
5213   } else {
5214     lea(rscratch1, src);
5215     vdivsd(dst, nds, Address(rscratch1, 0));
5216   }
5217 }
5218 
5219 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5220   if (reachable(src)) {
5221     vdivss(dst, nds, as_Address(src));
5222   } else {
5223     lea(rscratch1, src);
5224     vdivss(dst, nds, Address(rscratch1, 0));
5225   }
5226 }
5227 
5228 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5229   if (reachable(src)) {
5230     vmulsd(dst, nds, as_Address(src));
5231   } else {
5232     lea(rscratch1, src);
5233     vmulsd(dst, nds, Address(rscratch1, 0));
5234   }
5235 }
5236 
5237 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5238   if (reachable(src)) {
5239     vmulss(dst, nds, as_Address(src));
5240   } else {
5241     lea(rscratch1, src);
5242     vmulss(dst, nds, Address(rscratch1, 0));
5243   }
5244 }
5245 
5246 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5247   if (reachable(src)) {
5248     vsubsd(dst, nds, as_Address(src));
5249   } else {
5250     lea(rscratch1, src);
5251     vsubsd(dst, nds, Address(rscratch1, 0));
5252   }
5253 }
5254 
5255 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5256   if (reachable(src)) {
5257     vsubss(dst, nds, as_Address(src));
5258   } else {
5259     lea(rscratch1, src);
5260     vsubss(dst, nds, Address(rscratch1, 0));
5261   }
5262 }
5263 
5264 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5265   int nds_enc = nds->encoding();
5266   int dst_enc = dst->encoding();
5267   bool dst_upper_bank = (dst_enc > 15);
5268   bool nds_upper_bank = (nds_enc > 15);
5269   if (VM_Version::supports_avx512novl() &&
5270       (nds_upper_bank || dst_upper_bank)) {
5271     if (dst_upper_bank) {
5272       subptr(rsp, 64);
5273       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5274       movflt(xmm0, nds);
5275       vxorps(xmm0, xmm0, src, Assembler::AVX_128bit);
5276       movflt(dst, xmm0);
5277       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5278       addptr(rsp, 64);
5279     } else {
5280       movflt(dst, nds);
5281       vxorps(dst, dst, src, Assembler::AVX_128bit);
5282     }
5283   } else {
5284     vxorps(dst, nds, src, Assembler::AVX_128bit);
5285   }
5286 }
5287 
5288 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5289   int nds_enc = nds->encoding();
5290   int dst_enc = dst->encoding();
5291   bool dst_upper_bank = (dst_enc > 15);
5292   bool nds_upper_bank = (nds_enc > 15);
5293   if (VM_Version::supports_avx512novl() &&
5294       (nds_upper_bank || dst_upper_bank)) {
5295     if (dst_upper_bank) {
5296       subptr(rsp, 64);
5297       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5298       movdbl(xmm0, nds);
5299       vxorpd(xmm0, xmm0, src, Assembler::AVX_128bit);
5300       movdbl(dst, xmm0);
5301       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5302       addptr(rsp, 64);
5303     } else {
5304       movdbl(dst, nds);
5305       vxorpd(dst, dst, src, Assembler::AVX_128bit);
5306     }
5307   } else {
5308     vxorpd(dst, nds, src, Assembler::AVX_128bit);
5309   }
5310 }
5311 
5312 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5313   if (reachable(src)) {
5314     vxorpd(dst, nds, as_Address(src), vector_len);
5315   } else {
5316     lea(rscratch1, src);
5317     vxorpd(dst, nds, Address(rscratch1, 0), vector_len);
5318   }
5319 }
5320 
5321 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5322   if (reachable(src)) {
5323     vxorps(dst, nds, as_Address(src), vector_len);
5324   } else {
5325     lea(rscratch1, src);
5326     vxorps(dst, nds, Address(rscratch1, 0), vector_len);
5327   }
5328 }
5329 
5330 
5331 void MacroAssembler::resolve_jobject(Register value,
5332                                      Register thread,
5333                                      Register tmp) {
5334   assert_different_registers(value, thread, tmp);
5335   Label done, not_weak;
5336   testptr(value, value);
5337   jcc(Assembler::zero, done);                // Use NULL as-is.
5338   testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag.
5339   jcc(Assembler::zero, not_weak);
5340   // Resolve jweak.
5341   movptr(value, Address(value, -JNIHandles::weak_tag_value));
5342   verify_oop(value);
5343 #if INCLUDE_ALL_GCS
5344   if (UseG1GC) {
5345     g1_write_barrier_pre(noreg /* obj */,
5346                          value /* pre_val */,
5347                          thread /* thread */,
5348                          tmp /* tmp */,
5349                          true /* tosca_live */,
5350                          true /* expand_call */);
5351   }
5352 #endif // INCLUDE_ALL_GCS
5353   jmp(done);
5354   bind(not_weak);
5355   // Resolve (untagged) jobject.
5356   movptr(value, Address(value, 0));
5357   verify_oop(value);
5358   bind(done);
5359 }
5360 
5361 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) {
5362   const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask);
5363   STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code
5364   // The inverted mask is sign-extended
5365   andptr(possibly_jweak, inverted_jweak_mask);
5366 }
5367 
5368 //////////////////////////////////////////////////////////////////////////////////
5369 #if INCLUDE_ALL_GCS
5370 
5371 void MacroAssembler::g1_write_barrier_pre(Register obj,
5372                                           Register pre_val,
5373                                           Register thread,
5374                                           Register tmp,
5375                                           bool tosca_live,
5376                                           bool expand_call) {
5377 
5378   // If expand_call is true then we expand the call_VM_leaf macro
5379   // directly to skip generating the check by
5380   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
5381 
5382 #ifdef _LP64
5383   assert(thread == r15_thread, "must be");
5384 #endif // _LP64
5385 
5386   Label done;
5387   Label runtime;
5388 
5389   assert(pre_val != noreg, "check this code");
5390 
5391   if (obj != noreg) {
5392     assert_different_registers(obj, pre_val, tmp);
5393     assert(pre_val != rax, "check this code");
5394   }
5395 
5396   Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
5397                                        SATBMarkQueue::byte_offset_of_active()));
5398   Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
5399                                        SATBMarkQueue::byte_offset_of_index()));
5400   Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
5401                                        SATBMarkQueue::byte_offset_of_buf()));
5402 
5403 
5404   // Is marking active?
5405   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
5406     cmpl(in_progress, 0);
5407   } else {
5408     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
5409     cmpb(in_progress, 0);
5410   }
5411   jcc(Assembler::equal, done);
5412 
5413   // Do we need to load the previous value?
5414   if (obj != noreg) {
5415     load_heap_oop(pre_val, Address(obj, 0));
5416   }
5417 
5418   // Is the previous value null?
5419   cmpptr(pre_val, (int32_t) NULL_WORD);
5420   jcc(Assembler::equal, done);
5421 
5422   // Can we store original value in the thread's buffer?
5423   // Is index == 0?
5424   // (The index field is typed as size_t.)
5425 
5426   movptr(tmp, index);                   // tmp := *index_adr
5427   cmpptr(tmp, 0);                       // tmp == 0?
5428   jcc(Assembler::equal, runtime);       // If yes, goto runtime
5429 
5430   subptr(tmp, wordSize);                // tmp := tmp - wordSize
5431   movptr(index, tmp);                   // *index_adr := tmp
5432   addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
5433 
5434   // Record the previous value
5435   movptr(Address(tmp, 0), pre_val);
5436   jmp(done);
5437 
5438   bind(runtime);
5439   // save the live input values
5440   if(tosca_live) push(rax);
5441 
5442   if (obj != noreg && obj != rax)
5443     push(obj);
5444 
5445   if (pre_val != rax)
5446     push(pre_val);
5447 
5448   // Calling the runtime using the regular call_VM_leaf mechanism generates
5449   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
5450   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
5451   //
5452   // If we care generating the pre-barrier without a frame (e.g. in the
5453   // intrinsified Reference.get() routine) then ebp might be pointing to
5454   // the caller frame and so this check will most likely fail at runtime.
5455   //
5456   // Expanding the call directly bypasses the generation of the check.
5457   // So when we do not have have a full interpreter frame on the stack
5458   // expand_call should be passed true.
5459 
5460   NOT_LP64( push(thread); )
5461 
5462   if (expand_call) {
5463     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
5464     pass_arg1(this, thread);
5465     pass_arg0(this, pre_val);
5466     MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), 2);
5467   } else {
5468     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
5469   }
5470 
5471   NOT_LP64( pop(thread); )
5472 
5473   // save the live input values
5474   if (pre_val != rax)
5475     pop(pre_val);
5476 
5477   if (obj != noreg && obj != rax)
5478     pop(obj);
5479 
5480   if(tosca_live) pop(rax);
5481 
5482   bind(done);
5483 }
5484 
5485 void MacroAssembler::g1_write_barrier_post(Register store_addr,
5486                                            Register new_val,
5487                                            Register thread,
5488                                            Register tmp,
5489                                            Register tmp2) {
5490 #ifdef _LP64
5491   assert(thread == r15_thread, "must be");
5492 #endif // _LP64
5493 
5494   Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
5495                                        DirtyCardQueue::byte_offset_of_index()));
5496   Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
5497                                        DirtyCardQueue::byte_offset_of_buf()));
5498 
5499   CardTableModRefBS* ctbs =
5500     barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set());
5501   CardTable* ct = ctbs->card_table();
5502   assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
5503 
5504   Label done;
5505   Label runtime;
5506 
5507   // Does store cross heap regions?
5508 
5509   movptr(tmp, store_addr);
5510   xorptr(tmp, new_val);
5511   shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
5512   jcc(Assembler::equal, done);
5513 
5514   // crosses regions, storing NULL?
5515 
5516   cmpptr(new_val, (int32_t) NULL_WORD);
5517   jcc(Assembler::equal, done);
5518 
5519   // storing region crossing non-NULL, is card already dirty?
5520 
5521   const Register card_addr = tmp;
5522   const Register cardtable = tmp2;
5523 
5524   movptr(card_addr, store_addr);
5525   shrptr(card_addr, CardTable::card_shift);
5526   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
5527   // a valid address and therefore is not properly handled by the relocation code.
5528   movptr(cardtable, (intptr_t)ct->byte_map_base());
5529   addptr(card_addr, cardtable);
5530 
5531   cmpb(Address(card_addr, 0), (int)G1CardTable::g1_young_card_val());
5532   jcc(Assembler::equal, done);
5533 
5534   membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
5535   cmpb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
5536   jcc(Assembler::equal, done);
5537 
5538 
5539   // storing a region crossing, non-NULL oop, card is clean.
5540   // dirty card and log.
5541 
5542   movb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
5543 
5544   cmpl(queue_index, 0);
5545   jcc(Assembler::equal, runtime);
5546   subl(queue_index, wordSize);
5547   movptr(tmp2, buffer);
5548 #ifdef _LP64
5549   movslq(rscratch1, queue_index);
5550   addq(tmp2, rscratch1);
5551   movq(Address(tmp2, 0), card_addr);
5552 #else
5553   addl(tmp2, queue_index);
5554   movl(Address(tmp2, 0), card_addr);
5555 #endif
5556   jmp(done);
5557 
5558   bind(runtime);
5559   // save the live input values
5560   push(store_addr);
5561   push(new_val);
5562 #ifdef _LP64
5563   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread);
5564 #else
5565   push(thread);
5566   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
5567   pop(thread);
5568 #endif
5569   pop(new_val);
5570   pop(store_addr);
5571 
5572   bind(done);
5573 }
5574 
5575 #endif // INCLUDE_ALL_GCS
5576 //////////////////////////////////////////////////////////////////////////////////
5577 
5578 
5579 void MacroAssembler::store_check(Register obj, Address dst) {
5580   store_check(obj);
5581 }
5582 
5583 void MacroAssembler::store_check(Register obj) {
5584   // Does a store check for the oop in register obj. The content of
5585   // register obj is destroyed afterwards.
5586   BarrierSet* bs = Universe::heap()->barrier_set();
5587   assert(bs->kind() == BarrierSet::CardTableModRef,
5588          "Wrong barrier set kind");
5589 
5590   CardTableModRefBS* ctbs = barrier_set_cast<CardTableModRefBS>(bs);
5591   CardTable* ct = ctbs->card_table();
5592   assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
5593 
5594   shrptr(obj, CardTable::card_shift);
5595 
5596   Address card_addr;
5597 
5598   // The calculation for byte_map_base is as follows:
5599   // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
5600   // So this essentially converts an address to a displacement and it will
5601   // never need to be relocated. On 64bit however the value may be too
5602   // large for a 32bit displacement.
5603   intptr_t disp = (intptr_t) ct->byte_map_base();
5604   if (is_simm32(disp)) {
5605     card_addr = Address(noreg, obj, Address::times_1, disp);
5606   } else {
5607     // By doing it as an ExternalAddress 'disp' could be converted to a rip-relative
5608     // displacement and done in a single instruction given favorable mapping and a
5609     // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
5610     // entry and that entry is not properly handled by the relocation code.
5611     AddressLiteral cardtable((address)ct->byte_map_base(), relocInfo::none);
5612     Address index(noreg, obj, Address::times_1);
5613     card_addr = as_Address(ArrayAddress(cardtable, index));
5614   }
5615 
5616   int dirty = CardTable::dirty_card_val();
5617   if (UseCondCardMark) {
5618     Label L_already_dirty;
5619     if (UseConcMarkSweepGC) {
5620       membar(Assembler::StoreLoad);
5621     }
5622     cmpb(card_addr, dirty);
5623     jcc(Assembler::equal, L_already_dirty);
5624     movb(card_addr, dirty);
5625     bind(L_already_dirty);
5626   } else {
5627     movb(card_addr, dirty);
5628   }
5629 }
5630 
5631 void MacroAssembler::subptr(Register dst, int32_t imm32) {
5632   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
5633 }
5634 
5635 // Force generation of a 4 byte immediate value even if it fits into 8bit
5636 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
5637   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
5638 }
5639 
5640 void MacroAssembler::subptr(Register dst, Register src) {
5641   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
5642 }
5643 
5644 // C++ bool manipulation
5645 void MacroAssembler::testbool(Register dst) {
5646   if(sizeof(bool) == 1)
5647     testb(dst, 0xff);
5648   else if(sizeof(bool) == 2) {
5649     // testw implementation needed for two byte bools
5650     ShouldNotReachHere();
5651   } else if(sizeof(bool) == 4)
5652     testl(dst, dst);
5653   else
5654     // unsupported
5655     ShouldNotReachHere();
5656 }
5657 
5658 void MacroAssembler::testptr(Register dst, Register src) {
5659   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
5660 }
5661 
5662 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
5663 void MacroAssembler::tlab_allocate(Register obj,
5664                                    Register var_size_in_bytes,
5665                                    int con_size_in_bytes,
5666                                    Register t1,
5667                                    Register t2,
5668                                    Label& slow_case) {
5669   assert_different_registers(obj, t1, t2);
5670   assert_different_registers(obj, var_size_in_bytes, t1);
5671   Register end = t2;
5672   Register thread = NOT_LP64(t1) LP64_ONLY(r15_thread);
5673 
5674   verify_tlab();
5675 
5676   NOT_LP64(get_thread(thread));
5677 
5678   movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
5679   if (var_size_in_bytes == noreg) {
5680     lea(end, Address(obj, con_size_in_bytes));
5681   } else {
5682     lea(end, Address(obj, var_size_in_bytes, Address::times_1));
5683   }
5684   cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
5685   jcc(Assembler::above, slow_case);
5686 
5687   // update the tlab top pointer
5688   movptr(Address(thread, JavaThread::tlab_top_offset()), end);
5689 
5690   // recover var_size_in_bytes if necessary
5691   if (var_size_in_bytes == end) {
5692     subptr(var_size_in_bytes, obj);
5693   }
5694   verify_tlab();
5695 }
5696 
5697 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
5698 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
5699   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
5700   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
5701   Label done;
5702 
5703   testptr(length_in_bytes, length_in_bytes);
5704   jcc(Assembler::zero, done);
5705 
5706   // initialize topmost word, divide index by 2, check if odd and test if zero
5707   // note: for the remaining code to work, index must be a multiple of BytesPerWord
5708 #ifdef ASSERT
5709   {
5710     Label L;
5711     testptr(length_in_bytes, BytesPerWord - 1);
5712     jcc(Assembler::zero, L);
5713     stop("length must be a multiple of BytesPerWord");
5714     bind(L);
5715   }
5716 #endif
5717   Register index = length_in_bytes;
5718   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
5719   if (UseIncDec) {
5720     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
5721   } else {
5722     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
5723     shrptr(index, 1);
5724   }
5725 #ifndef _LP64
5726   // index could have not been a multiple of 8 (i.e., bit 2 was set)
5727   {
5728     Label even;
5729     // note: if index was a multiple of 8, then it cannot
5730     //       be 0 now otherwise it must have been 0 before
5731     //       => if it is even, we don't need to check for 0 again
5732     jcc(Assembler::carryClear, even);
5733     // clear topmost word (no jump would be needed if conditional assignment worked here)
5734     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
5735     // index could be 0 now, must check again
5736     jcc(Assembler::zero, done);
5737     bind(even);
5738   }
5739 #endif // !_LP64
5740   // initialize remaining object fields: index is a multiple of 2 now
5741   {
5742     Label loop;
5743     bind(loop);
5744     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
5745     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
5746     decrement(index);
5747     jcc(Assembler::notZero, loop);
5748   }
5749 
5750   bind(done);
5751 }
5752 
5753 void MacroAssembler::incr_allocated_bytes(Register thread,
5754                                           Register var_size_in_bytes,
5755                                           int con_size_in_bytes,
5756                                           Register t1) {
5757   if (!thread->is_valid()) {
5758 #ifdef _LP64
5759     thread = r15_thread;
5760 #else
5761     assert(t1->is_valid(), "need temp reg");
5762     thread = t1;
5763     get_thread(thread);
5764 #endif
5765   }
5766 
5767 #ifdef _LP64
5768   if (var_size_in_bytes->is_valid()) {
5769     addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
5770   } else {
5771     addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
5772   }
5773 #else
5774   if (var_size_in_bytes->is_valid()) {
5775     addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
5776   } else {
5777     addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
5778   }
5779   adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0);
5780 #endif
5781 }
5782 
5783 // Look up the method for a megamorphic invokeinterface call.
5784 // The target method is determined by <intf_klass, itable_index>.
5785 // The receiver klass is in recv_klass.
5786 // On success, the result will be in method_result, and execution falls through.
5787 // On failure, execution transfers to the given label.
5788 void MacroAssembler::lookup_interface_method(Register recv_klass,
5789                                              Register intf_klass,
5790                                              RegisterOrConstant itable_index,
5791                                              Register method_result,
5792                                              Register scan_temp,
5793                                              Label& L_no_such_interface,
5794                                              bool return_method) {
5795   assert_different_registers(recv_klass, intf_klass, scan_temp);
5796   assert_different_registers(method_result, intf_klass, scan_temp);
5797   assert(recv_klass != method_result || !return_method,
5798          "recv_klass can be destroyed when method isn't needed");
5799 
5800   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
5801          "caller must use same register for non-constant itable index as for method");
5802 
5803   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
5804   int vtable_base = in_bytes(Klass::vtable_start_offset());
5805   int itentry_off = itableMethodEntry::method_offset_in_bytes();
5806   int scan_step   = itableOffsetEntry::size() * wordSize;
5807   int vte_size    = vtableEntry::size_in_bytes();
5808   Address::ScaleFactor times_vte_scale = Address::times_ptr;
5809   assert(vte_size == wordSize, "else adjust times_vte_scale");
5810 
5811   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
5812 
5813   // %%% Could store the aligned, prescaled offset in the klassoop.
5814   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
5815 
5816   if (return_method) {
5817     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
5818     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
5819     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
5820   }
5821 
5822   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
5823   //   if (scan->interface() == intf) {
5824   //     result = (klass + scan->offset() + itable_index);
5825   //   }
5826   // }
5827   Label search, found_method;
5828 
5829   for (int peel = 1; peel >= 0; peel--) {
5830     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
5831     cmpptr(intf_klass, method_result);
5832 
5833     if (peel) {
5834       jccb(Assembler::equal, found_method);
5835     } else {
5836       jccb(Assembler::notEqual, search);
5837       // (invert the test to fall through to found_method...)
5838     }
5839 
5840     if (!peel)  break;
5841 
5842     bind(search);
5843 
5844     // Check that the previous entry is non-null.  A null entry means that
5845     // the receiver class doesn't implement the interface, and wasn't the
5846     // same as when the caller was compiled.
5847     testptr(method_result, method_result);
5848     jcc(Assembler::zero, L_no_such_interface);
5849     addptr(scan_temp, scan_step);
5850   }
5851 
5852   bind(found_method);
5853 
5854   if (return_method) {
5855     // Got a hit.
5856     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
5857     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
5858   }
5859 }
5860 
5861 
5862 // virtual method calling
5863 void MacroAssembler::lookup_virtual_method(Register recv_klass,
5864                                            RegisterOrConstant vtable_index,
5865                                            Register method_result) {
5866   const int base = in_bytes(Klass::vtable_start_offset());
5867   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
5868   Address vtable_entry_addr(recv_klass,
5869                             vtable_index, Address::times_ptr,
5870                             base + vtableEntry::method_offset_in_bytes());
5871   movptr(method_result, vtable_entry_addr);
5872 }
5873 
5874 
5875 void MacroAssembler::check_klass_subtype(Register sub_klass,
5876                            Register super_klass,
5877                            Register temp_reg,
5878                            Label& L_success) {
5879   Label L_failure;
5880   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, NULL);
5881   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
5882   bind(L_failure);
5883 }
5884 
5885 
5886 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
5887                                                    Register super_klass,
5888                                                    Register temp_reg,
5889                                                    Label* L_success,
5890                                                    Label* L_failure,
5891                                                    Label* L_slow_path,
5892                                         RegisterOrConstant super_check_offset) {
5893   assert_different_registers(sub_klass, super_klass, temp_reg);
5894   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
5895   if (super_check_offset.is_register()) {
5896     assert_different_registers(sub_klass, super_klass,
5897                                super_check_offset.as_register());
5898   } else if (must_load_sco) {
5899     assert(temp_reg != noreg, "supply either a temp or a register offset");
5900   }
5901 
5902   Label L_fallthrough;
5903   int label_nulls = 0;
5904   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
5905   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
5906   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
5907   assert(label_nulls <= 1, "at most one NULL in the batch");
5908 
5909   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
5910   int sco_offset = in_bytes(Klass::super_check_offset_offset());
5911   Address super_check_offset_addr(super_klass, sco_offset);
5912 
5913   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
5914   // range of a jccb.  If this routine grows larger, reconsider at
5915   // least some of these.
5916 #define local_jcc(assembler_cond, label)                                \
5917   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
5918   else                             jcc( assembler_cond, label) /*omit semi*/
5919 
5920   // Hacked jmp, which may only be used just before L_fallthrough.
5921 #define final_jmp(label)                                                \
5922   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
5923   else                            jmp(label)                /*omit semi*/
5924 
5925   // If the pointers are equal, we are done (e.g., String[] elements).
5926   // This self-check enables sharing of secondary supertype arrays among
5927   // non-primary types such as array-of-interface.  Otherwise, each such
5928   // type would need its own customized SSA.
5929   // We move this check to the front of the fast path because many
5930   // type checks are in fact trivially successful in this manner,
5931   // so we get a nicely predicted branch right at the start of the check.
5932   cmpptr(sub_klass, super_klass);
5933   local_jcc(Assembler::equal, *L_success);
5934 
5935   // Check the supertype display:
5936   if (must_load_sco) {
5937     // Positive movl does right thing on LP64.
5938     movl(temp_reg, super_check_offset_addr);
5939     super_check_offset = RegisterOrConstant(temp_reg);
5940   }
5941   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
5942   cmpptr(super_klass, super_check_addr); // load displayed supertype
5943 
5944   // This check has worked decisively for primary supers.
5945   // Secondary supers are sought in the super_cache ('super_cache_addr').
5946   // (Secondary supers are interfaces and very deeply nested subtypes.)
5947   // This works in the same check above because of a tricky aliasing
5948   // between the super_cache and the primary super display elements.
5949   // (The 'super_check_addr' can address either, as the case requires.)
5950   // Note that the cache is updated below if it does not help us find
5951   // what we need immediately.
5952   // So if it was a primary super, we can just fail immediately.
5953   // Otherwise, it's the slow path for us (no success at this point).
5954 
5955   if (super_check_offset.is_register()) {
5956     local_jcc(Assembler::equal, *L_success);
5957     cmpl(super_check_offset.as_register(), sc_offset);
5958     if (L_failure == &L_fallthrough) {
5959       local_jcc(Assembler::equal, *L_slow_path);
5960     } else {
5961       local_jcc(Assembler::notEqual, *L_failure);
5962       final_jmp(*L_slow_path);
5963     }
5964   } else if (super_check_offset.as_constant() == sc_offset) {
5965     // Need a slow path; fast failure is impossible.
5966     if (L_slow_path == &L_fallthrough) {
5967       local_jcc(Assembler::equal, *L_success);
5968     } else {
5969       local_jcc(Assembler::notEqual, *L_slow_path);
5970       final_jmp(*L_success);
5971     }
5972   } else {
5973     // No slow path; it's a fast decision.
5974     if (L_failure == &L_fallthrough) {
5975       local_jcc(Assembler::equal, *L_success);
5976     } else {
5977       local_jcc(Assembler::notEqual, *L_failure);
5978       final_jmp(*L_success);
5979     }
5980   }
5981 
5982   bind(L_fallthrough);
5983 
5984 #undef local_jcc
5985 #undef final_jmp
5986 }
5987 
5988 
5989 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
5990                                                    Register super_klass,
5991                                                    Register temp_reg,
5992                                                    Register temp2_reg,
5993                                                    Label* L_success,
5994                                                    Label* L_failure,
5995                                                    bool set_cond_codes) {
5996   assert_different_registers(sub_klass, super_klass, temp_reg);
5997   if (temp2_reg != noreg)
5998     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
5999 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
6000 
6001   Label L_fallthrough;
6002   int label_nulls = 0;
6003   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
6004   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
6005   assert(label_nulls <= 1, "at most one NULL in the batch");
6006 
6007   // a couple of useful fields in sub_klass:
6008   int ss_offset = in_bytes(Klass::secondary_supers_offset());
6009   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
6010   Address secondary_supers_addr(sub_klass, ss_offset);
6011   Address super_cache_addr(     sub_klass, sc_offset);
6012 
6013   // Do a linear scan of the secondary super-klass chain.
6014   // This code is rarely used, so simplicity is a virtue here.
6015   // The repne_scan instruction uses fixed registers, which we must spill.
6016   // Don't worry too much about pre-existing connections with the input regs.
6017 
6018   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
6019   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
6020 
6021   // Get super_klass value into rax (even if it was in rdi or rcx).
6022   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
6023   if (super_klass != rax || UseCompressedOops) {
6024     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
6025     mov(rax, super_klass);
6026   }
6027   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
6028   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
6029 
6030 #ifndef PRODUCT
6031   int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
6032   ExternalAddress pst_counter_addr((address) pst_counter);
6033   NOT_LP64(  incrementl(pst_counter_addr) );
6034   LP64_ONLY( lea(rcx, pst_counter_addr) );
6035   LP64_ONLY( incrementl(Address(rcx, 0)) );
6036 #endif //PRODUCT
6037 
6038   // We will consult the secondary-super array.
6039   movptr(rdi, secondary_supers_addr);
6040   // Load the array length.  (Positive movl does right thing on LP64.)
6041   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
6042   // Skip to start of data.
6043   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
6044 
6045   // Scan RCX words at [RDI] for an occurrence of RAX.
6046   // Set NZ/Z based on last compare.
6047   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
6048   // not change flags (only scas instruction which is repeated sets flags).
6049   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
6050 
6051     testptr(rax,rax); // Set Z = 0
6052     repne_scan();
6053 
6054   // Unspill the temp. registers:
6055   if (pushed_rdi)  pop(rdi);
6056   if (pushed_rcx)  pop(rcx);
6057   if (pushed_rax)  pop(rax);
6058 
6059   if (set_cond_codes) {
6060     // Special hack for the AD files:  rdi is guaranteed non-zero.
6061     assert(!pushed_rdi, "rdi must be left non-NULL");
6062     // Also, the condition codes are properly set Z/NZ on succeed/failure.
6063   }
6064 
6065   if (L_failure == &L_fallthrough)
6066         jccb(Assembler::notEqual, *L_failure);
6067   else  jcc(Assembler::notEqual, *L_failure);
6068 
6069   // Success.  Cache the super we found and proceed in triumph.
6070   movptr(super_cache_addr, super_klass);
6071 
6072   if (L_success != &L_fallthrough) {
6073     jmp(*L_success);
6074   }
6075 
6076 #undef IS_A_TEMP
6077 
6078   bind(L_fallthrough);
6079 }
6080 
6081 
6082 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
6083   if (VM_Version::supports_cmov()) {
6084     cmovl(cc, dst, src);
6085   } else {
6086     Label L;
6087     jccb(negate_condition(cc), L);
6088     movl(dst, src);
6089     bind(L);
6090   }
6091 }
6092 
6093 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
6094   if (VM_Version::supports_cmov()) {
6095     cmovl(cc, dst, src);
6096   } else {
6097     Label L;
6098     jccb(negate_condition(cc), L);
6099     movl(dst, src);
6100     bind(L);
6101   }
6102 }
6103 
6104 void MacroAssembler::verify_oop(Register reg, const char* s) {
6105   if (!VerifyOops || VerifyAdapterSharing) {
6106     // Below address of the code string confuses VerifyAdapterSharing
6107     // because it may differ between otherwise equivalent adapters.
6108     return;
6109   }
6110 
6111   // Pass register number to verify_oop_subroutine
6112   const char* b = NULL;
6113   {
6114     ResourceMark rm;
6115     stringStream ss;
6116     ss.print("verify_oop: %s: %s", reg->name(), s);
6117     b = code_string(ss.as_string());
6118   }
6119   BLOCK_COMMENT("verify_oop {");
6120 #ifdef _LP64
6121   push(rscratch1);                    // save r10, trashed by movptr()
6122 #endif
6123   push(rax);                          // save rax,
6124   push(reg);                          // pass register argument
6125   ExternalAddress buffer((address) b);
6126   // avoid using pushptr, as it modifies scratch registers
6127   // and our contract is not to modify anything
6128   movptr(rax, buffer.addr());
6129   push(rax);
6130   // call indirectly to solve generation ordering problem
6131   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
6132   call(rax);
6133   // Caller pops the arguments (oop, message) and restores rax, r10
6134   BLOCK_COMMENT("} verify_oop");
6135 }
6136 
6137 
6138 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
6139                                                       Register tmp,
6140                                                       int offset) {
6141   intptr_t value = *delayed_value_addr;
6142   if (value != 0)
6143     return RegisterOrConstant(value + offset);
6144 
6145   // load indirectly to solve generation ordering problem
6146   movptr(tmp, ExternalAddress((address) delayed_value_addr));
6147 
6148 #ifdef ASSERT
6149   { Label L;
6150     testptr(tmp, tmp);
6151     if (WizardMode) {
6152       const char* buf = NULL;
6153       {
6154         ResourceMark rm;
6155         stringStream ss;
6156         ss.print("DelayedValue=" INTPTR_FORMAT, delayed_value_addr[1]);
6157         buf = code_string(ss.as_string());
6158       }
6159       jcc(Assembler::notZero, L);
6160       STOP(buf);
6161     } else {
6162       jccb(Assembler::notZero, L);
6163       hlt();
6164     }
6165     bind(L);
6166   }
6167 #endif
6168 
6169   if (offset != 0)
6170     addptr(tmp, offset);
6171 
6172   return RegisterOrConstant(tmp);
6173 }
6174 
6175 
6176 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
6177                                          int extra_slot_offset) {
6178   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
6179   int stackElementSize = Interpreter::stackElementSize;
6180   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
6181 #ifdef ASSERT
6182   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
6183   assert(offset1 - offset == stackElementSize, "correct arithmetic");
6184 #endif
6185   Register             scale_reg    = noreg;
6186   Address::ScaleFactor scale_factor = Address::no_scale;
6187   if (arg_slot.is_constant()) {
6188     offset += arg_slot.as_constant() * stackElementSize;
6189   } else {
6190     scale_reg    = arg_slot.as_register();
6191     scale_factor = Address::times(stackElementSize);
6192   }
6193   offset += wordSize;           // return PC is on stack
6194   return Address(rsp, scale_reg, scale_factor, offset);
6195 }
6196 
6197 
6198 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
6199   if (!VerifyOops || VerifyAdapterSharing) {
6200     // Below address of the code string confuses VerifyAdapterSharing
6201     // because it may differ between otherwise equivalent adapters.
6202     return;
6203   }
6204 
6205   // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
6206   // Pass register number to verify_oop_subroutine
6207   const char* b = NULL;
6208   {
6209     ResourceMark rm;
6210     stringStream ss;
6211     ss.print("verify_oop_addr: %s", s);
6212     b = code_string(ss.as_string());
6213   }
6214 #ifdef _LP64
6215   push(rscratch1);                    // save r10, trashed by movptr()
6216 #endif
6217   push(rax);                          // save rax,
6218   // addr may contain rsp so we will have to adjust it based on the push
6219   // we just did (and on 64 bit we do two pushes)
6220   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
6221   // stores rax into addr which is backwards of what was intended.
6222   if (addr.uses(rsp)) {
6223     lea(rax, addr);
6224     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
6225   } else {
6226     pushptr(addr);
6227   }
6228 
6229   ExternalAddress buffer((address) b);
6230   // pass msg argument
6231   // avoid using pushptr, as it modifies scratch registers
6232   // and our contract is not to modify anything
6233   movptr(rax, buffer.addr());
6234   push(rax);
6235 
6236   // call indirectly to solve generation ordering problem
6237   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
6238   call(rax);
6239   // Caller pops the arguments (addr, message) and restores rax, r10.
6240 }
6241 
6242 void MacroAssembler::verify_tlab() {
6243 #ifdef ASSERT
6244   if (UseTLAB && VerifyOops) {
6245     Label next, ok;
6246     Register t1 = rsi;
6247     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
6248 
6249     push(t1);
6250     NOT_LP64(push(thread_reg));
6251     NOT_LP64(get_thread(thread_reg));
6252 
6253     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
6254     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
6255     jcc(Assembler::aboveEqual, next);
6256     STOP("assert(top >= start)");
6257     should_not_reach_here();
6258 
6259     bind(next);
6260     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
6261     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
6262     jcc(Assembler::aboveEqual, ok);
6263     STOP("assert(top <= end)");
6264     should_not_reach_here();
6265 
6266     bind(ok);
6267     NOT_LP64(pop(thread_reg));
6268     pop(t1);
6269   }
6270 #endif
6271 }
6272 
6273 class ControlWord {
6274  public:
6275   int32_t _value;
6276 
6277   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
6278   int  precision_control() const       { return  (_value >>  8) & 3      ; }
6279   bool precision() const               { return ((_value >>  5) & 1) != 0; }
6280   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
6281   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
6282   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
6283   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
6284   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
6285 
6286   void print() const {
6287     // rounding control
6288     const char* rc;
6289     switch (rounding_control()) {
6290       case 0: rc = "round near"; break;
6291       case 1: rc = "round down"; break;
6292       case 2: rc = "round up  "; break;
6293       case 3: rc = "chop      "; break;
6294     };
6295     // precision control
6296     const char* pc;
6297     switch (precision_control()) {
6298       case 0: pc = "24 bits "; break;
6299       case 1: pc = "reserved"; break;
6300       case 2: pc = "53 bits "; break;
6301       case 3: pc = "64 bits "; break;
6302     };
6303     // flags
6304     char f[9];
6305     f[0] = ' ';
6306     f[1] = ' ';
6307     f[2] = (precision   ()) ? 'P' : 'p';
6308     f[3] = (underflow   ()) ? 'U' : 'u';
6309     f[4] = (overflow    ()) ? 'O' : 'o';
6310     f[5] = (zero_divide ()) ? 'Z' : 'z';
6311     f[6] = (denormalized()) ? 'D' : 'd';
6312     f[7] = (invalid     ()) ? 'I' : 'i';
6313     f[8] = '\x0';
6314     // output
6315     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
6316   }
6317 
6318 };
6319 
6320 class StatusWord {
6321  public:
6322   int32_t _value;
6323 
6324   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
6325   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
6326   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
6327   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
6328   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
6329   int  top() const                     { return  (_value >> 11) & 7      ; }
6330   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
6331   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
6332   bool precision() const               { return ((_value >>  5) & 1) != 0; }
6333   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
6334   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
6335   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
6336   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
6337   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
6338 
6339   void print() const {
6340     // condition codes
6341     char c[5];
6342     c[0] = (C3()) ? '3' : '-';
6343     c[1] = (C2()) ? '2' : '-';
6344     c[2] = (C1()) ? '1' : '-';
6345     c[3] = (C0()) ? '0' : '-';
6346     c[4] = '\x0';
6347     // flags
6348     char f[9];
6349     f[0] = (error_status()) ? 'E' : '-';
6350     f[1] = (stack_fault ()) ? 'S' : '-';
6351     f[2] = (precision   ()) ? 'P' : '-';
6352     f[3] = (underflow   ()) ? 'U' : '-';
6353     f[4] = (overflow    ()) ? 'O' : '-';
6354     f[5] = (zero_divide ()) ? 'Z' : '-';
6355     f[6] = (denormalized()) ? 'D' : '-';
6356     f[7] = (invalid     ()) ? 'I' : '-';
6357     f[8] = '\x0';
6358     // output
6359     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
6360   }
6361 
6362 };
6363 
6364 class TagWord {
6365  public:
6366   int32_t _value;
6367 
6368   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
6369 
6370   void print() const {
6371     printf("%04x", _value & 0xFFFF);
6372   }
6373 
6374 };
6375 
6376 class FPU_Register {
6377  public:
6378   int32_t _m0;
6379   int32_t _m1;
6380   int16_t _ex;
6381 
6382   bool is_indefinite() const           {
6383     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
6384   }
6385 
6386   void print() const {
6387     char  sign = (_ex < 0) ? '-' : '+';
6388     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
6389     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
6390   };
6391 
6392 };
6393 
6394 class FPU_State {
6395  public:
6396   enum {
6397     register_size       = 10,
6398     number_of_registers =  8,
6399     register_mask       =  7
6400   };
6401 
6402   ControlWord  _control_word;
6403   StatusWord   _status_word;
6404   TagWord      _tag_word;
6405   int32_t      _error_offset;
6406   int32_t      _error_selector;
6407   int32_t      _data_offset;
6408   int32_t      _data_selector;
6409   int8_t       _register[register_size * number_of_registers];
6410 
6411   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
6412   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
6413 
6414   const char* tag_as_string(int tag) const {
6415     switch (tag) {
6416       case 0: return "valid";
6417       case 1: return "zero";
6418       case 2: return "special";
6419       case 3: return "empty";
6420     }
6421     ShouldNotReachHere();
6422     return NULL;
6423   }
6424 
6425   void print() const {
6426     // print computation registers
6427     { int t = _status_word.top();
6428       for (int i = 0; i < number_of_registers; i++) {
6429         int j = (i - t) & register_mask;
6430         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
6431         st(j)->print();
6432         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
6433       }
6434     }
6435     printf("\n");
6436     // print control registers
6437     printf("ctrl = "); _control_word.print(); printf("\n");
6438     printf("stat = "); _status_word .print(); printf("\n");
6439     printf("tags = "); _tag_word    .print(); printf("\n");
6440   }
6441 
6442 };
6443 
6444 class Flag_Register {
6445  public:
6446   int32_t _value;
6447 
6448   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
6449   bool direction() const               { return ((_value >> 10) & 1) != 0; }
6450   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
6451   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
6452   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
6453   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
6454   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
6455 
6456   void print() const {
6457     // flags
6458     char f[8];
6459     f[0] = (overflow       ()) ? 'O' : '-';
6460     f[1] = (direction      ()) ? 'D' : '-';
6461     f[2] = (sign           ()) ? 'S' : '-';
6462     f[3] = (zero           ()) ? 'Z' : '-';
6463     f[4] = (auxiliary_carry()) ? 'A' : '-';
6464     f[5] = (parity         ()) ? 'P' : '-';
6465     f[6] = (carry          ()) ? 'C' : '-';
6466     f[7] = '\x0';
6467     // output
6468     printf("%08x  flags = %s", _value, f);
6469   }
6470 
6471 };
6472 
6473 class IU_Register {
6474  public:
6475   int32_t _value;
6476 
6477   void print() const {
6478     printf("%08x  %11d", _value, _value);
6479   }
6480 
6481 };
6482 
6483 class IU_State {
6484  public:
6485   Flag_Register _eflags;
6486   IU_Register   _rdi;
6487   IU_Register   _rsi;
6488   IU_Register   _rbp;
6489   IU_Register   _rsp;
6490   IU_Register   _rbx;
6491   IU_Register   _rdx;
6492   IU_Register   _rcx;
6493   IU_Register   _rax;
6494 
6495   void print() const {
6496     // computation registers
6497     printf("rax,  = "); _rax.print(); printf("\n");
6498     printf("rbx,  = "); _rbx.print(); printf("\n");
6499     printf("rcx  = "); _rcx.print(); printf("\n");
6500     printf("rdx  = "); _rdx.print(); printf("\n");
6501     printf("rdi  = "); _rdi.print(); printf("\n");
6502     printf("rsi  = "); _rsi.print(); printf("\n");
6503     printf("rbp,  = "); _rbp.print(); printf("\n");
6504     printf("rsp  = "); _rsp.print(); printf("\n");
6505     printf("\n");
6506     // control registers
6507     printf("flgs = "); _eflags.print(); printf("\n");
6508   }
6509 };
6510 
6511 
6512 class CPU_State {
6513  public:
6514   FPU_State _fpu_state;
6515   IU_State  _iu_state;
6516 
6517   void print() const {
6518     printf("--------------------------------------------------\n");
6519     _iu_state .print();
6520     printf("\n");
6521     _fpu_state.print();
6522     printf("--------------------------------------------------\n");
6523   }
6524 
6525 };
6526 
6527 
6528 static void _print_CPU_state(CPU_State* state) {
6529   state->print();
6530 };
6531 
6532 
6533 void MacroAssembler::print_CPU_state() {
6534   push_CPU_state();
6535   push(rsp);                // pass CPU state
6536   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
6537   addptr(rsp, wordSize);       // discard argument
6538   pop_CPU_state();
6539 }
6540 
6541 
6542 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
6543   static int counter = 0;
6544   FPU_State* fs = &state->_fpu_state;
6545   counter++;
6546   // For leaf calls, only verify that the top few elements remain empty.
6547   // We only need 1 empty at the top for C2 code.
6548   if( stack_depth < 0 ) {
6549     if( fs->tag_for_st(7) != 3 ) {
6550       printf("FPR7 not empty\n");
6551       state->print();
6552       assert(false, "error");
6553       return false;
6554     }
6555     return true;                // All other stack states do not matter
6556   }
6557 
6558   assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std,
6559          "bad FPU control word");
6560 
6561   // compute stack depth
6562   int i = 0;
6563   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
6564   int d = i;
6565   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
6566   // verify findings
6567   if (i != FPU_State::number_of_registers) {
6568     // stack not contiguous
6569     printf("%s: stack not contiguous at ST%d\n", s, i);
6570     state->print();
6571     assert(false, "error");
6572     return false;
6573   }
6574   // check if computed stack depth corresponds to expected stack depth
6575   if (stack_depth < 0) {
6576     // expected stack depth is -stack_depth or less
6577     if (d > -stack_depth) {
6578       // too many elements on the stack
6579       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
6580       state->print();
6581       assert(false, "error");
6582       return false;
6583     }
6584   } else {
6585     // expected stack depth is stack_depth
6586     if (d != stack_depth) {
6587       // wrong stack depth
6588       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
6589       state->print();
6590       assert(false, "error");
6591       return false;
6592     }
6593   }
6594   // everything is cool
6595   return true;
6596 }
6597 
6598 
6599 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
6600   if (!VerifyFPU) return;
6601   push_CPU_state();
6602   push(rsp);                // pass CPU state
6603   ExternalAddress msg((address) s);
6604   // pass message string s
6605   pushptr(msg.addr());
6606   push(stack_depth);        // pass stack depth
6607   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
6608   addptr(rsp, 3 * wordSize);   // discard arguments
6609   // check for error
6610   { Label L;
6611     testl(rax, rax);
6612     jcc(Assembler::notZero, L);
6613     int3();                  // break if error condition
6614     bind(L);
6615   }
6616   pop_CPU_state();
6617 }
6618 
6619 void MacroAssembler::restore_cpu_control_state_after_jni() {
6620   // Either restore the MXCSR register after returning from the JNI Call
6621   // or verify that it wasn't changed (with -Xcheck:jni flag).
6622   if (VM_Version::supports_sse()) {
6623     if (RestoreMXCSROnJNICalls) {
6624       ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std()));
6625     } else if (CheckJNICalls) {
6626       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
6627     }
6628   }
6629   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
6630   vzeroupper();
6631   // Reset k1 to 0xffff.
6632   if (VM_Version::supports_evex()) {
6633     push(rcx);
6634     movl(rcx, 0xffff);
6635     kmovwl(k1, rcx);
6636     pop(rcx);
6637   }
6638 
6639 #ifndef _LP64
6640   // Either restore the x87 floating pointer control word after returning
6641   // from the JNI call or verify that it wasn't changed.
6642   if (CheckJNICalls) {
6643     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
6644   }
6645 #endif // _LP64
6646 }
6647 
6648 // ((OopHandle)result).resolve();
6649 void MacroAssembler::resolve_oop_handle(Register result) {
6650   // OopHandle::resolve is an indirection.
6651   movptr(result, Address(result, 0));
6652 }
6653 
6654 void MacroAssembler::load_mirror(Register mirror, Register method) {
6655   // get mirror
6656   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
6657   movptr(mirror, Address(method, Method::const_offset()));
6658   movptr(mirror, Address(mirror, ConstMethod::constants_offset()));
6659   movptr(mirror, Address(mirror, ConstantPool::pool_holder_offset_in_bytes()));
6660   movptr(mirror, Address(mirror, mirror_offset));
6661   resolve_oop_handle(mirror);
6662 }
6663 
6664 void MacroAssembler::load_klass(Register dst, Register src) {
6665 #ifdef _LP64
6666   if (UseCompressedClassPointers) {
6667     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
6668     decode_klass_not_null(dst);
6669   } else
6670 #endif
6671     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
6672 }
6673 
6674 void MacroAssembler::load_prototype_header(Register dst, Register src) {
6675   load_klass(dst, src);
6676   movptr(dst, Address(dst, Klass::prototype_header_offset()));
6677 }
6678 
6679 void MacroAssembler::store_klass(Register dst, Register src) {
6680 #ifdef _LP64
6681   if (UseCompressedClassPointers) {
6682     encode_klass_not_null(src);
6683     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
6684   } else
6685 #endif
6686     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
6687 }
6688 
6689 void MacroAssembler::load_heap_oop(Register dst, Address src) {
6690 #ifdef _LP64
6691   // FIXME: Must change all places where we try to load the klass.
6692   if (UseCompressedOops) {
6693     movl(dst, src);
6694     decode_heap_oop(dst);
6695   } else
6696 #endif
6697     movptr(dst, src);
6698 }
6699 
6700 // Doesn't do verfication, generates fixed size code
6701 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src) {
6702 #ifdef _LP64
6703   if (UseCompressedOops) {
6704     movl(dst, src);
6705     decode_heap_oop_not_null(dst);
6706   } else
6707 #endif
6708     movptr(dst, src);
6709 }
6710 
6711 void MacroAssembler::store_heap_oop(Address dst, Register src) {
6712 #ifdef _LP64
6713   if (UseCompressedOops) {
6714     assert(!dst.uses(src), "not enough registers");
6715     encode_heap_oop(src);
6716     movl(dst, src);
6717   } else
6718 #endif
6719     movptr(dst, src);
6720 }
6721 
6722 void MacroAssembler::cmp_heap_oop(Register src1, Address src2, Register tmp) {
6723   assert_different_registers(src1, tmp);
6724 #ifdef _LP64
6725   if (UseCompressedOops) {
6726     bool did_push = false;
6727     if (tmp == noreg) {
6728       tmp = rax;
6729       push(tmp);
6730       did_push = true;
6731       assert(!src2.uses(rsp), "can't push");
6732     }
6733     load_heap_oop(tmp, src2);
6734     cmpptr(src1, tmp);
6735     if (did_push)  pop(tmp);
6736   } else
6737 #endif
6738     cmpptr(src1, src2);
6739 }
6740 
6741 // Used for storing NULLs.
6742 void MacroAssembler::store_heap_oop_null(Address dst) {
6743 #ifdef _LP64
6744   if (UseCompressedOops) {
6745     movl(dst, (int32_t)NULL_WORD);
6746   } else {
6747     movslq(dst, (int32_t)NULL_WORD);
6748   }
6749 #else
6750   movl(dst, (int32_t)NULL_WORD);
6751 #endif
6752 }
6753 
6754 #ifdef _LP64
6755 void MacroAssembler::store_klass_gap(Register dst, Register src) {
6756   if (UseCompressedClassPointers) {
6757     // Store to klass gap in destination
6758     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
6759   }
6760 }
6761 
6762 #ifdef ASSERT
6763 void MacroAssembler::verify_heapbase(const char* msg) {
6764   assert (UseCompressedOops, "should be compressed");
6765   assert (Universe::heap() != NULL, "java heap should be initialized");
6766   if (CheckCompressedOops) {
6767     Label ok;
6768     push(rscratch1); // cmpptr trashes rscratch1
6769     cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6770     jcc(Assembler::equal, ok);
6771     STOP(msg);
6772     bind(ok);
6773     pop(rscratch1);
6774   }
6775 }
6776 #endif
6777 
6778 // Algorithm must match oop.inline.hpp encode_heap_oop.
6779 void MacroAssembler::encode_heap_oop(Register r) {
6780 #ifdef ASSERT
6781   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
6782 #endif
6783   verify_oop(r, "broken oop in encode_heap_oop");
6784   if (Universe::narrow_oop_base() == NULL) {
6785     if (Universe::narrow_oop_shift() != 0) {
6786       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6787       shrq(r, LogMinObjAlignmentInBytes);
6788     }
6789     return;
6790   }
6791   testq(r, r);
6792   cmovq(Assembler::equal, r, r12_heapbase);
6793   subq(r, r12_heapbase);
6794   shrq(r, LogMinObjAlignmentInBytes);
6795 }
6796 
6797 void MacroAssembler::encode_heap_oop_not_null(Register r) {
6798 #ifdef ASSERT
6799   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
6800   if (CheckCompressedOops) {
6801     Label ok;
6802     testq(r, r);
6803     jcc(Assembler::notEqual, ok);
6804     STOP("null oop passed to encode_heap_oop_not_null");
6805     bind(ok);
6806   }
6807 #endif
6808   verify_oop(r, "broken oop in encode_heap_oop_not_null");
6809   if (Universe::narrow_oop_base() != NULL) {
6810     subq(r, r12_heapbase);
6811   }
6812   if (Universe::narrow_oop_shift() != 0) {
6813     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6814     shrq(r, LogMinObjAlignmentInBytes);
6815   }
6816 }
6817 
6818 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
6819 #ifdef ASSERT
6820   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
6821   if (CheckCompressedOops) {
6822     Label ok;
6823     testq(src, src);
6824     jcc(Assembler::notEqual, ok);
6825     STOP("null oop passed to encode_heap_oop_not_null2");
6826     bind(ok);
6827   }
6828 #endif
6829   verify_oop(src, "broken oop in encode_heap_oop_not_null2");
6830   if (dst != src) {
6831     movq(dst, src);
6832   }
6833   if (Universe::narrow_oop_base() != NULL) {
6834     subq(dst, r12_heapbase);
6835   }
6836   if (Universe::narrow_oop_shift() != 0) {
6837     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6838     shrq(dst, LogMinObjAlignmentInBytes);
6839   }
6840 }
6841 
6842 void  MacroAssembler::decode_heap_oop(Register r) {
6843 #ifdef ASSERT
6844   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
6845 #endif
6846   if (Universe::narrow_oop_base() == NULL) {
6847     if (Universe::narrow_oop_shift() != 0) {
6848       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6849       shlq(r, LogMinObjAlignmentInBytes);
6850     }
6851   } else {
6852     Label done;
6853     shlq(r, LogMinObjAlignmentInBytes);
6854     jccb(Assembler::equal, done);
6855     addq(r, r12_heapbase);
6856     bind(done);
6857   }
6858   verify_oop(r, "broken oop in decode_heap_oop");
6859 }
6860 
6861 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
6862   // Note: it will change flags
6863   assert (UseCompressedOops, "should only be used for compressed headers");
6864   assert (Universe::heap() != NULL, "java heap should be initialized");
6865   // Cannot assert, unverified entry point counts instructions (see .ad file)
6866   // vtableStubs also counts instructions in pd_code_size_limit.
6867   // Also do not verify_oop as this is called by verify_oop.
6868   if (Universe::narrow_oop_shift() != 0) {
6869     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6870     shlq(r, LogMinObjAlignmentInBytes);
6871     if (Universe::narrow_oop_base() != NULL) {
6872       addq(r, r12_heapbase);
6873     }
6874   } else {
6875     assert (Universe::narrow_oop_base() == NULL, "sanity");
6876   }
6877 }
6878 
6879 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
6880   // Note: it will change flags
6881   assert (UseCompressedOops, "should only be used for compressed headers");
6882   assert (Universe::heap() != NULL, "java heap should be initialized");
6883   // Cannot assert, unverified entry point counts instructions (see .ad file)
6884   // vtableStubs also counts instructions in pd_code_size_limit.
6885   // Also do not verify_oop as this is called by verify_oop.
6886   if (Universe::narrow_oop_shift() != 0) {
6887     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6888     if (LogMinObjAlignmentInBytes == Address::times_8) {
6889       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
6890     } else {
6891       if (dst != src) {
6892         movq(dst, src);
6893       }
6894       shlq(dst, LogMinObjAlignmentInBytes);
6895       if (Universe::narrow_oop_base() != NULL) {
6896         addq(dst, r12_heapbase);
6897       }
6898     }
6899   } else {
6900     assert (Universe::narrow_oop_base() == NULL, "sanity");
6901     if (dst != src) {
6902       movq(dst, src);
6903     }
6904   }
6905 }
6906 
6907 void MacroAssembler::encode_klass_not_null(Register r) {
6908   if (Universe::narrow_klass_base() != NULL) {
6909     // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6910     assert(r != r12_heapbase, "Encoding a klass in r12");
6911     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6912     subq(r, r12_heapbase);
6913   }
6914   if (Universe::narrow_klass_shift() != 0) {
6915     assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6916     shrq(r, LogKlassAlignmentInBytes);
6917   }
6918   if (Universe::narrow_klass_base() != NULL) {
6919     reinit_heapbase();
6920   }
6921 }
6922 
6923 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
6924   if (dst == src) {
6925     encode_klass_not_null(src);
6926   } else {
6927     if (Universe::narrow_klass_base() != NULL) {
6928       mov64(dst, (int64_t)Universe::narrow_klass_base());
6929       negq(dst);
6930       addq(dst, src);
6931     } else {
6932       movptr(dst, src);
6933     }
6934     if (Universe::narrow_klass_shift() != 0) {
6935       assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6936       shrq(dst, LogKlassAlignmentInBytes);
6937     }
6938   }
6939 }
6940 
6941 // Function instr_size_for_decode_klass_not_null() counts the instructions
6942 // generated by decode_klass_not_null(register r) and reinit_heapbase(),
6943 // when (Universe::heap() != NULL).  Hence, if the instructions they
6944 // generate change, then this method needs to be updated.
6945 int MacroAssembler::instr_size_for_decode_klass_not_null() {
6946   assert (UseCompressedClassPointers, "only for compressed klass ptrs");
6947   if (Universe::narrow_klass_base() != NULL) {
6948     // mov64 + addq + shlq? + mov64  (for reinit_heapbase()).
6949     return (Universe::narrow_klass_shift() == 0 ? 20 : 24);
6950   } else {
6951     // longest load decode klass function, mov64, leaq
6952     return 16;
6953   }
6954 }
6955 
6956 // !!! If the instructions that get generated here change then function
6957 // instr_size_for_decode_klass_not_null() needs to get updated.
6958 void  MacroAssembler::decode_klass_not_null(Register r) {
6959   // Note: it will change flags
6960   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6961   assert(r != r12_heapbase, "Decoding a klass in r12");
6962   // Cannot assert, unverified entry point counts instructions (see .ad file)
6963   // vtableStubs also counts instructions in pd_code_size_limit.
6964   // Also do not verify_oop as this is called by verify_oop.
6965   if (Universe::narrow_klass_shift() != 0) {
6966     assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6967     shlq(r, LogKlassAlignmentInBytes);
6968   }
6969   // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6970   if (Universe::narrow_klass_base() != NULL) {
6971     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6972     addq(r, r12_heapbase);
6973     reinit_heapbase();
6974   }
6975 }
6976 
6977 void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
6978   // Note: it will change flags
6979   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6980   if (dst == src) {
6981     decode_klass_not_null(dst);
6982   } else {
6983     // Cannot assert, unverified entry point counts instructions (see .ad file)
6984     // vtableStubs also counts instructions in pd_code_size_limit.
6985     // Also do not verify_oop as this is called by verify_oop.
6986     mov64(dst, (int64_t)Universe::narrow_klass_base());
6987     if (Universe::narrow_klass_shift() != 0) {
6988       assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6989       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
6990       leaq(dst, Address(dst, src, Address::times_8, 0));
6991     } else {
6992       addq(dst, src);
6993     }
6994   }
6995 }
6996 
6997 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
6998   assert (UseCompressedOops, "should only be used for compressed headers");
6999   assert (Universe::heap() != NULL, "java heap should be initialized");
7000   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
7001   int oop_index = oop_recorder()->find_index(obj);
7002   RelocationHolder rspec = oop_Relocation::spec(oop_index);
7003   mov_narrow_oop(dst, oop_index, rspec);
7004 }
7005 
7006 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
7007   assert (UseCompressedOops, "should only be used for compressed headers");
7008   assert (Universe::heap() != NULL, "java heap should be initialized");
7009   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
7010   int oop_index = oop_recorder()->find_index(obj);
7011   RelocationHolder rspec = oop_Relocation::spec(oop_index);
7012   mov_narrow_oop(dst, oop_index, rspec);
7013 }
7014 
7015 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
7016   assert (UseCompressedClassPointers, "should only be used for compressed headers");
7017   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
7018   int klass_index = oop_recorder()->find_index(k);
7019   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
7020   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
7021 }
7022 
7023 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
7024   assert (UseCompressedClassPointers, "should only be used for compressed headers");
7025   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
7026   int klass_index = oop_recorder()->find_index(k);
7027   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
7028   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
7029 }
7030 
7031 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
7032   assert (UseCompressedOops, "should only be used for compressed headers");
7033   assert (Universe::heap() != NULL, "java heap should be initialized");
7034   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
7035   int oop_index = oop_recorder()->find_index(obj);
7036   RelocationHolder rspec = oop_Relocation::spec(oop_index);
7037   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
7038 }
7039 
7040 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
7041   assert (UseCompressedOops, "should only be used for compressed headers");
7042   assert (Universe::heap() != NULL, "java heap should be initialized");
7043   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
7044   int oop_index = oop_recorder()->find_index(obj);
7045   RelocationHolder rspec = oop_Relocation::spec(oop_index);
7046   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
7047 }
7048 
7049 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
7050   assert (UseCompressedClassPointers, "should only be used for compressed headers");
7051   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
7052   int klass_index = oop_recorder()->find_index(k);
7053   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
7054   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
7055 }
7056 
7057 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
7058   assert (UseCompressedClassPointers, "should only be used for compressed headers");
7059   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
7060   int klass_index = oop_recorder()->find_index(k);
7061   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
7062   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
7063 }
7064 
7065 void MacroAssembler::reinit_heapbase() {
7066   if (UseCompressedOops || UseCompressedClassPointers) {
7067     if (Universe::heap() != NULL) {
7068       if (Universe::narrow_oop_base() == NULL) {
7069         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
7070       } else {
7071         mov64(r12_heapbase, (int64_t)Universe::narrow_ptrs_base());
7072       }
7073     } else {
7074       movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
7075     }
7076   }
7077 }
7078 
7079 #endif // _LP64
7080 
7081 // C2 compiled method's prolog code.
7082 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b) {
7083 
7084   // WARNING: Initial instruction MUST be 5 bytes or longer so that
7085   // NativeJump::patch_verified_entry will be able to patch out the entry
7086   // code safely. The push to verify stack depth is ok at 5 bytes,
7087   // the frame allocation can be either 3 or 6 bytes. So if we don't do
7088   // stack bang then we must use the 6 byte frame allocation even if
7089   // we have no frame. :-(
7090   assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
7091 
7092   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
7093   // Remove word for return addr
7094   framesize -= wordSize;
7095   stack_bang_size -= wordSize;
7096 
7097   // Calls to C2R adapters often do not accept exceptional returns.
7098   // We require that their callers must bang for them.  But be careful, because
7099   // some VM calls (such as call site linkage) can use several kilobytes of
7100   // stack.  But the stack safety zone should account for that.
7101   // See bugs 4446381, 4468289, 4497237.
7102   if (stack_bang_size > 0) {
7103     generate_stack_overflow_check(stack_bang_size);
7104 
7105     // We always push rbp, so that on return to interpreter rbp, will be
7106     // restored correctly and we can correct the stack.
7107     push(rbp);
7108     // Save caller's stack pointer into RBP if the frame pointer is preserved.
7109     if (PreserveFramePointer) {
7110       mov(rbp, rsp);
7111     }
7112     // Remove word for ebp
7113     framesize -= wordSize;
7114 
7115     // Create frame
7116     if (framesize) {
7117       subptr(rsp, framesize);
7118     }
7119   } else {
7120     // Create frame (force generation of a 4 byte immediate value)
7121     subptr_imm32(rsp, framesize);
7122 
7123     // Save RBP register now.
7124     framesize -= wordSize;
7125     movptr(Address(rsp, framesize), rbp);
7126     // Save caller's stack pointer into RBP if the frame pointer is preserved.
7127     if (PreserveFramePointer) {
7128       movptr(rbp, rsp);
7129       if (framesize > 0) {
7130         addptr(rbp, framesize);
7131       }
7132     }
7133   }
7134 
7135   if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
7136     framesize -= wordSize;
7137     movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
7138   }
7139 
7140 #ifndef _LP64
7141   // If method sets FPU control word do it now
7142   if (fp_mode_24b) {
7143     fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
7144   }
7145   if (UseSSE >= 2 && VerifyFPU) {
7146     verify_FPU(0, "FPU stack must be clean on entry");
7147   }
7148 #endif
7149 
7150 #ifdef ASSERT
7151   if (VerifyStackAtCalls) {
7152     Label L;
7153     push(rax);
7154     mov(rax, rsp);
7155     andptr(rax, StackAlignmentInBytes-1);
7156     cmpptr(rax, StackAlignmentInBytes-wordSize);
7157     pop(rax);
7158     jcc(Assembler::equal, L);
7159     STOP("Stack is not properly aligned!");
7160     bind(L);
7161   }
7162 #endif
7163 
7164 }
7165 
7166 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, bool is_large) {
7167   // cnt - number of qwords (8-byte words).
7168   // base - start address, qword aligned.
7169   // is_large - if optimizers know cnt is larger than InitArrayShortSize
7170   assert(base==rdi, "base register must be edi for rep stos");
7171   assert(tmp==rax,   "tmp register must be eax for rep stos");
7172   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
7173   assert(InitArrayShortSize % BytesPerLong == 0,
7174     "InitArrayShortSize should be the multiple of BytesPerLong");
7175 
7176   Label DONE;
7177 
7178   xorptr(tmp, tmp);
7179 
7180   if (!is_large) {
7181     Label LOOP, LONG;
7182     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
7183     jccb(Assembler::greater, LONG);
7184 
7185     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
7186 
7187     decrement(cnt);
7188     jccb(Assembler::negative, DONE); // Zero length
7189 
7190     // Use individual pointer-sized stores for small counts:
7191     BIND(LOOP);
7192     movptr(Address(base, cnt, Address::times_ptr), tmp);
7193     decrement(cnt);
7194     jccb(Assembler::greaterEqual, LOOP);
7195     jmpb(DONE);
7196 
7197     BIND(LONG);
7198   }
7199 
7200   // Use longer rep-prefixed ops for non-small counts:
7201   if (UseFastStosb) {
7202     shlptr(cnt, 3); // convert to number of bytes
7203     rep_stosb();
7204   } else {
7205     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
7206     rep_stos();
7207   }
7208 
7209   BIND(DONE);
7210 }
7211 
7212 #ifdef COMPILER2
7213 
7214 // IndexOf for constant substrings with size >= 8 chars
7215 // which don't need to be loaded through stack.
7216 void MacroAssembler::string_indexofC8(Register str1, Register str2,
7217                                       Register cnt1, Register cnt2,
7218                                       int int_cnt2,  Register result,
7219                                       XMMRegister vec, Register tmp,
7220                                       int ae) {
7221   ShortBranchVerifier sbv(this);
7222   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7223   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
7224 
7225   // This method uses the pcmpestri instruction with bound registers
7226   //   inputs:
7227   //     xmm - substring
7228   //     rax - substring length (elements count)
7229   //     mem - scanned string
7230   //     rdx - string length (elements count)
7231   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
7232   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
7233   //   outputs:
7234   //     rcx - matched index in string
7235   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
7236   int mode   = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
7237   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
7238   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
7239   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
7240 
7241   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR,
7242         RET_FOUND, RET_NOT_FOUND, EXIT, FOUND_SUBSTR,
7243         MATCH_SUBSTR_HEAD, RELOAD_STR, FOUND_CANDIDATE;
7244 
7245   // Note, inline_string_indexOf() generates checks:
7246   // if (substr.count > string.count) return -1;
7247   // if (substr.count == 0) return 0;
7248   assert(int_cnt2 >= stride, "this code is used only for cnt2 >= 8 chars");
7249 
7250   // Load substring.
7251   if (ae == StrIntrinsicNode::UL) {
7252     pmovzxbw(vec, Address(str2, 0));
7253   } else {
7254     movdqu(vec, Address(str2, 0));
7255   }
7256   movl(cnt2, int_cnt2);
7257   movptr(result, str1); // string addr
7258 
7259   if (int_cnt2 > stride) {
7260     jmpb(SCAN_TO_SUBSTR);
7261 
7262     // Reload substr for rescan, this code
7263     // is executed only for large substrings (> 8 chars)
7264     bind(RELOAD_SUBSTR);
7265     if (ae == StrIntrinsicNode::UL) {
7266       pmovzxbw(vec, Address(str2, 0));
7267     } else {
7268       movdqu(vec, Address(str2, 0));
7269     }
7270     negptr(cnt2); // Jumped here with negative cnt2, convert to positive
7271 
7272     bind(RELOAD_STR);
7273     // We came here after the beginning of the substring was
7274     // matched but the rest of it was not so we need to search
7275     // again. Start from the next element after the previous match.
7276 
7277     // cnt2 is number of substring reminding elements and
7278     // cnt1 is number of string reminding elements when cmp failed.
7279     // Restored cnt1 = cnt1 - cnt2 + int_cnt2
7280     subl(cnt1, cnt2);
7281     addl(cnt1, int_cnt2);
7282     movl(cnt2, int_cnt2); // Now restore cnt2
7283 
7284     decrementl(cnt1);     // Shift to next element
7285     cmpl(cnt1, cnt2);
7286     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7287 
7288     addptr(result, (1<<scale1));
7289 
7290   } // (int_cnt2 > 8)
7291 
7292   // Scan string for start of substr in 16-byte vectors
7293   bind(SCAN_TO_SUBSTR);
7294   pcmpestri(vec, Address(result, 0), mode);
7295   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
7296   subl(cnt1, stride);
7297   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
7298   cmpl(cnt1, cnt2);
7299   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7300   addptr(result, 16);
7301   jmpb(SCAN_TO_SUBSTR);
7302 
7303   // Found a potential substr
7304   bind(FOUND_CANDIDATE);
7305   // Matched whole vector if first element matched (tmp(rcx) == 0).
7306   if (int_cnt2 == stride) {
7307     jccb(Assembler::overflow, RET_FOUND);    // OF == 1
7308   } else { // int_cnt2 > 8
7309     jccb(Assembler::overflow, FOUND_SUBSTR);
7310   }
7311   // After pcmpestri tmp(rcx) contains matched element index
7312   // Compute start addr of substr
7313   lea(result, Address(result, tmp, scale1));
7314 
7315   // Make sure string is still long enough
7316   subl(cnt1, tmp);
7317   cmpl(cnt1, cnt2);
7318   if (int_cnt2 == stride) {
7319     jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
7320   } else { // int_cnt2 > 8
7321     jccb(Assembler::greaterEqual, MATCH_SUBSTR_HEAD);
7322   }
7323   // Left less then substring.
7324 
7325   bind(RET_NOT_FOUND);
7326   movl(result, -1);
7327   jmp(EXIT);
7328 
7329   if (int_cnt2 > stride) {
7330     // This code is optimized for the case when whole substring
7331     // is matched if its head is matched.
7332     bind(MATCH_SUBSTR_HEAD);
7333     pcmpestri(vec, Address(result, 0), mode);
7334     // Reload only string if does not match
7335     jcc(Assembler::noOverflow, RELOAD_STR); // OF == 0
7336 
7337     Label CONT_SCAN_SUBSTR;
7338     // Compare the rest of substring (> 8 chars).
7339     bind(FOUND_SUBSTR);
7340     // First 8 chars are already matched.
7341     negptr(cnt2);
7342     addptr(cnt2, stride);
7343 
7344     bind(SCAN_SUBSTR);
7345     subl(cnt1, stride);
7346     cmpl(cnt2, -stride); // Do not read beyond substring
7347     jccb(Assembler::lessEqual, CONT_SCAN_SUBSTR);
7348     // Back-up strings to avoid reading beyond substring:
7349     // cnt1 = cnt1 - cnt2 + 8
7350     addl(cnt1, cnt2); // cnt2 is negative
7351     addl(cnt1, stride);
7352     movl(cnt2, stride); negptr(cnt2);
7353     bind(CONT_SCAN_SUBSTR);
7354     if (int_cnt2 < (int)G) {
7355       int tail_off1 = int_cnt2<<scale1;
7356       int tail_off2 = int_cnt2<<scale2;
7357       if (ae == StrIntrinsicNode::UL) {
7358         pmovzxbw(vec, Address(str2, cnt2, scale2, tail_off2));
7359       } else {
7360         movdqu(vec, Address(str2, cnt2, scale2, tail_off2));
7361       }
7362       pcmpestri(vec, Address(result, cnt2, scale1, tail_off1), mode);
7363     } else {
7364       // calculate index in register to avoid integer overflow (int_cnt2*2)
7365       movl(tmp, int_cnt2);
7366       addptr(tmp, cnt2);
7367       if (ae == StrIntrinsicNode::UL) {
7368         pmovzxbw(vec, Address(str2, tmp, scale2, 0));
7369       } else {
7370         movdqu(vec, Address(str2, tmp, scale2, 0));
7371       }
7372       pcmpestri(vec, Address(result, tmp, scale1, 0), mode);
7373     }
7374     // Need to reload strings pointers if not matched whole vector
7375     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
7376     addptr(cnt2, stride);
7377     jcc(Assembler::negative, SCAN_SUBSTR);
7378     // Fall through if found full substring
7379 
7380   } // (int_cnt2 > 8)
7381 
7382   bind(RET_FOUND);
7383   // Found result if we matched full small substring.
7384   // Compute substr offset
7385   subptr(result, str1);
7386   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7387     shrl(result, 1); // index
7388   }
7389   bind(EXIT);
7390 
7391 } // string_indexofC8
7392 
7393 // Small strings are loaded through stack if they cross page boundary.
7394 void MacroAssembler::string_indexof(Register str1, Register str2,
7395                                     Register cnt1, Register cnt2,
7396                                     int int_cnt2,  Register result,
7397                                     XMMRegister vec, Register tmp,
7398                                     int ae) {
7399   ShortBranchVerifier sbv(this);
7400   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7401   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
7402 
7403   //
7404   // int_cnt2 is length of small (< 8 chars) constant substring
7405   // or (-1) for non constant substring in which case its length
7406   // is in cnt2 register.
7407   //
7408   // Note, inline_string_indexOf() generates checks:
7409   // if (substr.count > string.count) return -1;
7410   // if (substr.count == 0) return 0;
7411   //
7412   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
7413   assert(int_cnt2 == -1 || (0 < int_cnt2 && int_cnt2 < stride), "should be != 0");
7414   // This method uses the pcmpestri instruction with bound registers
7415   //   inputs:
7416   //     xmm - substring
7417   //     rax - substring length (elements count)
7418   //     mem - scanned string
7419   //     rdx - string length (elements count)
7420   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
7421   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
7422   //   outputs:
7423   //     rcx - matched index in string
7424   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
7425   int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
7426   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
7427   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
7428 
7429   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, ADJUST_STR,
7430         RET_FOUND, RET_NOT_FOUND, CLEANUP, FOUND_SUBSTR,
7431         FOUND_CANDIDATE;
7432 
7433   { //========================================================
7434     // We don't know where these strings are located
7435     // and we can't read beyond them. Load them through stack.
7436     Label BIG_STRINGS, CHECK_STR, COPY_SUBSTR, COPY_STR;
7437 
7438     movptr(tmp, rsp); // save old SP
7439 
7440     if (int_cnt2 > 0) {     // small (< 8 chars) constant substring
7441       if (int_cnt2 == (1>>scale2)) { // One byte
7442         assert((ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL), "Only possible for latin1 encoding");
7443         load_unsigned_byte(result, Address(str2, 0));
7444         movdl(vec, result); // move 32 bits
7445       } else if (ae == StrIntrinsicNode::LL && int_cnt2 == 3) {  // Three bytes
7446         // Not enough header space in 32-bit VM: 12+3 = 15.
7447         movl(result, Address(str2, -1));
7448         shrl(result, 8);
7449         movdl(vec, result); // move 32 bits
7450       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (2>>scale2)) {  // One char
7451         load_unsigned_short(result, Address(str2, 0));
7452         movdl(vec, result); // move 32 bits
7453       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (4>>scale2)) { // Two chars
7454         movdl(vec, Address(str2, 0)); // move 32 bits
7455       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (8>>scale2)) { // Four chars
7456         movq(vec, Address(str2, 0));  // move 64 bits
7457       } else { // cnt2 = { 3, 5, 6, 7 } || (ae == StrIntrinsicNode::UL && cnt2 ={2, ..., 7})
7458         // Array header size is 12 bytes in 32-bit VM
7459         // + 6 bytes for 3 chars == 18 bytes,
7460         // enough space to load vec and shift.
7461         assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity");
7462         if (ae == StrIntrinsicNode::UL) {
7463           int tail_off = int_cnt2-8;
7464           pmovzxbw(vec, Address(str2, tail_off));
7465           psrldq(vec, -2*tail_off);
7466         }
7467         else {
7468           int tail_off = int_cnt2*(1<<scale2);
7469           movdqu(vec, Address(str2, tail_off-16));
7470           psrldq(vec, 16-tail_off);
7471         }
7472       }
7473     } else { // not constant substring
7474       cmpl(cnt2, stride);
7475       jccb(Assembler::aboveEqual, BIG_STRINGS); // Both strings are big enough
7476 
7477       // We can read beyond string if srt+16 does not cross page boundary
7478       // since heaps are aligned and mapped by pages.
7479       assert(os::vm_page_size() < (int)G, "default page should be small");
7480       movl(result, str2); // We need only low 32 bits
7481       andl(result, (os::vm_page_size()-1));
7482       cmpl(result, (os::vm_page_size()-16));
7483       jccb(Assembler::belowEqual, CHECK_STR);
7484 
7485       // Move small strings to stack to allow load 16 bytes into vec.
7486       subptr(rsp, 16);
7487       int stk_offset = wordSize-(1<<scale2);
7488       push(cnt2);
7489 
7490       bind(COPY_SUBSTR);
7491       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL) {
7492         load_unsigned_byte(result, Address(str2, cnt2, scale2, -1));
7493         movb(Address(rsp, cnt2, scale2, stk_offset), result);
7494       } else if (ae == StrIntrinsicNode::UU) {
7495         load_unsigned_short(result, Address(str2, cnt2, scale2, -2));
7496         movw(Address(rsp, cnt2, scale2, stk_offset), result);
7497       }
7498       decrement(cnt2);
7499       jccb(Assembler::notZero, COPY_SUBSTR);
7500 
7501       pop(cnt2);
7502       movptr(str2, rsp);  // New substring address
7503     } // non constant
7504 
7505     bind(CHECK_STR);
7506     cmpl(cnt1, stride);
7507     jccb(Assembler::aboveEqual, BIG_STRINGS);
7508 
7509     // Check cross page boundary.
7510     movl(result, str1); // We need only low 32 bits
7511     andl(result, (os::vm_page_size()-1));
7512     cmpl(result, (os::vm_page_size()-16));
7513     jccb(Assembler::belowEqual, BIG_STRINGS);
7514 
7515     subptr(rsp, 16);
7516     int stk_offset = -(1<<scale1);
7517     if (int_cnt2 < 0) { // not constant
7518       push(cnt2);
7519       stk_offset += wordSize;
7520     }
7521     movl(cnt2, cnt1);
7522 
7523     bind(COPY_STR);
7524     if (ae == StrIntrinsicNode::LL) {
7525       load_unsigned_byte(result, Address(str1, cnt2, scale1, -1));
7526       movb(Address(rsp, cnt2, scale1, stk_offset), result);
7527     } else {
7528       load_unsigned_short(result, Address(str1, cnt2, scale1, -2));
7529       movw(Address(rsp, cnt2, scale1, stk_offset), result);
7530     }
7531     decrement(cnt2);
7532     jccb(Assembler::notZero, COPY_STR);
7533 
7534     if (int_cnt2 < 0) { // not constant
7535       pop(cnt2);
7536     }
7537     movptr(str1, rsp);  // New string address
7538 
7539     bind(BIG_STRINGS);
7540     // Load substring.
7541     if (int_cnt2 < 0) { // -1
7542       if (ae == StrIntrinsicNode::UL) {
7543         pmovzxbw(vec, Address(str2, 0));
7544       } else {
7545         movdqu(vec, Address(str2, 0));
7546       }
7547       push(cnt2);       // substr count
7548       push(str2);       // substr addr
7549       push(str1);       // string addr
7550     } else {
7551       // Small (< 8 chars) constant substrings are loaded already.
7552       movl(cnt2, int_cnt2);
7553     }
7554     push(tmp);  // original SP
7555 
7556   } // Finished loading
7557 
7558   //========================================================
7559   // Start search
7560   //
7561 
7562   movptr(result, str1); // string addr
7563 
7564   if (int_cnt2  < 0) {  // Only for non constant substring
7565     jmpb(SCAN_TO_SUBSTR);
7566 
7567     // SP saved at sp+0
7568     // String saved at sp+1*wordSize
7569     // Substr saved at sp+2*wordSize
7570     // Substr count saved at sp+3*wordSize
7571 
7572     // Reload substr for rescan, this code
7573     // is executed only for large substrings (> 8 chars)
7574     bind(RELOAD_SUBSTR);
7575     movptr(str2, Address(rsp, 2*wordSize));
7576     movl(cnt2, Address(rsp, 3*wordSize));
7577     if (ae == StrIntrinsicNode::UL) {
7578       pmovzxbw(vec, Address(str2, 0));
7579     } else {
7580       movdqu(vec, Address(str2, 0));
7581     }
7582     // We came here after the beginning of the substring was
7583     // matched but the rest of it was not so we need to search
7584     // again. Start from the next element after the previous match.
7585     subptr(str1, result); // Restore counter
7586     if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7587       shrl(str1, 1);
7588     }
7589     addl(cnt1, str1);
7590     decrementl(cnt1);   // Shift to next element
7591     cmpl(cnt1, cnt2);
7592     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7593 
7594     addptr(result, (1<<scale1));
7595   } // non constant
7596 
7597   // Scan string for start of substr in 16-byte vectors
7598   bind(SCAN_TO_SUBSTR);
7599   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
7600   pcmpestri(vec, Address(result, 0), mode);
7601   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
7602   subl(cnt1, stride);
7603   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
7604   cmpl(cnt1, cnt2);
7605   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7606   addptr(result, 16);
7607 
7608   bind(ADJUST_STR);
7609   cmpl(cnt1, stride); // Do not read beyond string
7610   jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
7611   // Back-up string to avoid reading beyond string.
7612   lea(result, Address(result, cnt1, scale1, -16));
7613   movl(cnt1, stride);
7614   jmpb(SCAN_TO_SUBSTR);
7615 
7616   // Found a potential substr
7617   bind(FOUND_CANDIDATE);
7618   // After pcmpestri tmp(rcx) contains matched element index
7619 
7620   // Make sure string is still long enough
7621   subl(cnt1, tmp);
7622   cmpl(cnt1, cnt2);
7623   jccb(Assembler::greaterEqual, FOUND_SUBSTR);
7624   // Left less then substring.
7625 
7626   bind(RET_NOT_FOUND);
7627   movl(result, -1);
7628   jmpb(CLEANUP);
7629 
7630   bind(FOUND_SUBSTR);
7631   // Compute start addr of substr
7632   lea(result, Address(result, tmp, scale1));
7633   if (int_cnt2 > 0) { // Constant substring
7634     // Repeat search for small substring (< 8 chars)
7635     // from new point without reloading substring.
7636     // Have to check that we don't read beyond string.
7637     cmpl(tmp, stride-int_cnt2);
7638     jccb(Assembler::greater, ADJUST_STR);
7639     // Fall through if matched whole substring.
7640   } else { // non constant
7641     assert(int_cnt2 == -1, "should be != 0");
7642 
7643     addl(tmp, cnt2);
7644     // Found result if we matched whole substring.
7645     cmpl(tmp, stride);
7646     jccb(Assembler::lessEqual, RET_FOUND);
7647 
7648     // Repeat search for small substring (<= 8 chars)
7649     // from new point 'str1' without reloading substring.
7650     cmpl(cnt2, stride);
7651     // Have to check that we don't read beyond string.
7652     jccb(Assembler::lessEqual, ADJUST_STR);
7653 
7654     Label CHECK_NEXT, CONT_SCAN_SUBSTR, RET_FOUND_LONG;
7655     // Compare the rest of substring (> 8 chars).
7656     movptr(str1, result);
7657 
7658     cmpl(tmp, cnt2);
7659     // First 8 chars are already matched.
7660     jccb(Assembler::equal, CHECK_NEXT);
7661 
7662     bind(SCAN_SUBSTR);
7663     pcmpestri(vec, Address(str1, 0), mode);
7664     // Need to reload strings pointers if not matched whole vector
7665     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
7666 
7667     bind(CHECK_NEXT);
7668     subl(cnt2, stride);
7669     jccb(Assembler::lessEqual, RET_FOUND_LONG); // Found full substring
7670     addptr(str1, 16);
7671     if (ae == StrIntrinsicNode::UL) {
7672       addptr(str2, 8);
7673     } else {
7674       addptr(str2, 16);
7675     }
7676     subl(cnt1, stride);
7677     cmpl(cnt2, stride); // Do not read beyond substring
7678     jccb(Assembler::greaterEqual, CONT_SCAN_SUBSTR);
7679     // Back-up strings to avoid reading beyond substring.
7680 
7681     if (ae == StrIntrinsicNode::UL) {
7682       lea(str2, Address(str2, cnt2, scale2, -8));
7683       lea(str1, Address(str1, cnt2, scale1, -16));
7684     } else {
7685       lea(str2, Address(str2, cnt2, scale2, -16));
7686       lea(str1, Address(str1, cnt2, scale1, -16));
7687     }
7688     subl(cnt1, cnt2);
7689     movl(cnt2, stride);
7690     addl(cnt1, stride);
7691     bind(CONT_SCAN_SUBSTR);
7692     if (ae == StrIntrinsicNode::UL) {
7693       pmovzxbw(vec, Address(str2, 0));
7694     } else {
7695       movdqu(vec, Address(str2, 0));
7696     }
7697     jmp(SCAN_SUBSTR);
7698 
7699     bind(RET_FOUND_LONG);
7700     movptr(str1, Address(rsp, wordSize));
7701   } // non constant
7702 
7703   bind(RET_FOUND);
7704   // Compute substr offset
7705   subptr(result, str1);
7706   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7707     shrl(result, 1); // index
7708   }
7709   bind(CLEANUP);
7710   pop(rsp); // restore SP
7711 
7712 } // string_indexof
7713 
7714 void MacroAssembler::string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
7715                                          XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp) {
7716   ShortBranchVerifier sbv(this);
7717   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7718 
7719   int stride = 8;
7720 
7721   Label FOUND_CHAR, SCAN_TO_CHAR, SCAN_TO_CHAR_LOOP,
7722         SCAN_TO_8_CHAR, SCAN_TO_8_CHAR_LOOP, SCAN_TO_16_CHAR_LOOP,
7723         RET_NOT_FOUND, SCAN_TO_8_CHAR_INIT,
7724         FOUND_SEQ_CHAR, DONE_LABEL;
7725 
7726   movptr(result, str1);
7727   if (UseAVX >= 2) {
7728     cmpl(cnt1, stride);
7729     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7730     cmpl(cnt1, 2*stride);
7731     jcc(Assembler::less, SCAN_TO_8_CHAR_INIT);
7732     movdl(vec1, ch);
7733     vpbroadcastw(vec1, vec1);
7734     vpxor(vec2, vec2);
7735     movl(tmp, cnt1);
7736     andl(tmp, 0xFFFFFFF0);  //vector count (in chars)
7737     andl(cnt1,0x0000000F);  //tail count (in chars)
7738 
7739     bind(SCAN_TO_16_CHAR_LOOP);
7740     vmovdqu(vec3, Address(result, 0));
7741     vpcmpeqw(vec3, vec3, vec1, 1);
7742     vptest(vec2, vec3);
7743     jcc(Assembler::carryClear, FOUND_CHAR);
7744     addptr(result, 32);
7745     subl(tmp, 2*stride);
7746     jccb(Assembler::notZero, SCAN_TO_16_CHAR_LOOP);
7747     jmp(SCAN_TO_8_CHAR);
7748     bind(SCAN_TO_8_CHAR_INIT);
7749     movdl(vec1, ch);
7750     pshuflw(vec1, vec1, 0x00);
7751     pshufd(vec1, vec1, 0);
7752     pxor(vec2, vec2);
7753   }
7754   bind(SCAN_TO_8_CHAR);
7755   cmpl(cnt1, stride);
7756   if (UseAVX >= 2) {
7757     jcc(Assembler::less, SCAN_TO_CHAR);
7758   } else {
7759     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7760     movdl(vec1, ch);
7761     pshuflw(vec1, vec1, 0x00);
7762     pshufd(vec1, vec1, 0);
7763     pxor(vec2, vec2);
7764   }
7765   movl(tmp, cnt1);
7766   andl(tmp, 0xFFFFFFF8);  //vector count (in chars)
7767   andl(cnt1,0x00000007);  //tail count (in chars)
7768 
7769   bind(SCAN_TO_8_CHAR_LOOP);
7770   movdqu(vec3, Address(result, 0));
7771   pcmpeqw(vec3, vec1);
7772   ptest(vec2, vec3);
7773   jcc(Assembler::carryClear, FOUND_CHAR);
7774   addptr(result, 16);
7775   subl(tmp, stride);
7776   jccb(Assembler::notZero, SCAN_TO_8_CHAR_LOOP);
7777   bind(SCAN_TO_CHAR);
7778   testl(cnt1, cnt1);
7779   jcc(Assembler::zero, RET_NOT_FOUND);
7780   bind(SCAN_TO_CHAR_LOOP);
7781   load_unsigned_short(tmp, Address(result, 0));
7782   cmpl(ch, tmp);
7783   jccb(Assembler::equal, FOUND_SEQ_CHAR);
7784   addptr(result, 2);
7785   subl(cnt1, 1);
7786   jccb(Assembler::zero, RET_NOT_FOUND);
7787   jmp(SCAN_TO_CHAR_LOOP);
7788 
7789   bind(RET_NOT_FOUND);
7790   movl(result, -1);
7791   jmpb(DONE_LABEL);
7792 
7793   bind(FOUND_CHAR);
7794   if (UseAVX >= 2) {
7795     vpmovmskb(tmp, vec3);
7796   } else {
7797     pmovmskb(tmp, vec3);
7798   }
7799   bsfl(ch, tmp);
7800   addl(result, ch);
7801 
7802   bind(FOUND_SEQ_CHAR);
7803   subptr(result, str1);
7804   shrl(result, 1);
7805 
7806   bind(DONE_LABEL);
7807 } // string_indexof_char
7808 
7809 // helper function for string_compare
7810 void MacroAssembler::load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
7811                                         Address::ScaleFactor scale, Address::ScaleFactor scale1,
7812                                         Address::ScaleFactor scale2, Register index, int ae) {
7813   if (ae == StrIntrinsicNode::LL) {
7814     load_unsigned_byte(elem1, Address(str1, index, scale, 0));
7815     load_unsigned_byte(elem2, Address(str2, index, scale, 0));
7816   } else if (ae == StrIntrinsicNode::UU) {
7817     load_unsigned_short(elem1, Address(str1, index, scale, 0));
7818     load_unsigned_short(elem2, Address(str2, index, scale, 0));
7819   } else {
7820     load_unsigned_byte(elem1, Address(str1, index, scale1, 0));
7821     load_unsigned_short(elem2, Address(str2, index, scale2, 0));
7822   }
7823 }
7824 
7825 // Compare strings, used for char[] and byte[].
7826 void MacroAssembler::string_compare(Register str1, Register str2,
7827                                     Register cnt1, Register cnt2, Register result,
7828                                     XMMRegister vec1, int ae) {
7829   ShortBranchVerifier sbv(this);
7830   Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
7831   Label COMPARE_WIDE_VECTORS_LOOP_FAILED;  // used only _LP64 && AVX3
7832   int stride, stride2, adr_stride, adr_stride1, adr_stride2;
7833   int stride2x2 = 0x40;
7834   Address::ScaleFactor scale = Address::no_scale;
7835   Address::ScaleFactor scale1 = Address::no_scale;
7836   Address::ScaleFactor scale2 = Address::no_scale;
7837 
7838   if (ae != StrIntrinsicNode::LL) {
7839     stride2x2 = 0x20;
7840   }
7841 
7842   if (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL) {
7843     shrl(cnt2, 1);
7844   }
7845   // Compute the minimum of the string lengths and the
7846   // difference of the string lengths (stack).
7847   // Do the conditional move stuff
7848   movl(result, cnt1);
7849   subl(cnt1, cnt2);
7850   push(cnt1);
7851   cmov32(Assembler::lessEqual, cnt2, result);    // cnt2 = min(cnt1, cnt2)
7852 
7853   // Is the minimum length zero?
7854   testl(cnt2, cnt2);
7855   jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7856   if (ae == StrIntrinsicNode::LL) {
7857     // Load first bytes
7858     load_unsigned_byte(result, Address(str1, 0));  // result = str1[0]
7859     load_unsigned_byte(cnt1, Address(str2, 0));    // cnt1   = str2[0]
7860   } else if (ae == StrIntrinsicNode::UU) {
7861     // Load first characters
7862     load_unsigned_short(result, Address(str1, 0));
7863     load_unsigned_short(cnt1, Address(str2, 0));
7864   } else {
7865     load_unsigned_byte(result, Address(str1, 0));
7866     load_unsigned_short(cnt1, Address(str2, 0));
7867   }
7868   subl(result, cnt1);
7869   jcc(Assembler::notZero,  POP_LABEL);
7870 
7871   if (ae == StrIntrinsicNode::UU) {
7872     // Divide length by 2 to get number of chars
7873     shrl(cnt2, 1);
7874   }
7875   cmpl(cnt2, 1);
7876   jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7877 
7878   // Check if the strings start at the same location and setup scale and stride
7879   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7880     cmpptr(str1, str2);
7881     jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7882     if (ae == StrIntrinsicNode::LL) {
7883       scale = Address::times_1;
7884       stride = 16;
7885     } else {
7886       scale = Address::times_2;
7887       stride = 8;
7888     }
7889   } else {
7890     scale1 = Address::times_1;
7891     scale2 = Address::times_2;
7892     // scale not used
7893     stride = 8;
7894   }
7895 
7896   if (UseAVX >= 2 && UseSSE42Intrinsics) {
7897     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_WIDE_TAIL, COMPARE_SMALL_STR;
7898     Label COMPARE_WIDE_VECTORS_LOOP, COMPARE_16_CHARS, COMPARE_INDEX_CHAR;
7899     Label COMPARE_WIDE_VECTORS_LOOP_AVX2;
7900     Label COMPARE_TAIL_LONG;
7901     Label COMPARE_WIDE_VECTORS_LOOP_AVX3;  // used only _LP64 && AVX3
7902 
7903     int pcmpmask = 0x19;
7904     if (ae == StrIntrinsicNode::LL) {
7905       pcmpmask &= ~0x01;
7906     }
7907 
7908     // Setup to compare 16-chars (32-bytes) vectors,
7909     // start from first character again because it has aligned address.
7910     if (ae == StrIntrinsicNode::LL) {
7911       stride2 = 32;
7912     } else {
7913       stride2 = 16;
7914     }
7915     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7916       adr_stride = stride << scale;
7917     } else {
7918       adr_stride1 = 8;  //stride << scale1;
7919       adr_stride2 = 16; //stride << scale2;
7920     }
7921 
7922     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
7923     // rax and rdx are used by pcmpestri as elements counters
7924     movl(result, cnt2);
7925     andl(cnt2, ~(stride2-1));   // cnt2 holds the vector count
7926     jcc(Assembler::zero, COMPARE_TAIL_LONG);
7927 
7928     // fast path : compare first 2 8-char vectors.
7929     bind(COMPARE_16_CHARS);
7930     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7931       movdqu(vec1, Address(str1, 0));
7932     } else {
7933       pmovzxbw(vec1, Address(str1, 0));
7934     }
7935     pcmpestri(vec1, Address(str2, 0), pcmpmask);
7936     jccb(Assembler::below, COMPARE_INDEX_CHAR);
7937 
7938     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7939       movdqu(vec1, Address(str1, adr_stride));
7940       pcmpestri(vec1, Address(str2, adr_stride), pcmpmask);
7941     } else {
7942       pmovzxbw(vec1, Address(str1, adr_stride1));
7943       pcmpestri(vec1, Address(str2, adr_stride2), pcmpmask);
7944     }
7945     jccb(Assembler::aboveEqual, COMPARE_WIDE_VECTORS);
7946     addl(cnt1, stride);
7947 
7948     // Compare the characters at index in cnt1
7949     bind(COMPARE_INDEX_CHAR); // cnt1 has the offset of the mismatching character
7950     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
7951     subl(result, cnt2);
7952     jmp(POP_LABEL);
7953 
7954     // Setup the registers to start vector comparison loop
7955     bind(COMPARE_WIDE_VECTORS);
7956     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7957       lea(str1, Address(str1, result, scale));
7958       lea(str2, Address(str2, result, scale));
7959     } else {
7960       lea(str1, Address(str1, result, scale1));
7961       lea(str2, Address(str2, result, scale2));
7962     }
7963     subl(result, stride2);
7964     subl(cnt2, stride2);
7965     jcc(Assembler::zero, COMPARE_WIDE_TAIL);
7966     negptr(result);
7967 
7968     //  In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest)
7969     bind(COMPARE_WIDE_VECTORS_LOOP);
7970 
7971 #ifdef _LP64
7972     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
7973       cmpl(cnt2, stride2x2);
7974       jccb(Assembler::below, COMPARE_WIDE_VECTORS_LOOP_AVX2);
7975       testl(cnt2, stride2x2-1);   // cnt2 holds the vector count
7976       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX2);   // means we cannot subtract by 0x40
7977 
7978       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
7979       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7980         evmovdquq(vec1, Address(str1, result, scale), Assembler::AVX_512bit);
7981         evpcmpeqb(k7, vec1, Address(str2, result, scale), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7982       } else {
7983         vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_512bit);
7984         evpcmpeqb(k7, vec1, Address(str2, result, scale2), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7985       }
7986       kortestql(k7, k7);
7987       jcc(Assembler::aboveEqual, COMPARE_WIDE_VECTORS_LOOP_FAILED);     // miscompare
7988       addptr(result, stride2x2);  // update since we already compared at this addr
7989       subl(cnt2, stride2x2);      // and sub the size too
7990       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX3);
7991 
7992       vpxor(vec1, vec1);
7993       jmpb(COMPARE_WIDE_TAIL);
7994     }//if (VM_Version::supports_avx512vlbw())
7995 #endif // _LP64
7996 
7997 
7998     bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7999     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
8000       vmovdqu(vec1, Address(str1, result, scale));
8001       vpxor(vec1, Address(str2, result, scale));
8002     } else {
8003       vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_256bit);
8004       vpxor(vec1, Address(str2, result, scale2));
8005     }
8006     vptest(vec1, vec1);
8007     jcc(Assembler::notZero, VECTOR_NOT_EQUAL);
8008     addptr(result, stride2);
8009     subl(cnt2, stride2);
8010     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP);
8011     // clean upper bits of YMM registers
8012     vpxor(vec1, vec1);
8013 
8014     // compare wide vectors tail
8015     bind(COMPARE_WIDE_TAIL);
8016     testptr(result, result);
8017     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
8018 
8019     movl(result, stride2);
8020     movl(cnt2, result);
8021     negptr(result);
8022     jmp(COMPARE_WIDE_VECTORS_LOOP_AVX2);
8023 
8024     // Identifies the mismatching (higher or lower)16-bytes in the 32-byte vectors.
8025     bind(VECTOR_NOT_EQUAL);
8026     // clean upper bits of YMM registers
8027     vpxor(vec1, vec1);
8028     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
8029       lea(str1, Address(str1, result, scale));
8030       lea(str2, Address(str2, result, scale));
8031     } else {
8032       lea(str1, Address(str1, result, scale1));
8033       lea(str2, Address(str2, result, scale2));
8034     }
8035     jmp(COMPARE_16_CHARS);
8036 
8037     // Compare tail chars, length between 1 to 15 chars
8038     bind(COMPARE_TAIL_LONG);
8039     movl(cnt2, result);
8040     cmpl(cnt2, stride);
8041     jcc(Assembler::less, COMPARE_SMALL_STR);
8042 
8043     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
8044       movdqu(vec1, Address(str1, 0));
8045     } else {
8046       pmovzxbw(vec1, Address(str1, 0));
8047     }
8048     pcmpestri(vec1, Address(str2, 0), pcmpmask);
8049     jcc(Assembler::below, COMPARE_INDEX_CHAR);
8050     subptr(cnt2, stride);
8051     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
8052     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
8053       lea(str1, Address(str1, result, scale));
8054       lea(str2, Address(str2, result, scale));
8055     } else {
8056       lea(str1, Address(str1, result, scale1));
8057       lea(str2, Address(str2, result, scale2));
8058     }
8059     negptr(cnt2);
8060     jmpb(WHILE_HEAD_LABEL);
8061 
8062     bind(COMPARE_SMALL_STR);
8063   } else if (UseSSE42Intrinsics) {
8064     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
8065     int pcmpmask = 0x19;
8066     // Setup to compare 8-char (16-byte) vectors,
8067     // start from first character again because it has aligned address.
8068     movl(result, cnt2);
8069     andl(cnt2, ~(stride - 1));   // cnt2 holds the vector count
8070     if (ae == StrIntrinsicNode::LL) {
8071       pcmpmask &= ~0x01;
8072     }
8073     jcc(Assembler::zero, COMPARE_TAIL);
8074     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
8075       lea(str1, Address(str1, result, scale));
8076       lea(str2, Address(str2, result, scale));
8077     } else {
8078       lea(str1, Address(str1, result, scale1));
8079       lea(str2, Address(str2, result, scale2));
8080     }
8081     negptr(result);
8082 
8083     // pcmpestri
8084     //   inputs:
8085     //     vec1- substring
8086     //     rax - negative string length (elements count)
8087     //     mem - scanned string
8088     //     rdx - string length (elements count)
8089     //     pcmpmask - cmp mode: 11000 (string compare with negated result)
8090     //               + 00 (unsigned bytes) or  + 01 (unsigned shorts)
8091     //   outputs:
8092     //     rcx - first mismatched element index
8093     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
8094 
8095     bind(COMPARE_WIDE_VECTORS);
8096     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
8097       movdqu(vec1, Address(str1, result, scale));
8098       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
8099     } else {
8100       pmovzxbw(vec1, Address(str1, result, scale1));
8101       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
8102     }
8103     // After pcmpestri cnt1(rcx) contains mismatched element index
8104 
8105     jccb(Assembler::below, VECTOR_NOT_EQUAL);  // CF==1
8106     addptr(result, stride);
8107     subptr(cnt2, stride);
8108     jccb(Assembler::notZero, COMPARE_WIDE_VECTORS);
8109 
8110     // compare wide vectors tail
8111     testptr(result, result);
8112     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
8113 
8114     movl(cnt2, stride);
8115     movl(result, stride);
8116     negptr(result);
8117     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
8118       movdqu(vec1, Address(str1, result, scale));
8119       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
8120     } else {
8121       pmovzxbw(vec1, Address(str1, result, scale1));
8122       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
8123     }
8124     jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL);
8125 
8126     // Mismatched characters in the vectors
8127     bind(VECTOR_NOT_EQUAL);
8128     addptr(cnt1, result);
8129     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
8130     subl(result, cnt2);
8131     jmpb(POP_LABEL);
8132 
8133     bind(COMPARE_TAIL); // limit is zero
8134     movl(cnt2, result);
8135     // Fallthru to tail compare
8136   }
8137   // Shift str2 and str1 to the end of the arrays, negate min
8138   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
8139     lea(str1, Address(str1, cnt2, scale));
8140     lea(str2, Address(str2, cnt2, scale));
8141   } else {
8142     lea(str1, Address(str1, cnt2, scale1));
8143     lea(str2, Address(str2, cnt2, scale2));
8144   }
8145   decrementl(cnt2);  // first character was compared already
8146   negptr(cnt2);
8147 
8148   // Compare the rest of the elements
8149   bind(WHILE_HEAD_LABEL);
8150   load_next_elements(result, cnt1, str1, str2, scale, scale1, scale2, cnt2, ae);
8151   subl(result, cnt1);
8152   jccb(Assembler::notZero, POP_LABEL);
8153   increment(cnt2);
8154   jccb(Assembler::notZero, WHILE_HEAD_LABEL);
8155 
8156   // Strings are equal up to min length.  Return the length difference.
8157   bind(LENGTH_DIFF_LABEL);
8158   pop(result);
8159   if (ae == StrIntrinsicNode::UU) {
8160     // Divide diff by 2 to get number of chars
8161     sarl(result, 1);
8162   }
8163   jmpb(DONE_LABEL);
8164 
8165 #ifdef _LP64
8166   if (VM_Version::supports_avx512vlbw()) {
8167 
8168     bind(COMPARE_WIDE_VECTORS_LOOP_FAILED);
8169 
8170     kmovql(cnt1, k7);
8171     notq(cnt1);
8172     bsfq(cnt2, cnt1);
8173     if (ae != StrIntrinsicNode::LL) {
8174       // Divide diff by 2 to get number of chars
8175       sarl(cnt2, 1);
8176     }
8177     addq(result, cnt2);
8178     if (ae == StrIntrinsicNode::LL) {
8179       load_unsigned_byte(cnt1, Address(str2, result));
8180       load_unsigned_byte(result, Address(str1, result));
8181     } else if (ae == StrIntrinsicNode::UU) {
8182       load_unsigned_short(cnt1, Address(str2, result, scale));
8183       load_unsigned_short(result, Address(str1, result, scale));
8184     } else {
8185       load_unsigned_short(cnt1, Address(str2, result, scale2));
8186       load_unsigned_byte(result, Address(str1, result, scale1));
8187     }
8188     subl(result, cnt1);
8189     jmpb(POP_LABEL);
8190   }//if (VM_Version::supports_avx512vlbw())
8191 #endif // _LP64
8192 
8193   // Discard the stored length difference
8194   bind(POP_LABEL);
8195   pop(cnt1);
8196 
8197   // That's it
8198   bind(DONE_LABEL);
8199   if(ae == StrIntrinsicNode::UL) {
8200     negl(result);
8201   }
8202 
8203 }
8204 
8205 // Search for Non-ASCII character (Negative byte value) in a byte array,
8206 // return true if it has any and false otherwise.
8207 //   ..\jdk\src\java.base\share\classes\java\lang\StringCoding.java
8208 //   @HotSpotIntrinsicCandidate
8209 //   private static boolean hasNegatives(byte[] ba, int off, int len) {
8210 //     for (int i = off; i < off + len; i++) {
8211 //       if (ba[i] < 0) {
8212 //         return true;
8213 //       }
8214 //     }
8215 //     return false;
8216 //   }
8217 void MacroAssembler::has_negatives(Register ary1, Register len,
8218   Register result, Register tmp1,
8219   XMMRegister vec1, XMMRegister vec2) {
8220   // rsi: byte array
8221   // rcx: len
8222   // rax: result
8223   ShortBranchVerifier sbv(this);
8224   assert_different_registers(ary1, len, result, tmp1);
8225   assert_different_registers(vec1, vec2);
8226   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_CHAR, COMPARE_VECTORS, COMPARE_BYTE;
8227 
8228   // len == 0
8229   testl(len, len);
8230   jcc(Assembler::zero, FALSE_LABEL);
8231 
8232   if ((UseAVX > 2) && // AVX512
8233     VM_Version::supports_avx512vlbw() &&
8234     VM_Version::supports_bmi2()) {
8235 
8236     set_vector_masking();  // opening of the stub context for programming mask registers
8237 
8238     Label test_64_loop, test_tail;
8239     Register tmp3_aliased = len;
8240 
8241     movl(tmp1, len);
8242     vpxor(vec2, vec2, vec2, Assembler::AVX_512bit);
8243 
8244     andl(tmp1, 64 - 1);   // tail count (in chars) 0x3F
8245     andl(len, ~(64 - 1));    // vector count (in chars)
8246     jccb(Assembler::zero, test_tail);
8247 
8248     lea(ary1, Address(ary1, len, Address::times_1));
8249     negptr(len);
8250 
8251     bind(test_64_loop);
8252     // Check whether our 64 elements of size byte contain negatives
8253     evpcmpgtb(k2, vec2, Address(ary1, len, Address::times_1), Assembler::AVX_512bit);
8254     kortestql(k2, k2);
8255     jcc(Assembler::notZero, TRUE_LABEL);
8256 
8257     addptr(len, 64);
8258     jccb(Assembler::notZero, test_64_loop);
8259 
8260 
8261     bind(test_tail);
8262     // bail out when there is nothing to be done
8263     testl(tmp1, -1);
8264     jcc(Assembler::zero, FALSE_LABEL);
8265 
8266     // Save k1
8267     kmovql(k3, k1);
8268 
8269     // ~(~0 << len) applied up to two times (for 32-bit scenario)
8270 #ifdef _LP64
8271     mov64(tmp3_aliased, 0xFFFFFFFFFFFFFFFF);
8272     shlxq(tmp3_aliased, tmp3_aliased, tmp1);
8273     notq(tmp3_aliased);
8274     kmovql(k1, tmp3_aliased);
8275 #else
8276     Label k_init;
8277     jmp(k_init);
8278 
8279     // We could not read 64-bits from a general purpose register thus we move
8280     // data required to compose 64 1's to the instruction stream
8281     // We emit 64 byte wide series of elements from 0..63 which later on would
8282     // be used as a compare targets with tail count contained in tmp1 register.
8283     // Result would be a k1 register having tmp1 consecutive number or 1
8284     // counting from least significant bit.
8285     address tmp = pc();
8286     emit_int64(0x0706050403020100);
8287     emit_int64(0x0F0E0D0C0B0A0908);
8288     emit_int64(0x1716151413121110);
8289     emit_int64(0x1F1E1D1C1B1A1918);
8290     emit_int64(0x2726252423222120);
8291     emit_int64(0x2F2E2D2C2B2A2928);
8292     emit_int64(0x3736353433323130);
8293     emit_int64(0x3F3E3D3C3B3A3938);
8294 
8295     bind(k_init);
8296     lea(len, InternalAddress(tmp));
8297     // create mask to test for negative byte inside a vector
8298     evpbroadcastb(vec1, tmp1, Assembler::AVX_512bit);
8299     evpcmpgtb(k1, vec1, Address(len, 0), Assembler::AVX_512bit);
8300 
8301 #endif
8302     evpcmpgtb(k2, k1, vec2, Address(ary1, 0), Assembler::AVX_512bit);
8303     ktestq(k2, k1);
8304     // Restore k1
8305     kmovql(k1, k3);
8306     jcc(Assembler::notZero, TRUE_LABEL);
8307 
8308     jmp(FALSE_LABEL);
8309 
8310     clear_vector_masking();   // closing of the stub context for programming mask registers
8311   } else {
8312     movl(result, len); // copy
8313 
8314     if (UseAVX == 2 && UseSSE >= 2) {
8315       // With AVX2, use 32-byte vector compare
8316       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8317 
8318       // Compare 32-byte vectors
8319       andl(result, 0x0000001f);  //   tail count (in bytes)
8320       andl(len, 0xffffffe0);   // vector count (in bytes)
8321       jccb(Assembler::zero, COMPARE_TAIL);
8322 
8323       lea(ary1, Address(ary1, len, Address::times_1));
8324       negptr(len);
8325 
8326       movl(tmp1, 0x80808080);   // create mask to test for Unicode chars in vector
8327       movdl(vec2, tmp1);
8328       vpbroadcastd(vec2, vec2);
8329 
8330       bind(COMPARE_WIDE_VECTORS);
8331       vmovdqu(vec1, Address(ary1, len, Address::times_1));
8332       vptest(vec1, vec2);
8333       jccb(Assembler::notZero, TRUE_LABEL);
8334       addptr(len, 32);
8335       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8336 
8337       testl(result, result);
8338       jccb(Assembler::zero, FALSE_LABEL);
8339 
8340       vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
8341       vptest(vec1, vec2);
8342       jccb(Assembler::notZero, TRUE_LABEL);
8343       jmpb(FALSE_LABEL);
8344 
8345       bind(COMPARE_TAIL); // len is zero
8346       movl(len, result);
8347       // Fallthru to tail compare
8348     } else if (UseSSE42Intrinsics) {
8349       // With SSE4.2, use double quad vector compare
8350       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8351 
8352       // Compare 16-byte vectors
8353       andl(result, 0x0000000f);  //   tail count (in bytes)
8354       andl(len, 0xfffffff0);   // vector count (in bytes)
8355       jccb(Assembler::zero, COMPARE_TAIL);
8356 
8357       lea(ary1, Address(ary1, len, Address::times_1));
8358       negptr(len);
8359 
8360       movl(tmp1, 0x80808080);
8361       movdl(vec2, tmp1);
8362       pshufd(vec2, vec2, 0);
8363 
8364       bind(COMPARE_WIDE_VECTORS);
8365       movdqu(vec1, Address(ary1, len, Address::times_1));
8366       ptest(vec1, vec2);
8367       jccb(Assembler::notZero, TRUE_LABEL);
8368       addptr(len, 16);
8369       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8370 
8371       testl(result, result);
8372       jccb(Assembler::zero, FALSE_LABEL);
8373 
8374       movdqu(vec1, Address(ary1, result, Address::times_1, -16));
8375       ptest(vec1, vec2);
8376       jccb(Assembler::notZero, TRUE_LABEL);
8377       jmpb(FALSE_LABEL);
8378 
8379       bind(COMPARE_TAIL); // len is zero
8380       movl(len, result);
8381       // Fallthru to tail compare
8382     }
8383   }
8384   // Compare 4-byte vectors
8385   andl(len, 0xfffffffc); // vector count (in bytes)
8386   jccb(Assembler::zero, COMPARE_CHAR);
8387 
8388   lea(ary1, Address(ary1, len, Address::times_1));
8389   negptr(len);
8390 
8391   bind(COMPARE_VECTORS);
8392   movl(tmp1, Address(ary1, len, Address::times_1));
8393   andl(tmp1, 0x80808080);
8394   jccb(Assembler::notZero, TRUE_LABEL);
8395   addptr(len, 4);
8396   jcc(Assembler::notZero, COMPARE_VECTORS);
8397 
8398   // Compare trailing char (final 2 bytes), if any
8399   bind(COMPARE_CHAR);
8400   testl(result, 0x2);   // tail  char
8401   jccb(Assembler::zero, COMPARE_BYTE);
8402   load_unsigned_short(tmp1, Address(ary1, 0));
8403   andl(tmp1, 0x00008080);
8404   jccb(Assembler::notZero, TRUE_LABEL);
8405   subptr(result, 2);
8406   lea(ary1, Address(ary1, 2));
8407 
8408   bind(COMPARE_BYTE);
8409   testl(result, 0x1);   // tail  byte
8410   jccb(Assembler::zero, FALSE_LABEL);
8411   load_unsigned_byte(tmp1, Address(ary1, 0));
8412   andl(tmp1, 0x00000080);
8413   jccb(Assembler::notEqual, TRUE_LABEL);
8414   jmpb(FALSE_LABEL);
8415 
8416   bind(TRUE_LABEL);
8417   movl(result, 1);   // return true
8418   jmpb(DONE);
8419 
8420   bind(FALSE_LABEL);
8421   xorl(result, result); // return false
8422 
8423   // That's it
8424   bind(DONE);
8425   if (UseAVX >= 2 && UseSSE >= 2) {
8426     // clean upper bits of YMM registers
8427     vpxor(vec1, vec1);
8428     vpxor(vec2, vec2);
8429   }
8430 }
8431 // Compare char[] or byte[] arrays aligned to 4 bytes or substrings.
8432 void MacroAssembler::arrays_equals(bool is_array_equ, Register ary1, Register ary2,
8433                                    Register limit, Register result, Register chr,
8434                                    XMMRegister vec1, XMMRegister vec2, bool is_char) {
8435   ShortBranchVerifier sbv(this);
8436   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR, COMPARE_BYTE;
8437 
8438   int length_offset  = arrayOopDesc::length_offset_in_bytes();
8439   int base_offset    = arrayOopDesc::base_offset_in_bytes(is_char ? T_CHAR : T_BYTE);
8440 
8441   if (is_array_equ) {
8442     // Check the input args
8443     cmpoop(ary1, ary2);
8444     jcc(Assembler::equal, TRUE_LABEL);
8445 
8446     // Need additional checks for arrays_equals.
8447     testptr(ary1, ary1);
8448     jcc(Assembler::zero, FALSE_LABEL);
8449     testptr(ary2, ary2);
8450     jcc(Assembler::zero, FALSE_LABEL);
8451 
8452     // Check the lengths
8453     movl(limit, Address(ary1, length_offset));
8454     cmpl(limit, Address(ary2, length_offset));
8455     jcc(Assembler::notEqual, FALSE_LABEL);
8456   }
8457 
8458   // count == 0
8459   testl(limit, limit);
8460   jcc(Assembler::zero, TRUE_LABEL);
8461 
8462   if (is_array_equ) {
8463     // Load array address
8464     lea(ary1, Address(ary1, base_offset));
8465     lea(ary2, Address(ary2, base_offset));
8466   }
8467 
8468   if (is_array_equ && is_char) {
8469     // arrays_equals when used for char[].
8470     shll(limit, 1);      // byte count != 0
8471   }
8472   movl(result, limit); // copy
8473 
8474   if (UseAVX >= 2) {
8475     // With AVX2, use 32-byte vector compare
8476     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8477 
8478     // Compare 32-byte vectors
8479     andl(result, 0x0000001f);  //   tail count (in bytes)
8480     andl(limit, 0xffffffe0);   // vector count (in bytes)
8481     jcc(Assembler::zero, COMPARE_TAIL);
8482 
8483     lea(ary1, Address(ary1, limit, Address::times_1));
8484     lea(ary2, Address(ary2, limit, Address::times_1));
8485     negptr(limit);
8486 
8487     bind(COMPARE_WIDE_VECTORS);
8488 
8489 #ifdef _LP64
8490     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
8491       Label COMPARE_WIDE_VECTORS_LOOP_AVX2, COMPARE_WIDE_VECTORS_LOOP_AVX3;
8492 
8493       cmpl(limit, -64);
8494       jccb(Assembler::greater, COMPARE_WIDE_VECTORS_LOOP_AVX2);
8495 
8496       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
8497 
8498       evmovdquq(vec1, Address(ary1, limit, Address::times_1), Assembler::AVX_512bit);
8499       evpcmpeqb(k7, vec1, Address(ary2, limit, Address::times_1), Assembler::AVX_512bit);
8500       kortestql(k7, k7);
8501       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
8502       addptr(limit, 64);  // update since we already compared at this addr
8503       cmpl(limit, -64);
8504       jccb(Assembler::lessEqual, COMPARE_WIDE_VECTORS_LOOP_AVX3);
8505 
8506       // At this point we may still need to compare -limit+result bytes.
8507       // We could execute the next two instruction and just continue via non-wide path:
8508       //  cmpl(limit, 0);
8509       //  jcc(Assembler::equal, COMPARE_TAIL);  // true
8510       // But since we stopped at the points ary{1,2}+limit which are
8511       // not farther than 64 bytes from the ends of arrays ary{1,2}+result
8512       // (|limit| <= 32 and result < 32),
8513       // we may just compare the last 64 bytes.
8514       //
8515       addptr(result, -64);   // it is safe, bc we just came from this area
8516       evmovdquq(vec1, Address(ary1, result, Address::times_1), Assembler::AVX_512bit);
8517       evpcmpeqb(k7, vec1, Address(ary2, result, Address::times_1), Assembler::AVX_512bit);
8518       kortestql(k7, k7);
8519       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
8520 
8521       jmp(TRUE_LABEL);
8522 
8523       bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
8524 
8525     }//if (VM_Version::supports_avx512vlbw())
8526 #endif //_LP64
8527 
8528     vmovdqu(vec1, Address(ary1, limit, Address::times_1));
8529     vmovdqu(vec2, Address(ary2, limit, Address::times_1));
8530     vpxor(vec1, vec2);
8531 
8532     vptest(vec1, vec1);
8533     jcc(Assembler::notZero, FALSE_LABEL);
8534     addptr(limit, 32);
8535     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8536 
8537     testl(result, result);
8538     jcc(Assembler::zero, TRUE_LABEL);
8539 
8540     vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
8541     vmovdqu(vec2, Address(ary2, result, Address::times_1, -32));
8542     vpxor(vec1, vec2);
8543 
8544     vptest(vec1, vec1);
8545     jccb(Assembler::notZero, FALSE_LABEL);
8546     jmpb(TRUE_LABEL);
8547 
8548     bind(COMPARE_TAIL); // limit is zero
8549     movl(limit, result);
8550     // Fallthru to tail compare
8551   } else if (UseSSE42Intrinsics) {
8552     // With SSE4.2, use double quad vector compare
8553     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8554 
8555     // Compare 16-byte vectors
8556     andl(result, 0x0000000f);  //   tail count (in bytes)
8557     andl(limit, 0xfffffff0);   // vector count (in bytes)
8558     jcc(Assembler::zero, COMPARE_TAIL);
8559 
8560     lea(ary1, Address(ary1, limit, Address::times_1));
8561     lea(ary2, Address(ary2, limit, Address::times_1));
8562     negptr(limit);
8563 
8564     bind(COMPARE_WIDE_VECTORS);
8565     movdqu(vec1, Address(ary1, limit, Address::times_1));
8566     movdqu(vec2, Address(ary2, limit, Address::times_1));
8567     pxor(vec1, vec2);
8568 
8569     ptest(vec1, vec1);
8570     jcc(Assembler::notZero, FALSE_LABEL);
8571     addptr(limit, 16);
8572     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8573 
8574     testl(result, result);
8575     jcc(Assembler::zero, TRUE_LABEL);
8576 
8577     movdqu(vec1, Address(ary1, result, Address::times_1, -16));
8578     movdqu(vec2, Address(ary2, result, Address::times_1, -16));
8579     pxor(vec1, vec2);
8580 
8581     ptest(vec1, vec1);
8582     jccb(Assembler::notZero, FALSE_LABEL);
8583     jmpb(TRUE_LABEL);
8584 
8585     bind(COMPARE_TAIL); // limit is zero
8586     movl(limit, result);
8587     // Fallthru to tail compare
8588   }
8589 
8590   // Compare 4-byte vectors
8591   andl(limit, 0xfffffffc); // vector count (in bytes)
8592   jccb(Assembler::zero, COMPARE_CHAR);
8593 
8594   lea(ary1, Address(ary1, limit, Address::times_1));
8595   lea(ary2, Address(ary2, limit, Address::times_1));
8596   negptr(limit);
8597 
8598   bind(COMPARE_VECTORS);
8599   movl(chr, Address(ary1, limit, Address::times_1));
8600   cmpl(chr, Address(ary2, limit, Address::times_1));
8601   jccb(Assembler::notEqual, FALSE_LABEL);
8602   addptr(limit, 4);
8603   jcc(Assembler::notZero, COMPARE_VECTORS);
8604 
8605   // Compare trailing char (final 2 bytes), if any
8606   bind(COMPARE_CHAR);
8607   testl(result, 0x2);   // tail  char
8608   jccb(Assembler::zero, COMPARE_BYTE);
8609   load_unsigned_short(chr, Address(ary1, 0));
8610   load_unsigned_short(limit, Address(ary2, 0));
8611   cmpl(chr, limit);
8612   jccb(Assembler::notEqual, FALSE_LABEL);
8613 
8614   if (is_array_equ && is_char) {
8615     bind(COMPARE_BYTE);
8616   } else {
8617     lea(ary1, Address(ary1, 2));
8618     lea(ary2, Address(ary2, 2));
8619 
8620     bind(COMPARE_BYTE);
8621     testl(result, 0x1);   // tail  byte
8622     jccb(Assembler::zero, TRUE_LABEL);
8623     load_unsigned_byte(chr, Address(ary1, 0));
8624     load_unsigned_byte(limit, Address(ary2, 0));
8625     cmpl(chr, limit);
8626     jccb(Assembler::notEqual, FALSE_LABEL);
8627   }
8628   bind(TRUE_LABEL);
8629   movl(result, 1);   // return true
8630   jmpb(DONE);
8631 
8632   bind(FALSE_LABEL);
8633   xorl(result, result); // return false
8634 
8635   // That's it
8636   bind(DONE);
8637   if (UseAVX >= 2) {
8638     // clean upper bits of YMM registers
8639     vpxor(vec1, vec1);
8640     vpxor(vec2, vec2);
8641   }
8642 }
8643 
8644 #endif
8645 
8646 void MacroAssembler::generate_fill(BasicType t, bool aligned,
8647                                    Register to, Register value, Register count,
8648                                    Register rtmp, XMMRegister xtmp) {
8649   ShortBranchVerifier sbv(this);
8650   assert_different_registers(to, value, count, rtmp);
8651   Label L_exit, L_skip_align1, L_skip_align2, L_fill_byte;
8652   Label L_fill_2_bytes, L_fill_4_bytes;
8653 
8654   int shift = -1;
8655   switch (t) {
8656     case T_BYTE:
8657       shift = 2;
8658       break;
8659     case T_SHORT:
8660       shift = 1;
8661       break;
8662     case T_INT:
8663       shift = 0;
8664       break;
8665     default: ShouldNotReachHere();
8666   }
8667 
8668   if (t == T_BYTE) {
8669     andl(value, 0xff);
8670     movl(rtmp, value);
8671     shll(rtmp, 8);
8672     orl(value, rtmp);
8673   }
8674   if (t == T_SHORT) {
8675     andl(value, 0xffff);
8676   }
8677   if (t == T_BYTE || t == T_SHORT) {
8678     movl(rtmp, value);
8679     shll(rtmp, 16);
8680     orl(value, rtmp);
8681   }
8682 
8683   cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
8684   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
8685   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
8686     // align source address at 4 bytes address boundary
8687     if (t == T_BYTE) {
8688       // One byte misalignment happens only for byte arrays
8689       testptr(to, 1);
8690       jccb(Assembler::zero, L_skip_align1);
8691       movb(Address(to, 0), value);
8692       increment(to);
8693       decrement(count);
8694       BIND(L_skip_align1);
8695     }
8696     // Two bytes misalignment happens only for byte and short (char) arrays
8697     testptr(to, 2);
8698     jccb(Assembler::zero, L_skip_align2);
8699     movw(Address(to, 0), value);
8700     addptr(to, 2);
8701     subl(count, 1<<(shift-1));
8702     BIND(L_skip_align2);
8703   }
8704   if (UseSSE < 2) {
8705     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8706     // Fill 32-byte chunks
8707     subl(count, 8 << shift);
8708     jcc(Assembler::less, L_check_fill_8_bytes);
8709     align(16);
8710 
8711     BIND(L_fill_32_bytes_loop);
8712 
8713     for (int i = 0; i < 32; i += 4) {
8714       movl(Address(to, i), value);
8715     }
8716 
8717     addptr(to, 32);
8718     subl(count, 8 << shift);
8719     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8720     BIND(L_check_fill_8_bytes);
8721     addl(count, 8 << shift);
8722     jccb(Assembler::zero, L_exit);
8723     jmpb(L_fill_8_bytes);
8724 
8725     //
8726     // length is too short, just fill qwords
8727     //
8728     BIND(L_fill_8_bytes_loop);
8729     movl(Address(to, 0), value);
8730     movl(Address(to, 4), value);
8731     addptr(to, 8);
8732     BIND(L_fill_8_bytes);
8733     subl(count, 1 << (shift + 1));
8734     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8735     // fall through to fill 4 bytes
8736   } else {
8737     Label L_fill_32_bytes;
8738     if (!UseUnalignedLoadStores) {
8739       // align to 8 bytes, we know we are 4 byte aligned to start
8740       testptr(to, 4);
8741       jccb(Assembler::zero, L_fill_32_bytes);
8742       movl(Address(to, 0), value);
8743       addptr(to, 4);
8744       subl(count, 1<<shift);
8745     }
8746     BIND(L_fill_32_bytes);
8747     {
8748       assert( UseSSE >= 2, "supported cpu only" );
8749       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8750       if (UseAVX > 2) {
8751         movl(rtmp, 0xffff);
8752         kmovwl(k1, rtmp);
8753       }
8754       movdl(xtmp, value);
8755       if (UseAVX > 2 && UseUnalignedLoadStores) {
8756         // Fill 64-byte chunks
8757         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8758         evpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
8759 
8760         subl(count, 16 << shift);
8761         jcc(Assembler::less, L_check_fill_32_bytes);
8762         align(16);
8763 
8764         BIND(L_fill_64_bytes_loop);
8765         evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
8766         addptr(to, 64);
8767         subl(count, 16 << shift);
8768         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8769 
8770         BIND(L_check_fill_32_bytes);
8771         addl(count, 8 << shift);
8772         jccb(Assembler::less, L_check_fill_8_bytes);
8773         vmovdqu(Address(to, 0), xtmp);
8774         addptr(to, 32);
8775         subl(count, 8 << shift);
8776 
8777         BIND(L_check_fill_8_bytes);
8778       } else if (UseAVX == 2 && UseUnalignedLoadStores) {
8779         // Fill 64-byte chunks
8780         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8781         vpbroadcastd(xtmp, xtmp);
8782 
8783         subl(count, 16 << shift);
8784         jcc(Assembler::less, L_check_fill_32_bytes);
8785         align(16);
8786 
8787         BIND(L_fill_64_bytes_loop);
8788         vmovdqu(Address(to, 0), xtmp);
8789         vmovdqu(Address(to, 32), xtmp);
8790         addptr(to, 64);
8791         subl(count, 16 << shift);
8792         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8793 
8794         BIND(L_check_fill_32_bytes);
8795         addl(count, 8 << shift);
8796         jccb(Assembler::less, L_check_fill_8_bytes);
8797         vmovdqu(Address(to, 0), xtmp);
8798         addptr(to, 32);
8799         subl(count, 8 << shift);
8800 
8801         BIND(L_check_fill_8_bytes);
8802         // clean upper bits of YMM registers
8803         movdl(xtmp, value);
8804         pshufd(xtmp, xtmp, 0);
8805       } else {
8806         // Fill 32-byte chunks
8807         pshufd(xtmp, xtmp, 0);
8808 
8809         subl(count, 8 << shift);
8810         jcc(Assembler::less, L_check_fill_8_bytes);
8811         align(16);
8812 
8813         BIND(L_fill_32_bytes_loop);
8814 
8815         if (UseUnalignedLoadStores) {
8816           movdqu(Address(to, 0), xtmp);
8817           movdqu(Address(to, 16), xtmp);
8818         } else {
8819           movq(Address(to, 0), xtmp);
8820           movq(Address(to, 8), xtmp);
8821           movq(Address(to, 16), xtmp);
8822           movq(Address(to, 24), xtmp);
8823         }
8824 
8825         addptr(to, 32);
8826         subl(count, 8 << shift);
8827         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8828 
8829         BIND(L_check_fill_8_bytes);
8830       }
8831       addl(count, 8 << shift);
8832       jccb(Assembler::zero, L_exit);
8833       jmpb(L_fill_8_bytes);
8834 
8835       //
8836       // length is too short, just fill qwords
8837       //
8838       BIND(L_fill_8_bytes_loop);
8839       movq(Address(to, 0), xtmp);
8840       addptr(to, 8);
8841       BIND(L_fill_8_bytes);
8842       subl(count, 1 << (shift + 1));
8843       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8844     }
8845   }
8846   // fill trailing 4 bytes
8847   BIND(L_fill_4_bytes);
8848   testl(count, 1<<shift);
8849   jccb(Assembler::zero, L_fill_2_bytes);
8850   movl(Address(to, 0), value);
8851   if (t == T_BYTE || t == T_SHORT) {
8852     addptr(to, 4);
8853     BIND(L_fill_2_bytes);
8854     // fill trailing 2 bytes
8855     testl(count, 1<<(shift-1));
8856     jccb(Assembler::zero, L_fill_byte);
8857     movw(Address(to, 0), value);
8858     if (t == T_BYTE) {
8859       addptr(to, 2);
8860       BIND(L_fill_byte);
8861       // fill trailing byte
8862       testl(count, 1);
8863       jccb(Assembler::zero, L_exit);
8864       movb(Address(to, 0), value);
8865     } else {
8866       BIND(L_fill_byte);
8867     }
8868   } else {
8869     BIND(L_fill_2_bytes);
8870   }
8871   BIND(L_exit);
8872 }
8873 
8874 // encode char[] to byte[] in ISO_8859_1
8875    //@HotSpotIntrinsicCandidate
8876    //private static int implEncodeISOArray(byte[] sa, int sp,
8877    //byte[] da, int dp, int len) {
8878    //  int i = 0;
8879    //  for (; i < len; i++) {
8880    //    char c = StringUTF16.getChar(sa, sp++);
8881    //    if (c > '\u00FF')
8882    //      break;
8883    //    da[dp++] = (byte)c;
8884    //  }
8885    //  return i;
8886    //}
8887 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
8888   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
8889   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
8890   Register tmp5, Register result) {
8891 
8892   // rsi: src
8893   // rdi: dst
8894   // rdx: len
8895   // rcx: tmp5
8896   // rax: result
8897   ShortBranchVerifier sbv(this);
8898   assert_different_registers(src, dst, len, tmp5, result);
8899   Label L_done, L_copy_1_char, L_copy_1_char_exit;
8900 
8901   // set result
8902   xorl(result, result);
8903   // check for zero length
8904   testl(len, len);
8905   jcc(Assembler::zero, L_done);
8906 
8907   movl(result, len);
8908 
8909   // Setup pointers
8910   lea(src, Address(src, len, Address::times_2)); // char[]
8911   lea(dst, Address(dst, len, Address::times_1)); // byte[]
8912   negptr(len);
8913 
8914   if (UseSSE42Intrinsics || UseAVX >= 2) {
8915     Label L_chars_8_check, L_copy_8_chars, L_copy_8_chars_exit;
8916     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
8917 
8918     if (UseAVX >= 2) {
8919       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
8920       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8921       movdl(tmp1Reg, tmp5);
8922       vpbroadcastd(tmp1Reg, tmp1Reg);
8923       jmp(L_chars_32_check);
8924 
8925       bind(L_copy_32_chars);
8926       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
8927       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
8928       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8929       vptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8930       jccb(Assembler::notZero, L_copy_32_chars_exit);
8931       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8932       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
8933       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
8934 
8935       bind(L_chars_32_check);
8936       addptr(len, 32);
8937       jcc(Assembler::lessEqual, L_copy_32_chars);
8938 
8939       bind(L_copy_32_chars_exit);
8940       subptr(len, 16);
8941       jccb(Assembler::greater, L_copy_16_chars_exit);
8942 
8943     } else if (UseSSE42Intrinsics) {
8944       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8945       movdl(tmp1Reg, tmp5);
8946       pshufd(tmp1Reg, tmp1Reg, 0);
8947       jmpb(L_chars_16_check);
8948     }
8949 
8950     bind(L_copy_16_chars);
8951     if (UseAVX >= 2) {
8952       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
8953       vptest(tmp2Reg, tmp1Reg);
8954       jcc(Assembler::notZero, L_copy_16_chars_exit);
8955       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
8956       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
8957     } else {
8958       if (UseAVX > 0) {
8959         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8960         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8961         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
8962       } else {
8963         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8964         por(tmp2Reg, tmp3Reg);
8965         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8966         por(tmp2Reg, tmp4Reg);
8967       }
8968       ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8969       jccb(Assembler::notZero, L_copy_16_chars_exit);
8970       packuswb(tmp3Reg, tmp4Reg);
8971     }
8972     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
8973 
8974     bind(L_chars_16_check);
8975     addptr(len, 16);
8976     jcc(Assembler::lessEqual, L_copy_16_chars);
8977 
8978     bind(L_copy_16_chars_exit);
8979     if (UseAVX >= 2) {
8980       // clean upper bits of YMM registers
8981       vpxor(tmp2Reg, tmp2Reg);
8982       vpxor(tmp3Reg, tmp3Reg);
8983       vpxor(tmp4Reg, tmp4Reg);
8984       movdl(tmp1Reg, tmp5);
8985       pshufd(tmp1Reg, tmp1Reg, 0);
8986     }
8987     subptr(len, 8);
8988     jccb(Assembler::greater, L_copy_8_chars_exit);
8989 
8990     bind(L_copy_8_chars);
8991     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
8992     ptest(tmp3Reg, tmp1Reg);
8993     jccb(Assembler::notZero, L_copy_8_chars_exit);
8994     packuswb(tmp3Reg, tmp1Reg);
8995     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
8996     addptr(len, 8);
8997     jccb(Assembler::lessEqual, L_copy_8_chars);
8998 
8999     bind(L_copy_8_chars_exit);
9000     subptr(len, 8);
9001     jccb(Assembler::zero, L_done);
9002   }
9003 
9004   bind(L_copy_1_char);
9005   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
9006   testl(tmp5, 0xff00);      // check if Unicode char
9007   jccb(Assembler::notZero, L_copy_1_char_exit);
9008   movb(Address(dst, len, Address::times_1, 0), tmp5);
9009   addptr(len, 1);
9010   jccb(Assembler::less, L_copy_1_char);
9011 
9012   bind(L_copy_1_char_exit);
9013   addptr(result, len); // len is negative count of not processed elements
9014 
9015   bind(L_done);
9016 }
9017 
9018 #ifdef _LP64
9019 /**
9020  * Helper for multiply_to_len().
9021  */
9022 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
9023   addq(dest_lo, src1);
9024   adcq(dest_hi, 0);
9025   addq(dest_lo, src2);
9026   adcq(dest_hi, 0);
9027 }
9028 
9029 /**
9030  * Multiply 64 bit by 64 bit first loop.
9031  */
9032 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
9033                                            Register y, Register y_idx, Register z,
9034                                            Register carry, Register product,
9035                                            Register idx, Register kdx) {
9036   //
9037   //  jlong carry, x[], y[], z[];
9038   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
9039   //    huge_128 product = y[idx] * x[xstart] + carry;
9040   //    z[kdx] = (jlong)product;
9041   //    carry  = (jlong)(product >>> 64);
9042   //  }
9043   //  z[xstart] = carry;
9044   //
9045 
9046   Label L_first_loop, L_first_loop_exit;
9047   Label L_one_x, L_one_y, L_multiply;
9048 
9049   decrementl(xstart);
9050   jcc(Assembler::negative, L_one_x);
9051 
9052   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
9053   rorq(x_xstart, 32); // convert big-endian to little-endian
9054 
9055   bind(L_first_loop);
9056   decrementl(idx);
9057   jcc(Assembler::negative, L_first_loop_exit);
9058   decrementl(idx);
9059   jcc(Assembler::negative, L_one_y);
9060   movq(y_idx, Address(y, idx, Address::times_4,  0));
9061   rorq(y_idx, 32); // convert big-endian to little-endian
9062   bind(L_multiply);
9063   movq(product, x_xstart);
9064   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
9065   addq(product, carry);
9066   adcq(rdx, 0);
9067   subl(kdx, 2);
9068   movl(Address(z, kdx, Address::times_4,  4), product);
9069   shrq(product, 32);
9070   movl(Address(z, kdx, Address::times_4,  0), product);
9071   movq(carry, rdx);
9072   jmp(L_first_loop);
9073 
9074   bind(L_one_y);
9075   movl(y_idx, Address(y,  0));
9076   jmp(L_multiply);
9077 
9078   bind(L_one_x);
9079   movl(x_xstart, Address(x,  0));
9080   jmp(L_first_loop);
9081 
9082   bind(L_first_loop_exit);
9083 }
9084 
9085 /**
9086  * Multiply 64 bit by 64 bit and add 128 bit.
9087  */
9088 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
9089                                             Register yz_idx, Register idx,
9090                                             Register carry, Register product, int offset) {
9091   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
9092   //     z[kdx] = (jlong)product;
9093 
9094   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
9095   rorq(yz_idx, 32); // convert big-endian to little-endian
9096   movq(product, x_xstart);
9097   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
9098   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
9099   rorq(yz_idx, 32); // convert big-endian to little-endian
9100 
9101   add2_with_carry(rdx, product, carry, yz_idx);
9102 
9103   movl(Address(z, idx, Address::times_4,  offset+4), product);
9104   shrq(product, 32);
9105   movl(Address(z, idx, Address::times_4,  offset), product);
9106 
9107 }
9108 
9109 /**
9110  * Multiply 128 bit by 128 bit. Unrolled inner loop.
9111  */
9112 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
9113                                              Register yz_idx, Register idx, Register jdx,
9114                                              Register carry, Register product,
9115                                              Register carry2) {
9116   //   jlong carry, x[], y[], z[];
9117   //   int kdx = ystart+1;
9118   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
9119   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
9120   //     z[kdx+idx+1] = (jlong)product;
9121   //     jlong carry2  = (jlong)(product >>> 64);
9122   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
9123   //     z[kdx+idx] = (jlong)product;
9124   //     carry  = (jlong)(product >>> 64);
9125   //   }
9126   //   idx += 2;
9127   //   if (idx > 0) {
9128   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
9129   //     z[kdx+idx] = (jlong)product;
9130   //     carry  = (jlong)(product >>> 64);
9131   //   }
9132   //
9133 
9134   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
9135 
9136   movl(jdx, idx);
9137   andl(jdx, 0xFFFFFFFC);
9138   shrl(jdx, 2);
9139 
9140   bind(L_third_loop);
9141   subl(jdx, 1);
9142   jcc(Assembler::negative, L_third_loop_exit);
9143   subl(idx, 4);
9144 
9145   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
9146   movq(carry2, rdx);
9147 
9148   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
9149   movq(carry, rdx);
9150   jmp(L_third_loop);
9151 
9152   bind (L_third_loop_exit);
9153 
9154   andl (idx, 0x3);
9155   jcc(Assembler::zero, L_post_third_loop_done);
9156 
9157   Label L_check_1;
9158   subl(idx, 2);
9159   jcc(Assembler::negative, L_check_1);
9160 
9161   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
9162   movq(carry, rdx);
9163 
9164   bind (L_check_1);
9165   addl (idx, 0x2);
9166   andl (idx, 0x1);
9167   subl(idx, 1);
9168   jcc(Assembler::negative, L_post_third_loop_done);
9169 
9170   movl(yz_idx, Address(y, idx, Address::times_4,  0));
9171   movq(product, x_xstart);
9172   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
9173   movl(yz_idx, Address(z, idx, Address::times_4,  0));
9174 
9175   add2_with_carry(rdx, product, yz_idx, carry);
9176 
9177   movl(Address(z, idx, Address::times_4,  0), product);
9178   shrq(product, 32);
9179 
9180   shlq(rdx, 32);
9181   orq(product, rdx);
9182   movq(carry, product);
9183 
9184   bind(L_post_third_loop_done);
9185 }
9186 
9187 /**
9188  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
9189  *
9190  */
9191 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
9192                                                   Register carry, Register carry2,
9193                                                   Register idx, Register jdx,
9194                                                   Register yz_idx1, Register yz_idx2,
9195                                                   Register tmp, Register tmp3, Register tmp4) {
9196   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
9197 
9198   //   jlong carry, x[], y[], z[];
9199   //   int kdx = ystart+1;
9200   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
9201   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
9202   //     jlong carry2  = (jlong)(tmp3 >>> 64);
9203   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
9204   //     carry  = (jlong)(tmp4 >>> 64);
9205   //     z[kdx+idx+1] = (jlong)tmp3;
9206   //     z[kdx+idx] = (jlong)tmp4;
9207   //   }
9208   //   idx += 2;
9209   //   if (idx > 0) {
9210   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
9211   //     z[kdx+idx] = (jlong)yz_idx1;
9212   //     carry  = (jlong)(yz_idx1 >>> 64);
9213   //   }
9214   //
9215 
9216   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
9217 
9218   movl(jdx, idx);
9219   andl(jdx, 0xFFFFFFFC);
9220   shrl(jdx, 2);
9221 
9222   bind(L_third_loop);
9223   subl(jdx, 1);
9224   jcc(Assembler::negative, L_third_loop_exit);
9225   subl(idx, 4);
9226 
9227   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
9228   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
9229   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
9230   rorxq(yz_idx2, yz_idx2, 32);
9231 
9232   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
9233   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
9234 
9235   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
9236   rorxq(yz_idx1, yz_idx1, 32);
9237   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
9238   rorxq(yz_idx2, yz_idx2, 32);
9239 
9240   if (VM_Version::supports_adx()) {
9241     adcxq(tmp3, carry);
9242     adoxq(tmp3, yz_idx1);
9243 
9244     adcxq(tmp4, tmp);
9245     adoxq(tmp4, yz_idx2);
9246 
9247     movl(carry, 0); // does not affect flags
9248     adcxq(carry2, carry);
9249     adoxq(carry2, carry);
9250   } else {
9251     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
9252     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
9253   }
9254   movq(carry, carry2);
9255 
9256   movl(Address(z, idx, Address::times_4, 12), tmp3);
9257   shrq(tmp3, 32);
9258   movl(Address(z, idx, Address::times_4,  8), tmp3);
9259 
9260   movl(Address(z, idx, Address::times_4,  4), tmp4);
9261   shrq(tmp4, 32);
9262   movl(Address(z, idx, Address::times_4,  0), tmp4);
9263 
9264   jmp(L_third_loop);
9265 
9266   bind (L_third_loop_exit);
9267 
9268   andl (idx, 0x3);
9269   jcc(Assembler::zero, L_post_third_loop_done);
9270 
9271   Label L_check_1;
9272   subl(idx, 2);
9273   jcc(Assembler::negative, L_check_1);
9274 
9275   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
9276   rorxq(yz_idx1, yz_idx1, 32);
9277   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
9278   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
9279   rorxq(yz_idx2, yz_idx2, 32);
9280 
9281   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
9282 
9283   movl(Address(z, idx, Address::times_4,  4), tmp3);
9284   shrq(tmp3, 32);
9285   movl(Address(z, idx, Address::times_4,  0), tmp3);
9286   movq(carry, tmp4);
9287 
9288   bind (L_check_1);
9289   addl (idx, 0x2);
9290   andl (idx, 0x1);
9291   subl(idx, 1);
9292   jcc(Assembler::negative, L_post_third_loop_done);
9293   movl(tmp4, Address(y, idx, Address::times_4,  0));
9294   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
9295   movl(tmp4, Address(z, idx, Address::times_4,  0));
9296 
9297   add2_with_carry(carry2, tmp3, tmp4, carry);
9298 
9299   movl(Address(z, idx, Address::times_4,  0), tmp3);
9300   shrq(tmp3, 32);
9301 
9302   shlq(carry2, 32);
9303   orq(tmp3, carry2);
9304   movq(carry, tmp3);
9305 
9306   bind(L_post_third_loop_done);
9307 }
9308 
9309 /**
9310  * Code for BigInteger::multiplyToLen() instrinsic.
9311  *
9312  * rdi: x
9313  * rax: xlen
9314  * rsi: y
9315  * rcx: ylen
9316  * r8:  z
9317  * r11: zlen
9318  * r12: tmp1
9319  * r13: tmp2
9320  * r14: tmp3
9321  * r15: tmp4
9322  * rbx: tmp5
9323  *
9324  */
9325 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
9326                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
9327   ShortBranchVerifier sbv(this);
9328   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
9329 
9330   push(tmp1);
9331   push(tmp2);
9332   push(tmp3);
9333   push(tmp4);
9334   push(tmp5);
9335 
9336   push(xlen);
9337   push(zlen);
9338 
9339   const Register idx = tmp1;
9340   const Register kdx = tmp2;
9341   const Register xstart = tmp3;
9342 
9343   const Register y_idx = tmp4;
9344   const Register carry = tmp5;
9345   const Register product  = xlen;
9346   const Register x_xstart = zlen;  // reuse register
9347 
9348   // First Loop.
9349   //
9350   //  final static long LONG_MASK = 0xffffffffL;
9351   //  int xstart = xlen - 1;
9352   //  int ystart = ylen - 1;
9353   //  long carry = 0;
9354   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
9355   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
9356   //    z[kdx] = (int)product;
9357   //    carry = product >>> 32;
9358   //  }
9359   //  z[xstart] = (int)carry;
9360   //
9361 
9362   movl(idx, ylen);      // idx = ylen;
9363   movl(kdx, zlen);      // kdx = xlen+ylen;
9364   xorq(carry, carry);   // carry = 0;
9365 
9366   Label L_done;
9367 
9368   movl(xstart, xlen);
9369   decrementl(xstart);
9370   jcc(Assembler::negative, L_done);
9371 
9372   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
9373 
9374   Label L_second_loop;
9375   testl(kdx, kdx);
9376   jcc(Assembler::zero, L_second_loop);
9377 
9378   Label L_carry;
9379   subl(kdx, 1);
9380   jcc(Assembler::zero, L_carry);
9381 
9382   movl(Address(z, kdx, Address::times_4,  0), carry);
9383   shrq(carry, 32);
9384   subl(kdx, 1);
9385 
9386   bind(L_carry);
9387   movl(Address(z, kdx, Address::times_4,  0), carry);
9388 
9389   // Second and third (nested) loops.
9390   //
9391   // for (int i = xstart-1; i >= 0; i--) { // Second loop
9392   //   carry = 0;
9393   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
9394   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
9395   //                    (z[k] & LONG_MASK) + carry;
9396   //     z[k] = (int)product;
9397   //     carry = product >>> 32;
9398   //   }
9399   //   z[i] = (int)carry;
9400   // }
9401   //
9402   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
9403 
9404   const Register jdx = tmp1;
9405 
9406   bind(L_second_loop);
9407   xorl(carry, carry);    // carry = 0;
9408   movl(jdx, ylen);       // j = ystart+1
9409 
9410   subl(xstart, 1);       // i = xstart-1;
9411   jcc(Assembler::negative, L_done);
9412 
9413   push (z);
9414 
9415   Label L_last_x;
9416   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
9417   subl(xstart, 1);       // i = xstart-1;
9418   jcc(Assembler::negative, L_last_x);
9419 
9420   if (UseBMI2Instructions) {
9421     movq(rdx,  Address(x, xstart, Address::times_4,  0));
9422     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
9423   } else {
9424     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
9425     rorq(x_xstart, 32);  // convert big-endian to little-endian
9426   }
9427 
9428   Label L_third_loop_prologue;
9429   bind(L_third_loop_prologue);
9430 
9431   push (x);
9432   push (xstart);
9433   push (ylen);
9434 
9435 
9436   if (UseBMI2Instructions) {
9437     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
9438   } else { // !UseBMI2Instructions
9439     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
9440   }
9441 
9442   pop(ylen);
9443   pop(xlen);
9444   pop(x);
9445   pop(z);
9446 
9447   movl(tmp3, xlen);
9448   addl(tmp3, 1);
9449   movl(Address(z, tmp3, Address::times_4,  0), carry);
9450   subl(tmp3, 1);
9451   jccb(Assembler::negative, L_done);
9452 
9453   shrq(carry, 32);
9454   movl(Address(z, tmp3, Address::times_4,  0), carry);
9455   jmp(L_second_loop);
9456 
9457   // Next infrequent code is moved outside loops.
9458   bind(L_last_x);
9459   if (UseBMI2Instructions) {
9460     movl(rdx, Address(x,  0));
9461   } else {
9462     movl(x_xstart, Address(x,  0));
9463   }
9464   jmp(L_third_loop_prologue);
9465 
9466   bind(L_done);
9467 
9468   pop(zlen);
9469   pop(xlen);
9470 
9471   pop(tmp5);
9472   pop(tmp4);
9473   pop(tmp3);
9474   pop(tmp2);
9475   pop(tmp1);
9476 }
9477 
9478 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
9479   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
9480   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
9481   Label VECTOR64_LOOP, VECTOR64_TAIL, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
9482   Label VECTOR32_LOOP, VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
9483   Label VECTOR16_TAIL, VECTOR8_TAIL, VECTOR4_TAIL;
9484   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
9485   Label SAME_TILL_END, DONE;
9486   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
9487 
9488   //scale is in rcx in both Win64 and Unix
9489   ShortBranchVerifier sbv(this);
9490 
9491   shlq(length);
9492   xorq(result, result);
9493 
9494   if ((UseAVX > 2) &&
9495       VM_Version::supports_avx512vlbw()) {
9496     set_vector_masking();  // opening of the stub context for programming mask registers
9497     cmpq(length, 64);
9498     jcc(Assembler::less, VECTOR32_TAIL);
9499     movq(tmp1, length);
9500     andq(tmp1, 0x3F);      // tail count
9501     andq(length, ~(0x3F)); //vector count
9502 
9503     bind(VECTOR64_LOOP);
9504     // AVX512 code to compare 64 byte vectors.
9505     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
9506     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
9507     kortestql(k7, k7);
9508     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
9509     addq(result, 64);
9510     subq(length, 64);
9511     jccb(Assembler::notZero, VECTOR64_LOOP);
9512 
9513     //bind(VECTOR64_TAIL);
9514     testq(tmp1, tmp1);
9515     jcc(Assembler::zero, SAME_TILL_END);
9516 
9517     bind(VECTOR64_TAIL);
9518     // AVX512 code to compare upto 63 byte vectors.
9519     // Save k1
9520     kmovql(k3, k1);
9521     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
9522     shlxq(tmp2, tmp2, tmp1);
9523     notq(tmp2);
9524     kmovql(k1, tmp2);
9525 
9526     evmovdqub(rymm0, k1, Address(obja, result), Assembler::AVX_512bit);
9527     evpcmpeqb(k7, k1, rymm0, Address(objb, result), Assembler::AVX_512bit);
9528 
9529     ktestql(k7, k1);
9530     // Restore k1
9531     kmovql(k1, k3);
9532     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
9533 
9534     bind(VECTOR64_NOT_EQUAL);
9535     kmovql(tmp1, k7);
9536     notq(tmp1);
9537     tzcntq(tmp1, tmp1);
9538     addq(result, tmp1);
9539     shrq(result);
9540     jmp(DONE);
9541     bind(VECTOR32_TAIL);
9542     clear_vector_masking();   // closing of the stub context for programming mask registers
9543   }
9544 
9545   cmpq(length, 8);
9546   jcc(Assembler::equal, VECTOR8_LOOP);
9547   jcc(Assembler::less, VECTOR4_TAIL);
9548 
9549   if (UseAVX >= 2) {
9550 
9551     cmpq(length, 16);
9552     jcc(Assembler::equal, VECTOR16_LOOP);
9553     jcc(Assembler::less, VECTOR8_LOOP);
9554 
9555     cmpq(length, 32);
9556     jccb(Assembler::less, VECTOR16_TAIL);
9557 
9558     subq(length, 32);
9559     bind(VECTOR32_LOOP);
9560     vmovdqu(rymm0, Address(obja, result));
9561     vmovdqu(rymm1, Address(objb, result));
9562     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
9563     vptest(rymm2, rymm2);
9564     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
9565     addq(result, 32);
9566     subq(length, 32);
9567     jccb(Assembler::greaterEqual, VECTOR32_LOOP);
9568     addq(length, 32);
9569     jcc(Assembler::equal, SAME_TILL_END);
9570     //falling through if less than 32 bytes left //close the branch here.
9571 
9572     bind(VECTOR16_TAIL);
9573     cmpq(length, 16);
9574     jccb(Assembler::less, VECTOR8_TAIL);
9575     bind(VECTOR16_LOOP);
9576     movdqu(rymm0, Address(obja, result));
9577     movdqu(rymm1, Address(objb, result));
9578     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
9579     ptest(rymm2, rymm2);
9580     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
9581     addq(result, 16);
9582     subq(length, 16);
9583     jcc(Assembler::equal, SAME_TILL_END);
9584     //falling through if less than 16 bytes left
9585   } else {//regular intrinsics
9586 
9587     cmpq(length, 16);
9588     jccb(Assembler::less, VECTOR8_TAIL);
9589 
9590     subq(length, 16);
9591     bind(VECTOR16_LOOP);
9592     movdqu(rymm0, Address(obja, result));
9593     movdqu(rymm1, Address(objb, result));
9594     pxor(rymm0, rymm1);
9595     ptest(rymm0, rymm0);
9596     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
9597     addq(result, 16);
9598     subq(length, 16);
9599     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
9600     addq(length, 16);
9601     jcc(Assembler::equal, SAME_TILL_END);
9602     //falling through if less than 16 bytes left
9603   }
9604 
9605   bind(VECTOR8_TAIL);
9606   cmpq(length, 8);
9607   jccb(Assembler::less, VECTOR4_TAIL);
9608   bind(VECTOR8_LOOP);
9609   movq(tmp1, Address(obja, result));
9610   movq(tmp2, Address(objb, result));
9611   xorq(tmp1, tmp2);
9612   testq(tmp1, tmp1);
9613   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
9614   addq(result, 8);
9615   subq(length, 8);
9616   jcc(Assembler::equal, SAME_TILL_END);
9617   //falling through if less than 8 bytes left
9618 
9619   bind(VECTOR4_TAIL);
9620   cmpq(length, 4);
9621   jccb(Assembler::less, BYTES_TAIL);
9622   bind(VECTOR4_LOOP);
9623   movl(tmp1, Address(obja, result));
9624   xorl(tmp1, Address(objb, result));
9625   testl(tmp1, tmp1);
9626   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
9627   addq(result, 4);
9628   subq(length, 4);
9629   jcc(Assembler::equal, SAME_TILL_END);
9630   //falling through if less than 4 bytes left
9631 
9632   bind(BYTES_TAIL);
9633   bind(BYTES_LOOP);
9634   load_unsigned_byte(tmp1, Address(obja, result));
9635   load_unsigned_byte(tmp2, Address(objb, result));
9636   xorl(tmp1, tmp2);
9637   testl(tmp1, tmp1);
9638   jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9639   decq(length);
9640   jccb(Assembler::zero, SAME_TILL_END);
9641   incq(result);
9642   load_unsigned_byte(tmp1, Address(obja, result));
9643   load_unsigned_byte(tmp2, Address(objb, result));
9644   xorl(tmp1, tmp2);
9645   testl(tmp1, tmp1);
9646   jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9647   decq(length);
9648   jccb(Assembler::zero, SAME_TILL_END);
9649   incq(result);
9650   load_unsigned_byte(tmp1, Address(obja, result));
9651   load_unsigned_byte(tmp2, Address(objb, result));
9652   xorl(tmp1, tmp2);
9653   testl(tmp1, tmp1);
9654   jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9655   jmpb(SAME_TILL_END);
9656 
9657   if (UseAVX >= 2) {
9658     bind(VECTOR32_NOT_EQUAL);
9659     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
9660     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
9661     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
9662     vpmovmskb(tmp1, rymm0);
9663     bsfq(tmp1, tmp1);
9664     addq(result, tmp1);
9665     shrq(result);
9666     jmpb(DONE);
9667   }
9668 
9669   bind(VECTOR16_NOT_EQUAL);
9670   if (UseAVX >= 2) {
9671     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
9672     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
9673     pxor(rymm0, rymm2);
9674   } else {
9675     pcmpeqb(rymm2, rymm2);
9676     pxor(rymm0, rymm1);
9677     pcmpeqb(rymm0, rymm1);
9678     pxor(rymm0, rymm2);
9679   }
9680   pmovmskb(tmp1, rymm0);
9681   bsfq(tmp1, tmp1);
9682   addq(result, tmp1);
9683   shrq(result);
9684   jmpb(DONE);
9685 
9686   bind(VECTOR8_NOT_EQUAL);
9687   bind(VECTOR4_NOT_EQUAL);
9688   bsfq(tmp1, tmp1);
9689   shrq(tmp1, 3);
9690   addq(result, tmp1);
9691   bind(BYTES_NOT_EQUAL);
9692   shrq(result);
9693   jmpb(DONE);
9694 
9695   bind(SAME_TILL_END);
9696   mov64(result, -1);
9697 
9698   bind(DONE);
9699 }
9700 
9701 //Helper functions for square_to_len()
9702 
9703 /**
9704  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
9705  * Preserves x and z and modifies rest of the registers.
9706  */
9707 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9708   // Perform square and right shift by 1
9709   // Handle odd xlen case first, then for even xlen do the following
9710   // jlong carry = 0;
9711   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
9712   //     huge_128 product = x[j:j+1] * x[j:j+1];
9713   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
9714   //     z[i+2:i+3] = (jlong)(product >>> 1);
9715   //     carry = (jlong)product;
9716   // }
9717 
9718   xorq(tmp5, tmp5);     // carry
9719   xorq(rdxReg, rdxReg);
9720   xorl(tmp1, tmp1);     // index for x
9721   xorl(tmp4, tmp4);     // index for z
9722 
9723   Label L_first_loop, L_first_loop_exit;
9724 
9725   testl(xlen, 1);
9726   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
9727 
9728   // Square and right shift by 1 the odd element using 32 bit multiply
9729   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
9730   imulq(raxReg, raxReg);
9731   shrq(raxReg, 1);
9732   adcq(tmp5, 0);
9733   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
9734   incrementl(tmp1);
9735   addl(tmp4, 2);
9736 
9737   // Square and  right shift by 1 the rest using 64 bit multiply
9738   bind(L_first_loop);
9739   cmpptr(tmp1, xlen);
9740   jccb(Assembler::equal, L_first_loop_exit);
9741 
9742   // Square
9743   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
9744   rorq(raxReg, 32);    // convert big-endian to little-endian
9745   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
9746 
9747   // Right shift by 1 and save carry
9748   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
9749   rcrq(rdxReg, 1);
9750   rcrq(raxReg, 1);
9751   adcq(tmp5, 0);
9752 
9753   // Store result in z
9754   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
9755   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
9756 
9757   // Update indices for x and z
9758   addl(tmp1, 2);
9759   addl(tmp4, 4);
9760   jmp(L_first_loop);
9761 
9762   bind(L_first_loop_exit);
9763 }
9764 
9765 
9766 /**
9767  * Perform the following multiply add operation using BMI2 instructions
9768  * carry:sum = sum + op1*op2 + carry
9769  * op2 should be in rdx
9770  * op2 is preserved, all other registers are modified
9771  */
9772 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
9773   // assert op2 is rdx
9774   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
9775   addq(sum, carry);
9776   adcq(tmp2, 0);
9777   addq(sum, op1);
9778   adcq(tmp2, 0);
9779   movq(carry, tmp2);
9780 }
9781 
9782 /**
9783  * Perform the following multiply add operation:
9784  * carry:sum = sum + op1*op2 + carry
9785  * Preserves op1, op2 and modifies rest of registers
9786  */
9787 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
9788   // rdx:rax = op1 * op2
9789   movq(raxReg, op2);
9790   mulq(op1);
9791 
9792   //  rdx:rax = sum + carry + rdx:rax
9793   addq(sum, carry);
9794   adcq(rdxReg, 0);
9795   addq(sum, raxReg);
9796   adcq(rdxReg, 0);
9797 
9798   // carry:sum = rdx:sum
9799   movq(carry, rdxReg);
9800 }
9801 
9802 /**
9803  * Add 64 bit long carry into z[] with carry propogation.
9804  * Preserves z and carry register values and modifies rest of registers.
9805  *
9806  */
9807 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
9808   Label L_fourth_loop, L_fourth_loop_exit;
9809 
9810   movl(tmp1, 1);
9811   subl(zlen, 2);
9812   addq(Address(z, zlen, Address::times_4, 0), carry);
9813 
9814   bind(L_fourth_loop);
9815   jccb(Assembler::carryClear, L_fourth_loop_exit);
9816   subl(zlen, 2);
9817   jccb(Assembler::negative, L_fourth_loop_exit);
9818   addq(Address(z, zlen, Address::times_4, 0), tmp1);
9819   jmp(L_fourth_loop);
9820   bind(L_fourth_loop_exit);
9821 }
9822 
9823 /**
9824  * Shift z[] left by 1 bit.
9825  * Preserves x, len, z and zlen registers and modifies rest of the registers.
9826  *
9827  */
9828 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
9829 
9830   Label L_fifth_loop, L_fifth_loop_exit;
9831 
9832   // Fifth loop
9833   // Perform primitiveLeftShift(z, zlen, 1)
9834 
9835   const Register prev_carry = tmp1;
9836   const Register new_carry = tmp4;
9837   const Register value = tmp2;
9838   const Register zidx = tmp3;
9839 
9840   // int zidx, carry;
9841   // long value;
9842   // carry = 0;
9843   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
9844   //    (carry:value)  = (z[i] << 1) | carry ;
9845   //    z[i] = value;
9846   // }
9847 
9848   movl(zidx, zlen);
9849   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
9850 
9851   bind(L_fifth_loop);
9852   decl(zidx);  // Use decl to preserve carry flag
9853   decl(zidx);
9854   jccb(Assembler::negative, L_fifth_loop_exit);
9855 
9856   if (UseBMI2Instructions) {
9857      movq(value, Address(z, zidx, Address::times_4, 0));
9858      rclq(value, 1);
9859      rorxq(value, value, 32);
9860      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9861   }
9862   else {
9863     // clear new_carry
9864     xorl(new_carry, new_carry);
9865 
9866     // Shift z[i] by 1, or in previous carry and save new carry
9867     movq(value, Address(z, zidx, Address::times_4, 0));
9868     shlq(value, 1);
9869     adcl(new_carry, 0);
9870 
9871     orq(value, prev_carry);
9872     rorq(value, 0x20);
9873     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9874 
9875     // Set previous carry = new carry
9876     movl(prev_carry, new_carry);
9877   }
9878   jmp(L_fifth_loop);
9879 
9880   bind(L_fifth_loop_exit);
9881 }
9882 
9883 
9884 /**
9885  * Code for BigInteger::squareToLen() intrinsic
9886  *
9887  * rdi: x
9888  * rsi: len
9889  * r8:  z
9890  * rcx: zlen
9891  * r12: tmp1
9892  * r13: tmp2
9893  * r14: tmp3
9894  * r15: tmp4
9895  * rbx: tmp5
9896  *
9897  */
9898 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9899 
9900   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, fifth_loop, fifth_loop_exit, L_last_x, L_multiply;
9901   push(tmp1);
9902   push(tmp2);
9903   push(tmp3);
9904   push(tmp4);
9905   push(tmp5);
9906 
9907   // First loop
9908   // Store the squares, right shifted one bit (i.e., divided by 2).
9909   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
9910 
9911   // Add in off-diagonal sums.
9912   //
9913   // Second, third (nested) and fourth loops.
9914   // zlen +=2;
9915   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
9916   //    carry = 0;
9917   //    long op2 = x[xidx:xidx+1];
9918   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
9919   //       k -= 2;
9920   //       long op1 = x[j:j+1];
9921   //       long sum = z[k:k+1];
9922   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
9923   //       z[k:k+1] = sum;
9924   //    }
9925   //    add_one_64(z, k, carry, tmp_regs);
9926   // }
9927 
9928   const Register carry = tmp5;
9929   const Register sum = tmp3;
9930   const Register op1 = tmp4;
9931   Register op2 = tmp2;
9932 
9933   push(zlen);
9934   push(len);
9935   addl(zlen,2);
9936   bind(L_second_loop);
9937   xorq(carry, carry);
9938   subl(zlen, 4);
9939   subl(len, 2);
9940   push(zlen);
9941   push(len);
9942   cmpl(len, 0);
9943   jccb(Assembler::lessEqual, L_second_loop_exit);
9944 
9945   // Multiply an array by one 64 bit long.
9946   if (UseBMI2Instructions) {
9947     op2 = rdxReg;
9948     movq(op2, Address(x, len, Address::times_4,  0));
9949     rorxq(op2, op2, 32);
9950   }
9951   else {
9952     movq(op2, Address(x, len, Address::times_4,  0));
9953     rorq(op2, 32);
9954   }
9955 
9956   bind(L_third_loop);
9957   decrementl(len);
9958   jccb(Assembler::negative, L_third_loop_exit);
9959   decrementl(len);
9960   jccb(Assembler::negative, L_last_x);
9961 
9962   movq(op1, Address(x, len, Address::times_4,  0));
9963   rorq(op1, 32);
9964 
9965   bind(L_multiply);
9966   subl(zlen, 2);
9967   movq(sum, Address(z, zlen, Address::times_4,  0));
9968 
9969   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
9970   if (UseBMI2Instructions) {
9971     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
9972   }
9973   else {
9974     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9975   }
9976 
9977   movq(Address(z, zlen, Address::times_4, 0), sum);
9978 
9979   jmp(L_third_loop);
9980   bind(L_third_loop_exit);
9981 
9982   // Fourth loop
9983   // Add 64 bit long carry into z with carry propogation.
9984   // Uses offsetted zlen.
9985   add_one_64(z, zlen, carry, tmp1);
9986 
9987   pop(len);
9988   pop(zlen);
9989   jmp(L_second_loop);
9990 
9991   // Next infrequent code is moved outside loops.
9992   bind(L_last_x);
9993   movl(op1, Address(x, 0));
9994   jmp(L_multiply);
9995 
9996   bind(L_second_loop_exit);
9997   pop(len);
9998   pop(zlen);
9999   pop(len);
10000   pop(zlen);
10001 
10002   // Fifth loop
10003   // Shift z left 1 bit.
10004   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
10005 
10006   // z[zlen-1] |= x[len-1] & 1;
10007   movl(tmp3, Address(x, len, Address::times_4, -4));
10008   andl(tmp3, 1);
10009   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
10010 
10011   pop(tmp5);
10012   pop(tmp4);
10013   pop(tmp3);
10014   pop(tmp2);
10015   pop(tmp1);
10016 }
10017 
10018 /**
10019  * Helper function for mul_add()
10020  * Multiply the in[] by int k and add to out[] starting at offset offs using
10021  * 128 bit by 32 bit multiply and return the carry in tmp5.
10022  * Only quad int aligned length of in[] is operated on in this function.
10023  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
10024  * This function preserves out, in and k registers.
10025  * len and offset point to the appropriate index in "in" & "out" correspondingly
10026  * tmp5 has the carry.
10027  * other registers are temporary and are modified.
10028  *
10029  */
10030 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
10031   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
10032   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
10033 
10034   Label L_first_loop, L_first_loop_exit;
10035 
10036   movl(tmp1, len);
10037   shrl(tmp1, 2);
10038 
10039   bind(L_first_loop);
10040   subl(tmp1, 1);
10041   jccb(Assembler::negative, L_first_loop_exit);
10042 
10043   subl(len, 4);
10044   subl(offset, 4);
10045 
10046   Register op2 = tmp2;
10047   const Register sum = tmp3;
10048   const Register op1 = tmp4;
10049   const Register carry = tmp5;
10050 
10051   if (UseBMI2Instructions) {
10052     op2 = rdxReg;
10053   }
10054 
10055   movq(op1, Address(in, len, Address::times_4,  8));
10056   rorq(op1, 32);
10057   movq(sum, Address(out, offset, Address::times_4,  8));
10058   rorq(sum, 32);
10059   if (UseBMI2Instructions) {
10060     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
10061   }
10062   else {
10063     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
10064   }
10065   // Store back in big endian from little endian
10066   rorq(sum, 0x20);
10067   movq(Address(out, offset, Address::times_4,  8), sum);
10068 
10069   movq(op1, Address(in, len, Address::times_4,  0));
10070   rorq(op1, 32);
10071   movq(sum, Address(out, offset, Address::times_4,  0));
10072   rorq(sum, 32);
10073   if (UseBMI2Instructions) {
10074     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
10075   }
10076   else {
10077     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
10078   }
10079   // Store back in big endian from little endian
10080   rorq(sum, 0x20);
10081   movq(Address(out, offset, Address::times_4,  0), sum);
10082 
10083   jmp(L_first_loop);
10084   bind(L_first_loop_exit);
10085 }
10086 
10087 /**
10088  * Code for BigInteger::mulAdd() intrinsic
10089  *
10090  * rdi: out
10091  * rsi: in
10092  * r11: offs (out.length - offset)
10093  * rcx: len
10094  * r8:  k
10095  * r12: tmp1
10096  * r13: tmp2
10097  * r14: tmp3
10098  * r15: tmp4
10099  * rbx: tmp5
10100  * Multiply the in[] by word k and add to out[], return the carry in rax
10101  */
10102 void MacroAssembler::mul_add(Register out, Register in, Register offs,
10103    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
10104    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
10105 
10106   Label L_carry, L_last_in, L_done;
10107 
10108 // carry = 0;
10109 // for (int j=len-1; j >= 0; j--) {
10110 //    long product = (in[j] & LONG_MASK) * kLong +
10111 //                   (out[offs] & LONG_MASK) + carry;
10112 //    out[offs--] = (int)product;
10113 //    carry = product >>> 32;
10114 // }
10115 //
10116   push(tmp1);
10117   push(tmp2);
10118   push(tmp3);
10119   push(tmp4);
10120   push(tmp5);
10121 
10122   Register op2 = tmp2;
10123   const Register sum = tmp3;
10124   const Register op1 = tmp4;
10125   const Register carry =  tmp5;
10126 
10127   if (UseBMI2Instructions) {
10128     op2 = rdxReg;
10129     movl(op2, k);
10130   }
10131   else {
10132     movl(op2, k);
10133   }
10134 
10135   xorq(carry, carry);
10136 
10137   //First loop
10138 
10139   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
10140   //The carry is in tmp5
10141   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
10142 
10143   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
10144   decrementl(len);
10145   jccb(Assembler::negative, L_carry);
10146   decrementl(len);
10147   jccb(Assembler::negative, L_last_in);
10148 
10149   movq(op1, Address(in, len, Address::times_4,  0));
10150   rorq(op1, 32);
10151 
10152   subl(offs, 2);
10153   movq(sum, Address(out, offs, Address::times_4,  0));
10154   rorq(sum, 32);
10155 
10156   if (UseBMI2Instructions) {
10157     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
10158   }
10159   else {
10160     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
10161   }
10162 
10163   // Store back in big endian from little endian
10164   rorq(sum, 0x20);
10165   movq(Address(out, offs, Address::times_4,  0), sum);
10166 
10167   testl(len, len);
10168   jccb(Assembler::zero, L_carry);
10169 
10170   //Multiply the last in[] entry, if any
10171   bind(L_last_in);
10172   movl(op1, Address(in, 0));
10173   movl(sum, Address(out, offs, Address::times_4,  -4));
10174 
10175   movl(raxReg, k);
10176   mull(op1); //tmp4 * eax -> edx:eax
10177   addl(sum, carry);
10178   adcl(rdxReg, 0);
10179   addl(sum, raxReg);
10180   adcl(rdxReg, 0);
10181   movl(carry, rdxReg);
10182 
10183   movl(Address(out, offs, Address::times_4,  -4), sum);
10184 
10185   bind(L_carry);
10186   //return tmp5/carry as carry in rax
10187   movl(rax, carry);
10188 
10189   bind(L_done);
10190   pop(tmp5);
10191   pop(tmp4);
10192   pop(tmp3);
10193   pop(tmp2);
10194   pop(tmp1);
10195 }
10196 #endif
10197 
10198 /**
10199  * Emits code to update CRC-32 with a byte value according to constants in table
10200  *
10201  * @param [in,out]crc   Register containing the crc.
10202  * @param [in]val       Register containing the byte to fold into the CRC.
10203  * @param [in]table     Register containing the table of crc constants.
10204  *
10205  * uint32_t crc;
10206  * val = crc_table[(val ^ crc) & 0xFF];
10207  * crc = val ^ (crc >> 8);
10208  *
10209  */
10210 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
10211   xorl(val, crc);
10212   andl(val, 0xFF);
10213   shrl(crc, 8); // unsigned shift
10214   xorl(crc, Address(table, val, Address::times_4, 0));
10215 }
10216 
10217 /**
10218  * Fold 128-bit data chunk
10219  */
10220 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
10221   if (UseAVX > 0) {
10222     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
10223     vpclmulldq(xcrc, xK, xcrc); // [63:0]
10224     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
10225     pxor(xcrc, xtmp);
10226   } else {
10227     movdqa(xtmp, xcrc);
10228     pclmulhdq(xtmp, xK);   // [123:64]
10229     pclmulldq(xcrc, xK);   // [63:0]
10230     pxor(xcrc, xtmp);
10231     movdqu(xtmp, Address(buf, offset));
10232     pxor(xcrc, xtmp);
10233   }
10234 }
10235 
10236 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
10237   if (UseAVX > 0) {
10238     vpclmulhdq(xtmp, xK, xcrc);
10239     vpclmulldq(xcrc, xK, xcrc);
10240     pxor(xcrc, xbuf);
10241     pxor(xcrc, xtmp);
10242   } else {
10243     movdqa(xtmp, xcrc);
10244     pclmulhdq(xtmp, xK);
10245     pclmulldq(xcrc, xK);
10246     pxor(xcrc, xbuf);
10247     pxor(xcrc, xtmp);
10248   }
10249 }
10250 
10251 /**
10252  * 8-bit folds to compute 32-bit CRC
10253  *
10254  * uint64_t xcrc;
10255  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
10256  */
10257 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
10258   movdl(tmp, xcrc);
10259   andl(tmp, 0xFF);
10260   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
10261   psrldq(xcrc, 1); // unsigned shift one byte
10262   pxor(xcrc, xtmp);
10263 }
10264 
10265 /**
10266  * uint32_t crc;
10267  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
10268  */
10269 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
10270   movl(tmp, crc);
10271   andl(tmp, 0xFF);
10272   shrl(crc, 8);
10273   xorl(crc, Address(table, tmp, Address::times_4, 0));
10274 }
10275 
10276 /**
10277  * @param crc   register containing existing CRC (32-bit)
10278  * @param buf   register pointing to input byte buffer (byte*)
10279  * @param len   register containing number of bytes
10280  * @param table register that will contain address of CRC table
10281  * @param tmp   scratch register
10282  */
10283 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
10284   assert_different_registers(crc, buf, len, table, tmp, rax);
10285 
10286   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
10287   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
10288 
10289   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
10290   // context for the registers used, where all instructions below are using 128-bit mode
10291   // On EVEX without VL and BW, these instructions will all be AVX.
10292   if (VM_Version::supports_avx512vlbw()) {
10293     movl(tmp, 0xffff);
10294     kmovwl(k1, tmp);
10295   }
10296 
10297   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
10298   notl(crc); // ~crc
10299   cmpl(len, 16);
10300   jcc(Assembler::less, L_tail);
10301 
10302   // Align buffer to 16 bytes
10303   movl(tmp, buf);
10304   andl(tmp, 0xF);
10305   jccb(Assembler::zero, L_aligned);
10306   subl(tmp,  16);
10307   addl(len, tmp);
10308 
10309   align(4);
10310   BIND(L_align_loop);
10311   movsbl(rax, Address(buf, 0)); // load byte with sign extension
10312   update_byte_crc32(crc, rax, table);
10313   increment(buf);
10314   incrementl(tmp);
10315   jccb(Assembler::less, L_align_loop);
10316 
10317   BIND(L_aligned);
10318   movl(tmp, len); // save
10319   shrl(len, 4);
10320   jcc(Assembler::zero, L_tail_restore);
10321 
10322   // Fold crc into first bytes of vector
10323   movdqa(xmm1, Address(buf, 0));
10324   movdl(rax, xmm1);
10325   xorl(crc, rax);
10326   if (VM_Version::supports_sse4_1()) {
10327     pinsrd(xmm1, crc, 0);
10328   } else {
10329     pinsrw(xmm1, crc, 0);
10330     shrl(crc, 16);
10331     pinsrw(xmm1, crc, 1);
10332   }
10333   addptr(buf, 16);
10334   subl(len, 4); // len > 0
10335   jcc(Assembler::less, L_fold_tail);
10336 
10337   movdqa(xmm2, Address(buf,  0));
10338   movdqa(xmm3, Address(buf, 16));
10339   movdqa(xmm4, Address(buf, 32));
10340   addptr(buf, 48);
10341   subl(len, 3);
10342   jcc(Assembler::lessEqual, L_fold_512b);
10343 
10344   // Fold total 512 bits of polynomial on each iteration,
10345   // 128 bits per each of 4 parallel streams.
10346   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
10347 
10348   align(32);
10349   BIND(L_fold_512b_loop);
10350   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
10351   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
10352   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
10353   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
10354   addptr(buf, 64);
10355   subl(len, 4);
10356   jcc(Assembler::greater, L_fold_512b_loop);
10357 
10358   // Fold 512 bits to 128 bits.
10359   BIND(L_fold_512b);
10360   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
10361   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
10362   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
10363   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
10364 
10365   // Fold the rest of 128 bits data chunks
10366   BIND(L_fold_tail);
10367   addl(len, 3);
10368   jccb(Assembler::lessEqual, L_fold_128b);
10369   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
10370 
10371   BIND(L_fold_tail_loop);
10372   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
10373   addptr(buf, 16);
10374   decrementl(len);
10375   jccb(Assembler::greater, L_fold_tail_loop);
10376 
10377   // Fold 128 bits in xmm1 down into 32 bits in crc register.
10378   BIND(L_fold_128b);
10379   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()));
10380   if (UseAVX > 0) {
10381     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
10382     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
10383     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
10384   } else {
10385     movdqa(xmm2, xmm0);
10386     pclmulqdq(xmm2, xmm1, 0x1);
10387     movdqa(xmm3, xmm0);
10388     pand(xmm3, xmm2);
10389     pclmulqdq(xmm0, xmm3, 0x1);
10390   }
10391   psrldq(xmm1, 8);
10392   psrldq(xmm2, 4);
10393   pxor(xmm0, xmm1);
10394   pxor(xmm0, xmm2);
10395 
10396   // 8 8-bit folds to compute 32-bit CRC.
10397   for (int j = 0; j < 4; j++) {
10398     fold_8bit_crc32(xmm0, table, xmm1, rax);
10399   }
10400   movdl(crc, xmm0); // mov 32 bits to general register
10401   for (int j = 0; j < 4; j++) {
10402     fold_8bit_crc32(crc, table, rax);
10403   }
10404 
10405   BIND(L_tail_restore);
10406   movl(len, tmp); // restore
10407   BIND(L_tail);
10408   andl(len, 0xf);
10409   jccb(Assembler::zero, L_exit);
10410 
10411   // Fold the rest of bytes
10412   align(4);
10413   BIND(L_tail_loop);
10414   movsbl(rax, Address(buf, 0)); // load byte with sign extension
10415   update_byte_crc32(crc, rax, table);
10416   increment(buf);
10417   decrementl(len);
10418   jccb(Assembler::greater, L_tail_loop);
10419 
10420   BIND(L_exit);
10421   notl(crc); // ~c
10422 }
10423 
10424 #ifdef _LP64
10425 // S. Gueron / Information Processing Letters 112 (2012) 184
10426 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
10427 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
10428 // Output: the 64-bit carry-less product of B * CONST
10429 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
10430                                      Register tmp1, Register tmp2, Register tmp3) {
10431   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
10432   if (n > 0) {
10433     addq(tmp3, n * 256 * 8);
10434   }
10435   //    Q1 = TABLEExt[n][B & 0xFF];
10436   movl(tmp1, in);
10437   andl(tmp1, 0x000000FF);
10438   shll(tmp1, 3);
10439   addq(tmp1, tmp3);
10440   movq(tmp1, Address(tmp1, 0));
10441 
10442   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
10443   movl(tmp2, in);
10444   shrl(tmp2, 8);
10445   andl(tmp2, 0x000000FF);
10446   shll(tmp2, 3);
10447   addq(tmp2, tmp3);
10448   movq(tmp2, Address(tmp2, 0));
10449 
10450   shlq(tmp2, 8);
10451   xorq(tmp1, tmp2);
10452 
10453   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
10454   movl(tmp2, in);
10455   shrl(tmp2, 16);
10456   andl(tmp2, 0x000000FF);
10457   shll(tmp2, 3);
10458   addq(tmp2, tmp3);
10459   movq(tmp2, Address(tmp2, 0));
10460 
10461   shlq(tmp2, 16);
10462   xorq(tmp1, tmp2);
10463 
10464   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
10465   shrl(in, 24);
10466   andl(in, 0x000000FF);
10467   shll(in, 3);
10468   addq(in, tmp3);
10469   movq(in, Address(in, 0));
10470 
10471   shlq(in, 24);
10472   xorq(in, tmp1);
10473   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
10474 }
10475 
10476 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
10477                                       Register in_out,
10478                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
10479                                       XMMRegister w_xtmp2,
10480                                       Register tmp1,
10481                                       Register n_tmp2, Register n_tmp3) {
10482   if (is_pclmulqdq_supported) {
10483     movdl(w_xtmp1, in_out); // modified blindly
10484 
10485     movl(tmp1, const_or_pre_comp_const_index);
10486     movdl(w_xtmp2, tmp1);
10487     pclmulqdq(w_xtmp1, w_xtmp2, 0);
10488 
10489     movdq(in_out, w_xtmp1);
10490   } else {
10491     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
10492   }
10493 }
10494 
10495 // Recombination Alternative 2: No bit-reflections
10496 // T1 = (CRC_A * U1) << 1
10497 // T2 = (CRC_B * U2) << 1
10498 // C1 = T1 >> 32
10499 // C2 = T2 >> 32
10500 // T1 = T1 & 0xFFFFFFFF
10501 // T2 = T2 & 0xFFFFFFFF
10502 // T1 = CRC32(0, T1)
10503 // T2 = CRC32(0, T2)
10504 // C1 = C1 ^ T1
10505 // C2 = C2 ^ T2
10506 // CRC = C1 ^ C2 ^ CRC_C
10507 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
10508                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10509                                      Register tmp1, Register tmp2,
10510                                      Register n_tmp3) {
10511   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10512   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10513   shlq(in_out, 1);
10514   movl(tmp1, in_out);
10515   shrq(in_out, 32);
10516   xorl(tmp2, tmp2);
10517   crc32(tmp2, tmp1, 4);
10518   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
10519   shlq(in1, 1);
10520   movl(tmp1, in1);
10521   shrq(in1, 32);
10522   xorl(tmp2, tmp2);
10523   crc32(tmp2, tmp1, 4);
10524   xorl(in1, tmp2);
10525   xorl(in_out, in1);
10526   xorl(in_out, in2);
10527 }
10528 
10529 // Set N to predefined value
10530 // Subtract from a lenght of a buffer
10531 // execute in a loop:
10532 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
10533 // for i = 1 to N do
10534 //  CRC_A = CRC32(CRC_A, A[i])
10535 //  CRC_B = CRC32(CRC_B, B[i])
10536 //  CRC_C = CRC32(CRC_C, C[i])
10537 // end for
10538 // Recombine
10539 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
10540                                        Register in_out1, Register in_out2, Register in_out3,
10541                                        Register tmp1, Register tmp2, Register tmp3,
10542                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10543                                        Register tmp4, Register tmp5,
10544                                        Register n_tmp6) {
10545   Label L_processPartitions;
10546   Label L_processPartition;
10547   Label L_exit;
10548 
10549   bind(L_processPartitions);
10550   cmpl(in_out1, 3 * size);
10551   jcc(Assembler::less, L_exit);
10552     xorl(tmp1, tmp1);
10553     xorl(tmp2, tmp2);
10554     movq(tmp3, in_out2);
10555     addq(tmp3, size);
10556 
10557     bind(L_processPartition);
10558       crc32(in_out3, Address(in_out2, 0), 8);
10559       crc32(tmp1, Address(in_out2, size), 8);
10560       crc32(tmp2, Address(in_out2, size * 2), 8);
10561       addq(in_out2, 8);
10562       cmpq(in_out2, tmp3);
10563       jcc(Assembler::less, L_processPartition);
10564     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
10565             w_xtmp1, w_xtmp2, w_xtmp3,
10566             tmp4, tmp5,
10567             n_tmp6);
10568     addq(in_out2, 2 * size);
10569     subl(in_out1, 3 * size);
10570     jmp(L_processPartitions);
10571 
10572   bind(L_exit);
10573 }
10574 #else
10575 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
10576                                      Register tmp1, Register tmp2, Register tmp3,
10577                                      XMMRegister xtmp1, XMMRegister xtmp2) {
10578   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
10579   if (n > 0) {
10580     addl(tmp3, n * 256 * 8);
10581   }
10582   //    Q1 = TABLEExt[n][B & 0xFF];
10583   movl(tmp1, in_out);
10584   andl(tmp1, 0x000000FF);
10585   shll(tmp1, 3);
10586   addl(tmp1, tmp3);
10587   movq(xtmp1, Address(tmp1, 0));
10588 
10589   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
10590   movl(tmp2, in_out);
10591   shrl(tmp2, 8);
10592   andl(tmp2, 0x000000FF);
10593   shll(tmp2, 3);
10594   addl(tmp2, tmp3);
10595   movq(xtmp2, Address(tmp2, 0));
10596 
10597   psllq(xtmp2, 8);
10598   pxor(xtmp1, xtmp2);
10599 
10600   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
10601   movl(tmp2, in_out);
10602   shrl(tmp2, 16);
10603   andl(tmp2, 0x000000FF);
10604   shll(tmp2, 3);
10605   addl(tmp2, tmp3);
10606   movq(xtmp2, Address(tmp2, 0));
10607 
10608   psllq(xtmp2, 16);
10609   pxor(xtmp1, xtmp2);
10610 
10611   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
10612   shrl(in_out, 24);
10613   andl(in_out, 0x000000FF);
10614   shll(in_out, 3);
10615   addl(in_out, tmp3);
10616   movq(xtmp2, Address(in_out, 0));
10617 
10618   psllq(xtmp2, 24);
10619   pxor(xtmp1, xtmp2); // Result in CXMM
10620   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
10621 }
10622 
10623 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
10624                                       Register in_out,
10625                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
10626                                       XMMRegister w_xtmp2,
10627                                       Register tmp1,
10628                                       Register n_tmp2, Register n_tmp3) {
10629   if (is_pclmulqdq_supported) {
10630     movdl(w_xtmp1, in_out);
10631 
10632     movl(tmp1, const_or_pre_comp_const_index);
10633     movdl(w_xtmp2, tmp1);
10634     pclmulqdq(w_xtmp1, w_xtmp2, 0);
10635     // Keep result in XMM since GPR is 32 bit in length
10636   } else {
10637     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
10638   }
10639 }
10640 
10641 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
10642                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10643                                      Register tmp1, Register tmp2,
10644                                      Register n_tmp3) {
10645   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10646   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10647 
10648   psllq(w_xtmp1, 1);
10649   movdl(tmp1, w_xtmp1);
10650   psrlq(w_xtmp1, 32);
10651   movdl(in_out, w_xtmp1);
10652 
10653   xorl(tmp2, tmp2);
10654   crc32(tmp2, tmp1, 4);
10655   xorl(in_out, tmp2);
10656 
10657   psllq(w_xtmp2, 1);
10658   movdl(tmp1, w_xtmp2);
10659   psrlq(w_xtmp2, 32);
10660   movdl(in1, w_xtmp2);
10661 
10662   xorl(tmp2, tmp2);
10663   crc32(tmp2, tmp1, 4);
10664   xorl(in1, tmp2);
10665   xorl(in_out, in1);
10666   xorl(in_out, in2);
10667 }
10668 
10669 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
10670                                        Register in_out1, Register in_out2, Register in_out3,
10671                                        Register tmp1, Register tmp2, Register tmp3,
10672                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10673                                        Register tmp4, Register tmp5,
10674                                        Register n_tmp6) {
10675   Label L_processPartitions;
10676   Label L_processPartition;
10677   Label L_exit;
10678 
10679   bind(L_processPartitions);
10680   cmpl(in_out1, 3 * size);
10681   jcc(Assembler::less, L_exit);
10682     xorl(tmp1, tmp1);
10683     xorl(tmp2, tmp2);
10684     movl(tmp3, in_out2);
10685     addl(tmp3, size);
10686 
10687     bind(L_processPartition);
10688       crc32(in_out3, Address(in_out2, 0), 4);
10689       crc32(tmp1, Address(in_out2, size), 4);
10690       crc32(tmp2, Address(in_out2, size*2), 4);
10691       crc32(in_out3, Address(in_out2, 0+4), 4);
10692       crc32(tmp1, Address(in_out2, size+4), 4);
10693       crc32(tmp2, Address(in_out2, size*2+4), 4);
10694       addl(in_out2, 8);
10695       cmpl(in_out2, tmp3);
10696       jcc(Assembler::less, L_processPartition);
10697 
10698         push(tmp3);
10699         push(in_out1);
10700         push(in_out2);
10701         tmp4 = tmp3;
10702         tmp5 = in_out1;
10703         n_tmp6 = in_out2;
10704 
10705       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
10706             w_xtmp1, w_xtmp2, w_xtmp3,
10707             tmp4, tmp5,
10708             n_tmp6);
10709 
10710         pop(in_out2);
10711         pop(in_out1);
10712         pop(tmp3);
10713 
10714     addl(in_out2, 2 * size);
10715     subl(in_out1, 3 * size);
10716     jmp(L_processPartitions);
10717 
10718   bind(L_exit);
10719 }
10720 #endif //LP64
10721 
10722 #ifdef _LP64
10723 // Algorithm 2: Pipelined usage of the CRC32 instruction.
10724 // Input: A buffer I of L bytes.
10725 // Output: the CRC32C value of the buffer.
10726 // Notations:
10727 // Write L = 24N + r, with N = floor (L/24).
10728 // r = L mod 24 (0 <= r < 24).
10729 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
10730 // N quadwords, and R consists of r bytes.
10731 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
10732 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
10733 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
10734 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
10735 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10736                                           Register tmp1, Register tmp2, Register tmp3,
10737                                           Register tmp4, Register tmp5, Register tmp6,
10738                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10739                                           bool is_pclmulqdq_supported) {
10740   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10741   Label L_wordByWord;
10742   Label L_byteByByteProlog;
10743   Label L_byteByByte;
10744   Label L_exit;
10745 
10746   if (is_pclmulqdq_supported ) {
10747     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10748     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
10749 
10750     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10751     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10752 
10753     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10754     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10755     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
10756   } else {
10757     const_or_pre_comp_const_index[0] = 1;
10758     const_or_pre_comp_const_index[1] = 0;
10759 
10760     const_or_pre_comp_const_index[2] = 3;
10761     const_or_pre_comp_const_index[3] = 2;
10762 
10763     const_or_pre_comp_const_index[4] = 5;
10764     const_or_pre_comp_const_index[5] = 4;
10765    }
10766   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10767                     in2, in1, in_out,
10768                     tmp1, tmp2, tmp3,
10769                     w_xtmp1, w_xtmp2, w_xtmp3,
10770                     tmp4, tmp5,
10771                     tmp6);
10772   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10773                     in2, in1, in_out,
10774                     tmp1, tmp2, tmp3,
10775                     w_xtmp1, w_xtmp2, w_xtmp3,
10776                     tmp4, tmp5,
10777                     tmp6);
10778   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10779                     in2, in1, in_out,
10780                     tmp1, tmp2, tmp3,
10781                     w_xtmp1, w_xtmp2, w_xtmp3,
10782                     tmp4, tmp5,
10783                     tmp6);
10784   movl(tmp1, in2);
10785   andl(tmp1, 0x00000007);
10786   negl(tmp1);
10787   addl(tmp1, in2);
10788   addq(tmp1, in1);
10789 
10790   BIND(L_wordByWord);
10791   cmpq(in1, tmp1);
10792   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10793     crc32(in_out, Address(in1, 0), 4);
10794     addq(in1, 4);
10795     jmp(L_wordByWord);
10796 
10797   BIND(L_byteByByteProlog);
10798   andl(in2, 0x00000007);
10799   movl(tmp2, 1);
10800 
10801   BIND(L_byteByByte);
10802   cmpl(tmp2, in2);
10803   jccb(Assembler::greater, L_exit);
10804     crc32(in_out, Address(in1, 0), 1);
10805     incq(in1);
10806     incl(tmp2);
10807     jmp(L_byteByByte);
10808 
10809   BIND(L_exit);
10810 }
10811 #else
10812 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10813                                           Register tmp1, Register  tmp2, Register tmp3,
10814                                           Register tmp4, Register  tmp5, Register tmp6,
10815                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10816                                           bool is_pclmulqdq_supported) {
10817   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10818   Label L_wordByWord;
10819   Label L_byteByByteProlog;
10820   Label L_byteByByte;
10821   Label L_exit;
10822 
10823   if (is_pclmulqdq_supported) {
10824     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10825     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
10826 
10827     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10828     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10829 
10830     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10831     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10832   } else {
10833     const_or_pre_comp_const_index[0] = 1;
10834     const_or_pre_comp_const_index[1] = 0;
10835 
10836     const_or_pre_comp_const_index[2] = 3;
10837     const_or_pre_comp_const_index[3] = 2;
10838 
10839     const_or_pre_comp_const_index[4] = 5;
10840     const_or_pre_comp_const_index[5] = 4;
10841   }
10842   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10843                     in2, in1, in_out,
10844                     tmp1, tmp2, tmp3,
10845                     w_xtmp1, w_xtmp2, w_xtmp3,
10846                     tmp4, tmp5,
10847                     tmp6);
10848   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10849                     in2, in1, in_out,
10850                     tmp1, tmp2, tmp3,
10851                     w_xtmp1, w_xtmp2, w_xtmp3,
10852                     tmp4, tmp5,
10853                     tmp6);
10854   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10855                     in2, in1, in_out,
10856                     tmp1, tmp2, tmp3,
10857                     w_xtmp1, w_xtmp2, w_xtmp3,
10858                     tmp4, tmp5,
10859                     tmp6);
10860   movl(tmp1, in2);
10861   andl(tmp1, 0x00000007);
10862   negl(tmp1);
10863   addl(tmp1, in2);
10864   addl(tmp1, in1);
10865 
10866   BIND(L_wordByWord);
10867   cmpl(in1, tmp1);
10868   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10869     crc32(in_out, Address(in1,0), 4);
10870     addl(in1, 4);
10871     jmp(L_wordByWord);
10872 
10873   BIND(L_byteByByteProlog);
10874   andl(in2, 0x00000007);
10875   movl(tmp2, 1);
10876 
10877   BIND(L_byteByByte);
10878   cmpl(tmp2, in2);
10879   jccb(Assembler::greater, L_exit);
10880     movb(tmp1, Address(in1, 0));
10881     crc32(in_out, tmp1, 1);
10882     incl(in1);
10883     incl(tmp2);
10884     jmp(L_byteByByte);
10885 
10886   BIND(L_exit);
10887 }
10888 #endif // LP64
10889 #undef BIND
10890 #undef BLOCK_COMMENT
10891 
10892 // Compress char[] array to byte[].
10893 //   ..\jdk\src\java.base\share\classes\java\lang\StringUTF16.java
10894 //   @HotSpotIntrinsicCandidate
10895 //   private static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
10896 //     for (int i = 0; i < len; i++) {
10897 //       int c = src[srcOff++];
10898 //       if (c >>> 8 != 0) {
10899 //         return 0;
10900 //       }
10901 //       dst[dstOff++] = (byte)c;
10902 //     }
10903 //     return len;
10904 //   }
10905 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
10906   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
10907   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
10908   Register tmp5, Register result) {
10909   Label copy_chars_loop, return_length, return_zero, done, below_threshold;
10910 
10911   // rsi: src
10912   // rdi: dst
10913   // rdx: len
10914   // rcx: tmp5
10915   // rax: result
10916 
10917   // rsi holds start addr of source char[] to be compressed
10918   // rdi holds start addr of destination byte[]
10919   // rdx holds length
10920 
10921   assert(len != result, "");
10922 
10923   // save length for return
10924   push(len);
10925 
10926   if ((UseAVX > 2) && // AVX512
10927     VM_Version::supports_avx512vlbw() &&
10928     VM_Version::supports_bmi2()) {
10929 
10930     set_vector_masking();  // opening of the stub context for programming mask registers
10931 
10932     Label copy_32_loop, copy_loop_tail, restore_k1_return_zero;
10933 
10934     // alignement
10935     Label post_alignement;
10936 
10937     // if length of the string is less than 16, handle it in an old fashioned
10938     // way
10939     testl(len, -32);
10940     jcc(Assembler::zero, below_threshold);
10941 
10942     // First check whether a character is compressable ( <= 0xFF).
10943     // Create mask to test for Unicode chars inside zmm vector
10944     movl(result, 0x00FF);
10945     evpbroadcastw(tmp2Reg, result, Assembler::AVX_512bit);
10946 
10947     // Save k1
10948     kmovql(k3, k1);
10949 
10950     testl(len, -64);
10951     jcc(Assembler::zero, post_alignement);
10952 
10953     movl(tmp5, dst);
10954     andl(tmp5, (32 - 1));
10955     negl(tmp5);
10956     andl(tmp5, (32 - 1));
10957 
10958     // bail out when there is nothing to be done
10959     testl(tmp5, 0xFFFFFFFF);
10960     jcc(Assembler::zero, post_alignement);
10961 
10962     // ~(~0 << len), where len is the # of remaining elements to process
10963     movl(result, 0xFFFFFFFF);
10964     shlxl(result, result, tmp5);
10965     notl(result);
10966     kmovdl(k1, result);
10967 
10968     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
10969     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10970     ktestd(k2, k1);
10971     jcc(Assembler::carryClear, restore_k1_return_zero);
10972 
10973     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
10974 
10975     addptr(src, tmp5);
10976     addptr(src, tmp5);
10977     addptr(dst, tmp5);
10978     subl(len, tmp5);
10979 
10980     bind(post_alignement);
10981     // end of alignement
10982 
10983     movl(tmp5, len);
10984     andl(tmp5, (32 - 1));    // tail count (in chars)
10985     andl(len, ~(32 - 1));    // vector count (in chars)
10986     jcc(Assembler::zero, copy_loop_tail);
10987 
10988     lea(src, Address(src, len, Address::times_2));
10989     lea(dst, Address(dst, len, Address::times_1));
10990     negptr(len);
10991 
10992     bind(copy_32_loop);
10993     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
10994     evpcmpuw(k2, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10995     kortestdl(k2, k2);
10996     jcc(Assembler::carryClear, restore_k1_return_zero);
10997 
10998     // All elements in current processed chunk are valid candidates for
10999     // compression. Write a truncated byte elements to the memory.
11000     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
11001     addptr(len, 32);
11002     jcc(Assembler::notZero, copy_32_loop);
11003 
11004     bind(copy_loop_tail);
11005     // bail out when there is nothing to be done
11006     testl(tmp5, 0xFFFFFFFF);
11007     // Restore k1
11008     kmovql(k1, k3);
11009     jcc(Assembler::zero, return_length);
11010 
11011     movl(len, tmp5);
11012 
11013     // ~(~0 << len), where len is the # of remaining elements to process
11014     movl(result, 0xFFFFFFFF);
11015     shlxl(result, result, len);
11016     notl(result);
11017 
11018     kmovdl(k1, result);
11019 
11020     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
11021     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
11022     ktestd(k2, k1);
11023     jcc(Assembler::carryClear, restore_k1_return_zero);
11024 
11025     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
11026     // Restore k1
11027     kmovql(k1, k3);
11028     jmp(return_length);
11029 
11030     bind(restore_k1_return_zero);
11031     // Restore k1
11032     kmovql(k1, k3);
11033     jmp(return_zero);
11034 
11035     clear_vector_masking();   // closing of the stub context for programming mask registers
11036   }
11037   if (UseSSE42Intrinsics) {
11038     Label copy_32_loop, copy_16, copy_tail;
11039 
11040     bind(below_threshold);
11041 
11042     movl(result, len);
11043 
11044     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
11045 
11046     // vectored compression
11047     andl(len, 0xfffffff0);    // vector count (in chars)
11048     andl(result, 0x0000000f);    // tail count (in chars)
11049     testl(len, len);
11050     jccb(Assembler::zero, copy_16);
11051 
11052     // compress 16 chars per iter
11053     movdl(tmp1Reg, tmp5);
11054     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
11055     pxor(tmp4Reg, tmp4Reg);
11056 
11057     lea(src, Address(src, len, Address::times_2));
11058     lea(dst, Address(dst, len, Address::times_1));
11059     negptr(len);
11060 
11061     bind(copy_32_loop);
11062     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
11063     por(tmp4Reg, tmp2Reg);
11064     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
11065     por(tmp4Reg, tmp3Reg);
11066     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
11067     jcc(Assembler::notZero, return_zero);
11068     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
11069     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
11070     addptr(len, 16);
11071     jcc(Assembler::notZero, copy_32_loop);
11072 
11073     // compress next vector of 8 chars (if any)
11074     bind(copy_16);
11075     movl(len, result);
11076     andl(len, 0xfffffff8);    // vector count (in chars)
11077     andl(result, 0x00000007);    // tail count (in chars)
11078     testl(len, len);
11079     jccb(Assembler::zero, copy_tail);
11080 
11081     movdl(tmp1Reg, tmp5);
11082     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
11083     pxor(tmp3Reg, tmp3Reg);
11084 
11085     movdqu(tmp2Reg, Address(src, 0));
11086     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
11087     jccb(Assembler::notZero, return_zero);
11088     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
11089     movq(Address(dst, 0), tmp2Reg);
11090     addptr(src, 16);
11091     addptr(dst, 8);
11092 
11093     bind(copy_tail);
11094     movl(len, result);
11095   }
11096   // compress 1 char per iter
11097   testl(len, len);
11098   jccb(Assembler::zero, return_length);
11099   lea(src, Address(src, len, Address::times_2));
11100   lea(dst, Address(dst, len, Address::times_1));
11101   negptr(len);
11102 
11103   bind(copy_chars_loop);
11104   load_unsigned_short(result, Address(src, len, Address::times_2));
11105   testl(result, 0xff00);      // check if Unicode char
11106   jccb(Assembler::notZero, return_zero);
11107   movb(Address(dst, len, Address::times_1), result);  // ASCII char; compress to 1 byte
11108   increment(len);
11109   jcc(Assembler::notZero, copy_chars_loop);
11110 
11111   // if compression succeeded, return length
11112   bind(return_length);
11113   pop(result);
11114   jmpb(done);
11115 
11116   // if compression failed, return 0
11117   bind(return_zero);
11118   xorl(result, result);
11119   addptr(rsp, wordSize);
11120 
11121   bind(done);
11122 }
11123 
11124 // Inflate byte[] array to char[].
11125 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
11126 //   @HotSpotIntrinsicCandidate
11127 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
11128 //     for (int i = 0; i < len; i++) {
11129 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
11130 //     }
11131 //   }
11132 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
11133   XMMRegister tmp1, Register tmp2) {
11134   Label copy_chars_loop, done, below_threshold;
11135   // rsi: src
11136   // rdi: dst
11137   // rdx: len
11138   // rcx: tmp2
11139 
11140   // rsi holds start addr of source byte[] to be inflated
11141   // rdi holds start addr of destination char[]
11142   // rdx holds length
11143   assert_different_registers(src, dst, len, tmp2);
11144 
11145   if ((UseAVX > 2) && // AVX512
11146     VM_Version::supports_avx512vlbw() &&
11147     VM_Version::supports_bmi2()) {
11148 
11149     set_vector_masking();  // opening of the stub context for programming mask registers
11150 
11151     Label copy_32_loop, copy_tail;
11152     Register tmp3_aliased = len;
11153 
11154     // if length of the string is less than 16, handle it in an old fashioned
11155     // way
11156     testl(len, -16);
11157     jcc(Assembler::zero, below_threshold);
11158 
11159     // In order to use only one arithmetic operation for the main loop we use
11160     // this pre-calculation
11161     movl(tmp2, len);
11162     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
11163     andl(len, -32);     // vector count
11164     jccb(Assembler::zero, copy_tail);
11165 
11166     lea(src, Address(src, len, Address::times_1));
11167     lea(dst, Address(dst, len, Address::times_2));
11168     negptr(len);
11169 
11170 
11171     // inflate 32 chars per iter
11172     bind(copy_32_loop);
11173     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
11174     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
11175     addptr(len, 32);
11176     jcc(Assembler::notZero, copy_32_loop);
11177 
11178     bind(copy_tail);
11179     // bail out when there is nothing to be done
11180     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
11181     jcc(Assembler::zero, done);
11182 
11183     // Save k1
11184     kmovql(k2, k1);
11185 
11186     // ~(~0 << length), where length is the # of remaining elements to process
11187     movl(tmp3_aliased, -1);
11188     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
11189     notl(tmp3_aliased);
11190     kmovdl(k1, tmp3_aliased);
11191     evpmovzxbw(tmp1, k1, Address(src, 0), Assembler::AVX_512bit);
11192     evmovdquw(Address(dst, 0), k1, tmp1, Assembler::AVX_512bit);
11193 
11194     // Restore k1
11195     kmovql(k1, k2);
11196     jmp(done);
11197 
11198     clear_vector_masking();   // closing of the stub context for programming mask registers
11199   }
11200   if (UseSSE42Intrinsics) {
11201     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
11202 
11203     movl(tmp2, len);
11204 
11205     if (UseAVX > 1) {
11206       andl(tmp2, (16 - 1));
11207       andl(len, -16);
11208       jccb(Assembler::zero, copy_new_tail);
11209     } else {
11210       andl(tmp2, 0x00000007);   // tail count (in chars)
11211       andl(len, 0xfffffff8);    // vector count (in chars)
11212       jccb(Assembler::zero, copy_tail);
11213     }
11214 
11215     // vectored inflation
11216     lea(src, Address(src, len, Address::times_1));
11217     lea(dst, Address(dst, len, Address::times_2));
11218     negptr(len);
11219 
11220     if (UseAVX > 1) {
11221       bind(copy_16_loop);
11222       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
11223       vmovdqu(Address(dst, len, Address::times_2), tmp1);
11224       addptr(len, 16);
11225       jcc(Assembler::notZero, copy_16_loop);
11226 
11227       bind(below_threshold);
11228       bind(copy_new_tail);
11229       if ((UseAVX > 2) &&
11230         VM_Version::supports_avx512vlbw() &&
11231         VM_Version::supports_bmi2()) {
11232         movl(tmp2, len);
11233       } else {
11234         movl(len, tmp2);
11235       }
11236       andl(tmp2, 0x00000007);
11237       andl(len, 0xFFFFFFF8);
11238       jccb(Assembler::zero, copy_tail);
11239 
11240       pmovzxbw(tmp1, Address(src, 0));
11241       movdqu(Address(dst, 0), tmp1);
11242       addptr(src, 8);
11243       addptr(dst, 2 * 8);
11244 
11245       jmp(copy_tail, true);
11246     }
11247 
11248     // inflate 8 chars per iter
11249     bind(copy_8_loop);
11250     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
11251     movdqu(Address(dst, len, Address::times_2), tmp1);
11252     addptr(len, 8);
11253     jcc(Assembler::notZero, copy_8_loop);
11254 
11255     bind(copy_tail);
11256     movl(len, tmp2);
11257 
11258     cmpl(len, 4);
11259     jccb(Assembler::less, copy_bytes);
11260 
11261     movdl(tmp1, Address(src, 0));  // load 4 byte chars
11262     pmovzxbw(tmp1, tmp1);
11263     movq(Address(dst, 0), tmp1);
11264     subptr(len, 4);
11265     addptr(src, 4);
11266     addptr(dst, 8);
11267 
11268     bind(copy_bytes);
11269   }
11270   testl(len, len);
11271   jccb(Assembler::zero, done);
11272   lea(src, Address(src, len, Address::times_1));
11273   lea(dst, Address(dst, len, Address::times_2));
11274   negptr(len);
11275 
11276   // inflate 1 char per iter
11277   bind(copy_chars_loop);
11278   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
11279   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
11280   increment(len);
11281   jcc(Assembler::notZero, copy_chars_loop);
11282 
11283   bind(done);
11284 }
11285 
11286 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
11287   switch (cond) {
11288     // Note some conditions are synonyms for others
11289     case Assembler::zero:         return Assembler::notZero;
11290     case Assembler::notZero:      return Assembler::zero;
11291     case Assembler::less:         return Assembler::greaterEqual;
11292     case Assembler::lessEqual:    return Assembler::greater;
11293     case Assembler::greater:      return Assembler::lessEqual;
11294     case Assembler::greaterEqual: return Assembler::less;
11295     case Assembler::below:        return Assembler::aboveEqual;
11296     case Assembler::belowEqual:   return Assembler::above;
11297     case Assembler::above:        return Assembler::belowEqual;
11298     case Assembler::aboveEqual:   return Assembler::below;
11299     case Assembler::overflow:     return Assembler::noOverflow;
11300     case Assembler::noOverflow:   return Assembler::overflow;
11301     case Assembler::negative:     return Assembler::positive;
11302     case Assembler::positive:     return Assembler::negative;
11303     case Assembler::parity:       return Assembler::noParity;
11304     case Assembler::noParity:     return Assembler::parity;
11305   }
11306   ShouldNotReachHere(); return Assembler::overflow;
11307 }
11308 
11309 SkipIfEqual::SkipIfEqual(
11310     MacroAssembler* masm, const bool* flag_addr, bool value) {
11311   _masm = masm;
11312   _masm->cmp8(ExternalAddress((address)flag_addr), value);
11313   _masm->jcc(Assembler::equal, _label);
11314 }
11315 
11316 SkipIfEqual::~SkipIfEqual() {
11317   _masm->bind(_label);
11318 }
11319 
11320 // 32-bit Windows has its own fast-path implementation
11321 // of get_thread
11322 #if !defined(WIN32) || defined(_LP64)
11323 
11324 // This is simply a call to Thread::current()
11325 void MacroAssembler::get_thread(Register thread) {
11326   if (thread != rax) {
11327     push(rax);
11328   }
11329   LP64_ONLY(push(rdi);)
11330   LP64_ONLY(push(rsi);)
11331   push(rdx);
11332   push(rcx);
11333 #ifdef _LP64
11334   push(r8);
11335   push(r9);
11336   push(r10);
11337   push(r11);
11338 #endif
11339 
11340   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
11341 
11342 #ifdef _LP64
11343   pop(r11);
11344   pop(r10);
11345   pop(r9);
11346   pop(r8);
11347 #endif
11348   pop(rcx);
11349   pop(rdx);
11350   LP64_ONLY(pop(rsi);)
11351   LP64_ONLY(pop(rdi);)
11352   if (thread != rax) {
11353     mov(thread, rax);
11354     pop(rax);
11355   }
11356 }
11357 
11358 #endif