1 /*
   2  * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "asm/assembler.hpp"
  28 #include "asm/assembler.inline.hpp"
  29 #include "compiler/disassembler.hpp"
  30 #include "gc/shared/cardTable.hpp"
  31 #include "gc/shared/cardTableBarrierSet.hpp"
  32 #include "gc/shared/collectedHeap.inline.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "memory/universe.hpp"
  36 #include "oops/klass.inline.hpp"
  37 #include "prims/methodHandles.hpp"
  38 #include "runtime/biasedLocking.hpp"
  39 #include "runtime/interfaceSupport.inline.hpp"
  40 #include "runtime/objectMonitor.hpp"
  41 #include "runtime/os.hpp"
  42 #include "runtime/safepoint.hpp"
  43 #include "runtime/safepointMechanism.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 #include "runtime/thread.hpp"
  47 #include "utilities/macros.hpp"
  48 #if INCLUDE_ALL_GCS
  49 #include "gc/g1/g1BarrierSet.hpp"
  50 #include "gc/g1/g1CardTable.hpp"
  51 #include "gc/g1/heapRegion.hpp"
  52 #endif // INCLUDE_ALL_GCS
  53 #include "crc32c.h"
  54 #ifdef COMPILER2
  55 #include "opto/intrinsicnode.hpp"
  56 #endif
  57 
  58 #ifdef PRODUCT
  59 #define BLOCK_COMMENT(str) /* nothing */
  60 #define STOP(error) stop(error)
  61 #else
  62 #define BLOCK_COMMENT(str) block_comment(str)
  63 #define STOP(error) block_comment(error); stop(error)
  64 #endif
  65 
  66 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  67 
  68 #ifdef ASSERT
  69 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  70 #endif
  71 
  72 static Assembler::Condition reverse[] = {
  73     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  74     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  75     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  76     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  77     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
  78     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
  79     Assembler::above          /* belowEqual    = 0x6 */ ,
  80     Assembler::belowEqual     /* above         = 0x7 */ ,
  81     Assembler::positive       /* negative      = 0x8 */ ,
  82     Assembler::negative       /* positive      = 0x9 */ ,
  83     Assembler::noParity       /* parity        = 0xa */ ,
  84     Assembler::parity         /* noParity      = 0xb */ ,
  85     Assembler::greaterEqual   /* less          = 0xc */ ,
  86     Assembler::less           /* greaterEqual  = 0xd */ ,
  87     Assembler::greater        /* lessEqual     = 0xe */ ,
  88     Assembler::lessEqual      /* greater       = 0xf, */
  89 
  90 };
  91 
  92 
  93 // Implementation of MacroAssembler
  94 
  95 // First all the versions that have distinct versions depending on 32/64 bit
  96 // Unless the difference is trivial (1 line or so).
  97 
  98 #ifndef _LP64
  99 
 100 // 32bit versions
 101 
 102 Address MacroAssembler::as_Address(AddressLiteral adr) {
 103   return Address(adr.target(), adr.rspec());
 104 }
 105 
 106 Address MacroAssembler::as_Address(ArrayAddress adr) {
 107   return Address::make_array(adr);
 108 }
 109 
 110 void MacroAssembler::call_VM_leaf_base(address entry_point,
 111                                        int number_of_arguments) {
 112   call(RuntimeAddress(entry_point));
 113   increment(rsp, number_of_arguments * wordSize);
 114 }
 115 
 116 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
 117   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 118 }
 119 
 120 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
 121   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 122 }
 123 
 124 void MacroAssembler::cmpoop(Address src1, jobject obj) {
 125   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 126 }
 127 
 128 void MacroAssembler::cmpoop(Register src1, jobject obj) {
 129   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 130 }
 131 
 132 void MacroAssembler::extend_sign(Register hi, Register lo) {
 133   // According to Intel Doc. AP-526, "Integer Divide", p.18.
 134   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
 135     cdql();
 136   } else {
 137     movl(hi, lo);
 138     sarl(hi, 31);
 139   }
 140 }
 141 
 142 void MacroAssembler::jC2(Register tmp, Label& L) {
 143   // set parity bit if FPU flag C2 is set (via rax)
 144   save_rax(tmp);
 145   fwait(); fnstsw_ax();
 146   sahf();
 147   restore_rax(tmp);
 148   // branch
 149   jcc(Assembler::parity, L);
 150 }
 151 
 152 void MacroAssembler::jnC2(Register tmp, Label& L) {
 153   // set parity bit if FPU flag C2 is set (via rax)
 154   save_rax(tmp);
 155   fwait(); fnstsw_ax();
 156   sahf();
 157   restore_rax(tmp);
 158   // branch
 159   jcc(Assembler::noParity, L);
 160 }
 161 
 162 // 32bit can do a case table jump in one instruction but we no longer allow the base
 163 // to be installed in the Address class
 164 void MacroAssembler::jump(ArrayAddress entry) {
 165   jmp(as_Address(entry));
 166 }
 167 
 168 // Note: y_lo will be destroyed
 169 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 170   // Long compare for Java (semantics as described in JVM spec.)
 171   Label high, low, done;
 172 
 173   cmpl(x_hi, y_hi);
 174   jcc(Assembler::less, low);
 175   jcc(Assembler::greater, high);
 176   // x_hi is the return register
 177   xorl(x_hi, x_hi);
 178   cmpl(x_lo, y_lo);
 179   jcc(Assembler::below, low);
 180   jcc(Assembler::equal, done);
 181 
 182   bind(high);
 183   xorl(x_hi, x_hi);
 184   increment(x_hi);
 185   jmp(done);
 186 
 187   bind(low);
 188   xorl(x_hi, x_hi);
 189   decrementl(x_hi);
 190 
 191   bind(done);
 192 }
 193 
 194 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 195     mov_literal32(dst, (int32_t)src.target(), src.rspec());
 196 }
 197 
 198 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 199   // leal(dst, as_Address(adr));
 200   // see note in movl as to why we must use a move
 201   mov_literal32(dst, (int32_t) adr.target(), adr.rspec());
 202 }
 203 
 204 void MacroAssembler::leave() {
 205   mov(rsp, rbp);
 206   pop(rbp);
 207 }
 208 
 209 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
 210   // Multiplication of two Java long values stored on the stack
 211   // as illustrated below. Result is in rdx:rax.
 212   //
 213   // rsp ---> [  ??  ] \               \
 214   //            ....    | y_rsp_offset  |
 215   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
 216   //          [ y_hi ]                  | (in bytes)
 217   //            ....                    |
 218   //          [ x_lo ]                 /
 219   //          [ x_hi ]
 220   //            ....
 221   //
 222   // Basic idea: lo(result) = lo(x_lo * y_lo)
 223   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
 224   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
 225   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
 226   Label quick;
 227   // load x_hi, y_hi and check if quick
 228   // multiplication is possible
 229   movl(rbx, x_hi);
 230   movl(rcx, y_hi);
 231   movl(rax, rbx);
 232   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
 233   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
 234   // do full multiplication
 235   // 1st step
 236   mull(y_lo);                                    // x_hi * y_lo
 237   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
 238   // 2nd step
 239   movl(rax, x_lo);
 240   mull(rcx);                                     // x_lo * y_hi
 241   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
 242   // 3rd step
 243   bind(quick);                                   // note: rbx, = 0 if quick multiply!
 244   movl(rax, x_lo);
 245   mull(y_lo);                                    // x_lo * y_lo
 246   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
 247 }
 248 
 249 void MacroAssembler::lneg(Register hi, Register lo) {
 250   negl(lo);
 251   adcl(hi, 0);
 252   negl(hi);
 253 }
 254 
 255 void MacroAssembler::lshl(Register hi, Register lo) {
 256   // Java shift left long support (semantics as described in JVM spec., p.305)
 257   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
 258   // shift value is in rcx !
 259   assert(hi != rcx, "must not use rcx");
 260   assert(lo != rcx, "must not use rcx");
 261   const Register s = rcx;                        // shift count
 262   const int      n = BitsPerWord;
 263   Label L;
 264   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 265   cmpl(s, n);                                    // if (s < n)
 266   jcc(Assembler::less, L);                       // else (s >= n)
 267   movl(hi, lo);                                  // x := x << n
 268   xorl(lo, lo);
 269   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 270   bind(L);                                       // s (mod n) < n
 271   shldl(hi, lo);                                 // x := x << s
 272   shll(lo);
 273 }
 274 
 275 
 276 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
 277   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
 278   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
 279   assert(hi != rcx, "must not use rcx");
 280   assert(lo != rcx, "must not use rcx");
 281   const Register s = rcx;                        // shift count
 282   const int      n = BitsPerWord;
 283   Label L;
 284   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 285   cmpl(s, n);                                    // if (s < n)
 286   jcc(Assembler::less, L);                       // else (s >= n)
 287   movl(lo, hi);                                  // x := x >> n
 288   if (sign_extension) sarl(hi, 31);
 289   else                xorl(hi, hi);
 290   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 291   bind(L);                                       // s (mod n) < n
 292   shrdl(lo, hi);                                 // x := x >> s
 293   if (sign_extension) sarl(hi);
 294   else                shrl(hi);
 295 }
 296 
 297 void MacroAssembler::movoop(Register dst, jobject obj) {
 298   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 299 }
 300 
 301 void MacroAssembler::movoop(Address dst, jobject obj) {
 302   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 303 }
 304 
 305 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 306   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 307 }
 308 
 309 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 310   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 311 }
 312 
 313 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 314   // scratch register is not used,
 315   // it is defined to match parameters of 64-bit version of this method.
 316   if (src.is_lval()) {
 317     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
 318   } else {
 319     movl(dst, as_Address(src));
 320   }
 321 }
 322 
 323 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 324   movl(as_Address(dst), src);
 325 }
 326 
 327 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 328   movl(dst, as_Address(src));
 329 }
 330 
 331 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 332 void MacroAssembler::movptr(Address dst, intptr_t src) {
 333   movl(dst, src);
 334 }
 335 
 336 
 337 void MacroAssembler::pop_callee_saved_registers() {
 338   pop(rcx);
 339   pop(rdx);
 340   pop(rdi);
 341   pop(rsi);
 342 }
 343 
 344 void MacroAssembler::pop_fTOS() {
 345   fld_d(Address(rsp, 0));
 346   addl(rsp, 2 * wordSize);
 347 }
 348 
 349 void MacroAssembler::push_callee_saved_registers() {
 350   push(rsi);
 351   push(rdi);
 352   push(rdx);
 353   push(rcx);
 354 }
 355 
 356 void MacroAssembler::push_fTOS() {
 357   subl(rsp, 2 * wordSize);
 358   fstp_d(Address(rsp, 0));
 359 }
 360 
 361 
 362 void MacroAssembler::pushoop(jobject obj) {
 363   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
 364 }
 365 
 366 void MacroAssembler::pushklass(Metadata* obj) {
 367   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
 368 }
 369 
 370 void MacroAssembler::pushptr(AddressLiteral src) {
 371   if (src.is_lval()) {
 372     push_literal32((int32_t)src.target(), src.rspec());
 373   } else {
 374     pushl(as_Address(src));
 375   }
 376 }
 377 
 378 void MacroAssembler::set_word_if_not_zero(Register dst) {
 379   xorl(dst, dst);
 380   set_byte_if_not_zero(dst);
 381 }
 382 
 383 static void pass_arg0(MacroAssembler* masm, Register arg) {
 384   masm->push(arg);
 385 }
 386 
 387 static void pass_arg1(MacroAssembler* masm, Register arg) {
 388   masm->push(arg);
 389 }
 390 
 391 static void pass_arg2(MacroAssembler* masm, Register arg) {
 392   masm->push(arg);
 393 }
 394 
 395 static void pass_arg3(MacroAssembler* masm, Register arg) {
 396   masm->push(arg);
 397 }
 398 
 399 #ifndef PRODUCT
 400 extern "C" void findpc(intptr_t x);
 401 #endif
 402 
 403 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
 404   // In order to get locks to work, we need to fake a in_VM state
 405   JavaThread* thread = JavaThread::current();
 406   JavaThreadState saved_state = thread->thread_state();
 407   thread->set_thread_state(_thread_in_vm);
 408   if (ShowMessageBoxOnError) {
 409     JavaThread* thread = JavaThread::current();
 410     JavaThreadState saved_state = thread->thread_state();
 411     thread->set_thread_state(_thread_in_vm);
 412     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 413       ttyLocker ttyl;
 414       BytecodeCounter::print();
 415     }
 416     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 417     // This is the value of eip which points to where verify_oop will return.
 418     if (os::message_box(msg, "Execution stopped, print registers?")) {
 419       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
 420       BREAKPOINT;
 421     }
 422   } else {
 423     ttyLocker ttyl;
 424     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
 425   }
 426   // Don't assert holding the ttyLock
 427     assert(false, "DEBUG MESSAGE: %s", msg);
 428   ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 429 }
 430 
 431 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
 432   ttyLocker ttyl;
 433   FlagSetting fs(Debugging, true);
 434   tty->print_cr("eip = 0x%08x", eip);
 435 #ifndef PRODUCT
 436   if ((WizardMode || Verbose) && PrintMiscellaneous) {
 437     tty->cr();
 438     findpc(eip);
 439     tty->cr();
 440   }
 441 #endif
 442 #define PRINT_REG(rax) \
 443   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
 444   PRINT_REG(rax);
 445   PRINT_REG(rbx);
 446   PRINT_REG(rcx);
 447   PRINT_REG(rdx);
 448   PRINT_REG(rdi);
 449   PRINT_REG(rsi);
 450   PRINT_REG(rbp);
 451   PRINT_REG(rsp);
 452 #undef PRINT_REG
 453   // Print some words near top of staack.
 454   int* dump_sp = (int*) rsp;
 455   for (int col1 = 0; col1 < 8; col1++) {
 456     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 457     os::print_location(tty, *dump_sp++);
 458   }
 459   for (int row = 0; row < 16; row++) {
 460     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 461     for (int col = 0; col < 8; col++) {
 462       tty->print(" 0x%08x", *dump_sp++);
 463     }
 464     tty->cr();
 465   }
 466   // Print some instructions around pc:
 467   Disassembler::decode((address)eip-64, (address)eip);
 468   tty->print_cr("--------");
 469   Disassembler::decode((address)eip, (address)eip+32);
 470 }
 471 
 472 void MacroAssembler::stop(const char* msg) {
 473   ExternalAddress message((address)msg);
 474   // push address of message
 475   pushptr(message.addr());
 476   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 477   pusha();                                            // push registers
 478   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 479   hlt();
 480 }
 481 
 482 void MacroAssembler::warn(const char* msg) {
 483   push_CPU_state();
 484 
 485   ExternalAddress message((address) msg);
 486   // push address of message
 487   pushptr(message.addr());
 488 
 489   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
 490   addl(rsp, wordSize);       // discard argument
 491   pop_CPU_state();
 492 }
 493 
 494 void MacroAssembler::print_state() {
 495   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 496   pusha();                                            // push registers
 497 
 498   push_CPU_state();
 499   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
 500   pop_CPU_state();
 501 
 502   popa();
 503   addl(rsp, wordSize);
 504 }
 505 
 506 #else // _LP64
 507 
 508 // 64 bit versions
 509 
 510 Address MacroAssembler::as_Address(AddressLiteral adr) {
 511   // amd64 always does this as a pc-rel
 512   // we can be absolute or disp based on the instruction type
 513   // jmp/call are displacements others are absolute
 514   assert(!adr.is_lval(), "must be rval");
 515   assert(reachable(adr), "must be");
 516   return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
 517 
 518 }
 519 
 520 Address MacroAssembler::as_Address(ArrayAddress adr) {
 521   AddressLiteral base = adr.base();
 522   lea(rscratch1, base);
 523   Address index = adr.index();
 524   assert(index._disp == 0, "must not have disp"); // maybe it can?
 525   Address array(rscratch1, index._index, index._scale, index._disp);
 526   return array;
 527 }
 528 
 529 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
 530   Label L, E;
 531 
 532 #ifdef _WIN64
 533   // Windows always allocates space for it's register args
 534   assert(num_args <= 4, "only register arguments supported");
 535   subq(rsp,  frame::arg_reg_save_area_bytes);
 536 #endif
 537 
 538   // Align stack if necessary
 539   testl(rsp, 15);
 540   jcc(Assembler::zero, L);
 541 
 542   subq(rsp, 8);
 543   {
 544     call(RuntimeAddress(entry_point));
 545   }
 546   addq(rsp, 8);
 547   jmp(E);
 548 
 549   bind(L);
 550   {
 551     call(RuntimeAddress(entry_point));
 552   }
 553 
 554   bind(E);
 555 
 556 #ifdef _WIN64
 557   // restore stack pointer
 558   addq(rsp, frame::arg_reg_save_area_bytes);
 559 #endif
 560 
 561 }
 562 
 563 void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
 564   assert(!src2.is_lval(), "should use cmpptr");
 565 
 566   if (reachable(src2)) {
 567     cmpq(src1, as_Address(src2));
 568   } else {
 569     lea(rscratch1, src2);
 570     Assembler::cmpq(src1, Address(rscratch1, 0));
 571   }
 572 }
 573 
 574 int MacroAssembler::corrected_idivq(Register reg) {
 575   // Full implementation of Java ldiv and lrem; checks for special
 576   // case as described in JVM spec., p.243 & p.271.  The function
 577   // returns the (pc) offset of the idivl instruction - may be needed
 578   // for implicit exceptions.
 579   //
 580   //         normal case                           special case
 581   //
 582   // input : rax: dividend                         min_long
 583   //         reg: divisor   (may not be eax/edx)   -1
 584   //
 585   // output: rax: quotient  (= rax idiv reg)       min_long
 586   //         rdx: remainder (= rax irem reg)       0
 587   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
 588   static const int64_t min_long = 0x8000000000000000;
 589   Label normal_case, special_case;
 590 
 591   // check for special case
 592   cmp64(rax, ExternalAddress((address) &min_long));
 593   jcc(Assembler::notEqual, normal_case);
 594   xorl(rdx, rdx); // prepare rdx for possible special case (where
 595                   // remainder = 0)
 596   cmpq(reg, -1);
 597   jcc(Assembler::equal, special_case);
 598 
 599   // handle normal case
 600   bind(normal_case);
 601   cdqq();
 602   int idivq_offset = offset();
 603   idivq(reg);
 604 
 605   // normal and special case exit
 606   bind(special_case);
 607 
 608   return idivq_offset;
 609 }
 610 
 611 void MacroAssembler::decrementq(Register reg, int value) {
 612   if (value == min_jint) { subq(reg, value); return; }
 613   if (value <  0) { incrementq(reg, -value); return; }
 614   if (value == 0) {                        ; return; }
 615   if (value == 1 && UseIncDec) { decq(reg) ; return; }
 616   /* else */      { subq(reg, value)       ; return; }
 617 }
 618 
 619 void MacroAssembler::decrementq(Address dst, int value) {
 620   if (value == min_jint) { subq(dst, value); return; }
 621   if (value <  0) { incrementq(dst, -value); return; }
 622   if (value == 0) {                        ; return; }
 623   if (value == 1 && UseIncDec) { decq(dst) ; return; }
 624   /* else */      { subq(dst, value)       ; return; }
 625 }
 626 
 627 void MacroAssembler::incrementq(AddressLiteral dst) {
 628   if (reachable(dst)) {
 629     incrementq(as_Address(dst));
 630   } else {
 631     lea(rscratch1, dst);
 632     incrementq(Address(rscratch1, 0));
 633   }
 634 }
 635 
 636 void MacroAssembler::incrementq(Register reg, int value) {
 637   if (value == min_jint) { addq(reg, value); return; }
 638   if (value <  0) { decrementq(reg, -value); return; }
 639   if (value == 0) {                        ; return; }
 640   if (value == 1 && UseIncDec) { incq(reg) ; return; }
 641   /* else */      { addq(reg, value)       ; return; }
 642 }
 643 
 644 void MacroAssembler::incrementq(Address dst, int value) {
 645   if (value == min_jint) { addq(dst, value); return; }
 646   if (value <  0) { decrementq(dst, -value); return; }
 647   if (value == 0) {                        ; return; }
 648   if (value == 1 && UseIncDec) { incq(dst) ; return; }
 649   /* else */      { addq(dst, value)       ; return; }
 650 }
 651 
 652 // 32bit can do a case table jump in one instruction but we no longer allow the base
 653 // to be installed in the Address class
 654 void MacroAssembler::jump(ArrayAddress entry) {
 655   lea(rscratch1, entry.base());
 656   Address dispatch = entry.index();
 657   assert(dispatch._base == noreg, "must be");
 658   dispatch._base = rscratch1;
 659   jmp(dispatch);
 660 }
 661 
 662 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 663   ShouldNotReachHere(); // 64bit doesn't use two regs
 664   cmpq(x_lo, y_lo);
 665 }
 666 
 667 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 668     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 669 }
 670 
 671 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 672   mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());
 673   movptr(dst, rscratch1);
 674 }
 675 
 676 void MacroAssembler::leave() {
 677   // %%% is this really better? Why not on 32bit too?
 678   emit_int8((unsigned char)0xC9); // LEAVE
 679 }
 680 
 681 void MacroAssembler::lneg(Register hi, Register lo) {
 682   ShouldNotReachHere(); // 64bit doesn't use two regs
 683   negq(lo);
 684 }
 685 
 686 void MacroAssembler::movoop(Register dst, jobject obj) {
 687   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 688 }
 689 
 690 void MacroAssembler::movoop(Address dst, jobject obj) {
 691   mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 692   movq(dst, rscratch1);
 693 }
 694 
 695 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 696   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 697 }
 698 
 699 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 700   mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 701   movq(dst, rscratch1);
 702 }
 703 
 704 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 705   if (src.is_lval()) {
 706     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 707   } else {
 708     if (reachable(src)) {
 709       movq(dst, as_Address(src));
 710     } else {
 711       lea(scratch, src);
 712       movq(dst, Address(scratch, 0));
 713     }
 714   }
 715 }
 716 
 717 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 718   movq(as_Address(dst), src);
 719 }
 720 
 721 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 722   movq(dst, as_Address(src));
 723 }
 724 
 725 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 726 void MacroAssembler::movptr(Address dst, intptr_t src) {
 727   mov64(rscratch1, src);
 728   movq(dst, rscratch1);
 729 }
 730 
 731 // These are mostly for initializing NULL
 732 void MacroAssembler::movptr(Address dst, int32_t src) {
 733   movslq(dst, src);
 734 }
 735 
 736 void MacroAssembler::movptr(Register dst, int32_t src) {
 737   mov64(dst, (intptr_t)src);
 738 }
 739 
 740 void MacroAssembler::pushoop(jobject obj) {
 741   movoop(rscratch1, obj);
 742   push(rscratch1);
 743 }
 744 
 745 void MacroAssembler::pushklass(Metadata* obj) {
 746   mov_metadata(rscratch1, obj);
 747   push(rscratch1);
 748 }
 749 
 750 void MacroAssembler::pushptr(AddressLiteral src) {
 751   lea(rscratch1, src);
 752   if (src.is_lval()) {
 753     push(rscratch1);
 754   } else {
 755     pushq(Address(rscratch1, 0));
 756   }
 757 }
 758 
 759 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
 760   // we must set sp to zero to clear frame
 761   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 762   // must clear fp, so that compiled frames are not confused; it is
 763   // possible that we need it only for debugging
 764   if (clear_fp) {
 765     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 766   }
 767 
 768   // Always clear the pc because it could have been set by make_walkable()
 769   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 770   vzeroupper();
 771 }
 772 
 773 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 774                                          Register last_java_fp,
 775                                          address  last_java_pc) {
 776   vzeroupper();
 777   // determine last_java_sp register
 778   if (!last_java_sp->is_valid()) {
 779     last_java_sp = rsp;
 780   }
 781 
 782   // last_java_fp is optional
 783   if (last_java_fp->is_valid()) {
 784     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),
 785            last_java_fp);
 786   }
 787 
 788   // last_java_pc is optional
 789   if (last_java_pc != NULL) {
 790     Address java_pc(r15_thread,
 791                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 792     lea(rscratch1, InternalAddress(last_java_pc));
 793     movptr(java_pc, rscratch1);
 794   }
 795 
 796   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 797 }
 798 
 799 static void pass_arg0(MacroAssembler* masm, Register arg) {
 800   if (c_rarg0 != arg ) {
 801     masm->mov(c_rarg0, arg);
 802   }
 803 }
 804 
 805 static void pass_arg1(MacroAssembler* masm, Register arg) {
 806   if (c_rarg1 != arg ) {
 807     masm->mov(c_rarg1, arg);
 808   }
 809 }
 810 
 811 static void pass_arg2(MacroAssembler* masm, Register arg) {
 812   if (c_rarg2 != arg ) {
 813     masm->mov(c_rarg2, arg);
 814   }
 815 }
 816 
 817 static void pass_arg3(MacroAssembler* masm, Register arg) {
 818   if (c_rarg3 != arg ) {
 819     masm->mov(c_rarg3, arg);
 820   }
 821 }
 822 
 823 void MacroAssembler::stop(const char* msg) {
 824   address rip = pc();
 825   pusha(); // get regs on stack
 826   lea(c_rarg0, ExternalAddress((address) msg));
 827   lea(c_rarg1, InternalAddress(rip));
 828   movq(c_rarg2, rsp); // pass pointer to regs array
 829   andq(rsp, -16); // align stack as required by ABI
 830   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
 831   hlt();
 832 }
 833 
 834 void MacroAssembler::warn(const char* msg) {
 835   push(rbp);
 836   movq(rbp, rsp);
 837   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 838   push_CPU_state();   // keeps alignment at 16 bytes
 839   lea(c_rarg0, ExternalAddress((address) msg));
 840   lea(rax, ExternalAddress(CAST_FROM_FN_PTR(address, warning)));
 841   call(rax);
 842   pop_CPU_state();
 843   mov(rsp, rbp);
 844   pop(rbp);
 845 }
 846 
 847 void MacroAssembler::print_state() {
 848   address rip = pc();
 849   pusha();            // get regs on stack
 850   push(rbp);
 851   movq(rbp, rsp);
 852   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 853   push_CPU_state();   // keeps alignment at 16 bytes
 854 
 855   lea(c_rarg0, InternalAddress(rip));
 856   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
 857   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
 858 
 859   pop_CPU_state();
 860   mov(rsp, rbp);
 861   pop(rbp);
 862   popa();
 863 }
 864 
 865 #ifndef PRODUCT
 866 extern "C" void findpc(intptr_t x);
 867 #endif
 868 
 869 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
 870   // In order to get locks to work, we need to fake a in_VM state
 871   if (ShowMessageBoxOnError) {
 872     JavaThread* thread = JavaThread::current();
 873     JavaThreadState saved_state = thread->thread_state();
 874     thread->set_thread_state(_thread_in_vm);
 875 #ifndef PRODUCT
 876     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 877       ttyLocker ttyl;
 878       BytecodeCounter::print();
 879     }
 880 #endif
 881     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 882     // XXX correct this offset for amd64
 883     // This is the value of eip which points to where verify_oop will return.
 884     if (os::message_box(msg, "Execution stopped, print registers?")) {
 885       print_state64(pc, regs);
 886       BREAKPOINT;
 887       assert(false, "start up GDB");
 888     }
 889     ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 890   } else {
 891     ttyLocker ttyl;
 892     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
 893                     msg);
 894     assert(false, "DEBUG MESSAGE: %s", msg);
 895   }
 896 }
 897 
 898 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
 899   ttyLocker ttyl;
 900   FlagSetting fs(Debugging, true);
 901   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
 902 #ifndef PRODUCT
 903   tty->cr();
 904   findpc(pc);
 905   tty->cr();
 906 #endif
 907 #define PRINT_REG(rax, value) \
 908   { tty->print("%s = ", #rax); os::print_location(tty, value); }
 909   PRINT_REG(rax, regs[15]);
 910   PRINT_REG(rbx, regs[12]);
 911   PRINT_REG(rcx, regs[14]);
 912   PRINT_REG(rdx, regs[13]);
 913   PRINT_REG(rdi, regs[8]);
 914   PRINT_REG(rsi, regs[9]);
 915   PRINT_REG(rbp, regs[10]);
 916   PRINT_REG(rsp, regs[11]);
 917   PRINT_REG(r8 , regs[7]);
 918   PRINT_REG(r9 , regs[6]);
 919   PRINT_REG(r10, regs[5]);
 920   PRINT_REG(r11, regs[4]);
 921   PRINT_REG(r12, regs[3]);
 922   PRINT_REG(r13, regs[2]);
 923   PRINT_REG(r14, regs[1]);
 924   PRINT_REG(r15, regs[0]);
 925 #undef PRINT_REG
 926   // Print some words near top of staack.
 927   int64_t* rsp = (int64_t*) regs[11];
 928   int64_t* dump_sp = rsp;
 929   for (int col1 = 0; col1 < 8; col1++) {
 930     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 931     os::print_location(tty, *dump_sp++);
 932   }
 933   for (int row = 0; row < 25; row++) {
 934     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 935     for (int col = 0; col < 4; col++) {
 936       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
 937     }
 938     tty->cr();
 939   }
 940   // Print some instructions around pc:
 941   Disassembler::decode((address)pc-64, (address)pc);
 942   tty->print_cr("--------");
 943   Disassembler::decode((address)pc, (address)pc+32);
 944 }
 945 
 946 #endif // _LP64
 947 
 948 // Now versions that are common to 32/64 bit
 949 
 950 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 951   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 952 }
 953 
 954 void MacroAssembler::addptr(Register dst, Register src) {
 955   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 956 }
 957 
 958 void MacroAssembler::addptr(Address dst, Register src) {
 959   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 960 }
 961 
 962 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {
 963   if (reachable(src)) {
 964     Assembler::addsd(dst, as_Address(src));
 965   } else {
 966     lea(rscratch1, src);
 967     Assembler::addsd(dst, Address(rscratch1, 0));
 968   }
 969 }
 970 
 971 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {
 972   if (reachable(src)) {
 973     addss(dst, as_Address(src));
 974   } else {
 975     lea(rscratch1, src);
 976     addss(dst, Address(rscratch1, 0));
 977   }
 978 }
 979 
 980 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) {
 981   if (reachable(src)) {
 982     Assembler::addpd(dst, as_Address(src));
 983   } else {
 984     lea(rscratch1, src);
 985     Assembler::addpd(dst, Address(rscratch1, 0));
 986   }
 987 }
 988 
 989 void MacroAssembler::align(int modulus) {
 990   align(modulus, offset());
 991 }
 992 
 993 void MacroAssembler::align(int modulus, int target) {
 994   if (target % modulus != 0) {
 995     nop(modulus - (target % modulus));
 996   }
 997 }
 998 
 999 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {
1000   // Used in sign-masking with aligned address.
1001   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1002   if (reachable(src)) {
1003     Assembler::andpd(dst, as_Address(src));
1004   } else {
1005     lea(rscratch1, src);
1006     Assembler::andpd(dst, Address(rscratch1, 0));
1007   }
1008 }
1009 
1010 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {
1011   // Used in sign-masking with aligned address.
1012   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1013   if (reachable(src)) {
1014     Assembler::andps(dst, as_Address(src));
1015   } else {
1016     lea(rscratch1, src);
1017     Assembler::andps(dst, Address(rscratch1, 0));
1018   }
1019 }
1020 
1021 void MacroAssembler::andptr(Register dst, int32_t imm32) {
1022   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
1023 }
1024 
1025 void MacroAssembler::atomic_incl(Address counter_addr) {
1026   if (os::is_MP())
1027     lock();
1028   incrementl(counter_addr);
1029 }
1030 
1031 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
1032   if (reachable(counter_addr)) {
1033     atomic_incl(as_Address(counter_addr));
1034   } else {
1035     lea(scr, counter_addr);
1036     atomic_incl(Address(scr, 0));
1037   }
1038 }
1039 
1040 #ifdef _LP64
1041 void MacroAssembler::atomic_incq(Address counter_addr) {
1042   if (os::is_MP())
1043     lock();
1044   incrementq(counter_addr);
1045 }
1046 
1047 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) {
1048   if (reachable(counter_addr)) {
1049     atomic_incq(as_Address(counter_addr));
1050   } else {
1051     lea(scr, counter_addr);
1052     atomic_incq(Address(scr, 0));
1053   }
1054 }
1055 #endif
1056 
1057 // Writes to stack successive pages until offset reached to check for
1058 // stack overflow + shadow pages.  This clobbers tmp.
1059 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
1060   movptr(tmp, rsp);
1061   // Bang stack for total size given plus shadow page size.
1062   // Bang one page at a time because large size can bang beyond yellow and
1063   // red zones.
1064   Label loop;
1065   bind(loop);
1066   movl(Address(tmp, (-os::vm_page_size())), size );
1067   subptr(tmp, os::vm_page_size());
1068   subl(size, os::vm_page_size());
1069   jcc(Assembler::greater, loop);
1070 
1071   // Bang down shadow pages too.
1072   // At this point, (tmp-0) is the last address touched, so don't
1073   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
1074   // was post-decremented.)  Skip this address by starting at i=1, and
1075   // touch a few more pages below.  N.B.  It is important to touch all
1076   // the way down including all pages in the shadow zone.
1077   for (int i = 1; i < ((int)JavaThread::stack_shadow_zone_size() / os::vm_page_size()); i++) {
1078     // this could be any sized move but this is can be a debugging crumb
1079     // so the bigger the better.
1080     movptr(Address(tmp, (-i*os::vm_page_size())), size );
1081   }
1082 }
1083 
1084 void MacroAssembler::reserved_stack_check() {
1085     // testing if reserved zone needs to be enabled
1086     Label no_reserved_zone_enabling;
1087     Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
1088     NOT_LP64(get_thread(rsi);)
1089 
1090     cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
1091     jcc(Assembler::below, no_reserved_zone_enabling);
1092 
1093     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
1094     jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
1095     should_not_reach_here();
1096 
1097     bind(no_reserved_zone_enabling);
1098 }
1099 
1100 int MacroAssembler::biased_locking_enter(Register lock_reg,
1101                                          Register obj_reg,
1102                                          Register swap_reg,
1103                                          Register tmp_reg,
1104                                          bool swap_reg_contains_mark,
1105                                          Label& done,
1106                                          Label* slow_case,
1107                                          BiasedLockingCounters* counters) {
1108   assert(UseBiasedLocking, "why call this otherwise?");
1109   assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
1110   assert(tmp_reg != noreg, "tmp_reg must be supplied");
1111   assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
1112   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
1113   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
1114   NOT_LP64( Address saved_mark_addr(lock_reg, 0); )
1115 
1116   if (PrintBiasedLockingStatistics && counters == NULL) {
1117     counters = BiasedLocking::counters();
1118   }
1119   // Biased locking
1120   // See whether the lock is currently biased toward our thread and
1121   // whether the epoch is still valid
1122   // Note that the runtime guarantees sufficient alignment of JavaThread
1123   // pointers to allow age to be placed into low bits
1124   // First check to see whether biasing is even enabled for this object
1125   Label cas_label;
1126   int null_check_offset = -1;
1127   if (!swap_reg_contains_mark) {
1128     null_check_offset = offset();
1129     movptr(swap_reg, mark_addr);
1130   }
1131   movptr(tmp_reg, swap_reg);
1132   andptr(tmp_reg, markOopDesc::biased_lock_mask_in_place);
1133   cmpptr(tmp_reg, markOopDesc::biased_lock_pattern);
1134   jcc(Assembler::notEqual, cas_label);
1135   // The bias pattern is present in the object's header. Need to check
1136   // whether the bias owner and the epoch are both still current.
1137 #ifndef _LP64
1138   // Note that because there is no current thread register on x86_32 we
1139   // need to store off the mark word we read out of the object to
1140   // avoid reloading it and needing to recheck invariants below. This
1141   // store is unfortunate but it makes the overall code shorter and
1142   // simpler.
1143   movptr(saved_mark_addr, swap_reg);
1144 #endif
1145   if (swap_reg_contains_mark) {
1146     null_check_offset = offset();
1147   }
1148   load_prototype_header(tmp_reg, obj_reg);
1149 #ifdef _LP64
1150   orptr(tmp_reg, r15_thread);
1151   xorptr(tmp_reg, swap_reg);
1152   Register header_reg = tmp_reg;
1153 #else
1154   xorptr(tmp_reg, swap_reg);
1155   get_thread(swap_reg);
1156   xorptr(swap_reg, tmp_reg);
1157   Register header_reg = swap_reg;
1158 #endif
1159   andptr(header_reg, ~((int) markOopDesc::age_mask_in_place));
1160   if (counters != NULL) {
1161     cond_inc32(Assembler::zero,
1162                ExternalAddress((address) counters->biased_lock_entry_count_addr()));
1163   }
1164   jcc(Assembler::equal, done);
1165 
1166   Label try_revoke_bias;
1167   Label try_rebias;
1168 
1169   // At this point we know that the header has the bias pattern and
1170   // that we are not the bias owner in the current epoch. We need to
1171   // figure out more details about the state of the header in order to
1172   // know what operations can be legally performed on the object's
1173   // header.
1174 
1175   // If the low three bits in the xor result aren't clear, that means
1176   // the prototype header is no longer biased and we have to revoke
1177   // the bias on this object.
1178   testptr(header_reg, markOopDesc::biased_lock_mask_in_place);
1179   jccb(Assembler::notZero, try_revoke_bias);
1180 
1181   // Biasing is still enabled for this data type. See whether the
1182   // epoch of the current bias is still valid, meaning that the epoch
1183   // bits of the mark word are equal to the epoch bits of the
1184   // prototype header. (Note that the prototype header's epoch bits
1185   // only change at a safepoint.) If not, attempt to rebias the object
1186   // toward the current thread. Note that we must be absolutely sure
1187   // that the current epoch is invalid in order to do this because
1188   // otherwise the manipulations it performs on the mark word are
1189   // illegal.
1190   testptr(header_reg, markOopDesc::epoch_mask_in_place);
1191   jccb(Assembler::notZero, try_rebias);
1192 
1193   // The epoch of the current bias is still valid but we know nothing
1194   // about the owner; it might be set or it might be clear. Try to
1195   // acquire the bias of the object using an atomic operation. If this
1196   // fails we will go in to the runtime to revoke the object's bias.
1197   // Note that we first construct the presumed unbiased header so we
1198   // don't accidentally blow away another thread's valid bias.
1199   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1200   andptr(swap_reg,
1201          markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
1202 #ifdef _LP64
1203   movptr(tmp_reg, swap_reg);
1204   orptr(tmp_reg, r15_thread);
1205 #else
1206   get_thread(tmp_reg);
1207   orptr(tmp_reg, swap_reg);
1208 #endif
1209   if (os::is_MP()) {
1210     lock();
1211   }
1212   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1213   // If the biasing toward our thread failed, this means that
1214   // another thread succeeded in biasing it toward itself and we
1215   // need to revoke that bias. The revocation will occur in the
1216   // interpreter runtime in the slow case.
1217   if (counters != NULL) {
1218     cond_inc32(Assembler::zero,
1219                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
1220   }
1221   if (slow_case != NULL) {
1222     jcc(Assembler::notZero, *slow_case);
1223   }
1224   jmp(done);
1225 
1226   bind(try_rebias);
1227   // At this point we know the epoch has expired, meaning that the
1228   // current "bias owner", if any, is actually invalid. Under these
1229   // circumstances _only_, we are allowed to use the current header's
1230   // value as the comparison value when doing the cas to acquire the
1231   // bias in the current epoch. In other words, we allow transfer of
1232   // the bias from one thread to another directly in this situation.
1233   //
1234   // FIXME: due to a lack of registers we currently blow away the age
1235   // bits in this situation. Should attempt to preserve them.
1236   load_prototype_header(tmp_reg, obj_reg);
1237 #ifdef _LP64
1238   orptr(tmp_reg, r15_thread);
1239 #else
1240   get_thread(swap_reg);
1241   orptr(tmp_reg, swap_reg);
1242   movptr(swap_reg, saved_mark_addr);
1243 #endif
1244   if (os::is_MP()) {
1245     lock();
1246   }
1247   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1248   // If the biasing toward our thread failed, then another thread
1249   // succeeded in biasing it toward itself and we need to revoke that
1250   // bias. The revocation will occur in the runtime in the slow case.
1251   if (counters != NULL) {
1252     cond_inc32(Assembler::zero,
1253                ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
1254   }
1255   if (slow_case != NULL) {
1256     jcc(Assembler::notZero, *slow_case);
1257   }
1258   jmp(done);
1259 
1260   bind(try_revoke_bias);
1261   // The prototype mark in the klass doesn't have the bias bit set any
1262   // more, indicating that objects of this data type are not supposed
1263   // to be biased any more. We are going to try to reset the mark of
1264   // this object to the prototype value and fall through to the
1265   // CAS-based locking scheme. Note that if our CAS fails, it means
1266   // that another thread raced us for the privilege of revoking the
1267   // bias of this particular object, so it's okay to continue in the
1268   // normal locking code.
1269   //
1270   // FIXME: due to a lack of registers we currently blow away the age
1271   // bits in this situation. Should attempt to preserve them.
1272   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1273   load_prototype_header(tmp_reg, obj_reg);
1274   if (os::is_MP()) {
1275     lock();
1276   }
1277   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1278   // Fall through to the normal CAS-based lock, because no matter what
1279   // the result of the above CAS, some thread must have succeeded in
1280   // removing the bias bit from the object's header.
1281   if (counters != NULL) {
1282     cond_inc32(Assembler::zero,
1283                ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
1284   }
1285 
1286   bind(cas_label);
1287 
1288   return null_check_offset;
1289 }
1290 
1291 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
1292   assert(UseBiasedLocking, "why call this otherwise?");
1293 
1294   // Check for biased locking unlock case, which is a no-op
1295   // Note: we do not have to check the thread ID for two reasons.
1296   // First, the interpreter checks for IllegalMonitorStateException at
1297   // a higher level. Second, if the bias was revoked while we held the
1298   // lock, the object could not be rebiased toward another thread, so
1299   // the bias bit would be clear.
1300   movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1301   andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);
1302   cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
1303   jcc(Assembler::equal, done);
1304 }
1305 
1306 #ifdef COMPILER2
1307 
1308 #if INCLUDE_RTM_OPT
1309 
1310 // Update rtm_counters based on abort status
1311 // input: abort_status
1312 //        rtm_counters (RTMLockingCounters*)
1313 // flags are killed
1314 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) {
1315 
1316   atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset()));
1317   if (PrintPreciseRTMLockingStatistics) {
1318     for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
1319       Label check_abort;
1320       testl(abort_status, (1<<i));
1321       jccb(Assembler::equal, check_abort);
1322       atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx))));
1323       bind(check_abort);
1324     }
1325   }
1326 }
1327 
1328 // Branch if (random & (count-1) != 0), count is 2^n
1329 // tmp, scr and flags are killed
1330 void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) {
1331   assert(tmp == rax, "");
1332   assert(scr == rdx, "");
1333   rdtsc(); // modifies EDX:EAX
1334   andptr(tmp, count-1);
1335   jccb(Assembler::notZero, brLabel);
1336 }
1337 
1338 // Perform abort ratio calculation, set no_rtm bit if high ratio
1339 // input:  rtm_counters_Reg (RTMLockingCounters* address)
1340 // tmpReg, rtm_counters_Reg and flags are killed
1341 void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg,
1342                                                  Register rtm_counters_Reg,
1343                                                  RTMLockingCounters* rtm_counters,
1344                                                  Metadata* method_data) {
1345   Label L_done, L_check_always_rtm1, L_check_always_rtm2;
1346 
1347   if (RTMLockingCalculationDelay > 0) {
1348     // Delay calculation
1349     movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg);
1350     testptr(tmpReg, tmpReg);
1351     jccb(Assembler::equal, L_done);
1352   }
1353   // Abort ratio calculation only if abort_count > RTMAbortThreshold
1354   //   Aborted transactions = abort_count * 100
1355   //   All transactions = total_count *  RTMTotalCountIncrRate
1356   //   Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio)
1357 
1358   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset()));
1359   cmpptr(tmpReg, RTMAbortThreshold);
1360   jccb(Assembler::below, L_check_always_rtm2);
1361   imulptr(tmpReg, tmpReg, 100);
1362 
1363   Register scrReg = rtm_counters_Reg;
1364   movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1365   imulptr(scrReg, scrReg, RTMTotalCountIncrRate);
1366   imulptr(scrReg, scrReg, RTMAbortRatio);
1367   cmpptr(tmpReg, scrReg);
1368   jccb(Assembler::below, L_check_always_rtm1);
1369   if (method_data != NULL) {
1370     // set rtm_state to "no rtm" in MDO
1371     mov_metadata(tmpReg, method_data);
1372     if (os::is_MP()) {
1373       lock();
1374     }
1375     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM);
1376   }
1377   jmpb(L_done);
1378   bind(L_check_always_rtm1);
1379   // Reload RTMLockingCounters* address
1380   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1381   bind(L_check_always_rtm2);
1382   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1383   cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate);
1384   jccb(Assembler::below, L_done);
1385   if (method_data != NULL) {
1386     // set rtm_state to "always rtm" in MDO
1387     mov_metadata(tmpReg, method_data);
1388     if (os::is_MP()) {
1389       lock();
1390     }
1391     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM);
1392   }
1393   bind(L_done);
1394 }
1395 
1396 // Update counters and perform abort ratio calculation
1397 // input:  abort_status_Reg
1398 // rtm_counters_Reg, flags are killed
1399 void MacroAssembler::rtm_profiling(Register abort_status_Reg,
1400                                    Register rtm_counters_Reg,
1401                                    RTMLockingCounters* rtm_counters,
1402                                    Metadata* method_data,
1403                                    bool profile_rtm) {
1404 
1405   assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1406   // update rtm counters based on rax value at abort
1407   // reads abort_status_Reg, updates flags
1408   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1409   rtm_counters_update(abort_status_Reg, rtm_counters_Reg);
1410   if (profile_rtm) {
1411     // Save abort status because abort_status_Reg is used by following code.
1412     if (RTMRetryCount > 0) {
1413       push(abort_status_Reg);
1414     }
1415     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1416     rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data);
1417     // restore abort status
1418     if (RTMRetryCount > 0) {
1419       pop(abort_status_Reg);
1420     }
1421   }
1422 }
1423 
1424 // Retry on abort if abort's status is 0x6: can retry (0x2) | memory conflict (0x4)
1425 // inputs: retry_count_Reg
1426 //       : abort_status_Reg
1427 // output: retry_count_Reg decremented by 1
1428 // flags are killed
1429 void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) {
1430   Label doneRetry;
1431   assert(abort_status_Reg == rax, "");
1432   // The abort reason bits are in eax (see all states in rtmLocking.hpp)
1433   // 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4)
1434   // if reason is in 0x6 and retry count != 0 then retry
1435   andptr(abort_status_Reg, 0x6);
1436   jccb(Assembler::zero, doneRetry);
1437   testl(retry_count_Reg, retry_count_Reg);
1438   jccb(Assembler::zero, doneRetry);
1439   pause();
1440   decrementl(retry_count_Reg);
1441   jmp(retryLabel);
1442   bind(doneRetry);
1443 }
1444 
1445 // Spin and retry if lock is busy,
1446 // inputs: box_Reg (monitor address)
1447 //       : retry_count_Reg
1448 // output: retry_count_Reg decremented by 1
1449 //       : clear z flag if retry count exceeded
1450 // tmp_Reg, scr_Reg, flags are killed
1451 void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg,
1452                                             Register tmp_Reg, Register scr_Reg, Label& retryLabel) {
1453   Label SpinLoop, SpinExit, doneRetry;
1454   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1455 
1456   testl(retry_count_Reg, retry_count_Reg);
1457   jccb(Assembler::zero, doneRetry);
1458   decrementl(retry_count_Reg);
1459   movptr(scr_Reg, RTMSpinLoopCount);
1460 
1461   bind(SpinLoop);
1462   pause();
1463   decrementl(scr_Reg);
1464   jccb(Assembler::lessEqual, SpinExit);
1465   movptr(tmp_Reg, Address(box_Reg, owner_offset));
1466   testptr(tmp_Reg, tmp_Reg);
1467   jccb(Assembler::notZero, SpinLoop);
1468 
1469   bind(SpinExit);
1470   jmp(retryLabel);
1471   bind(doneRetry);
1472   incrementl(retry_count_Reg); // clear z flag
1473 }
1474 
1475 // Use RTM for normal stack locks
1476 // Input: objReg (object to lock)
1477 void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg,
1478                                        Register retry_on_abort_count_Reg,
1479                                        RTMLockingCounters* stack_rtm_counters,
1480                                        Metadata* method_data, bool profile_rtm,
1481                                        Label& DONE_LABEL, Label& IsInflated) {
1482   assert(UseRTMForStackLocks, "why call this otherwise?");
1483   assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1484   assert(tmpReg == rax, "");
1485   assert(scrReg == rdx, "");
1486   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1487 
1488   if (RTMRetryCount > 0) {
1489     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1490     bind(L_rtm_retry);
1491   }
1492   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1493   testptr(tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased
1494   jcc(Assembler::notZero, IsInflated);
1495 
1496   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1497     Label L_noincrement;
1498     if (RTMTotalCountIncrRate > 1) {
1499       // tmpReg, scrReg and flags are killed
1500       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1501     }
1502     assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM");
1503     atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg);
1504     bind(L_noincrement);
1505   }
1506   xbegin(L_on_abort);
1507   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));       // fetch markword
1508   andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
1509   cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
1510   jcc(Assembler::equal, DONE_LABEL);        // all done if unlocked
1511 
1512   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1513   if (UseRTMXendForLockBusy) {
1514     xend();
1515     movptr(abort_status_Reg, 0x2);   // Set the abort status to 2 (so we can retry)
1516     jmp(L_decrement_retry);
1517   }
1518   else {
1519     xabort(0);
1520   }
1521   bind(L_on_abort);
1522   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1523     rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm);
1524   }
1525   bind(L_decrement_retry);
1526   if (RTMRetryCount > 0) {
1527     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1528     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1529   }
1530 }
1531 
1532 // Use RTM for inflating locks
1533 // inputs: objReg (object to lock)
1534 //         boxReg (on-stack box address (displaced header location) - KILLED)
1535 //         tmpReg (ObjectMonitor address + markOopDesc::monitor_value)
1536 void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg,
1537                                           Register scrReg, Register retry_on_busy_count_Reg,
1538                                           Register retry_on_abort_count_Reg,
1539                                           RTMLockingCounters* rtm_counters,
1540                                           Metadata* method_data, bool profile_rtm,
1541                                           Label& DONE_LABEL) {
1542   assert(UseRTMLocking, "why call this otherwise?");
1543   assert(tmpReg == rax, "");
1544   assert(scrReg == rdx, "");
1545   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1546   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1547 
1548   // Without cast to int32_t a movptr will destroy r10 which is typically obj
1549   movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1550   movptr(boxReg, tmpReg); // Save ObjectMonitor address
1551 
1552   if (RTMRetryCount > 0) {
1553     movl(retry_on_busy_count_Reg, RTMRetryCount);  // Retry on lock busy
1554     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1555     bind(L_rtm_retry);
1556   }
1557   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1558     Label L_noincrement;
1559     if (RTMTotalCountIncrRate > 1) {
1560       // tmpReg, scrReg and flags are killed
1561       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1562     }
1563     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1564     atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);
1565     bind(L_noincrement);
1566   }
1567   xbegin(L_on_abort);
1568   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1569   movptr(tmpReg, Address(tmpReg, owner_offset));
1570   testptr(tmpReg, tmpReg);
1571   jcc(Assembler::zero, DONE_LABEL);
1572   if (UseRTMXendForLockBusy) {
1573     xend();
1574     jmp(L_decrement_retry);
1575   }
1576   else {
1577     xabort(0);
1578   }
1579   bind(L_on_abort);
1580   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1581   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1582     rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm);
1583   }
1584   if (RTMRetryCount > 0) {
1585     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1586     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1587   }
1588 
1589   movptr(tmpReg, Address(boxReg, owner_offset)) ;
1590   testptr(tmpReg, tmpReg) ;
1591   jccb(Assembler::notZero, L_decrement_retry) ;
1592 
1593   // Appears unlocked - try to swing _owner from null to non-null.
1594   // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1595 #ifdef _LP64
1596   Register threadReg = r15_thread;
1597 #else
1598   get_thread(scrReg);
1599   Register threadReg = scrReg;
1600 #endif
1601   if (os::is_MP()) {
1602     lock();
1603   }
1604   cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg
1605 
1606   if (RTMRetryCount > 0) {
1607     // success done else retry
1608     jccb(Assembler::equal, DONE_LABEL) ;
1609     bind(L_decrement_retry);
1610     // Spin and retry if lock is busy.
1611     rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
1612   }
1613   else {
1614     bind(L_decrement_retry);
1615   }
1616 }
1617 
1618 #endif //  INCLUDE_RTM_OPT
1619 
1620 // Fast_Lock and Fast_Unlock used by C2
1621 
1622 // Because the transitions from emitted code to the runtime
1623 // monitorenter/exit helper stubs are so slow it's critical that
1624 // we inline both the stack-locking fast-path and the inflated fast path.
1625 //
1626 // See also: cmpFastLock and cmpFastUnlock.
1627 //
1628 // What follows is a specialized inline transliteration of the code
1629 // in slow_enter() and slow_exit().  If we're concerned about I$ bloat
1630 // another option would be to emit TrySlowEnter and TrySlowExit methods
1631 // at startup-time.  These methods would accept arguments as
1632 // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure
1633 // indications in the icc.ZFlag.  Fast_Lock and Fast_Unlock would simply
1634 // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.
1635 // In practice, however, the # of lock sites is bounded and is usually small.
1636 // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer
1637 // if the processor uses simple bimodal branch predictors keyed by EIP
1638 // Since the helper routines would be called from multiple synchronization
1639 // sites.
1640 //
1641 // An even better approach would be write "MonitorEnter()" and "MonitorExit()"
1642 // in java - using j.u.c and unsafe - and just bind the lock and unlock sites
1643 // to those specialized methods.  That'd give us a mostly platform-independent
1644 // implementation that the JITs could optimize and inline at their pleasure.
1645 // Done correctly, the only time we'd need to cross to native could would be
1646 // to park() or unpark() threads.  We'd also need a few more unsafe operators
1647 // to (a) prevent compiler-JIT reordering of non-volatile accesses, and
1648 // (b) explicit barriers or fence operations.
1649 //
1650 // TODO:
1651 //
1652 // *  Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr).
1653 //    This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals.
1654 //    Given TLAB allocation, Self is usually manifested in a register, so passing it into
1655 //    the lock operators would typically be faster than reifying Self.
1656 //
1657 // *  Ideally I'd define the primitives as:
1658 //       fast_lock   (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.
1659 //       fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED
1660 //    Unfortunately ADLC bugs prevent us from expressing the ideal form.
1661 //    Instead, we're stuck with a rather awkward and brittle register assignments below.
1662 //    Furthermore the register assignments are overconstrained, possibly resulting in
1663 //    sub-optimal code near the synchronization site.
1664 //
1665 // *  Eliminate the sp-proximity tests and just use "== Self" tests instead.
1666 //    Alternately, use a better sp-proximity test.
1667 //
1668 // *  Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.
1669 //    Either one is sufficient to uniquely identify a thread.
1670 //    TODO: eliminate use of sp in _owner and use get_thread(tr) instead.
1671 //
1672 // *  Intrinsify notify() and notifyAll() for the common cases where the
1673 //    object is locked by the calling thread but the waitlist is empty.
1674 //    avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().
1675 //
1676 // *  use jccb and jmpb instead of jcc and jmp to improve code density.
1677 //    But beware of excessive branch density on AMD Opterons.
1678 //
1679 // *  Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success
1680 //    or failure of the fast-path.  If the fast-path fails then we pass
1681 //    control to the slow-path, typically in C.  In Fast_Lock and
1682 //    Fast_Unlock we often branch to DONE_LABEL, just to find that C2
1683 //    will emit a conditional branch immediately after the node.
1684 //    So we have branches to branches and lots of ICC.ZF games.
1685 //    Instead, it might be better to have C2 pass a "FailureLabel"
1686 //    into Fast_Lock and Fast_Unlock.  In the case of success, control
1687 //    will drop through the node.  ICC.ZF is undefined at exit.
1688 //    In the case of failure, the node will branch directly to the
1689 //    FailureLabel
1690 
1691 
1692 // obj: object to lock
1693 // box: on-stack box address (displaced header location) - KILLED
1694 // rax,: tmp -- KILLED
1695 // scr: tmp -- KILLED
1696 void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
1697                                Register scrReg, Register cx1Reg, Register cx2Reg,
1698                                BiasedLockingCounters* counters,
1699                                RTMLockingCounters* rtm_counters,
1700                                RTMLockingCounters* stack_rtm_counters,
1701                                Metadata* method_data,
1702                                bool use_rtm, bool profile_rtm) {
1703   // Ensure the register assignments are disjoint
1704   assert(tmpReg == rax, "");
1705 
1706   if (use_rtm) {
1707     assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg);
1708   } else {
1709     assert(cx1Reg == noreg, "");
1710     assert(cx2Reg == noreg, "");
1711     assert_different_registers(objReg, boxReg, tmpReg, scrReg);
1712   }
1713 
1714   if (counters != NULL) {
1715     atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);
1716   }
1717   if (EmitSync & 1) {
1718       // set box->dhw = markOopDesc::unused_mark()
1719       // Force all sync thru slow-path: slow_enter() and slow_exit()
1720       movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1721       cmpptr (rsp, (int32_t)NULL_WORD);
1722   } else {
1723     // Possible cases that we'll encounter in fast_lock
1724     // ------------------------------------------------
1725     // * Inflated
1726     //    -- unlocked
1727     //    -- Locked
1728     //       = by self
1729     //       = by other
1730     // * biased
1731     //    -- by Self
1732     //    -- by other
1733     // * neutral
1734     // * stack-locked
1735     //    -- by self
1736     //       = sp-proximity test hits
1737     //       = sp-proximity test generates false-negative
1738     //    -- by other
1739     //
1740 
1741     Label IsInflated, DONE_LABEL;
1742 
1743     // it's stack-locked, biased or neutral
1744     // TODO: optimize away redundant LDs of obj->mark and improve the markword triage
1745     // order to reduce the number of conditional branches in the most common cases.
1746     // Beware -- there's a subtle invariant that fetch of the markword
1747     // at [FETCH], below, will never observe a biased encoding (*101b).
1748     // If this invariant is not held we risk exclusion (safety) failure.
1749     if (UseBiasedLocking && !UseOptoBiasInlining) {
1750       biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters);
1751     }
1752 
1753 #if INCLUDE_RTM_OPT
1754     if (UseRTMForStackLocks && use_rtm) {
1755       rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
1756                         stack_rtm_counters, method_data, profile_rtm,
1757                         DONE_LABEL, IsInflated);
1758     }
1759 #endif // INCLUDE_RTM_OPT
1760 
1761     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));          // [FETCH]
1762     testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
1763     jccb(Assembler::notZero, IsInflated);
1764 
1765     // Attempt stack-locking ...
1766     orptr (tmpReg, markOopDesc::unlocked_value);
1767     movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
1768     if (os::is_MP()) {
1769       lock();
1770     }
1771     cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes()));      // Updates tmpReg
1772     if (counters != NULL) {
1773       cond_inc32(Assembler::equal,
1774                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
1775     }
1776     jcc(Assembler::equal, DONE_LABEL);           // Success
1777 
1778     // Recursive locking.
1779     // The object is stack-locked: markword contains stack pointer to BasicLock.
1780     // Locked by current thread if difference with current SP is less than one page.
1781     subptr(tmpReg, rsp);
1782     // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
1783     andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
1784     movptr(Address(boxReg, 0), tmpReg);
1785     if (counters != NULL) {
1786       cond_inc32(Assembler::equal,
1787                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
1788     }
1789     jmp(DONE_LABEL);
1790 
1791     bind(IsInflated);
1792     // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markOopDesc::monitor_value
1793 
1794 #if INCLUDE_RTM_OPT
1795     // Use the same RTM locking code in 32- and 64-bit VM.
1796     if (use_rtm) {
1797       rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg,
1798                            rtm_counters, method_data, profile_rtm, DONE_LABEL);
1799     } else {
1800 #endif // INCLUDE_RTM_OPT
1801 
1802 #ifndef _LP64
1803     // The object is inflated.
1804 
1805     // boxReg refers to the on-stack BasicLock in the current frame.
1806     // We'd like to write:
1807     //   set box->_displaced_header = markOopDesc::unused_mark().  Any non-0 value suffices.
1808     // This is convenient but results a ST-before-CAS penalty.  The following CAS suffers
1809     // additional latency as we have another ST in the store buffer that must drain.
1810 
1811     if (EmitSync & 8192) {
1812        movptr(Address(boxReg, 0), 3);            // results in ST-before-CAS penalty
1813        get_thread (scrReg);
1814        movptr(boxReg, tmpReg);                    // consider: LEA box, [tmp-2]
1815        movptr(tmpReg, NULL_WORD);                 // consider: xor vs mov
1816        if (os::is_MP()) {
1817          lock();
1818        }
1819        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1820     } else
1821     if ((EmitSync & 128) == 0) {                      // avoid ST-before-CAS
1822        // register juggle because we need tmpReg for cmpxchgptr below
1823        movptr(scrReg, boxReg);
1824        movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2]
1825 
1826        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
1827        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
1828           // prefetchw [eax + Offset(_owner)-2]
1829           prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1830        }
1831 
1832        if ((EmitSync & 64) == 0) {
1833          // Optimistic form: consider XORL tmpReg,tmpReg
1834          movptr(tmpReg, NULL_WORD);
1835        } else {
1836          // Can suffer RTS->RTO upgrades on shared or cold $ lines
1837          // Test-And-CAS instead of CAS
1838          movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));   // rax, = m->_owner
1839          testptr(tmpReg, tmpReg);                   // Locked ?
1840          jccb  (Assembler::notZero, DONE_LABEL);
1841        }
1842 
1843        // Appears unlocked - try to swing _owner from null to non-null.
1844        // Ideally, I'd manifest "Self" with get_thread and then attempt
1845        // to CAS the register containing Self into m->Owner.
1846        // But we don't have enough registers, so instead we can either try to CAS
1847        // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
1848        // we later store "Self" into m->Owner.  Transiently storing a stack address
1849        // (rsp or the address of the box) into  m->owner is harmless.
1850        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1851        if (os::is_MP()) {
1852          lock();
1853        }
1854        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1855        movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3
1856        // If we weren't able to swing _owner from NULL to the BasicLock
1857        // then take the slow path.
1858        jccb  (Assembler::notZero, DONE_LABEL);
1859        // update _owner from BasicLock to thread
1860        get_thread (scrReg);                    // beware: clobbers ICCs
1861        movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
1862        xorptr(boxReg, boxReg);                 // set icc.ZFlag = 1 to indicate success
1863 
1864        // If the CAS fails we can either retry or pass control to the slow-path.
1865        // We use the latter tactic.
1866        // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1867        // If the CAS was successful ...
1868        //   Self has acquired the lock
1869        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1870        // Intentional fall-through into DONE_LABEL ...
1871     } else {
1872        movptr(Address(boxReg, 0), intptr_t(markOopDesc::unused_mark()));  // results in ST-before-CAS penalty
1873        movptr(boxReg, tmpReg);
1874 
1875        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
1876        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
1877           // prefetchw [eax + Offset(_owner)-2]
1878           prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1879        }
1880 
1881        if ((EmitSync & 64) == 0) {
1882          // Optimistic form
1883          xorptr  (tmpReg, tmpReg);
1884        } else {
1885          // Can suffer RTS->RTO upgrades on shared or cold $ lines
1886          movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));   // rax, = m->_owner
1887          testptr(tmpReg, tmpReg);                   // Locked ?
1888          jccb  (Assembler::notZero, DONE_LABEL);
1889        }
1890 
1891        // Appears unlocked - try to swing _owner from null to non-null.
1892        // Use either "Self" (in scr) or rsp as thread identity in _owner.
1893        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1894        get_thread (scrReg);
1895        if (os::is_MP()) {
1896          lock();
1897        }
1898        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1899 
1900        // If the CAS fails we can either retry or pass control to the slow-path.
1901        // We use the latter tactic.
1902        // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1903        // If the CAS was successful ...
1904        //   Self has acquired the lock
1905        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1906        // Intentional fall-through into DONE_LABEL ...
1907     }
1908 #else // _LP64
1909     // It's inflated
1910     movq(scrReg, tmpReg);
1911     xorq(tmpReg, tmpReg);
1912 
1913     if (os::is_MP()) {
1914       lock();
1915     }
1916     cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1917     // Unconditionally set box->_displaced_header = markOopDesc::unused_mark().
1918     // Without cast to int32_t movptr will destroy r10 which is typically obj.
1919     movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1920     // Intentional fall-through into DONE_LABEL ...
1921     // Propagate ICC.ZF from CAS above into DONE_LABEL.
1922 #endif // _LP64
1923 #if INCLUDE_RTM_OPT
1924     } // use_rtm()
1925 #endif
1926     // DONE_LABEL is a hot target - we'd really like to place it at the
1927     // start of cache line by padding with NOPs.
1928     // See the AMD and Intel software optimization manuals for the
1929     // most efficient "long" NOP encodings.
1930     // Unfortunately none of our alignment mechanisms suffice.
1931     bind(DONE_LABEL);
1932 
1933     // At DONE_LABEL the icc ZFlag is set as follows ...
1934     // Fast_Unlock uses the same protocol.
1935     // ZFlag == 1 -> Success
1936     // ZFlag == 0 -> Failure - force control through the slow-path
1937   }
1938 }
1939 
1940 // obj: object to unlock
1941 // box: box address (displaced header location), killed.  Must be EAX.
1942 // tmp: killed, cannot be obj nor box.
1943 //
1944 // Some commentary on balanced locking:
1945 //
1946 // Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites.
1947 // Methods that don't have provably balanced locking are forced to run in the
1948 // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
1949 // The interpreter provides two properties:
1950 // I1:  At return-time the interpreter automatically and quietly unlocks any
1951 //      objects acquired the current activation (frame).  Recall that the
1952 //      interpreter maintains an on-stack list of locks currently held by
1953 //      a frame.
1954 // I2:  If a method attempts to unlock an object that is not held by the
1955 //      the frame the interpreter throws IMSX.
1956 //
1957 // Lets say A(), which has provably balanced locking, acquires O and then calls B().
1958 // B() doesn't have provably balanced locking so it runs in the interpreter.
1959 // Control returns to A() and A() unlocks O.  By I1 and I2, above, we know that O
1960 // is still locked by A().
1961 //
1962 // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:
1963 // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
1964 // should not be unlocked by "normal" java-level locking and vice-versa.  The specification
1965 // doesn't specify what will occur if a program engages in such mixed-mode locking, however.
1966 // Arguably given that the spec legislates the JNI case as undefined our implementation
1967 // could reasonably *avoid* checking owner in Fast_Unlock().
1968 // In the interest of performance we elide m->Owner==Self check in unlock.
1969 // A perfectly viable alternative is to elide the owner check except when
1970 // Xcheck:jni is enabled.
1971 
1972 void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
1973   assert(boxReg == rax, "");
1974   assert_different_registers(objReg, boxReg, tmpReg);
1975 
1976   if (EmitSync & 4) {
1977     // Disable - inhibit all inlining.  Force control through the slow-path
1978     cmpptr (rsp, 0);
1979   } else {
1980     Label DONE_LABEL, Stacked, CheckSucc;
1981 
1982     // Critically, the biased locking test must have precedence over
1983     // and appear before the (box->dhw == 0) recursive stack-lock test.
1984     if (UseBiasedLocking && !UseOptoBiasInlining) {
1985        biased_locking_exit(objReg, tmpReg, DONE_LABEL);
1986     }
1987 
1988 #if INCLUDE_RTM_OPT
1989     if (UseRTMForStackLocks && use_rtm) {
1990       assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1991       Label L_regular_unlock;
1992       movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));           // fetch markword
1993       andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
1994       cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
1995       jccb(Assembler::notEqual, L_regular_unlock);  // if !HLE RegularLock
1996       xend();                                       // otherwise end...
1997       jmp(DONE_LABEL);                              // ... and we're done
1998       bind(L_regular_unlock);
1999     }
2000 #endif
2001 
2002     cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
2003     jcc   (Assembler::zero, DONE_LABEL);            // 0 indicates recursive stack-lock
2004     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));             // Examine the object's markword
2005     testptr(tmpReg, markOopDesc::monitor_value);    // Inflated?
2006     jccb  (Assembler::zero, Stacked);
2007 
2008     // It's inflated.
2009 #if INCLUDE_RTM_OPT
2010     if (use_rtm) {
2011       Label L_regular_inflated_unlock;
2012       int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
2013       movptr(boxReg, Address(tmpReg, owner_offset));
2014       testptr(boxReg, boxReg);
2015       jccb(Assembler::notZero, L_regular_inflated_unlock);
2016       xend();
2017       jmpb(DONE_LABEL);
2018       bind(L_regular_inflated_unlock);
2019     }
2020 #endif
2021 
2022     // Despite our balanced locking property we still check that m->_owner == Self
2023     // as java routines or native JNI code called by this thread might
2024     // have released the lock.
2025     // Refer to the comments in synchronizer.cpp for how we might encode extra
2026     // state in _succ so we can avoid fetching EntryList|cxq.
2027     //
2028     // I'd like to add more cases in fast_lock() and fast_unlock() --
2029     // such as recursive enter and exit -- but we have to be wary of
2030     // I$ bloat, T$ effects and BP$ effects.
2031     //
2032     // If there's no contention try a 1-0 exit.  That is, exit without
2033     // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how
2034     // we detect and recover from the race that the 1-0 exit admits.
2035     //
2036     // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier
2037     // before it STs null into _owner, releasing the lock.  Updates
2038     // to data protected by the critical section must be visible before
2039     // we drop the lock (and thus before any other thread could acquire
2040     // the lock and observe the fields protected by the lock).
2041     // IA32's memory-model is SPO, so STs are ordered with respect to
2042     // each other and there's no need for an explicit barrier (fence).
2043     // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
2044 #ifndef _LP64
2045     get_thread (boxReg);
2046     if ((EmitSync & 4096) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
2047       // prefetchw [ebx + Offset(_owner)-2]
2048       prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2049     }
2050 
2051     // Note that we could employ various encoding schemes to reduce
2052     // the number of loads below (currently 4) to just 2 or 3.
2053     // Refer to the comments in synchronizer.cpp.
2054     // In practice the chain of fetches doesn't seem to impact performance, however.
2055     xorptr(boxReg, boxReg);
2056     if ((EmitSync & 65536) == 0 && (EmitSync & 256)) {
2057        // Attempt to reduce branch density - AMD's branch predictor.
2058        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2059        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2060        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2061        jccb  (Assembler::notZero, DONE_LABEL);
2062        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2063        jmpb  (DONE_LABEL);
2064     } else {
2065        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2066        jccb  (Assembler::notZero, DONE_LABEL);
2067        movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2068        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2069        jccb  (Assembler::notZero, CheckSucc);
2070        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2071        jmpb  (DONE_LABEL);
2072     }
2073 
2074     // The Following code fragment (EmitSync & 65536) improves the performance of
2075     // contended applications and contended synchronization microbenchmarks.
2076     // Unfortunately the emission of the code - even though not executed - causes regressions
2077     // in scimark and jetstream, evidently because of $ effects.  Replacing the code
2078     // with an equal number of never-executed NOPs results in the same regression.
2079     // We leave it off by default.
2080 
2081     if ((EmitSync & 65536) != 0) {
2082        Label LSuccess, LGoSlowPath ;
2083 
2084        bind  (CheckSucc);
2085 
2086        // Optional pre-test ... it's safe to elide this
2087        cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2088        jccb(Assembler::zero, LGoSlowPath);
2089 
2090        // We have a classic Dekker-style idiom:
2091        //    ST m->_owner = 0 ; MEMBAR; LD m->_succ
2092        // There are a number of ways to implement the barrier:
2093        // (1) lock:andl &m->_owner, 0
2094        //     is fast, but mask doesn't currently support the "ANDL M,IMM32" form.
2095        //     LOCK: ANDL [ebx+Offset(_Owner)-2], 0
2096        //     Encodes as 81 31 OFF32 IMM32 or 83 63 OFF8 IMM8
2097        // (2) If supported, an explicit MFENCE is appealing.
2098        //     In older IA32 processors MFENCE is slower than lock:add or xchg
2099        //     particularly if the write-buffer is full as might be the case if
2100        //     if stores closely precede the fence or fence-equivalent instruction.
2101        //     See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences
2102        //     as the situation has changed with Nehalem and Shanghai.
2103        // (3) In lieu of an explicit fence, use lock:addl to the top-of-stack
2104        //     The $lines underlying the top-of-stack should be in M-state.
2105        //     The locked add instruction is serializing, of course.
2106        // (4) Use xchg, which is serializing
2107        //     mov boxReg, 0; xchgl boxReg, [tmpReg + Offset(_owner)-2] also works
2108        // (5) ST m->_owner = 0 and then execute lock:orl &m->_succ, 0.
2109        //     The integer condition codes will tell us if succ was 0.
2110        //     Since _succ and _owner should reside in the same $line and
2111        //     we just stored into _owner, it's likely that the $line
2112        //     remains in M-state for the lock:orl.
2113        //
2114        // We currently use (3), although it's likely that switching to (2)
2115        // is correct for the future.
2116 
2117        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2118        if (os::is_MP()) {
2119          lock(); addptr(Address(rsp, 0), 0);
2120        }
2121        // Ratify _succ remains non-null
2122        cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), 0);
2123        jccb  (Assembler::notZero, LSuccess);
2124 
2125        xorptr(boxReg, boxReg);                  // box is really EAX
2126        if (os::is_MP()) { lock(); }
2127        cmpxchgptr(rsp, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2128        // There's no successor so we tried to regrab the lock with the
2129        // placeholder value. If that didn't work, then another thread
2130        // grabbed the lock so we're done (and exit was a success).
2131        jccb  (Assembler::notEqual, LSuccess);
2132        // Since we're low on registers we installed rsp as a placeholding in _owner.
2133        // Now install Self over rsp.  This is safe as we're transitioning from
2134        // non-null to non=null
2135        get_thread (boxReg);
2136        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), boxReg);
2137        // Intentional fall-through into LGoSlowPath ...
2138 
2139        bind  (LGoSlowPath);
2140        orptr(boxReg, 1);                      // set ICC.ZF=0 to indicate failure
2141        jmpb  (DONE_LABEL);
2142 
2143        bind  (LSuccess);
2144        xorptr(boxReg, boxReg);                 // set ICC.ZF=1 to indicate success
2145        jmpb  (DONE_LABEL);
2146     }
2147 
2148     bind (Stacked);
2149     // It's not inflated and it's not recursively stack-locked and it's not biased.
2150     // It must be stack-locked.
2151     // Try to reset the header to displaced header.
2152     // The "box" value on the stack is stable, so we can reload
2153     // and be assured we observe the same value as above.
2154     movptr(tmpReg, Address(boxReg, 0));
2155     if (os::is_MP()) {
2156       lock();
2157     }
2158     cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
2159     // Intention fall-thru into DONE_LABEL
2160 
2161     // DONE_LABEL is a hot target - we'd really like to place it at the
2162     // start of cache line by padding with NOPs.
2163     // See the AMD and Intel software optimization manuals for the
2164     // most efficient "long" NOP encodings.
2165     // Unfortunately none of our alignment mechanisms suffice.
2166     if ((EmitSync & 65536) == 0) {
2167        bind (CheckSucc);
2168     }
2169 #else // _LP64
2170     // It's inflated
2171     if (EmitSync & 1024) {
2172       // Emit code to check that _owner == Self
2173       // We could fold the _owner test into subsequent code more efficiently
2174       // than using a stand-alone check, but since _owner checking is off by
2175       // default we don't bother. We also might consider predicating the
2176       // _owner==Self check on Xcheck:jni or running on a debug build.
2177       movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2178       xorptr(boxReg, r15_thread);
2179     } else {
2180       xorptr(boxReg, boxReg);
2181     }
2182     orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2183     jccb  (Assembler::notZero, DONE_LABEL);
2184     movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2185     orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2186     jccb  (Assembler::notZero, CheckSucc);
2187     movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2188     jmpb  (DONE_LABEL);
2189 
2190     if ((EmitSync & 65536) == 0) {
2191       // Try to avoid passing control into the slow_path ...
2192       Label LSuccess, LGoSlowPath ;
2193       bind  (CheckSucc);
2194 
2195       // The following optional optimization can be elided if necessary
2196       // Effectively: if (succ == null) goto SlowPath
2197       // The code reduces the window for a race, however,
2198       // and thus benefits performance.
2199       cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2200       jccb  (Assembler::zero, LGoSlowPath);
2201 
2202       xorptr(boxReg, boxReg);
2203       if ((EmitSync & 16) && os::is_MP()) {
2204         xchgptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2205       } else {
2206         movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2207         if (os::is_MP()) {
2208           // Memory barrier/fence
2209           // Dekker pivot point -- fulcrum : ST Owner; MEMBAR; LD Succ
2210           // Instead of MFENCE we use a dummy locked add of 0 to the top-of-stack.
2211           // This is faster on Nehalem and AMD Shanghai/Barcelona.
2212           // See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences
2213           // We might also restructure (ST Owner=0;barrier;LD _Succ) to
2214           // (mov box,0; xchgq box, &m->Owner; LD _succ) .
2215           lock(); addl(Address(rsp, 0), 0);
2216         }
2217       }
2218       cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2219       jccb  (Assembler::notZero, LSuccess);
2220 
2221       // Rare inopportune interleaving - race.
2222       // The successor vanished in the small window above.
2223       // The lock is contended -- (cxq|EntryList) != null -- and there's no apparent successor.
2224       // We need to ensure progress and succession.
2225       // Try to reacquire the lock.
2226       // If that fails then the new owner is responsible for succession and this
2227       // thread needs to take no further action and can exit via the fast path (success).
2228       // If the re-acquire succeeds then pass control into the slow path.
2229       // As implemented, this latter mode is horrible because we generated more
2230       // coherence traffic on the lock *and* artifically extended the critical section
2231       // length while by virtue of passing control into the slow path.
2232 
2233       // box is really RAX -- the following CMPXCHG depends on that binding
2234       // cmpxchg R,[M] is equivalent to rax = CAS(M,rax,R)
2235       if (os::is_MP()) { lock(); }
2236       cmpxchgptr(r15_thread, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2237       // There's no successor so we tried to regrab the lock.
2238       // If that didn't work, then another thread grabbed the
2239       // lock so we're done (and exit was a success).
2240       jccb  (Assembler::notEqual, LSuccess);
2241       // Intentional fall-through into slow-path
2242 
2243       bind  (LGoSlowPath);
2244       orl   (boxReg, 1);                      // set ICC.ZF=0 to indicate failure
2245       jmpb  (DONE_LABEL);
2246 
2247       bind  (LSuccess);
2248       testl (boxReg, 0);                      // set ICC.ZF=1 to indicate success
2249       jmpb  (DONE_LABEL);
2250     }
2251 
2252     bind  (Stacked);
2253     movptr(tmpReg, Address (boxReg, 0));      // re-fetch
2254     if (os::is_MP()) { lock(); }
2255     cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
2256 
2257     if (EmitSync & 65536) {
2258        bind (CheckSucc);
2259     }
2260 #endif
2261     bind(DONE_LABEL);
2262   }
2263 }
2264 #endif // COMPILER2
2265 
2266 void MacroAssembler::c2bool(Register x) {
2267   // implements x == 0 ? 0 : 1
2268   // note: must only look at least-significant byte of x
2269   //       since C-style booleans are stored in one byte
2270   //       only! (was bug)
2271   andl(x, 0xFF);
2272   setb(Assembler::notZero, x);
2273 }
2274 
2275 // Wouldn't need if AddressLiteral version had new name
2276 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
2277   Assembler::call(L, rtype);
2278 }
2279 
2280 void MacroAssembler::call(Register entry) {
2281   Assembler::call(entry);
2282 }
2283 
2284 void MacroAssembler::call(AddressLiteral entry) {
2285   if (reachable(entry)) {
2286     Assembler::call_literal(entry.target(), entry.rspec());
2287   } else {
2288     lea(rscratch1, entry);
2289     Assembler::call(rscratch1);
2290   }
2291 }
2292 
2293 void MacroAssembler::ic_call(address entry, jint method_index) {
2294   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
2295   movptr(rax, (intptr_t)Universe::non_oop_word());
2296   call(AddressLiteral(entry, rh));
2297 }
2298 
2299 // Implementation of call_VM versions
2300 
2301 void MacroAssembler::call_VM(Register oop_result,
2302                              address entry_point,
2303                              bool check_exceptions) {
2304   Label C, E;
2305   call(C, relocInfo::none);
2306   jmp(E);
2307 
2308   bind(C);
2309   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
2310   ret(0);
2311 
2312   bind(E);
2313 }
2314 
2315 void MacroAssembler::call_VM(Register oop_result,
2316                              address entry_point,
2317                              Register arg_1,
2318                              bool check_exceptions) {
2319   Label C, E;
2320   call(C, relocInfo::none);
2321   jmp(E);
2322 
2323   bind(C);
2324   pass_arg1(this, arg_1);
2325   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
2326   ret(0);
2327 
2328   bind(E);
2329 }
2330 
2331 void MacroAssembler::call_VM(Register oop_result,
2332                              address entry_point,
2333                              Register arg_1,
2334                              Register arg_2,
2335                              bool check_exceptions) {
2336   Label C, E;
2337   call(C, relocInfo::none);
2338   jmp(E);
2339 
2340   bind(C);
2341 
2342   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2343 
2344   pass_arg2(this, arg_2);
2345   pass_arg1(this, arg_1);
2346   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
2347   ret(0);
2348 
2349   bind(E);
2350 }
2351 
2352 void MacroAssembler::call_VM(Register oop_result,
2353                              address entry_point,
2354                              Register arg_1,
2355                              Register arg_2,
2356                              Register arg_3,
2357                              bool check_exceptions) {
2358   Label C, E;
2359   call(C, relocInfo::none);
2360   jmp(E);
2361 
2362   bind(C);
2363 
2364   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2365   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2366   pass_arg3(this, arg_3);
2367 
2368   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2369   pass_arg2(this, arg_2);
2370 
2371   pass_arg1(this, arg_1);
2372   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
2373   ret(0);
2374 
2375   bind(E);
2376 }
2377 
2378 void MacroAssembler::call_VM(Register oop_result,
2379                              Register last_java_sp,
2380                              address entry_point,
2381                              int number_of_arguments,
2382                              bool check_exceptions) {
2383   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2384   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2385 }
2386 
2387 void MacroAssembler::call_VM(Register oop_result,
2388                              Register last_java_sp,
2389                              address entry_point,
2390                              Register arg_1,
2391                              bool check_exceptions) {
2392   pass_arg1(this, arg_1);
2393   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2394 }
2395 
2396 void MacroAssembler::call_VM(Register oop_result,
2397                              Register last_java_sp,
2398                              address entry_point,
2399                              Register arg_1,
2400                              Register arg_2,
2401                              bool check_exceptions) {
2402 
2403   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2404   pass_arg2(this, arg_2);
2405   pass_arg1(this, arg_1);
2406   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2407 }
2408 
2409 void MacroAssembler::call_VM(Register oop_result,
2410                              Register last_java_sp,
2411                              address entry_point,
2412                              Register arg_1,
2413                              Register arg_2,
2414                              Register arg_3,
2415                              bool check_exceptions) {
2416   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2417   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2418   pass_arg3(this, arg_3);
2419   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2420   pass_arg2(this, arg_2);
2421   pass_arg1(this, arg_1);
2422   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2423 }
2424 
2425 void MacroAssembler::super_call_VM(Register oop_result,
2426                                    Register last_java_sp,
2427                                    address entry_point,
2428                                    int number_of_arguments,
2429                                    bool check_exceptions) {
2430   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2431   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2432 }
2433 
2434 void MacroAssembler::super_call_VM(Register oop_result,
2435                                    Register last_java_sp,
2436                                    address entry_point,
2437                                    Register arg_1,
2438                                    bool check_exceptions) {
2439   pass_arg1(this, arg_1);
2440   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2441 }
2442 
2443 void MacroAssembler::super_call_VM(Register oop_result,
2444                                    Register last_java_sp,
2445                                    address entry_point,
2446                                    Register arg_1,
2447                                    Register arg_2,
2448                                    bool check_exceptions) {
2449 
2450   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2451   pass_arg2(this, arg_2);
2452   pass_arg1(this, arg_1);
2453   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2454 }
2455 
2456 void MacroAssembler::super_call_VM(Register oop_result,
2457                                    Register last_java_sp,
2458                                    address entry_point,
2459                                    Register arg_1,
2460                                    Register arg_2,
2461                                    Register arg_3,
2462                                    bool check_exceptions) {
2463   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2464   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2465   pass_arg3(this, arg_3);
2466   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2467   pass_arg2(this, arg_2);
2468   pass_arg1(this, arg_1);
2469   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2470 }
2471 
2472 void MacroAssembler::call_VM_base(Register oop_result,
2473                                   Register java_thread,
2474                                   Register last_java_sp,
2475                                   address  entry_point,
2476                                   int      number_of_arguments,
2477                                   bool     check_exceptions) {
2478   // determine java_thread register
2479   if (!java_thread->is_valid()) {
2480 #ifdef _LP64
2481     java_thread = r15_thread;
2482 #else
2483     java_thread = rdi;
2484     get_thread(java_thread);
2485 #endif // LP64
2486   }
2487   // determine last_java_sp register
2488   if (!last_java_sp->is_valid()) {
2489     last_java_sp = rsp;
2490   }
2491   // debugging support
2492   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
2493   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
2494 #ifdef ASSERT
2495   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
2496   // r12 is the heapbase.
2497   LP64_ONLY(if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
2498 #endif // ASSERT
2499 
2500   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
2501   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
2502 
2503   // push java thread (becomes first argument of C function)
2504 
2505   NOT_LP64(push(java_thread); number_of_arguments++);
2506   LP64_ONLY(mov(c_rarg0, r15_thread));
2507 
2508   // set last Java frame before call
2509   assert(last_java_sp != rbp, "can't use ebp/rbp");
2510 
2511   // Only interpreter should have to set fp
2512   set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);
2513 
2514   // do the call, remove parameters
2515   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
2516 
2517   // restore the thread (cannot use the pushed argument since arguments
2518   // may be overwritten by C code generated by an optimizing compiler);
2519   // however can use the register value directly if it is callee saved.
2520   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
2521     // rdi & rsi (also r15) are callee saved -> nothing to do
2522 #ifdef ASSERT
2523     guarantee(java_thread != rax, "change this code");
2524     push(rax);
2525     { Label L;
2526       get_thread(rax);
2527       cmpptr(java_thread, rax);
2528       jcc(Assembler::equal, L);
2529       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
2530       bind(L);
2531     }
2532     pop(rax);
2533 #endif
2534   } else {
2535     get_thread(java_thread);
2536   }
2537   // reset last Java frame
2538   // Only interpreter should have to clear fp
2539   reset_last_Java_frame(java_thread, true);
2540 
2541    // C++ interp handles this in the interpreter
2542   check_and_handle_popframe(java_thread);
2543   check_and_handle_earlyret(java_thread);
2544 
2545   if (check_exceptions) {
2546     // check for pending exceptions (java_thread is set upon return)
2547     cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
2548 #ifndef _LP64
2549     jump_cc(Assembler::notEqual,
2550             RuntimeAddress(StubRoutines::forward_exception_entry()));
2551 #else
2552     // This used to conditionally jump to forward_exception however it is
2553     // possible if we relocate that the branch will not reach. So we must jump
2554     // around so we can always reach
2555 
2556     Label ok;
2557     jcc(Assembler::equal, ok);
2558     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2559     bind(ok);
2560 #endif // LP64
2561   }
2562 
2563   // get oop result if there is one and reset the value in the thread
2564   if (oop_result->is_valid()) {
2565     get_vm_result(oop_result, java_thread);
2566   }
2567 }
2568 
2569 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
2570 
2571   // Calculate the value for last_Java_sp
2572   // somewhat subtle. call_VM does an intermediate call
2573   // which places a return address on the stack just under the
2574   // stack pointer as the user finsihed with it. This allows
2575   // use to retrieve last_Java_pc from last_Java_sp[-1].
2576   // On 32bit we then have to push additional args on the stack to accomplish
2577   // the actual requested call. On 64bit call_VM only can use register args
2578   // so the only extra space is the return address that call_VM created.
2579   // This hopefully explains the calculations here.
2580 
2581 #ifdef _LP64
2582   // We've pushed one address, correct last_Java_sp
2583   lea(rax, Address(rsp, wordSize));
2584 #else
2585   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
2586 #endif // LP64
2587 
2588   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
2589 
2590 }
2591 
2592 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
2593 void MacroAssembler::call_VM_leaf0(address entry_point) {
2594   MacroAssembler::call_VM_leaf_base(entry_point, 0);
2595 }
2596 
2597 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
2598   call_VM_leaf_base(entry_point, number_of_arguments);
2599 }
2600 
2601 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
2602   pass_arg0(this, arg_0);
2603   call_VM_leaf(entry_point, 1);
2604 }
2605 
2606 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2607 
2608   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2609   pass_arg1(this, arg_1);
2610   pass_arg0(this, arg_0);
2611   call_VM_leaf(entry_point, 2);
2612 }
2613 
2614 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2615   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2616   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2617   pass_arg2(this, arg_2);
2618   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2619   pass_arg1(this, arg_1);
2620   pass_arg0(this, arg_0);
2621   call_VM_leaf(entry_point, 3);
2622 }
2623 
2624 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
2625   pass_arg0(this, arg_0);
2626   MacroAssembler::call_VM_leaf_base(entry_point, 1);
2627 }
2628 
2629 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2630 
2631   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2632   pass_arg1(this, arg_1);
2633   pass_arg0(this, arg_0);
2634   MacroAssembler::call_VM_leaf_base(entry_point, 2);
2635 }
2636 
2637 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2638   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2639   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2640   pass_arg2(this, arg_2);
2641   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2642   pass_arg1(this, arg_1);
2643   pass_arg0(this, arg_0);
2644   MacroAssembler::call_VM_leaf_base(entry_point, 3);
2645 }
2646 
2647 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
2648   LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg"));
2649   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2650   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2651   pass_arg3(this, arg_3);
2652   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2653   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2654   pass_arg2(this, arg_2);
2655   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2656   pass_arg1(this, arg_1);
2657   pass_arg0(this, arg_0);
2658   MacroAssembler::call_VM_leaf_base(entry_point, 4);
2659 }
2660 
2661 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
2662   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
2663   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
2664   verify_oop(oop_result, "broken oop in call_VM_base");
2665 }
2666 
2667 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
2668   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
2669   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
2670 }
2671 
2672 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
2673 }
2674 
2675 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
2676 }
2677 
2678 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) {
2679   if (reachable(src1)) {
2680     cmpl(as_Address(src1), imm);
2681   } else {
2682     lea(rscratch1, src1);
2683     cmpl(Address(rscratch1, 0), imm);
2684   }
2685 }
2686 
2687 void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
2688   assert(!src2.is_lval(), "use cmpptr");
2689   if (reachable(src2)) {
2690     cmpl(src1, as_Address(src2));
2691   } else {
2692     lea(rscratch1, src2);
2693     cmpl(src1, Address(rscratch1, 0));
2694   }
2695 }
2696 
2697 void MacroAssembler::cmp32(Register src1, int32_t imm) {
2698   Assembler::cmpl(src1, imm);
2699 }
2700 
2701 void MacroAssembler::cmp32(Register src1, Address src2) {
2702   Assembler::cmpl(src1, src2);
2703 }
2704 
2705 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2706   ucomisd(opr1, opr2);
2707 
2708   Label L;
2709   if (unordered_is_less) {
2710     movl(dst, -1);
2711     jcc(Assembler::parity, L);
2712     jcc(Assembler::below , L);
2713     movl(dst, 0);
2714     jcc(Assembler::equal , L);
2715     increment(dst);
2716   } else { // unordered is greater
2717     movl(dst, 1);
2718     jcc(Assembler::parity, L);
2719     jcc(Assembler::above , L);
2720     movl(dst, 0);
2721     jcc(Assembler::equal , L);
2722     decrementl(dst);
2723   }
2724   bind(L);
2725 }
2726 
2727 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2728   ucomiss(opr1, opr2);
2729 
2730   Label L;
2731   if (unordered_is_less) {
2732     movl(dst, -1);
2733     jcc(Assembler::parity, L);
2734     jcc(Assembler::below , L);
2735     movl(dst, 0);
2736     jcc(Assembler::equal , L);
2737     increment(dst);
2738   } else { // unordered is greater
2739     movl(dst, 1);
2740     jcc(Assembler::parity, L);
2741     jcc(Assembler::above , L);
2742     movl(dst, 0);
2743     jcc(Assembler::equal , L);
2744     decrementl(dst);
2745   }
2746   bind(L);
2747 }
2748 
2749 
2750 void MacroAssembler::cmp8(AddressLiteral src1, int imm) {
2751   if (reachable(src1)) {
2752     cmpb(as_Address(src1), imm);
2753   } else {
2754     lea(rscratch1, src1);
2755     cmpb(Address(rscratch1, 0), imm);
2756   }
2757 }
2758 
2759 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
2760 #ifdef _LP64
2761   if (src2.is_lval()) {
2762     movptr(rscratch1, src2);
2763     Assembler::cmpq(src1, rscratch1);
2764   } else if (reachable(src2)) {
2765     cmpq(src1, as_Address(src2));
2766   } else {
2767     lea(rscratch1, src2);
2768     Assembler::cmpq(src1, Address(rscratch1, 0));
2769   }
2770 #else
2771   if (src2.is_lval()) {
2772     cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2773   } else {
2774     cmpl(src1, as_Address(src2));
2775   }
2776 #endif // _LP64
2777 }
2778 
2779 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
2780   assert(src2.is_lval(), "not a mem-mem compare");
2781 #ifdef _LP64
2782   // moves src2's literal address
2783   movptr(rscratch1, src2);
2784   Assembler::cmpq(src1, rscratch1);
2785 #else
2786   cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2787 #endif // _LP64
2788 }
2789 
2790 void MacroAssembler::cmpoop(Register src1, Register src2) {
2791   cmpptr(src1, src2);
2792 }
2793 
2794 void MacroAssembler::cmpoop(Register src1, Address src2) {
2795   cmpptr(src1, src2);
2796 }
2797 
2798 #ifdef _LP64
2799 void MacroAssembler::cmpoop(Register src1, jobject src2) {
2800   movoop(rscratch1, src2);
2801   cmpptr(src1, rscratch1);
2802 }
2803 #endif
2804 
2805 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) {
2806   if (reachable(adr)) {
2807     if (os::is_MP())
2808       lock();
2809     cmpxchgptr(reg, as_Address(adr));
2810   } else {
2811     lea(rscratch1, adr);
2812     if (os::is_MP())
2813       lock();
2814     cmpxchgptr(reg, Address(rscratch1, 0));
2815   }
2816 }
2817 
2818 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
2819   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
2820 }
2821 
2822 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {
2823   if (reachable(src)) {
2824     Assembler::comisd(dst, as_Address(src));
2825   } else {
2826     lea(rscratch1, src);
2827     Assembler::comisd(dst, Address(rscratch1, 0));
2828   }
2829 }
2830 
2831 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {
2832   if (reachable(src)) {
2833     Assembler::comiss(dst, as_Address(src));
2834   } else {
2835     lea(rscratch1, src);
2836     Assembler::comiss(dst, Address(rscratch1, 0));
2837   }
2838 }
2839 
2840 
2841 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
2842   Condition negated_cond = negate_condition(cond);
2843   Label L;
2844   jcc(negated_cond, L);
2845   pushf(); // Preserve flags
2846   atomic_incl(counter_addr);
2847   popf();
2848   bind(L);
2849 }
2850 
2851 int MacroAssembler::corrected_idivl(Register reg) {
2852   // Full implementation of Java idiv and irem; checks for
2853   // special case as described in JVM spec., p.243 & p.271.
2854   // The function returns the (pc) offset of the idivl
2855   // instruction - may be needed for implicit exceptions.
2856   //
2857   //         normal case                           special case
2858   //
2859   // input : rax,: dividend                         min_int
2860   //         reg: divisor   (may not be rax,/rdx)   -1
2861   //
2862   // output: rax,: quotient  (= rax, idiv reg)       min_int
2863   //         rdx: remainder (= rax, irem reg)       0
2864   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
2865   const int min_int = 0x80000000;
2866   Label normal_case, special_case;
2867 
2868   // check for special case
2869   cmpl(rax, min_int);
2870   jcc(Assembler::notEqual, normal_case);
2871   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
2872   cmpl(reg, -1);
2873   jcc(Assembler::equal, special_case);
2874 
2875   // handle normal case
2876   bind(normal_case);
2877   cdql();
2878   int idivl_offset = offset();
2879   idivl(reg);
2880 
2881   // normal and special case exit
2882   bind(special_case);
2883 
2884   return idivl_offset;
2885 }
2886 
2887 
2888 
2889 void MacroAssembler::decrementl(Register reg, int value) {
2890   if (value == min_jint) {subl(reg, value) ; return; }
2891   if (value <  0) { incrementl(reg, -value); return; }
2892   if (value == 0) {                        ; return; }
2893   if (value == 1 && UseIncDec) { decl(reg) ; return; }
2894   /* else */      { subl(reg, value)       ; return; }
2895 }
2896 
2897 void MacroAssembler::decrementl(Address dst, int value) {
2898   if (value == min_jint) {subl(dst, value) ; return; }
2899   if (value <  0) { incrementl(dst, -value); return; }
2900   if (value == 0) {                        ; return; }
2901   if (value == 1 && UseIncDec) { decl(dst) ; return; }
2902   /* else */      { subl(dst, value)       ; return; }
2903 }
2904 
2905 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
2906   assert (shift_value > 0, "illegal shift value");
2907   Label _is_positive;
2908   testl (reg, reg);
2909   jcc (Assembler::positive, _is_positive);
2910   int offset = (1 << shift_value) - 1 ;
2911 
2912   if (offset == 1) {
2913     incrementl(reg);
2914   } else {
2915     addl(reg, offset);
2916   }
2917 
2918   bind (_is_positive);
2919   sarl(reg, shift_value);
2920 }
2921 
2922 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
2923   if (reachable(src)) {
2924     Assembler::divsd(dst, as_Address(src));
2925   } else {
2926     lea(rscratch1, src);
2927     Assembler::divsd(dst, Address(rscratch1, 0));
2928   }
2929 }
2930 
2931 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
2932   if (reachable(src)) {
2933     Assembler::divss(dst, as_Address(src));
2934   } else {
2935     lea(rscratch1, src);
2936     Assembler::divss(dst, Address(rscratch1, 0));
2937   }
2938 }
2939 
2940 // !defined(COMPILER2) is because of stupid core builds
2941 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) || INCLUDE_JVMCI
2942 void MacroAssembler::empty_FPU_stack() {
2943   if (VM_Version::supports_mmx()) {
2944     emms();
2945   } else {
2946     for (int i = 8; i-- > 0; ) ffree(i);
2947   }
2948 }
2949 #endif // !LP64 || C1 || !C2 || INCLUDE_JVMCI
2950 
2951 
2952 // Defines obj, preserves var_size_in_bytes
2953 void MacroAssembler::eden_allocate(Register obj,
2954                                    Register var_size_in_bytes,
2955                                    int con_size_in_bytes,
2956                                    Register t1,
2957                                    Label& slow_case) {
2958   assert(obj == rax, "obj must be in rax, for cmpxchg");
2959   assert_different_registers(obj, var_size_in_bytes, t1);
2960   if (!Universe::heap()->supports_inline_contig_alloc()) {
2961     jmp(slow_case);
2962   } else {
2963     Register end = t1;
2964     Label retry;
2965     bind(retry);
2966     ExternalAddress heap_top((address) Universe::heap()->top_addr());
2967     movptr(obj, heap_top);
2968     if (var_size_in_bytes == noreg) {
2969       lea(end, Address(obj, con_size_in_bytes));
2970     } else {
2971       lea(end, Address(obj, var_size_in_bytes, Address::times_1));
2972     }
2973     // if end < obj then we wrapped around => object too long => slow case
2974     cmpptr(end, obj);
2975     jcc(Assembler::below, slow_case);
2976     cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
2977     jcc(Assembler::above, slow_case);
2978     // Compare obj with the top addr, and if still equal, store the new top addr in
2979     // end at the address of the top addr pointer. Sets ZF if was equal, and clears
2980     // it otherwise. Use lock prefix for atomicity on MPs.
2981     locked_cmpxchgptr(end, heap_top);
2982     jcc(Assembler::notEqual, retry);
2983   }
2984 }
2985 
2986 void MacroAssembler::enter() {
2987   push(rbp);
2988   mov(rbp, rsp);
2989 }
2990 
2991 // A 5 byte nop that is safe for patching (see patch_verified_entry)
2992 void MacroAssembler::fat_nop() {
2993   if (UseAddressNop) {
2994     addr_nop_5();
2995   } else {
2996     emit_int8(0x26); // es:
2997     emit_int8(0x2e); // cs:
2998     emit_int8(0x64); // fs:
2999     emit_int8(0x65); // gs:
3000     emit_int8((unsigned char)0x90);
3001   }
3002 }
3003 
3004 void MacroAssembler::fcmp(Register tmp) {
3005   fcmp(tmp, 1, true, true);
3006 }
3007 
3008 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
3009   assert(!pop_right || pop_left, "usage error");
3010   if (VM_Version::supports_cmov()) {
3011     assert(tmp == noreg, "unneeded temp");
3012     if (pop_left) {
3013       fucomip(index);
3014     } else {
3015       fucomi(index);
3016     }
3017     if (pop_right) {
3018       fpop();
3019     }
3020   } else {
3021     assert(tmp != noreg, "need temp");
3022     if (pop_left) {
3023       if (pop_right) {
3024         fcompp();
3025       } else {
3026         fcomp(index);
3027       }
3028     } else {
3029       fcom(index);
3030     }
3031     // convert FPU condition into eflags condition via rax,
3032     save_rax(tmp);
3033     fwait(); fnstsw_ax();
3034     sahf();
3035     restore_rax(tmp);
3036   }
3037   // condition codes set as follows:
3038   //
3039   // CF (corresponds to C0) if x < y
3040   // PF (corresponds to C2) if unordered
3041   // ZF (corresponds to C3) if x = y
3042 }
3043 
3044 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
3045   fcmp2int(dst, unordered_is_less, 1, true, true);
3046 }
3047 
3048 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
3049   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
3050   Label L;
3051   if (unordered_is_less) {
3052     movl(dst, -1);
3053     jcc(Assembler::parity, L);
3054     jcc(Assembler::below , L);
3055     movl(dst, 0);
3056     jcc(Assembler::equal , L);
3057     increment(dst);
3058   } else { // unordered is greater
3059     movl(dst, 1);
3060     jcc(Assembler::parity, L);
3061     jcc(Assembler::above , L);
3062     movl(dst, 0);
3063     jcc(Assembler::equal , L);
3064     decrementl(dst);
3065   }
3066   bind(L);
3067 }
3068 
3069 void MacroAssembler::fld_d(AddressLiteral src) {
3070   fld_d(as_Address(src));
3071 }
3072 
3073 void MacroAssembler::fld_s(AddressLiteral src) {
3074   fld_s(as_Address(src));
3075 }
3076 
3077 void MacroAssembler::fld_x(AddressLiteral src) {
3078   Assembler::fld_x(as_Address(src));
3079 }
3080 
3081 void MacroAssembler::fldcw(AddressLiteral src) {
3082   Assembler::fldcw(as_Address(src));
3083 }
3084 
3085 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src) {
3086   if (reachable(src)) {
3087     Assembler::mulpd(dst, as_Address(src));
3088   } else {
3089     lea(rscratch1, src);
3090     Assembler::mulpd(dst, Address(rscratch1, 0));
3091   }
3092 }
3093 
3094 void MacroAssembler::increase_precision() {
3095   subptr(rsp, BytesPerWord);
3096   fnstcw(Address(rsp, 0));
3097   movl(rax, Address(rsp, 0));
3098   orl(rax, 0x300);
3099   push(rax);
3100   fldcw(Address(rsp, 0));
3101   pop(rax);
3102 }
3103 
3104 void MacroAssembler::restore_precision() {
3105   fldcw(Address(rsp, 0));
3106   addptr(rsp, BytesPerWord);
3107 }
3108 
3109 void MacroAssembler::fpop() {
3110   ffree();
3111   fincstp();
3112 }
3113 
3114 void MacroAssembler::load_float(Address src) {
3115   if (UseSSE >= 1) {
3116     movflt(xmm0, src);
3117   } else {
3118     LP64_ONLY(ShouldNotReachHere());
3119     NOT_LP64(fld_s(src));
3120   }
3121 }
3122 
3123 void MacroAssembler::store_float(Address dst) {
3124   if (UseSSE >= 1) {
3125     movflt(dst, xmm0);
3126   } else {
3127     LP64_ONLY(ShouldNotReachHere());
3128     NOT_LP64(fstp_s(dst));
3129   }
3130 }
3131 
3132 void MacroAssembler::load_double(Address src) {
3133   if (UseSSE >= 2) {
3134     movdbl(xmm0, src);
3135   } else {
3136     LP64_ONLY(ShouldNotReachHere());
3137     NOT_LP64(fld_d(src));
3138   }
3139 }
3140 
3141 void MacroAssembler::store_double(Address dst) {
3142   if (UseSSE >= 2) {
3143     movdbl(dst, xmm0);
3144   } else {
3145     LP64_ONLY(ShouldNotReachHere());
3146     NOT_LP64(fstp_d(dst));
3147   }
3148 }
3149 
3150 void MacroAssembler::fremr(Register tmp) {
3151   save_rax(tmp);
3152   { Label L;
3153     bind(L);
3154     fprem();
3155     fwait(); fnstsw_ax();
3156 #ifdef _LP64
3157     testl(rax, 0x400);
3158     jcc(Assembler::notEqual, L);
3159 #else
3160     sahf();
3161     jcc(Assembler::parity, L);
3162 #endif // _LP64
3163   }
3164   restore_rax(tmp);
3165   // Result is in ST0.
3166   // Note: fxch & fpop to get rid of ST1
3167   // (otherwise FPU stack could overflow eventually)
3168   fxch(1);
3169   fpop();
3170 }
3171 
3172 // dst = c = a * b + c
3173 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
3174   Assembler::vfmadd231sd(c, a, b);
3175   if (dst != c) {
3176     movdbl(dst, c);
3177   }
3178 }
3179 
3180 // dst = c = a * b + c
3181 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
3182   Assembler::vfmadd231ss(c, a, b);
3183   if (dst != c) {
3184     movflt(dst, c);
3185   }
3186 }
3187 
3188 // dst = c = a * b + c
3189 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
3190   Assembler::vfmadd231pd(c, a, b, vector_len);
3191   if (dst != c) {
3192     vmovdqu(dst, c);
3193   }
3194 }
3195 
3196 // dst = c = a * b + c
3197 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
3198   Assembler::vfmadd231ps(c, a, b, vector_len);
3199   if (dst != c) {
3200     vmovdqu(dst, c);
3201   }
3202 }
3203 
3204 // dst = c = a * b + c
3205 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
3206   Assembler::vfmadd231pd(c, a, b, vector_len);
3207   if (dst != c) {
3208     vmovdqu(dst, c);
3209   }
3210 }
3211 
3212 // dst = c = a * b + c
3213 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
3214   Assembler::vfmadd231ps(c, a, b, vector_len);
3215   if (dst != c) {
3216     vmovdqu(dst, c);
3217   }
3218 }
3219 
3220 void MacroAssembler::incrementl(AddressLiteral dst) {
3221   if (reachable(dst)) {
3222     incrementl(as_Address(dst));
3223   } else {
3224     lea(rscratch1, dst);
3225     incrementl(Address(rscratch1, 0));
3226   }
3227 }
3228 
3229 void MacroAssembler::incrementl(ArrayAddress dst) {
3230   incrementl(as_Address(dst));
3231 }
3232 
3233 void MacroAssembler::incrementl(Register reg, int value) {
3234   if (value == min_jint) {addl(reg, value) ; return; }
3235   if (value <  0) { decrementl(reg, -value); return; }
3236   if (value == 0) {                        ; return; }
3237   if (value == 1 && UseIncDec) { incl(reg) ; return; }
3238   /* else */      { addl(reg, value)       ; return; }
3239 }
3240 
3241 void MacroAssembler::incrementl(Address dst, int value) {
3242   if (value == min_jint) {addl(dst, value) ; return; }
3243   if (value <  0) { decrementl(dst, -value); return; }
3244   if (value == 0) {                        ; return; }
3245   if (value == 1 && UseIncDec) { incl(dst) ; return; }
3246   /* else */      { addl(dst, value)       ; return; }
3247 }
3248 
3249 void MacroAssembler::jump(AddressLiteral dst) {
3250   if (reachable(dst)) {
3251     jmp_literal(dst.target(), dst.rspec());
3252   } else {
3253     lea(rscratch1, dst);
3254     jmp(rscratch1);
3255   }
3256 }
3257 
3258 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
3259   if (reachable(dst)) {
3260     InstructionMark im(this);
3261     relocate(dst.reloc());
3262     const int short_size = 2;
3263     const int long_size = 6;
3264     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
3265     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
3266       // 0111 tttn #8-bit disp
3267       emit_int8(0x70 | cc);
3268       emit_int8((offs - short_size) & 0xFF);
3269     } else {
3270       // 0000 1111 1000 tttn #32-bit disp
3271       emit_int8(0x0F);
3272       emit_int8((unsigned char)(0x80 | cc));
3273       emit_int32(offs - long_size);
3274     }
3275   } else {
3276 #ifdef ASSERT
3277     warning("reversing conditional branch");
3278 #endif /* ASSERT */
3279     Label skip;
3280     jccb(reverse[cc], skip);
3281     lea(rscratch1, dst);
3282     Assembler::jmp(rscratch1);
3283     bind(skip);
3284   }
3285 }
3286 
3287 void MacroAssembler::ldmxcsr(AddressLiteral src) {
3288   if (reachable(src)) {
3289     Assembler::ldmxcsr(as_Address(src));
3290   } else {
3291     lea(rscratch1, src);
3292     Assembler::ldmxcsr(Address(rscratch1, 0));
3293   }
3294 }
3295 
3296 int MacroAssembler::load_signed_byte(Register dst, Address src) {
3297   int off;
3298   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3299     off = offset();
3300     movsbl(dst, src); // movsxb
3301   } else {
3302     off = load_unsigned_byte(dst, src);
3303     shll(dst, 24);
3304     sarl(dst, 24);
3305   }
3306   return off;
3307 }
3308 
3309 // Note: load_signed_short used to be called load_signed_word.
3310 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
3311 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
3312 // The term "word" in HotSpot means a 32- or 64-bit machine word.
3313 int MacroAssembler::load_signed_short(Register dst, Address src) {
3314   int off;
3315   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3316     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
3317     // version but this is what 64bit has always done. This seems to imply
3318     // that users are only using 32bits worth.
3319     off = offset();
3320     movswl(dst, src); // movsxw
3321   } else {
3322     off = load_unsigned_short(dst, src);
3323     shll(dst, 16);
3324     sarl(dst, 16);
3325   }
3326   return off;
3327 }
3328 
3329 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
3330   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3331   // and "3.9 Partial Register Penalties", p. 22).
3332   int off;
3333   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
3334     off = offset();
3335     movzbl(dst, src); // movzxb
3336   } else {
3337     xorl(dst, dst);
3338     off = offset();
3339     movb(dst, src);
3340   }
3341   return off;
3342 }
3343 
3344 // Note: load_unsigned_short used to be called load_unsigned_word.
3345 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
3346   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3347   // and "3.9 Partial Register Penalties", p. 22).
3348   int off;
3349   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
3350     off = offset();
3351     movzwl(dst, src); // movzxw
3352   } else {
3353     xorl(dst, dst);
3354     off = offset();
3355     movw(dst, src);
3356   }
3357   return off;
3358 }
3359 
3360 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
3361   switch (size_in_bytes) {
3362 #ifndef _LP64
3363   case  8:
3364     assert(dst2 != noreg, "second dest register required");
3365     movl(dst,  src);
3366     movl(dst2, src.plus_disp(BytesPerInt));
3367     break;
3368 #else
3369   case  8:  movq(dst, src); break;
3370 #endif
3371   case  4:  movl(dst, src); break;
3372   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
3373   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
3374   default:  ShouldNotReachHere();
3375   }
3376 }
3377 
3378 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
3379   switch (size_in_bytes) {
3380 #ifndef _LP64
3381   case  8:
3382     assert(src2 != noreg, "second source register required");
3383     movl(dst,                        src);
3384     movl(dst.plus_disp(BytesPerInt), src2);
3385     break;
3386 #else
3387   case  8:  movq(dst, src); break;
3388 #endif
3389   case  4:  movl(dst, src); break;
3390   case  2:  movw(dst, src); break;
3391   case  1:  movb(dst, src); break;
3392   default:  ShouldNotReachHere();
3393   }
3394 }
3395 
3396 void MacroAssembler::mov32(AddressLiteral dst, Register src) {
3397   if (reachable(dst)) {
3398     movl(as_Address(dst), src);
3399   } else {
3400     lea(rscratch1, dst);
3401     movl(Address(rscratch1, 0), src);
3402   }
3403 }
3404 
3405 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
3406   if (reachable(src)) {
3407     movl(dst, as_Address(src));
3408   } else {
3409     lea(rscratch1, src);
3410     movl(dst, Address(rscratch1, 0));
3411   }
3412 }
3413 
3414 // C++ bool manipulation
3415 
3416 void MacroAssembler::movbool(Register dst, Address src) {
3417   if(sizeof(bool) == 1)
3418     movb(dst, src);
3419   else if(sizeof(bool) == 2)
3420     movw(dst, src);
3421   else if(sizeof(bool) == 4)
3422     movl(dst, src);
3423   else
3424     // unsupported
3425     ShouldNotReachHere();
3426 }
3427 
3428 void MacroAssembler::movbool(Address dst, bool boolconst) {
3429   if(sizeof(bool) == 1)
3430     movb(dst, (int) boolconst);
3431   else if(sizeof(bool) == 2)
3432     movw(dst, (int) boolconst);
3433   else if(sizeof(bool) == 4)
3434     movl(dst, (int) boolconst);
3435   else
3436     // unsupported
3437     ShouldNotReachHere();
3438 }
3439 
3440 void MacroAssembler::movbool(Address dst, Register src) {
3441   if(sizeof(bool) == 1)
3442     movb(dst, src);
3443   else if(sizeof(bool) == 2)
3444     movw(dst, src);
3445   else if(sizeof(bool) == 4)
3446     movl(dst, src);
3447   else
3448     // unsupported
3449     ShouldNotReachHere();
3450 }
3451 
3452 void MacroAssembler::movbyte(ArrayAddress dst, int src) {
3453   movb(as_Address(dst), src);
3454 }
3455 
3456 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src) {
3457   if (reachable(src)) {
3458     movdl(dst, as_Address(src));
3459   } else {
3460     lea(rscratch1, src);
3461     movdl(dst, Address(rscratch1, 0));
3462   }
3463 }
3464 
3465 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src) {
3466   if (reachable(src)) {
3467     movq(dst, as_Address(src));
3468   } else {
3469     lea(rscratch1, src);
3470     movq(dst, Address(rscratch1, 0));
3471   }
3472 }
3473 
3474 void MacroAssembler::setvectmask(Register dst, Register src) {
3475   Assembler::movl(dst, 1);
3476   Assembler::shlxl(dst, dst, src);
3477   Assembler::decl(dst);
3478   Assembler::kmovdl(k1, dst);
3479   Assembler::movl(dst, src);
3480 }
3481 
3482 void MacroAssembler::restorevectmask() {
3483   Assembler::knotwl(k1, k0);
3484 }
3485 
3486 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
3487   if (reachable(src)) {
3488     if (UseXmmLoadAndClearUpper) {
3489       movsd (dst, as_Address(src));
3490     } else {
3491       movlpd(dst, as_Address(src));
3492     }
3493   } else {
3494     lea(rscratch1, src);
3495     if (UseXmmLoadAndClearUpper) {
3496       movsd (dst, Address(rscratch1, 0));
3497     } else {
3498       movlpd(dst, Address(rscratch1, 0));
3499     }
3500   }
3501 }
3502 
3503 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
3504   if (reachable(src)) {
3505     movss(dst, as_Address(src));
3506   } else {
3507     lea(rscratch1, src);
3508     movss(dst, Address(rscratch1, 0));
3509   }
3510 }
3511 
3512 void MacroAssembler::movptr(Register dst, Register src) {
3513   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3514 }
3515 
3516 void MacroAssembler::movptr(Register dst, Address src) {
3517   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3518 }
3519 
3520 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
3521 void MacroAssembler::movptr(Register dst, intptr_t src) {
3522   LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src));
3523 }
3524 
3525 void MacroAssembler::movptr(Address dst, Register src) {
3526   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3527 }
3528 
3529 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
3530   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3531     Assembler::vextractf32x4(dst, src, 0);
3532   } else {
3533     Assembler::movdqu(dst, src);
3534   }
3535 }
3536 
3537 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
3538   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3539     Assembler::vinsertf32x4(dst, dst, src, 0);
3540   } else {
3541     Assembler::movdqu(dst, src);
3542   }
3543 }
3544 
3545 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
3546   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3547     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3548   } else {
3549     Assembler::movdqu(dst, src);
3550   }
3551 }
3552 
3553 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register scratchReg) {
3554   if (reachable(src)) {
3555     movdqu(dst, as_Address(src));
3556   } else {
3557     lea(scratchReg, src);
3558     movdqu(dst, Address(scratchReg, 0));
3559   }
3560 }
3561 
3562 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
3563   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3564     vextractf64x4_low(dst, src);
3565   } else {
3566     Assembler::vmovdqu(dst, src);
3567   }
3568 }
3569 
3570 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
3571   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3572     vinsertf64x4_low(dst, src);
3573   } else {
3574     Assembler::vmovdqu(dst, src);
3575   }
3576 }
3577 
3578 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3579   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3580     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3581   }
3582   else {
3583     Assembler::vmovdqu(dst, src);
3584   }
3585 }
3586 
3587 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src) {
3588   if (reachable(src)) {
3589     vmovdqu(dst, as_Address(src));
3590   }
3591   else {
3592     lea(rscratch1, src);
3593     vmovdqu(dst, Address(rscratch1, 0));
3594   }
3595 }
3596 
3597 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) {
3598   if (reachable(src)) {
3599     Assembler::movdqa(dst, as_Address(src));
3600   } else {
3601     lea(rscratch1, src);
3602     Assembler::movdqa(dst, Address(rscratch1, 0));
3603   }
3604 }
3605 
3606 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
3607   if (reachable(src)) {
3608     Assembler::movsd(dst, as_Address(src));
3609   } else {
3610     lea(rscratch1, src);
3611     Assembler::movsd(dst, Address(rscratch1, 0));
3612   }
3613 }
3614 
3615 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
3616   if (reachable(src)) {
3617     Assembler::movss(dst, as_Address(src));
3618   } else {
3619     lea(rscratch1, src);
3620     Assembler::movss(dst, Address(rscratch1, 0));
3621   }
3622 }
3623 
3624 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) {
3625   if (reachable(src)) {
3626     Assembler::mulsd(dst, as_Address(src));
3627   } else {
3628     lea(rscratch1, src);
3629     Assembler::mulsd(dst, Address(rscratch1, 0));
3630   }
3631 }
3632 
3633 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) {
3634   if (reachable(src)) {
3635     Assembler::mulss(dst, as_Address(src));
3636   } else {
3637     lea(rscratch1, src);
3638     Assembler::mulss(dst, Address(rscratch1, 0));
3639   }
3640 }
3641 
3642 void MacroAssembler::null_check(Register reg, int offset) {
3643   if (needs_explicit_null_check(offset)) {
3644     // provoke OS NULL exception if reg = NULL by
3645     // accessing M[reg] w/o changing any (non-CC) registers
3646     // NOTE: cmpl is plenty here to provoke a segv
3647     cmpptr(rax, Address(reg, 0));
3648     // Note: should probably use testl(rax, Address(reg, 0));
3649     //       may be shorter code (however, this version of
3650     //       testl needs to be implemented first)
3651   } else {
3652     // nothing to do, (later) access of M[reg + offset]
3653     // will provoke OS NULL exception if reg = NULL
3654   }
3655 }
3656 
3657 void MacroAssembler::os_breakpoint() {
3658   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
3659   // (e.g., MSVC can't call ps() otherwise)
3660   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
3661 }
3662 
3663 void MacroAssembler::unimplemented(const char* what) {
3664   const char* buf = NULL;
3665   {
3666     ResourceMark rm;
3667     stringStream ss;
3668     ss.print("unimplemented: %s", what);
3669     buf = code_string(ss.as_string());
3670   }
3671   stop(buf);
3672 }
3673 
3674 #ifdef _LP64
3675 #define XSTATE_BV 0x200
3676 #endif
3677 
3678 void MacroAssembler::pop_CPU_state() {
3679   pop_FPU_state();
3680   pop_IU_state();
3681 }
3682 
3683 void MacroAssembler::pop_FPU_state() {
3684 #ifndef _LP64
3685   frstor(Address(rsp, 0));
3686 #else
3687   fxrstor(Address(rsp, 0));
3688 #endif
3689   addptr(rsp, FPUStateSizeInWords * wordSize);
3690 }
3691 
3692 void MacroAssembler::pop_IU_state() {
3693   popa();
3694   LP64_ONLY(addq(rsp, 8));
3695   popf();
3696 }
3697 
3698 // Save Integer and Float state
3699 // Warning: Stack must be 16 byte aligned (64bit)
3700 void MacroAssembler::push_CPU_state() {
3701   push_IU_state();
3702   push_FPU_state();
3703 }
3704 
3705 void MacroAssembler::push_FPU_state() {
3706   subptr(rsp, FPUStateSizeInWords * wordSize);
3707 #ifndef _LP64
3708   fnsave(Address(rsp, 0));
3709   fwait();
3710 #else
3711   fxsave(Address(rsp, 0));
3712 #endif // LP64
3713 }
3714 
3715 void MacroAssembler::push_IU_state() {
3716   // Push flags first because pusha kills them
3717   pushf();
3718   // Make sure rsp stays 16-byte aligned
3719   LP64_ONLY(subq(rsp, 8));
3720   pusha();
3721 }
3722 
3723 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
3724   if (!java_thread->is_valid()) {
3725     java_thread = rdi;
3726     get_thread(java_thread);
3727   }
3728   // we must set sp to zero to clear frame
3729   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
3730   if (clear_fp) {
3731     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
3732   }
3733 
3734   // Always clear the pc because it could have been set by make_walkable()
3735   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
3736 
3737   vzeroupper();
3738 }
3739 
3740 void MacroAssembler::restore_rax(Register tmp) {
3741   if (tmp == noreg) pop(rax);
3742   else if (tmp != rax) mov(rax, tmp);
3743 }
3744 
3745 void MacroAssembler::round_to(Register reg, int modulus) {
3746   addptr(reg, modulus - 1);
3747   andptr(reg, -modulus);
3748 }
3749 
3750 void MacroAssembler::save_rax(Register tmp) {
3751   if (tmp == noreg) push(rax);
3752   else if (tmp != rax) mov(tmp, rax);
3753 }
3754 
3755 // Write serialization page so VM thread can do a pseudo remote membar.
3756 // We use the current thread pointer to calculate a thread specific
3757 // offset to write to within the page. This minimizes bus traffic
3758 // due to cache line collision.
3759 void MacroAssembler::serialize_memory(Register thread, Register tmp) {
3760   movl(tmp, thread);
3761   shrl(tmp, os::get_serialize_page_shift_count());
3762   andl(tmp, (os::vm_page_size() - sizeof(int)));
3763 
3764   Address index(noreg, tmp, Address::times_1);
3765   ExternalAddress page(os::get_memory_serialize_page());
3766 
3767   // Size of store must match masking code above
3768   movl(as_Address(ArrayAddress(page, index)), tmp);
3769 }
3770 
3771 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, Register temp_reg) {
3772   if (SafepointMechanism::uses_thread_local_poll()) {
3773 #ifdef _LP64
3774     assert(thread_reg == r15_thread, "should be");
3775 #else
3776     if (thread_reg == noreg) {
3777       thread_reg = temp_reg;
3778       get_thread(thread_reg);
3779     }
3780 #endif
3781     testb(Address(thread_reg, Thread::polling_page_offset()), SafepointMechanism::poll_bit());
3782     jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
3783   } else {
3784     cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
3785         SafepointSynchronize::_not_synchronized);
3786     jcc(Assembler::notEqual, slow_path);
3787   }
3788 }
3789 
3790 // Calls to C land
3791 //
3792 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
3793 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
3794 // has to be reset to 0. This is required to allow proper stack traversal.
3795 void MacroAssembler::set_last_Java_frame(Register java_thread,
3796                                          Register last_java_sp,
3797                                          Register last_java_fp,
3798                                          address  last_java_pc) {
3799   vzeroupper();
3800   // determine java_thread register
3801   if (!java_thread->is_valid()) {
3802     java_thread = rdi;
3803     get_thread(java_thread);
3804   }
3805   // determine last_java_sp register
3806   if (!last_java_sp->is_valid()) {
3807     last_java_sp = rsp;
3808   }
3809 
3810   // last_java_fp is optional
3811 
3812   if (last_java_fp->is_valid()) {
3813     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
3814   }
3815 
3816   // last_java_pc is optional
3817 
3818   if (last_java_pc != NULL) {
3819     lea(Address(java_thread,
3820                  JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()),
3821         InternalAddress(last_java_pc));
3822 
3823   }
3824   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
3825 }
3826 
3827 void MacroAssembler::shlptr(Register dst, int imm8) {
3828   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
3829 }
3830 
3831 void MacroAssembler::shrptr(Register dst, int imm8) {
3832   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
3833 }
3834 
3835 void MacroAssembler::sign_extend_byte(Register reg) {
3836   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
3837     movsbl(reg, reg); // movsxb
3838   } else {
3839     shll(reg, 24);
3840     sarl(reg, 24);
3841   }
3842 }
3843 
3844 void MacroAssembler::sign_extend_short(Register reg) {
3845   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3846     movswl(reg, reg); // movsxw
3847   } else {
3848     shll(reg, 16);
3849     sarl(reg, 16);
3850   }
3851 }
3852 
3853 void MacroAssembler::testl(Register dst, AddressLiteral src) {
3854   assert(reachable(src), "Address should be reachable");
3855   testl(dst, as_Address(src));
3856 }
3857 
3858 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3859   int dst_enc = dst->encoding();
3860   int src_enc = src->encoding();
3861   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3862     Assembler::pcmpeqb(dst, src);
3863   } else if ((dst_enc < 16) && (src_enc < 16)) {
3864     Assembler::pcmpeqb(dst, src);
3865   } else if (src_enc < 16) {
3866     subptr(rsp, 64);
3867     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3868     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3869     Assembler::pcmpeqb(xmm0, src);
3870     movdqu(dst, xmm0);
3871     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3872     addptr(rsp, 64);
3873   } else if (dst_enc < 16) {
3874     subptr(rsp, 64);
3875     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3876     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3877     Assembler::pcmpeqb(dst, xmm0);
3878     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3879     addptr(rsp, 64);
3880   } else {
3881     subptr(rsp, 64);
3882     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3883     subptr(rsp, 64);
3884     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
3885     movdqu(xmm0, src);
3886     movdqu(xmm1, dst);
3887     Assembler::pcmpeqb(xmm1, xmm0);
3888     movdqu(dst, xmm1);
3889     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
3890     addptr(rsp, 64);
3891     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3892     addptr(rsp, 64);
3893   }
3894 }
3895 
3896 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3897   int dst_enc = dst->encoding();
3898   int src_enc = src->encoding();
3899   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3900     Assembler::pcmpeqw(dst, src);
3901   } else if ((dst_enc < 16) && (src_enc < 16)) {
3902     Assembler::pcmpeqw(dst, src);
3903   } else if (src_enc < 16) {
3904     subptr(rsp, 64);
3905     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3906     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3907     Assembler::pcmpeqw(xmm0, src);
3908     movdqu(dst, xmm0);
3909     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3910     addptr(rsp, 64);
3911   } else if (dst_enc < 16) {
3912     subptr(rsp, 64);
3913     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3914     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3915     Assembler::pcmpeqw(dst, xmm0);
3916     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3917     addptr(rsp, 64);
3918   } else {
3919     subptr(rsp, 64);
3920     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3921     subptr(rsp, 64);
3922     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
3923     movdqu(xmm0, src);
3924     movdqu(xmm1, dst);
3925     Assembler::pcmpeqw(xmm1, xmm0);
3926     movdqu(dst, xmm1);
3927     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
3928     addptr(rsp, 64);
3929     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3930     addptr(rsp, 64);
3931   }
3932 }
3933 
3934 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3935   int dst_enc = dst->encoding();
3936   if (dst_enc < 16) {
3937     Assembler::pcmpestri(dst, src, imm8);
3938   } else {
3939     subptr(rsp, 64);
3940     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3941     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3942     Assembler::pcmpestri(xmm0, src, imm8);
3943     movdqu(dst, xmm0);
3944     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3945     addptr(rsp, 64);
3946   }
3947 }
3948 
3949 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3950   int dst_enc = dst->encoding();
3951   int src_enc = src->encoding();
3952   if ((dst_enc < 16) && (src_enc < 16)) {
3953     Assembler::pcmpestri(dst, src, imm8);
3954   } else if (src_enc < 16) {
3955     subptr(rsp, 64);
3956     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3957     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3958     Assembler::pcmpestri(xmm0, src, imm8);
3959     movdqu(dst, xmm0);
3960     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3961     addptr(rsp, 64);
3962   } else if (dst_enc < 16) {
3963     subptr(rsp, 64);
3964     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3965     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3966     Assembler::pcmpestri(dst, xmm0, imm8);
3967     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3968     addptr(rsp, 64);
3969   } else {
3970     subptr(rsp, 64);
3971     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3972     subptr(rsp, 64);
3973     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
3974     movdqu(xmm0, src);
3975     movdqu(xmm1, dst);
3976     Assembler::pcmpestri(xmm1, xmm0, imm8);
3977     movdqu(dst, xmm1);
3978     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
3979     addptr(rsp, 64);
3980     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3981     addptr(rsp, 64);
3982   }
3983 }
3984 
3985 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3986   int dst_enc = dst->encoding();
3987   int src_enc = src->encoding();
3988   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3989     Assembler::pmovzxbw(dst, src);
3990   } else if ((dst_enc < 16) && (src_enc < 16)) {
3991     Assembler::pmovzxbw(dst, src);
3992   } else if (src_enc < 16) {
3993     subptr(rsp, 64);
3994     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3995     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3996     Assembler::pmovzxbw(xmm0, src);
3997     movdqu(dst, xmm0);
3998     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3999     addptr(rsp, 64);
4000   } else if (dst_enc < 16) {
4001     subptr(rsp, 64);
4002     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4003     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4004     Assembler::pmovzxbw(dst, xmm0);
4005     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4006     addptr(rsp, 64);
4007   } else {
4008     subptr(rsp, 64);
4009     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4010     subptr(rsp, 64);
4011     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4012     movdqu(xmm0, src);
4013     movdqu(xmm1, dst);
4014     Assembler::pmovzxbw(xmm1, xmm0);
4015     movdqu(dst, xmm1);
4016     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4017     addptr(rsp, 64);
4018     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4019     addptr(rsp, 64);
4020   }
4021 }
4022 
4023 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
4024   int dst_enc = dst->encoding();
4025   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4026     Assembler::pmovzxbw(dst, src);
4027   } else if (dst_enc < 16) {
4028     Assembler::pmovzxbw(dst, src);
4029   } else {
4030     subptr(rsp, 64);
4031     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4032     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4033     Assembler::pmovzxbw(xmm0, src);
4034     movdqu(dst, xmm0);
4035     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4036     addptr(rsp, 64);
4037   }
4038 }
4039 
4040 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
4041   int src_enc = src->encoding();
4042   if (src_enc < 16) {
4043     Assembler::pmovmskb(dst, src);
4044   } else {
4045     subptr(rsp, 64);
4046     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4047     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4048     Assembler::pmovmskb(dst, xmm0);
4049     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4050     addptr(rsp, 64);
4051   }
4052 }
4053 
4054 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
4055   int dst_enc = dst->encoding();
4056   int src_enc = src->encoding();
4057   if ((dst_enc < 16) && (src_enc < 16)) {
4058     Assembler::ptest(dst, src);
4059   } else if (src_enc < 16) {
4060     subptr(rsp, 64);
4061     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4062     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4063     Assembler::ptest(xmm0, src);
4064     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4065     addptr(rsp, 64);
4066   } else if (dst_enc < 16) {
4067     subptr(rsp, 64);
4068     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4069     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4070     Assembler::ptest(dst, xmm0);
4071     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4072     addptr(rsp, 64);
4073   } else {
4074     subptr(rsp, 64);
4075     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4076     subptr(rsp, 64);
4077     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4078     movdqu(xmm0, src);
4079     movdqu(xmm1, dst);
4080     Assembler::ptest(xmm1, xmm0);
4081     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4082     addptr(rsp, 64);
4083     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4084     addptr(rsp, 64);
4085   }
4086 }
4087 
4088 void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) {
4089   if (reachable(src)) {
4090     Assembler::sqrtsd(dst, as_Address(src));
4091   } else {
4092     lea(rscratch1, src);
4093     Assembler::sqrtsd(dst, Address(rscratch1, 0));
4094   }
4095 }
4096 
4097 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) {
4098   if (reachable(src)) {
4099     Assembler::sqrtss(dst, as_Address(src));
4100   } else {
4101     lea(rscratch1, src);
4102     Assembler::sqrtss(dst, Address(rscratch1, 0));
4103   }
4104 }
4105 
4106 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) {
4107   if (reachable(src)) {
4108     Assembler::subsd(dst, as_Address(src));
4109   } else {
4110     lea(rscratch1, src);
4111     Assembler::subsd(dst, Address(rscratch1, 0));
4112   }
4113 }
4114 
4115 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) {
4116   if (reachable(src)) {
4117     Assembler::subss(dst, as_Address(src));
4118   } else {
4119     lea(rscratch1, src);
4120     Assembler::subss(dst, Address(rscratch1, 0));
4121   }
4122 }
4123 
4124 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
4125   if (reachable(src)) {
4126     Assembler::ucomisd(dst, as_Address(src));
4127   } else {
4128     lea(rscratch1, src);
4129     Assembler::ucomisd(dst, Address(rscratch1, 0));
4130   }
4131 }
4132 
4133 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
4134   if (reachable(src)) {
4135     Assembler::ucomiss(dst, as_Address(src));
4136   } else {
4137     lea(rscratch1, src);
4138     Assembler::ucomiss(dst, Address(rscratch1, 0));
4139   }
4140 }
4141 
4142 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
4143   // Used in sign-bit flipping with aligned address.
4144   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
4145   if (reachable(src)) {
4146     Assembler::xorpd(dst, as_Address(src));
4147   } else {
4148     lea(rscratch1, src);
4149     Assembler::xorpd(dst, Address(rscratch1, 0));
4150   }
4151 }
4152 
4153 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
4154   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
4155     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
4156   }
4157   else {
4158     Assembler::xorpd(dst, src);
4159   }
4160 }
4161 
4162 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
4163   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
4164     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
4165   } else {
4166     Assembler::xorps(dst, src);
4167   }
4168 }
4169 
4170 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
4171   // Used in sign-bit flipping with aligned address.
4172   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
4173   if (reachable(src)) {
4174     Assembler::xorps(dst, as_Address(src));
4175   } else {
4176     lea(rscratch1, src);
4177     Assembler::xorps(dst, Address(rscratch1, 0));
4178   }
4179 }
4180 
4181 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) {
4182   // Used in sign-bit flipping with aligned address.
4183   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
4184   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
4185   if (reachable(src)) {
4186     Assembler::pshufb(dst, as_Address(src));
4187   } else {
4188     lea(rscratch1, src);
4189     Assembler::pshufb(dst, Address(rscratch1, 0));
4190   }
4191 }
4192 
4193 // AVX 3-operands instructions
4194 
4195 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4196   if (reachable(src)) {
4197     vaddsd(dst, nds, as_Address(src));
4198   } else {
4199     lea(rscratch1, src);
4200     vaddsd(dst, nds, Address(rscratch1, 0));
4201   }
4202 }
4203 
4204 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4205   if (reachable(src)) {
4206     vaddss(dst, nds, as_Address(src));
4207   } else {
4208     lea(rscratch1, src);
4209     vaddss(dst, nds, Address(rscratch1, 0));
4210   }
4211 }
4212 
4213 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
4214   int dst_enc = dst->encoding();
4215   int nds_enc = nds->encoding();
4216   int src_enc = src->encoding();
4217   if ((dst_enc < 16) && (nds_enc < 16)) {
4218     vandps(dst, nds, negate_field, vector_len);
4219   } else if ((src_enc < 16) && (dst_enc < 16)) {
4220     evmovdqul(src, nds, Assembler::AVX_512bit);
4221     vandps(dst, src, negate_field, vector_len);
4222   } else if (src_enc < 16) {
4223     evmovdqul(src, nds, Assembler::AVX_512bit);
4224     vandps(src, src, negate_field, vector_len);
4225     evmovdqul(dst, src, Assembler::AVX_512bit);
4226   } else if (dst_enc < 16) {
4227     evmovdqul(src, xmm0, Assembler::AVX_512bit);
4228     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4229     vandps(dst, xmm0, negate_field, vector_len);
4230     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4231   } else {
4232     if (src_enc != dst_enc) {
4233       evmovdqul(src, xmm0, Assembler::AVX_512bit);
4234       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4235       vandps(xmm0, xmm0, negate_field, vector_len);
4236       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4237       evmovdqul(xmm0, src, Assembler::AVX_512bit);
4238     } else {
4239       subptr(rsp, 64);
4240       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4241       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4242       vandps(xmm0, xmm0, negate_field, vector_len);
4243       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4244       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4245       addptr(rsp, 64);
4246     }
4247   }
4248 }
4249 
4250 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
4251   int dst_enc = dst->encoding();
4252   int nds_enc = nds->encoding();
4253   int src_enc = src->encoding();
4254   if ((dst_enc < 16) && (nds_enc < 16)) {
4255     vandpd(dst, nds, negate_field, vector_len);
4256   } else if ((src_enc < 16) && (dst_enc < 16)) {
4257     evmovdqul(src, nds, Assembler::AVX_512bit);
4258     vandpd(dst, src, negate_field, vector_len);
4259   } else if (src_enc < 16) {
4260     evmovdqul(src, nds, Assembler::AVX_512bit);
4261     vandpd(src, src, negate_field, vector_len);
4262     evmovdqul(dst, src, Assembler::AVX_512bit);
4263   } else if (dst_enc < 16) {
4264     evmovdqul(src, xmm0, Assembler::AVX_512bit);
4265     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4266     vandpd(dst, xmm0, negate_field, vector_len);
4267     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4268   } else {
4269     if (src_enc != dst_enc) {
4270       evmovdqul(src, xmm0, Assembler::AVX_512bit);
4271       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4272       vandpd(xmm0, xmm0, negate_field, vector_len);
4273       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4274       evmovdqul(xmm0, src, Assembler::AVX_512bit);
4275     } else {
4276       subptr(rsp, 64);
4277       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4278       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4279       vandpd(xmm0, xmm0, negate_field, vector_len);
4280       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4281       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4282       addptr(rsp, 64);
4283     }
4284   }
4285 }
4286 
4287 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4288   int dst_enc = dst->encoding();
4289   int nds_enc = nds->encoding();
4290   int src_enc = src->encoding();
4291   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4292     Assembler::vpaddb(dst, nds, src, vector_len);
4293   } else if ((dst_enc < 16) && (src_enc < 16)) {
4294     Assembler::vpaddb(dst, dst, src, vector_len);
4295   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4296     // use nds as scratch for src
4297     evmovdqul(nds, src, Assembler::AVX_512bit);
4298     Assembler::vpaddb(dst, dst, nds, vector_len);
4299   } else if ((src_enc < 16) && (nds_enc < 16)) {
4300     // use nds as scratch for dst
4301     evmovdqul(nds, dst, Assembler::AVX_512bit);
4302     Assembler::vpaddb(nds, nds, src, vector_len);
4303     evmovdqul(dst, nds, Assembler::AVX_512bit);
4304   } else if (dst_enc < 16) {
4305     // use nds as scatch for xmm0 to hold src
4306     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4307     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4308     Assembler::vpaddb(dst, dst, xmm0, vector_len);
4309     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4310   } else {
4311     // worse case scenario, all regs are in the upper bank
4312     subptr(rsp, 64);
4313     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4314     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4315     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4316     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4317     Assembler::vpaddb(xmm0, xmm0, xmm1, vector_len);
4318     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4319     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4320     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4321     addptr(rsp, 64);
4322   }
4323 }
4324 
4325 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4326   int dst_enc = dst->encoding();
4327   int nds_enc = nds->encoding();
4328   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4329     Assembler::vpaddb(dst, nds, src, vector_len);
4330   } else if (dst_enc < 16) {
4331     Assembler::vpaddb(dst, dst, src, vector_len);
4332   } else if (nds_enc < 16) {
4333     // implies dst_enc in upper bank with src as scratch
4334     evmovdqul(nds, dst, Assembler::AVX_512bit);
4335     Assembler::vpaddb(nds, nds, src, vector_len);
4336     evmovdqul(dst, nds, Assembler::AVX_512bit);
4337   } else {
4338     // worse case scenario, all regs in upper bank
4339     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4340     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4341     Assembler::vpaddb(xmm0, xmm0, src, vector_len);
4342     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4343   }
4344 }
4345 
4346 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4347   int dst_enc = dst->encoding();
4348   int nds_enc = nds->encoding();
4349   int src_enc = src->encoding();
4350   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4351     Assembler::vpaddw(dst, nds, src, vector_len);
4352   } else if ((dst_enc < 16) && (src_enc < 16)) {
4353     Assembler::vpaddw(dst, dst, src, vector_len);
4354   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4355     // use nds as scratch for src
4356     evmovdqul(nds, src, Assembler::AVX_512bit);
4357     Assembler::vpaddw(dst, dst, nds, vector_len);
4358   } else if ((src_enc < 16) && (nds_enc < 16)) {
4359     // use nds as scratch for dst
4360     evmovdqul(nds, dst, Assembler::AVX_512bit);
4361     Assembler::vpaddw(nds, nds, src, vector_len);
4362     evmovdqul(dst, nds, Assembler::AVX_512bit);
4363   } else if (dst_enc < 16) {
4364     // use nds as scatch for xmm0 to hold src
4365     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4366     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4367     Assembler::vpaddw(dst, dst, xmm0, vector_len);
4368     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4369   } else {
4370     // worse case scenario, all regs are in the upper bank
4371     subptr(rsp, 64);
4372     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4373     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4374     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4375     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4376     Assembler::vpaddw(xmm0, xmm0, xmm1, vector_len);
4377     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4378     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4379     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4380     addptr(rsp, 64);
4381   }
4382 }
4383 
4384 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4385   int dst_enc = dst->encoding();
4386   int nds_enc = nds->encoding();
4387   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4388     Assembler::vpaddw(dst, nds, src, vector_len);
4389   } else if (dst_enc < 16) {
4390     Assembler::vpaddw(dst, dst, src, vector_len);
4391   } else if (nds_enc < 16) {
4392     // implies dst_enc in upper bank with src as scratch
4393     evmovdqul(nds, dst, Assembler::AVX_512bit);
4394     Assembler::vpaddw(nds, nds, src, vector_len);
4395     evmovdqul(dst, nds, Assembler::AVX_512bit);
4396   } else {
4397     // worse case scenario, all regs in upper bank
4398     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4399     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4400     Assembler::vpaddw(xmm0, xmm0, src, vector_len);
4401     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4402   }
4403 }
4404 
4405 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4406   if (reachable(src)) {
4407     Assembler::vpand(dst, nds, as_Address(src), vector_len);
4408   } else {
4409     lea(rscratch1, src);
4410     Assembler::vpand(dst, nds, Address(rscratch1, 0), vector_len);
4411   }
4412 }
4413 
4414 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
4415   int dst_enc = dst->encoding();
4416   int src_enc = src->encoding();
4417   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4418     Assembler::vpbroadcastw(dst, src);
4419   } else if ((dst_enc < 16) && (src_enc < 16)) {
4420     Assembler::vpbroadcastw(dst, src);
4421   } else if (src_enc < 16) {
4422     subptr(rsp, 64);
4423     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4424     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4425     Assembler::vpbroadcastw(xmm0, src);
4426     movdqu(dst, xmm0);
4427     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4428     addptr(rsp, 64);
4429   } else if (dst_enc < 16) {
4430     subptr(rsp, 64);
4431     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4432     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4433     Assembler::vpbroadcastw(dst, xmm0);
4434     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4435     addptr(rsp, 64);
4436   } else {
4437     subptr(rsp, 64);
4438     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4439     subptr(rsp, 64);
4440     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4441     movdqu(xmm0, src);
4442     movdqu(xmm1, dst);
4443     Assembler::vpbroadcastw(xmm1, xmm0);
4444     movdqu(dst, xmm1);
4445     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4446     addptr(rsp, 64);
4447     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4448     addptr(rsp, 64);
4449   }
4450 }
4451 
4452 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4453   int dst_enc = dst->encoding();
4454   int nds_enc = nds->encoding();
4455   int src_enc = src->encoding();
4456   assert(dst_enc == nds_enc, "");
4457   if ((dst_enc < 16) && (src_enc < 16)) {
4458     Assembler::vpcmpeqb(dst, nds, src, vector_len);
4459   } else if (src_enc < 16) {
4460     subptr(rsp, 64);
4461     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4462     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4463     Assembler::vpcmpeqb(xmm0, xmm0, src, vector_len);
4464     movdqu(dst, xmm0);
4465     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4466     addptr(rsp, 64);
4467   } else if (dst_enc < 16) {
4468     subptr(rsp, 64);
4469     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4470     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4471     Assembler::vpcmpeqb(dst, dst, xmm0, vector_len);
4472     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4473     addptr(rsp, 64);
4474   } else {
4475     subptr(rsp, 64);
4476     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4477     subptr(rsp, 64);
4478     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4479     movdqu(xmm0, src);
4480     movdqu(xmm1, dst);
4481     Assembler::vpcmpeqb(xmm1, xmm1, xmm0, vector_len);
4482     movdqu(dst, xmm1);
4483     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4484     addptr(rsp, 64);
4485     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4486     addptr(rsp, 64);
4487   }
4488 }
4489 
4490 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4491   int dst_enc = dst->encoding();
4492   int nds_enc = nds->encoding();
4493   int src_enc = src->encoding();
4494   assert(dst_enc == nds_enc, "");
4495   if ((dst_enc < 16) && (src_enc < 16)) {
4496     Assembler::vpcmpeqw(dst, nds, src, vector_len);
4497   } else if (src_enc < 16) {
4498     subptr(rsp, 64);
4499     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4500     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4501     Assembler::vpcmpeqw(xmm0, xmm0, src, vector_len);
4502     movdqu(dst, xmm0);
4503     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4504     addptr(rsp, 64);
4505   } else if (dst_enc < 16) {
4506     subptr(rsp, 64);
4507     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4508     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4509     Assembler::vpcmpeqw(dst, dst, xmm0, vector_len);
4510     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4511     addptr(rsp, 64);
4512   } else {
4513     subptr(rsp, 64);
4514     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4515     subptr(rsp, 64);
4516     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4517     movdqu(xmm0, src);
4518     movdqu(xmm1, dst);
4519     Assembler::vpcmpeqw(xmm1, xmm1, xmm0, vector_len);
4520     movdqu(dst, xmm1);
4521     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4522     addptr(rsp, 64);
4523     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4524     addptr(rsp, 64);
4525   }
4526 }
4527 
4528 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
4529   int dst_enc = dst->encoding();
4530   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4531     Assembler::vpmovzxbw(dst, src, vector_len);
4532   } else if (dst_enc < 16) {
4533     Assembler::vpmovzxbw(dst, src, vector_len);
4534   } else {
4535     subptr(rsp, 64);
4536     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4537     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4538     Assembler::vpmovzxbw(xmm0, src, vector_len);
4539     movdqu(dst, xmm0);
4540     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4541     addptr(rsp, 64);
4542   }
4543 }
4544 
4545 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src) {
4546   int src_enc = src->encoding();
4547   if (src_enc < 16) {
4548     Assembler::vpmovmskb(dst, src);
4549   } else {
4550     subptr(rsp, 64);
4551     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4552     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4553     Assembler::vpmovmskb(dst, xmm0);
4554     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4555     addptr(rsp, 64);
4556   }
4557 }
4558 
4559 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4560   int dst_enc = dst->encoding();
4561   int nds_enc = nds->encoding();
4562   int src_enc = src->encoding();
4563   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4564     Assembler::vpmullw(dst, nds, src, vector_len);
4565   } else if ((dst_enc < 16) && (src_enc < 16)) {
4566     Assembler::vpmullw(dst, dst, src, vector_len);
4567   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4568     // use nds as scratch for src
4569     evmovdqul(nds, src, Assembler::AVX_512bit);
4570     Assembler::vpmullw(dst, dst, nds, vector_len);
4571   } else if ((src_enc < 16) && (nds_enc < 16)) {
4572     // use nds as scratch for dst
4573     evmovdqul(nds, dst, Assembler::AVX_512bit);
4574     Assembler::vpmullw(nds, nds, src, vector_len);
4575     evmovdqul(dst, nds, Assembler::AVX_512bit);
4576   } else if (dst_enc < 16) {
4577     // use nds as scatch for xmm0 to hold src
4578     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4579     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4580     Assembler::vpmullw(dst, dst, xmm0, vector_len);
4581     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4582   } else {
4583     // worse case scenario, all regs are in the upper bank
4584     subptr(rsp, 64);
4585     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4586     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4587     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4588     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4589     Assembler::vpmullw(xmm0, xmm0, xmm1, vector_len);
4590     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4591     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4592     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4593     addptr(rsp, 64);
4594   }
4595 }
4596 
4597 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4598   int dst_enc = dst->encoding();
4599   int nds_enc = nds->encoding();
4600   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4601     Assembler::vpmullw(dst, nds, src, vector_len);
4602   } else if (dst_enc < 16) {
4603     Assembler::vpmullw(dst, dst, src, vector_len);
4604   } else if (nds_enc < 16) {
4605     // implies dst_enc in upper bank with src as scratch
4606     evmovdqul(nds, dst, Assembler::AVX_512bit);
4607     Assembler::vpmullw(nds, nds, src, vector_len);
4608     evmovdqul(dst, nds, Assembler::AVX_512bit);
4609   } else {
4610     // worse case scenario, all regs in upper bank
4611     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4612     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4613     Assembler::vpmullw(xmm0, xmm0, src, vector_len);
4614     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4615   }
4616 }
4617 
4618 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4619   int dst_enc = dst->encoding();
4620   int nds_enc = nds->encoding();
4621   int src_enc = src->encoding();
4622   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4623     Assembler::vpsubb(dst, nds, src, vector_len);
4624   } else if ((dst_enc < 16) && (src_enc < 16)) {
4625     Assembler::vpsubb(dst, dst, src, vector_len);
4626   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4627     // use nds as scratch for src
4628     evmovdqul(nds, src, Assembler::AVX_512bit);
4629     Assembler::vpsubb(dst, dst, nds, vector_len);
4630   } else if ((src_enc < 16) && (nds_enc < 16)) {
4631     // use nds as scratch for dst
4632     evmovdqul(nds, dst, Assembler::AVX_512bit);
4633     Assembler::vpsubb(nds, nds, src, vector_len);
4634     evmovdqul(dst, nds, Assembler::AVX_512bit);
4635   } else if (dst_enc < 16) {
4636     // use nds as scatch for xmm0 to hold src
4637     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4638     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4639     Assembler::vpsubb(dst, dst, xmm0, vector_len);
4640     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4641   } else {
4642     // worse case scenario, all regs are in the upper bank
4643     subptr(rsp, 64);
4644     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4645     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4646     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4647     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4648     Assembler::vpsubb(xmm0, xmm0, xmm1, vector_len);
4649     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4650     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4651     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4652     addptr(rsp, 64);
4653   }
4654 }
4655 
4656 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4657   int dst_enc = dst->encoding();
4658   int nds_enc = nds->encoding();
4659   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4660     Assembler::vpsubb(dst, nds, src, vector_len);
4661   } else if (dst_enc < 16) {
4662     Assembler::vpsubb(dst, dst, src, vector_len);
4663   } else if (nds_enc < 16) {
4664     // implies dst_enc in upper bank with src as scratch
4665     evmovdqul(nds, dst, Assembler::AVX_512bit);
4666     Assembler::vpsubb(nds, nds, src, vector_len);
4667     evmovdqul(dst, nds, Assembler::AVX_512bit);
4668   } else {
4669     // worse case scenario, all regs in upper bank
4670     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4671     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4672     Assembler::vpsubw(xmm0, xmm0, src, vector_len);
4673     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4674   }
4675 }
4676 
4677 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4678   int dst_enc = dst->encoding();
4679   int nds_enc = nds->encoding();
4680   int src_enc = src->encoding();
4681   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4682     Assembler::vpsubw(dst, nds, src, vector_len);
4683   } else if ((dst_enc < 16) && (src_enc < 16)) {
4684     Assembler::vpsubw(dst, dst, src, vector_len);
4685   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4686     // use nds as scratch for src
4687     evmovdqul(nds, src, Assembler::AVX_512bit);
4688     Assembler::vpsubw(dst, dst, nds, vector_len);
4689   } else if ((src_enc < 16) && (nds_enc < 16)) {
4690     // use nds as scratch for dst
4691     evmovdqul(nds, dst, Assembler::AVX_512bit);
4692     Assembler::vpsubw(nds, nds, src, vector_len);
4693     evmovdqul(dst, nds, Assembler::AVX_512bit);
4694   } else if (dst_enc < 16) {
4695     // use nds as scatch for xmm0 to hold src
4696     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4697     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4698     Assembler::vpsubw(dst, dst, xmm0, vector_len);
4699     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4700   } else {
4701     // worse case scenario, all regs are in the upper bank
4702     subptr(rsp, 64);
4703     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4704     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4705     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4706     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4707     Assembler::vpsubw(xmm0, xmm0, xmm1, vector_len);
4708     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4709     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4710     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4711     addptr(rsp, 64);
4712   }
4713 }
4714 
4715 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4716   int dst_enc = dst->encoding();
4717   int nds_enc = nds->encoding();
4718   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4719     Assembler::vpsubw(dst, nds, src, vector_len);
4720   } else if (dst_enc < 16) {
4721     Assembler::vpsubw(dst, dst, src, vector_len);
4722   } else if (nds_enc < 16) {
4723     // implies dst_enc in upper bank with src as scratch
4724     evmovdqul(nds, dst, Assembler::AVX_512bit);
4725     Assembler::vpsubw(nds, nds, src, vector_len);
4726     evmovdqul(dst, nds, Assembler::AVX_512bit);
4727   } else {
4728     // worse case scenario, all regs in upper bank
4729     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4730     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4731     Assembler::vpsubw(xmm0, xmm0, src, vector_len);
4732     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4733   }
4734 }
4735 
4736 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4737   int dst_enc = dst->encoding();
4738   int nds_enc = nds->encoding();
4739   int shift_enc = shift->encoding();
4740   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4741     Assembler::vpsraw(dst, nds, shift, vector_len);
4742   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4743     Assembler::vpsraw(dst, dst, shift, vector_len);
4744   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4745     // use nds_enc as scratch with shift
4746     evmovdqul(nds, shift, Assembler::AVX_512bit);
4747     Assembler::vpsraw(dst, dst, nds, vector_len);
4748   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4749     // use nds as scratch with dst
4750     evmovdqul(nds, dst, Assembler::AVX_512bit);
4751     Assembler::vpsraw(nds, nds, shift, vector_len);
4752     evmovdqul(dst, nds, Assembler::AVX_512bit);
4753   } else if (dst_enc < 16) {
4754     // use nds to save a copy of xmm0 and hold shift
4755     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4756     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4757     Assembler::vpsraw(dst, dst, xmm0, vector_len);
4758     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4759   } else if (nds_enc < 16) {
4760     // use nds as dest as temps
4761     evmovdqul(nds, dst, Assembler::AVX_512bit);
4762     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4763     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4764     Assembler::vpsraw(nds, nds, xmm0, vector_len);
4765     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4766     evmovdqul(dst, nds, Assembler::AVX_512bit);
4767   } else {
4768     // worse case scenario, all regs are in the upper bank
4769     subptr(rsp, 64);
4770     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4771     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4772     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4773     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4774     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4775     evmovdqul(xmm1, dst, Assembler::AVX_512bit);
4776     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4777     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4778     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4779     addptr(rsp, 64);
4780   }
4781 }
4782 
4783 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4784   int dst_enc = dst->encoding();
4785   int nds_enc = nds->encoding();
4786   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4787     Assembler::vpsraw(dst, nds, shift, vector_len);
4788   } else if (dst_enc < 16) {
4789     Assembler::vpsraw(dst, dst, shift, vector_len);
4790   } else if (nds_enc < 16) {
4791     // use nds as scratch
4792     evmovdqul(nds, dst, Assembler::AVX_512bit);
4793     Assembler::vpsraw(nds, nds, shift, vector_len);
4794     evmovdqul(dst, nds, Assembler::AVX_512bit);
4795   } else {
4796     // use nds as scratch for xmm0
4797     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4798     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4799     Assembler::vpsraw(xmm0, xmm0, shift, vector_len);
4800     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4801   }
4802 }
4803 
4804 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4805   int dst_enc = dst->encoding();
4806   int nds_enc = nds->encoding();
4807   int shift_enc = shift->encoding();
4808   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4809     Assembler::vpsrlw(dst, nds, shift, vector_len);
4810   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4811     Assembler::vpsrlw(dst, dst, shift, vector_len);
4812   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4813     // use nds_enc as scratch with shift
4814     evmovdqul(nds, shift, Assembler::AVX_512bit);
4815     Assembler::vpsrlw(dst, dst, nds, vector_len);
4816   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4817     // use nds as scratch with dst
4818     evmovdqul(nds, dst, Assembler::AVX_512bit);
4819     Assembler::vpsrlw(nds, nds, shift, vector_len);
4820     evmovdqul(dst, nds, Assembler::AVX_512bit);
4821   } else if (dst_enc < 16) {
4822     // use nds to save a copy of xmm0 and hold shift
4823     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4824     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4825     Assembler::vpsrlw(dst, dst, xmm0, vector_len);
4826     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4827   } else if (nds_enc < 16) {
4828     // use nds as dest as temps
4829     evmovdqul(nds, dst, Assembler::AVX_512bit);
4830     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4831     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4832     Assembler::vpsrlw(nds, nds, xmm0, vector_len);
4833     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4834     evmovdqul(dst, nds, Assembler::AVX_512bit);
4835   } else {
4836     // worse case scenario, all regs are in the upper bank
4837     subptr(rsp, 64);
4838     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4839     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4840     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4841     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4842     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4843     evmovdqul(xmm1, dst, Assembler::AVX_512bit);
4844     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4845     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4846     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4847     addptr(rsp, 64);
4848   }
4849 }
4850 
4851 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4852   int dst_enc = dst->encoding();
4853   int nds_enc = nds->encoding();
4854   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4855     Assembler::vpsrlw(dst, nds, shift, vector_len);
4856   } else if (dst_enc < 16) {
4857     Assembler::vpsrlw(dst, dst, shift, vector_len);
4858   } else if (nds_enc < 16) {
4859     // use nds as scratch
4860     evmovdqul(nds, dst, Assembler::AVX_512bit);
4861     Assembler::vpsrlw(nds, nds, shift, vector_len);
4862     evmovdqul(dst, nds, Assembler::AVX_512bit);
4863   } else {
4864     // use nds as scratch for xmm0
4865     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4866     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4867     Assembler::vpsrlw(xmm0, xmm0, shift, vector_len);
4868     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4869   }
4870 }
4871 
4872 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4873   int dst_enc = dst->encoding();
4874   int nds_enc = nds->encoding();
4875   int shift_enc = shift->encoding();
4876   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4877     Assembler::vpsllw(dst, nds, shift, vector_len);
4878   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4879     Assembler::vpsllw(dst, dst, shift, vector_len);
4880   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4881     // use nds_enc as scratch with shift
4882     evmovdqul(nds, shift, Assembler::AVX_512bit);
4883     Assembler::vpsllw(dst, dst, nds, vector_len);
4884   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4885     // use nds as scratch with dst
4886     evmovdqul(nds, dst, Assembler::AVX_512bit);
4887     Assembler::vpsllw(nds, nds, shift, vector_len);
4888     evmovdqul(dst, nds, Assembler::AVX_512bit);
4889   } else if (dst_enc < 16) {
4890     // use nds to save a copy of xmm0 and hold shift
4891     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4892     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4893     Assembler::vpsllw(dst, dst, xmm0, vector_len);
4894     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4895   } else if (nds_enc < 16) {
4896     // use nds as dest as temps
4897     evmovdqul(nds, dst, Assembler::AVX_512bit);
4898     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4899     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4900     Assembler::vpsllw(nds, nds, xmm0, vector_len);
4901     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4902     evmovdqul(dst, nds, Assembler::AVX_512bit);
4903   } else {
4904     // worse case scenario, all regs are in the upper bank
4905     subptr(rsp, 64);
4906     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4907     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4908     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4909     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4910     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4911     evmovdqul(xmm1, dst, Assembler::AVX_512bit);
4912     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4913     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4914     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4915     addptr(rsp, 64);
4916   }
4917 }
4918 
4919 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4920   int dst_enc = dst->encoding();
4921   int nds_enc = nds->encoding();
4922   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4923     Assembler::vpsllw(dst, nds, shift, vector_len);
4924   } else if (dst_enc < 16) {
4925     Assembler::vpsllw(dst, dst, shift, vector_len);
4926   } else if (nds_enc < 16) {
4927     // use nds as scratch
4928     evmovdqul(nds, dst, Assembler::AVX_512bit);
4929     Assembler::vpsllw(nds, nds, shift, vector_len);
4930     evmovdqul(dst, nds, Assembler::AVX_512bit);
4931   } else {
4932     // use nds as scratch for xmm0
4933     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4934     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4935     Assembler::vpsllw(xmm0, xmm0, shift, vector_len);
4936     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4937   }
4938 }
4939 
4940 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
4941   int dst_enc = dst->encoding();
4942   int src_enc = src->encoding();
4943   if ((dst_enc < 16) && (src_enc < 16)) {
4944     Assembler::vptest(dst, src);
4945   } else if (src_enc < 16) {
4946     subptr(rsp, 64);
4947     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4948     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4949     Assembler::vptest(xmm0, src);
4950     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4951     addptr(rsp, 64);
4952   } else if (dst_enc < 16) {
4953     subptr(rsp, 64);
4954     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4955     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4956     Assembler::vptest(dst, xmm0);
4957     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4958     addptr(rsp, 64);
4959   } else {
4960     subptr(rsp, 64);
4961     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4962     subptr(rsp, 64);
4963     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4964     movdqu(xmm0, src);
4965     movdqu(xmm1, dst);
4966     Assembler::vptest(xmm1, xmm0);
4967     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4968     addptr(rsp, 64);
4969     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4970     addptr(rsp, 64);
4971   }
4972 }
4973 
4974 // This instruction exists within macros, ergo we cannot control its input
4975 // when emitted through those patterns.
4976 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4977   if (VM_Version::supports_avx512nobw()) {
4978     int dst_enc = dst->encoding();
4979     int src_enc = src->encoding();
4980     if (dst_enc == src_enc) {
4981       if (dst_enc < 16) {
4982         Assembler::punpcklbw(dst, src);
4983       } else {
4984         subptr(rsp, 64);
4985         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4986         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4987         Assembler::punpcklbw(xmm0, xmm0);
4988         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4989         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4990         addptr(rsp, 64);
4991       }
4992     } else {
4993       if ((src_enc < 16) && (dst_enc < 16)) {
4994         Assembler::punpcklbw(dst, src);
4995       } else if (src_enc < 16) {
4996         subptr(rsp, 64);
4997         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4998         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4999         Assembler::punpcklbw(xmm0, src);
5000         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5001         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5002         addptr(rsp, 64);
5003       } else if (dst_enc < 16) {
5004         subptr(rsp, 64);
5005         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5006         evmovdqul(xmm0, src, Assembler::AVX_512bit);
5007         Assembler::punpcklbw(dst, xmm0);
5008         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5009         addptr(rsp, 64);
5010       } else {
5011         subptr(rsp, 64);
5012         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5013         subptr(rsp, 64);
5014         evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
5015         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5016         evmovdqul(xmm1, src, Assembler::AVX_512bit);
5017         Assembler::punpcklbw(xmm0, xmm1);
5018         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5019         evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
5020         addptr(rsp, 64);
5021         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5022         addptr(rsp, 64);
5023       }
5024     }
5025   } else {
5026     Assembler::punpcklbw(dst, src);
5027   }
5028 }
5029 
5030 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
5031   if (VM_Version::supports_avx512vl()) {
5032     Assembler::pshufd(dst, src, mode);
5033   } else {
5034     int dst_enc = dst->encoding();
5035     if (dst_enc < 16) {
5036       Assembler::pshufd(dst, src, mode);
5037     } else {
5038       subptr(rsp, 64);
5039       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5040       Assembler::pshufd(xmm0, src, mode);
5041       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5042       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5043       addptr(rsp, 64);
5044     }
5045   }
5046 }
5047 
5048 // This instruction exists within macros, ergo we cannot control its input
5049 // when emitted through those patterns.
5050 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
5051   if (VM_Version::supports_avx512nobw()) {
5052     int dst_enc = dst->encoding();
5053     int src_enc = src->encoding();
5054     if (dst_enc == src_enc) {
5055       if (dst_enc < 16) {
5056         Assembler::pshuflw(dst, src, mode);
5057       } else {
5058         subptr(rsp, 64);
5059         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5060         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5061         Assembler::pshuflw(xmm0, xmm0, mode);
5062         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5063         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5064         addptr(rsp, 64);
5065       }
5066     } else {
5067       if ((src_enc < 16) && (dst_enc < 16)) {
5068         Assembler::pshuflw(dst, src, mode);
5069       } else if (src_enc < 16) {
5070         subptr(rsp, 64);
5071         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5072         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5073         Assembler::pshuflw(xmm0, src, mode);
5074         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5075         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5076         addptr(rsp, 64);
5077       } else if (dst_enc < 16) {
5078         subptr(rsp, 64);
5079         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5080         evmovdqul(xmm0, src, Assembler::AVX_512bit);
5081         Assembler::pshuflw(dst, xmm0, mode);
5082         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5083         addptr(rsp, 64);
5084       } else {
5085         subptr(rsp, 64);
5086         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5087         subptr(rsp, 64);
5088         evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
5089         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5090         evmovdqul(xmm1, src, Assembler::AVX_512bit);
5091         Assembler::pshuflw(xmm0, xmm1, mode);
5092         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5093         evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
5094         addptr(rsp, 64);
5095         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5096         addptr(rsp, 64);
5097       }
5098     }
5099   } else {
5100     Assembler::pshuflw(dst, src, mode);
5101   }
5102 }
5103 
5104 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5105   if (reachable(src)) {
5106     vandpd(dst, nds, as_Address(src), vector_len);
5107   } else {
5108     lea(rscratch1, src);
5109     vandpd(dst, nds, Address(rscratch1, 0), vector_len);
5110   }
5111 }
5112 
5113 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5114   if (reachable(src)) {
5115     vandps(dst, nds, as_Address(src), vector_len);
5116   } else {
5117     lea(rscratch1, src);
5118     vandps(dst, nds, Address(rscratch1, 0), vector_len);
5119   }
5120 }
5121 
5122 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5123   if (reachable(src)) {
5124     vdivsd(dst, nds, as_Address(src));
5125   } else {
5126     lea(rscratch1, src);
5127     vdivsd(dst, nds, Address(rscratch1, 0));
5128   }
5129 }
5130 
5131 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5132   if (reachable(src)) {
5133     vdivss(dst, nds, as_Address(src));
5134   } else {
5135     lea(rscratch1, src);
5136     vdivss(dst, nds, Address(rscratch1, 0));
5137   }
5138 }
5139 
5140 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5141   if (reachable(src)) {
5142     vmulsd(dst, nds, as_Address(src));
5143   } else {
5144     lea(rscratch1, src);
5145     vmulsd(dst, nds, Address(rscratch1, 0));
5146   }
5147 }
5148 
5149 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5150   if (reachable(src)) {
5151     vmulss(dst, nds, as_Address(src));
5152   } else {
5153     lea(rscratch1, src);
5154     vmulss(dst, nds, Address(rscratch1, 0));
5155   }
5156 }
5157 
5158 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5159   if (reachable(src)) {
5160     vsubsd(dst, nds, as_Address(src));
5161   } else {
5162     lea(rscratch1, src);
5163     vsubsd(dst, nds, Address(rscratch1, 0));
5164   }
5165 }
5166 
5167 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5168   if (reachable(src)) {
5169     vsubss(dst, nds, as_Address(src));
5170   } else {
5171     lea(rscratch1, src);
5172     vsubss(dst, nds, Address(rscratch1, 0));
5173   }
5174 }
5175 
5176 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5177   int nds_enc = nds->encoding();
5178   int dst_enc = dst->encoding();
5179   bool dst_upper_bank = (dst_enc > 15);
5180   bool nds_upper_bank = (nds_enc > 15);
5181   if (VM_Version::supports_avx512novl() &&
5182       (nds_upper_bank || dst_upper_bank)) {
5183     if (dst_upper_bank) {
5184       subptr(rsp, 64);
5185       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5186       movflt(xmm0, nds);
5187       vxorps(xmm0, xmm0, src, Assembler::AVX_128bit);
5188       movflt(dst, xmm0);
5189       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5190       addptr(rsp, 64);
5191     } else {
5192       movflt(dst, nds);
5193       vxorps(dst, dst, src, Assembler::AVX_128bit);
5194     }
5195   } else {
5196     vxorps(dst, nds, src, Assembler::AVX_128bit);
5197   }
5198 }
5199 
5200 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5201   int nds_enc = nds->encoding();
5202   int dst_enc = dst->encoding();
5203   bool dst_upper_bank = (dst_enc > 15);
5204   bool nds_upper_bank = (nds_enc > 15);
5205   if (VM_Version::supports_avx512novl() &&
5206       (nds_upper_bank || dst_upper_bank)) {
5207     if (dst_upper_bank) {
5208       subptr(rsp, 64);
5209       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5210       movdbl(xmm0, nds);
5211       vxorpd(xmm0, xmm0, src, Assembler::AVX_128bit);
5212       movdbl(dst, xmm0);
5213       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5214       addptr(rsp, 64);
5215     } else {
5216       movdbl(dst, nds);
5217       vxorpd(dst, dst, src, Assembler::AVX_128bit);
5218     }
5219   } else {
5220     vxorpd(dst, nds, src, Assembler::AVX_128bit);
5221   }
5222 }
5223 
5224 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5225   if (reachable(src)) {
5226     vxorpd(dst, nds, as_Address(src), vector_len);
5227   } else {
5228     lea(rscratch1, src);
5229     vxorpd(dst, nds, Address(rscratch1, 0), vector_len);
5230   }
5231 }
5232 
5233 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5234   if (reachable(src)) {
5235     vxorps(dst, nds, as_Address(src), vector_len);
5236   } else {
5237     lea(rscratch1, src);
5238     vxorps(dst, nds, Address(rscratch1, 0), vector_len);
5239   }
5240 }
5241 
5242 
5243 void MacroAssembler::resolve_jobject(Register value,
5244                                      Register thread,
5245                                      Register tmp) {
5246   assert_different_registers(value, thread, tmp);
5247   Label done, not_weak;
5248   testptr(value, value);
5249   jcc(Assembler::zero, done);                // Use NULL as-is.
5250   testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag.
5251   jcc(Assembler::zero, not_weak);
5252   // Resolve jweak.
5253   movptr(value, Address(value, -JNIHandles::weak_tag_value));
5254   verify_oop(value);
5255 #if INCLUDE_ALL_GCS
5256   if (UseG1GC) {
5257     g1_write_barrier_pre(noreg /* obj */,
5258                          value /* pre_val */,
5259                          thread /* thread */,
5260                          tmp /* tmp */,
5261                          true /* tosca_live */,
5262                          true /* expand_call */);
5263   }
5264 #endif // INCLUDE_ALL_GCS
5265   jmp(done);
5266   bind(not_weak);
5267   // Resolve (untagged) jobject.
5268   movptr(value, Address(value, 0));
5269   verify_oop(value);
5270   bind(done);
5271 }
5272 
5273 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) {
5274   const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask);
5275   STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code
5276   // The inverted mask is sign-extended
5277   andptr(possibly_jweak, inverted_jweak_mask);
5278 }
5279 
5280 //////////////////////////////////////////////////////////////////////////////////
5281 #if INCLUDE_ALL_GCS
5282 
5283 void MacroAssembler::g1_write_barrier_pre(Register obj,
5284                                           Register pre_val,
5285                                           Register thread,
5286                                           Register tmp,
5287                                           bool tosca_live,
5288                                           bool expand_call) {
5289 
5290   // If expand_call is true then we expand the call_VM_leaf macro
5291   // directly to skip generating the check by
5292   // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
5293 
5294 #ifdef _LP64
5295   assert(thread == r15_thread, "must be");
5296 #endif // _LP64
5297 
5298   Label done;
5299   Label runtime;
5300 
5301   assert(pre_val != noreg, "check this code");
5302 
5303   if (obj != noreg) {
5304     assert_different_registers(obj, pre_val, tmp);
5305     assert(pre_val != rax, "check this code");
5306   }
5307 
5308   Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
5309                                        SATBMarkQueue::byte_offset_of_active()));
5310   Address index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
5311                                        SATBMarkQueue::byte_offset_of_index()));
5312   Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
5313                                        SATBMarkQueue::byte_offset_of_buf()));
5314 
5315 
5316   // Is marking active?
5317   if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
5318     cmpl(in_progress, 0);
5319   } else {
5320     assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
5321     cmpb(in_progress, 0);
5322   }
5323   jcc(Assembler::equal, done);
5324 
5325   // Do we need to load the previous value?
5326   if (obj != noreg) {
5327     load_heap_oop(pre_val, Address(obj, 0));
5328   }
5329 
5330   // Is the previous value null?
5331   cmpptr(pre_val, (int32_t) NULL_WORD);
5332   jcc(Assembler::equal, done);
5333 
5334   // Can we store original value in the thread's buffer?
5335   // Is index == 0?
5336   // (The index field is typed as size_t.)
5337 
5338   movptr(tmp, index);                   // tmp := *index_adr
5339   cmpptr(tmp, 0);                       // tmp == 0?
5340   jcc(Assembler::equal, runtime);       // If yes, goto runtime
5341 
5342   subptr(tmp, wordSize);                // tmp := tmp - wordSize
5343   movptr(index, tmp);                   // *index_adr := tmp
5344   addptr(tmp, buffer);                  // tmp := tmp + *buffer_adr
5345 
5346   // Record the previous value
5347   movptr(Address(tmp, 0), pre_val);
5348   jmp(done);
5349 
5350   bind(runtime);
5351   // save the live input values
5352   if(tosca_live) push(rax);
5353 
5354   if (obj != noreg && obj != rax)
5355     push(obj);
5356 
5357   if (pre_val != rax)
5358     push(pre_val);
5359 
5360   // Calling the runtime using the regular call_VM_leaf mechanism generates
5361   // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
5362   // that checks that the *(ebp+frame::interpreter_frame_last_sp) == NULL.
5363   //
5364   // If we care generating the pre-barrier without a frame (e.g. in the
5365   // intrinsified Reference.get() routine) then ebp might be pointing to
5366   // the caller frame and so this check will most likely fail at runtime.
5367   //
5368   // Expanding the call directly bypasses the generation of the check.
5369   // So when we do not have have a full interpreter frame on the stack
5370   // expand_call should be passed true.
5371 
5372   NOT_LP64( push(thread); )
5373 
5374   if (expand_call) {
5375     LP64_ONLY( assert(pre_val != c_rarg1, "smashed arg"); )
5376     pass_arg1(this, thread);
5377     pass_arg0(this, pre_val);
5378     MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), 2);
5379   } else {
5380     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
5381   }
5382 
5383   NOT_LP64( pop(thread); )
5384 
5385   // save the live input values
5386   if (pre_val != rax)
5387     pop(pre_val);
5388 
5389   if (obj != noreg && obj != rax)
5390     pop(obj);
5391 
5392   if(tosca_live) pop(rax);
5393 
5394   bind(done);
5395 }
5396 
5397 void MacroAssembler::g1_write_barrier_post(Register store_addr,
5398                                            Register new_val,
5399                                            Register thread,
5400                                            Register tmp,
5401                                            Register tmp2) {
5402 #ifdef _LP64
5403   assert(thread == r15_thread, "must be");
5404 #endif // _LP64
5405 
5406   Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
5407                                        DirtyCardQueue::byte_offset_of_index()));
5408   Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
5409                                        DirtyCardQueue::byte_offset_of_buf()));
5410 
5411   CardTableBarrierSet* ctbs =
5412     barrier_set_cast<CardTableBarrierSet>(Universe::heap()->barrier_set());
5413   CardTable* ct = ctbs->card_table();
5414   assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
5415 
5416   Label done;
5417   Label runtime;
5418 
5419   // Does store cross heap regions?
5420 
5421   movptr(tmp, store_addr);
5422   xorptr(tmp, new_val);
5423   shrptr(tmp, HeapRegion::LogOfHRGrainBytes);
5424   jcc(Assembler::equal, done);
5425 
5426   // crosses regions, storing NULL?
5427 
5428   cmpptr(new_val, (int32_t) NULL_WORD);
5429   jcc(Assembler::equal, done);
5430 
5431   // storing region crossing non-NULL, is card already dirty?
5432 
5433   const Register card_addr = tmp;
5434   const Register cardtable = tmp2;
5435 
5436   movptr(card_addr, store_addr);
5437   shrptr(card_addr, CardTable::card_shift);
5438   // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
5439   // a valid address and therefore is not properly handled by the relocation code.
5440   movptr(cardtable, (intptr_t)ct->byte_map_base());
5441   addptr(card_addr, cardtable);
5442 
5443   cmpb(Address(card_addr, 0), (int)G1CardTable::g1_young_card_val());
5444   jcc(Assembler::equal, done);
5445 
5446   membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
5447   cmpb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
5448   jcc(Assembler::equal, done);
5449 
5450 
5451   // storing a region crossing, non-NULL oop, card is clean.
5452   // dirty card and log.
5453 
5454   movb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
5455 
5456   cmpl(queue_index, 0);
5457   jcc(Assembler::equal, runtime);
5458   subl(queue_index, wordSize);
5459   movptr(tmp2, buffer);
5460 #ifdef _LP64
5461   movslq(rscratch1, queue_index);
5462   addq(tmp2, rscratch1);
5463   movq(Address(tmp2, 0), card_addr);
5464 #else
5465   addl(tmp2, queue_index);
5466   movl(Address(tmp2, 0), card_addr);
5467 #endif
5468   jmp(done);
5469 
5470   bind(runtime);
5471   // save the live input values
5472   push(store_addr);
5473   push(new_val);
5474 #ifdef _LP64
5475   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, r15_thread);
5476 #else
5477   push(thread);
5478   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
5479   pop(thread);
5480 #endif
5481   pop(new_val);
5482   pop(store_addr);
5483 
5484   bind(done);
5485 }
5486 
5487 #endif // INCLUDE_ALL_GCS
5488 //////////////////////////////////////////////////////////////////////////////////
5489 
5490 
5491 void MacroAssembler::store_check(Register obj, Address dst) {
5492   store_check(obj);
5493 }
5494 
5495 void MacroAssembler::store_check(Register obj) {
5496   // Does a store check for the oop in register obj. The content of
5497   // register obj is destroyed afterwards.
5498   BarrierSet* bs = Universe::heap()->barrier_set();
5499   assert(bs->kind() == BarrierSet::CardTableBarrierSet,
5500          "Wrong barrier set kind");
5501 
5502   CardTableBarrierSet* ctbs = barrier_set_cast<CardTableBarrierSet>(bs);
5503   CardTable* ct = ctbs->card_table();
5504   assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
5505 
5506   shrptr(obj, CardTable::card_shift);
5507 
5508   Address card_addr;
5509 
5510   // The calculation for byte_map_base is as follows:
5511   // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
5512   // So this essentially converts an address to a displacement and it will
5513   // never need to be relocated. On 64bit however the value may be too
5514   // large for a 32bit displacement.
5515   intptr_t disp = (intptr_t) ct->byte_map_base();
5516   if (is_simm32(disp)) {
5517     card_addr = Address(noreg, obj, Address::times_1, disp);
5518   } else {
5519     // By doing it as an ExternalAddress 'disp' could be converted to a rip-relative
5520     // displacement and done in a single instruction given favorable mapping and a
5521     // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
5522     // entry and that entry is not properly handled by the relocation code.
5523     AddressLiteral cardtable((address)ct->byte_map_base(), relocInfo::none);
5524     Address index(noreg, obj, Address::times_1);
5525     card_addr = as_Address(ArrayAddress(cardtable, index));
5526   }
5527 
5528   int dirty = CardTable::dirty_card_val();
5529   if (UseCondCardMark) {
5530     Label L_already_dirty;
5531     if (UseConcMarkSweepGC) {
5532       membar(Assembler::StoreLoad);
5533     }
5534     cmpb(card_addr, dirty);
5535     jcc(Assembler::equal, L_already_dirty);
5536     movb(card_addr, dirty);
5537     bind(L_already_dirty);
5538   } else {
5539     movb(card_addr, dirty);
5540   }
5541 }
5542 
5543 void MacroAssembler::subptr(Register dst, int32_t imm32) {
5544   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
5545 }
5546 
5547 // Force generation of a 4 byte immediate value even if it fits into 8bit
5548 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
5549   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
5550 }
5551 
5552 void MacroAssembler::subptr(Register dst, Register src) {
5553   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
5554 }
5555 
5556 // C++ bool manipulation
5557 void MacroAssembler::testbool(Register dst) {
5558   if(sizeof(bool) == 1)
5559     testb(dst, 0xff);
5560   else if(sizeof(bool) == 2) {
5561     // testw implementation needed for two byte bools
5562     ShouldNotReachHere();
5563   } else if(sizeof(bool) == 4)
5564     testl(dst, dst);
5565   else
5566     // unsupported
5567     ShouldNotReachHere();
5568 }
5569 
5570 void MacroAssembler::testptr(Register dst, Register src) {
5571   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
5572 }
5573 
5574 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
5575 void MacroAssembler::tlab_allocate(Register obj,
5576                                    Register var_size_in_bytes,
5577                                    int con_size_in_bytes,
5578                                    Register t1,
5579                                    Register t2,
5580                                    Label& slow_case) {
5581   assert_different_registers(obj, t1, t2);
5582   assert_different_registers(obj, var_size_in_bytes, t1);
5583   Register end = t2;
5584   Register thread = NOT_LP64(t1) LP64_ONLY(r15_thread);
5585 
5586   verify_tlab();
5587 
5588   NOT_LP64(get_thread(thread));
5589 
5590   movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
5591   if (var_size_in_bytes == noreg) {
5592     lea(end, Address(obj, con_size_in_bytes));
5593   } else {
5594     lea(end, Address(obj, var_size_in_bytes, Address::times_1));
5595   }
5596   cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
5597   jcc(Assembler::above, slow_case);
5598 
5599   // update the tlab top pointer
5600   movptr(Address(thread, JavaThread::tlab_top_offset()), end);
5601 
5602   // recover var_size_in_bytes if necessary
5603   if (var_size_in_bytes == end) {
5604     subptr(var_size_in_bytes, obj);
5605   }
5606   verify_tlab();
5607 }
5608 
5609 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
5610 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
5611   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
5612   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
5613   Label done;
5614 
5615   testptr(length_in_bytes, length_in_bytes);
5616   jcc(Assembler::zero, done);
5617 
5618   // initialize topmost word, divide index by 2, check if odd and test if zero
5619   // note: for the remaining code to work, index must be a multiple of BytesPerWord
5620 #ifdef ASSERT
5621   {
5622     Label L;
5623     testptr(length_in_bytes, BytesPerWord - 1);
5624     jcc(Assembler::zero, L);
5625     stop("length must be a multiple of BytesPerWord");
5626     bind(L);
5627   }
5628 #endif
5629   Register index = length_in_bytes;
5630   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
5631   if (UseIncDec) {
5632     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
5633   } else {
5634     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
5635     shrptr(index, 1);
5636   }
5637 #ifndef _LP64
5638   // index could have not been a multiple of 8 (i.e., bit 2 was set)
5639   {
5640     Label even;
5641     // note: if index was a multiple of 8, then it cannot
5642     //       be 0 now otherwise it must have been 0 before
5643     //       => if it is even, we don't need to check for 0 again
5644     jcc(Assembler::carryClear, even);
5645     // clear topmost word (no jump would be needed if conditional assignment worked here)
5646     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
5647     // index could be 0 now, must check again
5648     jcc(Assembler::zero, done);
5649     bind(even);
5650   }
5651 #endif // !_LP64
5652   // initialize remaining object fields: index is a multiple of 2 now
5653   {
5654     Label loop;
5655     bind(loop);
5656     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
5657     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
5658     decrement(index);
5659     jcc(Assembler::notZero, loop);
5660   }
5661 
5662   bind(done);
5663 }
5664 
5665 void MacroAssembler::incr_allocated_bytes(Register thread,
5666                                           Register var_size_in_bytes,
5667                                           int con_size_in_bytes,
5668                                           Register t1) {
5669   if (!thread->is_valid()) {
5670 #ifdef _LP64
5671     thread = r15_thread;
5672 #else
5673     assert(t1->is_valid(), "need temp reg");
5674     thread = t1;
5675     get_thread(thread);
5676 #endif
5677   }
5678 
5679 #ifdef _LP64
5680   if (var_size_in_bytes->is_valid()) {
5681     addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
5682   } else {
5683     addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
5684   }
5685 #else
5686   if (var_size_in_bytes->is_valid()) {
5687     addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
5688   } else {
5689     addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
5690   }
5691   adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0);
5692 #endif
5693 }
5694 
5695 // Look up the method for a megamorphic invokeinterface call.
5696 // The target method is determined by <intf_klass, itable_index>.
5697 // The receiver klass is in recv_klass.
5698 // On success, the result will be in method_result, and execution falls through.
5699 // On failure, execution transfers to the given label.
5700 void MacroAssembler::lookup_interface_method(Register recv_klass,
5701                                              Register intf_klass,
5702                                              RegisterOrConstant itable_index,
5703                                              Register method_result,
5704                                              Register scan_temp,
5705                                              Label& L_no_such_interface,
5706                                              bool return_method) {
5707   assert_different_registers(recv_klass, intf_klass, scan_temp);
5708   assert_different_registers(method_result, intf_klass, scan_temp);
5709   assert(recv_klass != method_result || !return_method,
5710          "recv_klass can be destroyed when method isn't needed");
5711 
5712   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
5713          "caller must use same register for non-constant itable index as for method");
5714 
5715   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
5716   int vtable_base = in_bytes(Klass::vtable_start_offset());
5717   int itentry_off = itableMethodEntry::method_offset_in_bytes();
5718   int scan_step   = itableOffsetEntry::size() * wordSize;
5719   int vte_size    = vtableEntry::size_in_bytes();
5720   Address::ScaleFactor times_vte_scale = Address::times_ptr;
5721   assert(vte_size == wordSize, "else adjust times_vte_scale");
5722 
5723   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
5724 
5725   // %%% Could store the aligned, prescaled offset in the klassoop.
5726   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
5727 
5728   if (return_method) {
5729     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
5730     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
5731     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
5732   }
5733 
5734   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
5735   //   if (scan->interface() == intf) {
5736   //     result = (klass + scan->offset() + itable_index);
5737   //   }
5738   // }
5739   Label search, found_method;
5740 
5741   for (int peel = 1; peel >= 0; peel--) {
5742     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
5743     cmpptr(intf_klass, method_result);
5744 
5745     if (peel) {
5746       jccb(Assembler::equal, found_method);
5747     } else {
5748       jccb(Assembler::notEqual, search);
5749       // (invert the test to fall through to found_method...)
5750     }
5751 
5752     if (!peel)  break;
5753 
5754     bind(search);
5755 
5756     // Check that the previous entry is non-null.  A null entry means that
5757     // the receiver class doesn't implement the interface, and wasn't the
5758     // same as when the caller was compiled.
5759     testptr(method_result, method_result);
5760     jcc(Assembler::zero, L_no_such_interface);
5761     addptr(scan_temp, scan_step);
5762   }
5763 
5764   bind(found_method);
5765 
5766   if (return_method) {
5767     // Got a hit.
5768     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
5769     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
5770   }
5771 }
5772 
5773 
5774 // virtual method calling
5775 void MacroAssembler::lookup_virtual_method(Register recv_klass,
5776                                            RegisterOrConstant vtable_index,
5777                                            Register method_result) {
5778   const int base = in_bytes(Klass::vtable_start_offset());
5779   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
5780   Address vtable_entry_addr(recv_klass,
5781                             vtable_index, Address::times_ptr,
5782                             base + vtableEntry::method_offset_in_bytes());
5783   movptr(method_result, vtable_entry_addr);
5784 }
5785 
5786 
5787 void MacroAssembler::check_klass_subtype(Register sub_klass,
5788                            Register super_klass,
5789                            Register temp_reg,
5790                            Label& L_success) {
5791   Label L_failure;
5792   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, NULL);
5793   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
5794   bind(L_failure);
5795 }
5796 
5797 
5798 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
5799                                                    Register super_klass,
5800                                                    Register temp_reg,
5801                                                    Label* L_success,
5802                                                    Label* L_failure,
5803                                                    Label* L_slow_path,
5804                                         RegisterOrConstant super_check_offset) {
5805   assert_different_registers(sub_klass, super_klass, temp_reg);
5806   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
5807   if (super_check_offset.is_register()) {
5808     assert_different_registers(sub_klass, super_klass,
5809                                super_check_offset.as_register());
5810   } else if (must_load_sco) {
5811     assert(temp_reg != noreg, "supply either a temp or a register offset");
5812   }
5813 
5814   Label L_fallthrough;
5815   int label_nulls = 0;
5816   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
5817   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
5818   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
5819   assert(label_nulls <= 1, "at most one NULL in the batch");
5820 
5821   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
5822   int sco_offset = in_bytes(Klass::super_check_offset_offset());
5823   Address super_check_offset_addr(super_klass, sco_offset);
5824 
5825   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
5826   // range of a jccb.  If this routine grows larger, reconsider at
5827   // least some of these.
5828 #define local_jcc(assembler_cond, label)                                \
5829   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
5830   else                             jcc( assembler_cond, label) /*omit semi*/
5831 
5832   // Hacked jmp, which may only be used just before L_fallthrough.
5833 #define final_jmp(label)                                                \
5834   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
5835   else                            jmp(label)                /*omit semi*/
5836 
5837   // If the pointers are equal, we are done (e.g., String[] elements).
5838   // This self-check enables sharing of secondary supertype arrays among
5839   // non-primary types such as array-of-interface.  Otherwise, each such
5840   // type would need its own customized SSA.
5841   // We move this check to the front of the fast path because many
5842   // type checks are in fact trivially successful in this manner,
5843   // so we get a nicely predicted branch right at the start of the check.
5844   cmpptr(sub_klass, super_klass);
5845   local_jcc(Assembler::equal, *L_success);
5846 
5847   // Check the supertype display:
5848   if (must_load_sco) {
5849     // Positive movl does right thing on LP64.
5850     movl(temp_reg, super_check_offset_addr);
5851     super_check_offset = RegisterOrConstant(temp_reg);
5852   }
5853   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
5854   cmpptr(super_klass, super_check_addr); // load displayed supertype
5855 
5856   // This check has worked decisively for primary supers.
5857   // Secondary supers are sought in the super_cache ('super_cache_addr').
5858   // (Secondary supers are interfaces and very deeply nested subtypes.)
5859   // This works in the same check above because of a tricky aliasing
5860   // between the super_cache and the primary super display elements.
5861   // (The 'super_check_addr' can address either, as the case requires.)
5862   // Note that the cache is updated below if it does not help us find
5863   // what we need immediately.
5864   // So if it was a primary super, we can just fail immediately.
5865   // Otherwise, it's the slow path for us (no success at this point).
5866 
5867   if (super_check_offset.is_register()) {
5868     local_jcc(Assembler::equal, *L_success);
5869     cmpl(super_check_offset.as_register(), sc_offset);
5870     if (L_failure == &L_fallthrough) {
5871       local_jcc(Assembler::equal, *L_slow_path);
5872     } else {
5873       local_jcc(Assembler::notEqual, *L_failure);
5874       final_jmp(*L_slow_path);
5875     }
5876   } else if (super_check_offset.as_constant() == sc_offset) {
5877     // Need a slow path; fast failure is impossible.
5878     if (L_slow_path == &L_fallthrough) {
5879       local_jcc(Assembler::equal, *L_success);
5880     } else {
5881       local_jcc(Assembler::notEqual, *L_slow_path);
5882       final_jmp(*L_success);
5883     }
5884   } else {
5885     // No slow path; it's a fast decision.
5886     if (L_failure == &L_fallthrough) {
5887       local_jcc(Assembler::equal, *L_success);
5888     } else {
5889       local_jcc(Assembler::notEqual, *L_failure);
5890       final_jmp(*L_success);
5891     }
5892   }
5893 
5894   bind(L_fallthrough);
5895 
5896 #undef local_jcc
5897 #undef final_jmp
5898 }
5899 
5900 
5901 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
5902                                                    Register super_klass,
5903                                                    Register temp_reg,
5904                                                    Register temp2_reg,
5905                                                    Label* L_success,
5906                                                    Label* L_failure,
5907                                                    bool set_cond_codes) {
5908   assert_different_registers(sub_klass, super_klass, temp_reg);
5909   if (temp2_reg != noreg)
5910     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
5911 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
5912 
5913   Label L_fallthrough;
5914   int label_nulls = 0;
5915   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
5916   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
5917   assert(label_nulls <= 1, "at most one NULL in the batch");
5918 
5919   // a couple of useful fields in sub_klass:
5920   int ss_offset = in_bytes(Klass::secondary_supers_offset());
5921   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
5922   Address secondary_supers_addr(sub_klass, ss_offset);
5923   Address super_cache_addr(     sub_klass, sc_offset);
5924 
5925   // Do a linear scan of the secondary super-klass chain.
5926   // This code is rarely used, so simplicity is a virtue here.
5927   // The repne_scan instruction uses fixed registers, which we must spill.
5928   // Don't worry too much about pre-existing connections with the input regs.
5929 
5930   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
5931   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
5932 
5933   // Get super_klass value into rax (even if it was in rdi or rcx).
5934   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
5935   if (super_klass != rax || UseCompressedOops) {
5936     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
5937     mov(rax, super_klass);
5938   }
5939   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
5940   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
5941 
5942 #ifndef PRODUCT
5943   int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
5944   ExternalAddress pst_counter_addr((address) pst_counter);
5945   NOT_LP64(  incrementl(pst_counter_addr) );
5946   LP64_ONLY( lea(rcx, pst_counter_addr) );
5947   LP64_ONLY( incrementl(Address(rcx, 0)) );
5948 #endif //PRODUCT
5949 
5950   // We will consult the secondary-super array.
5951   movptr(rdi, secondary_supers_addr);
5952   // Load the array length.  (Positive movl does right thing on LP64.)
5953   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
5954   // Skip to start of data.
5955   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
5956 
5957   // Scan RCX words at [RDI] for an occurrence of RAX.
5958   // Set NZ/Z based on last compare.
5959   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
5960   // not change flags (only scas instruction which is repeated sets flags).
5961   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
5962 
5963     testptr(rax,rax); // Set Z = 0
5964     repne_scan();
5965 
5966   // Unspill the temp. registers:
5967   if (pushed_rdi)  pop(rdi);
5968   if (pushed_rcx)  pop(rcx);
5969   if (pushed_rax)  pop(rax);
5970 
5971   if (set_cond_codes) {
5972     // Special hack for the AD files:  rdi is guaranteed non-zero.
5973     assert(!pushed_rdi, "rdi must be left non-NULL");
5974     // Also, the condition codes are properly set Z/NZ on succeed/failure.
5975   }
5976 
5977   if (L_failure == &L_fallthrough)
5978         jccb(Assembler::notEqual, *L_failure);
5979   else  jcc(Assembler::notEqual, *L_failure);
5980 
5981   // Success.  Cache the super we found and proceed in triumph.
5982   movptr(super_cache_addr, super_klass);
5983 
5984   if (L_success != &L_fallthrough) {
5985     jmp(*L_success);
5986   }
5987 
5988 #undef IS_A_TEMP
5989 
5990   bind(L_fallthrough);
5991 }
5992 
5993 
5994 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
5995   if (VM_Version::supports_cmov()) {
5996     cmovl(cc, dst, src);
5997   } else {
5998     Label L;
5999     jccb(negate_condition(cc), L);
6000     movl(dst, src);
6001     bind(L);
6002   }
6003 }
6004 
6005 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
6006   if (VM_Version::supports_cmov()) {
6007     cmovl(cc, dst, src);
6008   } else {
6009     Label L;
6010     jccb(negate_condition(cc), L);
6011     movl(dst, src);
6012     bind(L);
6013   }
6014 }
6015 
6016 void MacroAssembler::verify_oop(Register reg, const char* s) {
6017   if (!VerifyOops) return;
6018 
6019   // Pass register number to verify_oop_subroutine
6020   const char* b = NULL;
6021   {
6022     ResourceMark rm;
6023     stringStream ss;
6024     ss.print("verify_oop: %s: %s", reg->name(), s);
6025     b = code_string(ss.as_string());
6026   }
6027   BLOCK_COMMENT("verify_oop {");
6028 #ifdef _LP64
6029   push(rscratch1);                    // save r10, trashed by movptr()
6030 #endif
6031   push(rax);                          // save rax,
6032   push(reg);                          // pass register argument
6033   ExternalAddress buffer((address) b);
6034   // avoid using pushptr, as it modifies scratch registers
6035   // and our contract is not to modify anything
6036   movptr(rax, buffer.addr());
6037   push(rax);
6038   // call indirectly to solve generation ordering problem
6039   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
6040   call(rax);
6041   // Caller pops the arguments (oop, message) and restores rax, r10
6042   BLOCK_COMMENT("} verify_oop");
6043 }
6044 
6045 
6046 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
6047                                                       Register tmp,
6048                                                       int offset) {
6049   intptr_t value = *delayed_value_addr;
6050   if (value != 0)
6051     return RegisterOrConstant(value + offset);
6052 
6053   // load indirectly to solve generation ordering problem
6054   movptr(tmp, ExternalAddress((address) delayed_value_addr));
6055 
6056 #ifdef ASSERT
6057   { Label L;
6058     testptr(tmp, tmp);
6059     if (WizardMode) {
6060       const char* buf = NULL;
6061       {
6062         ResourceMark rm;
6063         stringStream ss;
6064         ss.print("DelayedValue=" INTPTR_FORMAT, delayed_value_addr[1]);
6065         buf = code_string(ss.as_string());
6066       }
6067       jcc(Assembler::notZero, L);
6068       STOP(buf);
6069     } else {
6070       jccb(Assembler::notZero, L);
6071       hlt();
6072     }
6073     bind(L);
6074   }
6075 #endif
6076 
6077   if (offset != 0)
6078     addptr(tmp, offset);
6079 
6080   return RegisterOrConstant(tmp);
6081 }
6082 
6083 
6084 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
6085                                          int extra_slot_offset) {
6086   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
6087   int stackElementSize = Interpreter::stackElementSize;
6088   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
6089 #ifdef ASSERT
6090   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
6091   assert(offset1 - offset == stackElementSize, "correct arithmetic");
6092 #endif
6093   Register             scale_reg    = noreg;
6094   Address::ScaleFactor scale_factor = Address::no_scale;
6095   if (arg_slot.is_constant()) {
6096     offset += arg_slot.as_constant() * stackElementSize;
6097   } else {
6098     scale_reg    = arg_slot.as_register();
6099     scale_factor = Address::times(stackElementSize);
6100   }
6101   offset += wordSize;           // return PC is on stack
6102   return Address(rsp, scale_reg, scale_factor, offset);
6103 }
6104 
6105 
6106 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
6107   if (!VerifyOops) return;
6108 
6109   // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
6110   // Pass register number to verify_oop_subroutine
6111   const char* b = NULL;
6112   {
6113     ResourceMark rm;
6114     stringStream ss;
6115     ss.print("verify_oop_addr: %s", s);
6116     b = code_string(ss.as_string());
6117   }
6118 #ifdef _LP64
6119   push(rscratch1);                    // save r10, trashed by movptr()
6120 #endif
6121   push(rax);                          // save rax,
6122   // addr may contain rsp so we will have to adjust it based on the push
6123   // we just did (and on 64 bit we do two pushes)
6124   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
6125   // stores rax into addr which is backwards of what was intended.
6126   if (addr.uses(rsp)) {
6127     lea(rax, addr);
6128     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
6129   } else {
6130     pushptr(addr);
6131   }
6132 
6133   ExternalAddress buffer((address) b);
6134   // pass msg argument
6135   // avoid using pushptr, as it modifies scratch registers
6136   // and our contract is not to modify anything
6137   movptr(rax, buffer.addr());
6138   push(rax);
6139 
6140   // call indirectly to solve generation ordering problem
6141   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
6142   call(rax);
6143   // Caller pops the arguments (addr, message) and restores rax, r10.
6144 }
6145 
6146 void MacroAssembler::verify_tlab() {
6147 #ifdef ASSERT
6148   if (UseTLAB && VerifyOops) {
6149     Label next, ok;
6150     Register t1 = rsi;
6151     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
6152 
6153     push(t1);
6154     NOT_LP64(push(thread_reg));
6155     NOT_LP64(get_thread(thread_reg));
6156 
6157     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
6158     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
6159     jcc(Assembler::aboveEqual, next);
6160     STOP("assert(top >= start)");
6161     should_not_reach_here();
6162 
6163     bind(next);
6164     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
6165     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
6166     jcc(Assembler::aboveEqual, ok);
6167     STOP("assert(top <= end)");
6168     should_not_reach_here();
6169 
6170     bind(ok);
6171     NOT_LP64(pop(thread_reg));
6172     pop(t1);
6173   }
6174 #endif
6175 }
6176 
6177 class ControlWord {
6178  public:
6179   int32_t _value;
6180 
6181   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
6182   int  precision_control() const       { return  (_value >>  8) & 3      ; }
6183   bool precision() const               { return ((_value >>  5) & 1) != 0; }
6184   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
6185   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
6186   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
6187   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
6188   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
6189 
6190   void print() const {
6191     // rounding control
6192     const char* rc;
6193     switch (rounding_control()) {
6194       case 0: rc = "round near"; break;
6195       case 1: rc = "round down"; break;
6196       case 2: rc = "round up  "; break;
6197       case 3: rc = "chop      "; break;
6198     };
6199     // precision control
6200     const char* pc;
6201     switch (precision_control()) {
6202       case 0: pc = "24 bits "; break;
6203       case 1: pc = "reserved"; break;
6204       case 2: pc = "53 bits "; break;
6205       case 3: pc = "64 bits "; break;
6206     };
6207     // flags
6208     char f[9];
6209     f[0] = ' ';
6210     f[1] = ' ';
6211     f[2] = (precision   ()) ? 'P' : 'p';
6212     f[3] = (underflow   ()) ? 'U' : 'u';
6213     f[4] = (overflow    ()) ? 'O' : 'o';
6214     f[5] = (zero_divide ()) ? 'Z' : 'z';
6215     f[6] = (denormalized()) ? 'D' : 'd';
6216     f[7] = (invalid     ()) ? 'I' : 'i';
6217     f[8] = '\x0';
6218     // output
6219     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
6220   }
6221 
6222 };
6223 
6224 class StatusWord {
6225  public:
6226   int32_t _value;
6227 
6228   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
6229   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
6230   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
6231   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
6232   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
6233   int  top() const                     { return  (_value >> 11) & 7      ; }
6234   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
6235   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
6236   bool precision() const               { return ((_value >>  5) & 1) != 0; }
6237   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
6238   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
6239   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
6240   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
6241   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
6242 
6243   void print() const {
6244     // condition codes
6245     char c[5];
6246     c[0] = (C3()) ? '3' : '-';
6247     c[1] = (C2()) ? '2' : '-';
6248     c[2] = (C1()) ? '1' : '-';
6249     c[3] = (C0()) ? '0' : '-';
6250     c[4] = '\x0';
6251     // flags
6252     char f[9];
6253     f[0] = (error_status()) ? 'E' : '-';
6254     f[1] = (stack_fault ()) ? 'S' : '-';
6255     f[2] = (precision   ()) ? 'P' : '-';
6256     f[3] = (underflow   ()) ? 'U' : '-';
6257     f[4] = (overflow    ()) ? 'O' : '-';
6258     f[5] = (zero_divide ()) ? 'Z' : '-';
6259     f[6] = (denormalized()) ? 'D' : '-';
6260     f[7] = (invalid     ()) ? 'I' : '-';
6261     f[8] = '\x0';
6262     // output
6263     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
6264   }
6265 
6266 };
6267 
6268 class TagWord {
6269  public:
6270   int32_t _value;
6271 
6272   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
6273 
6274   void print() const {
6275     printf("%04x", _value & 0xFFFF);
6276   }
6277 
6278 };
6279 
6280 class FPU_Register {
6281  public:
6282   int32_t _m0;
6283   int32_t _m1;
6284   int16_t _ex;
6285 
6286   bool is_indefinite() const           {
6287     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
6288   }
6289 
6290   void print() const {
6291     char  sign = (_ex < 0) ? '-' : '+';
6292     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
6293     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
6294   };
6295 
6296 };
6297 
6298 class FPU_State {
6299  public:
6300   enum {
6301     register_size       = 10,
6302     number_of_registers =  8,
6303     register_mask       =  7
6304   };
6305 
6306   ControlWord  _control_word;
6307   StatusWord   _status_word;
6308   TagWord      _tag_word;
6309   int32_t      _error_offset;
6310   int32_t      _error_selector;
6311   int32_t      _data_offset;
6312   int32_t      _data_selector;
6313   int8_t       _register[register_size * number_of_registers];
6314 
6315   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
6316   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
6317 
6318   const char* tag_as_string(int tag) const {
6319     switch (tag) {
6320       case 0: return "valid";
6321       case 1: return "zero";
6322       case 2: return "special";
6323       case 3: return "empty";
6324     }
6325     ShouldNotReachHere();
6326     return NULL;
6327   }
6328 
6329   void print() const {
6330     // print computation registers
6331     { int t = _status_word.top();
6332       for (int i = 0; i < number_of_registers; i++) {
6333         int j = (i - t) & register_mask;
6334         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
6335         st(j)->print();
6336         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
6337       }
6338     }
6339     printf("\n");
6340     // print control registers
6341     printf("ctrl = "); _control_word.print(); printf("\n");
6342     printf("stat = "); _status_word .print(); printf("\n");
6343     printf("tags = "); _tag_word    .print(); printf("\n");
6344   }
6345 
6346 };
6347 
6348 class Flag_Register {
6349  public:
6350   int32_t _value;
6351 
6352   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
6353   bool direction() const               { return ((_value >> 10) & 1) != 0; }
6354   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
6355   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
6356   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
6357   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
6358   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
6359 
6360   void print() const {
6361     // flags
6362     char f[8];
6363     f[0] = (overflow       ()) ? 'O' : '-';
6364     f[1] = (direction      ()) ? 'D' : '-';
6365     f[2] = (sign           ()) ? 'S' : '-';
6366     f[3] = (zero           ()) ? 'Z' : '-';
6367     f[4] = (auxiliary_carry()) ? 'A' : '-';
6368     f[5] = (parity         ()) ? 'P' : '-';
6369     f[6] = (carry          ()) ? 'C' : '-';
6370     f[7] = '\x0';
6371     // output
6372     printf("%08x  flags = %s", _value, f);
6373   }
6374 
6375 };
6376 
6377 class IU_Register {
6378  public:
6379   int32_t _value;
6380 
6381   void print() const {
6382     printf("%08x  %11d", _value, _value);
6383   }
6384 
6385 };
6386 
6387 class IU_State {
6388  public:
6389   Flag_Register _eflags;
6390   IU_Register   _rdi;
6391   IU_Register   _rsi;
6392   IU_Register   _rbp;
6393   IU_Register   _rsp;
6394   IU_Register   _rbx;
6395   IU_Register   _rdx;
6396   IU_Register   _rcx;
6397   IU_Register   _rax;
6398 
6399   void print() const {
6400     // computation registers
6401     printf("rax,  = "); _rax.print(); printf("\n");
6402     printf("rbx,  = "); _rbx.print(); printf("\n");
6403     printf("rcx  = "); _rcx.print(); printf("\n");
6404     printf("rdx  = "); _rdx.print(); printf("\n");
6405     printf("rdi  = "); _rdi.print(); printf("\n");
6406     printf("rsi  = "); _rsi.print(); printf("\n");
6407     printf("rbp,  = "); _rbp.print(); printf("\n");
6408     printf("rsp  = "); _rsp.print(); printf("\n");
6409     printf("\n");
6410     // control registers
6411     printf("flgs = "); _eflags.print(); printf("\n");
6412   }
6413 };
6414 
6415 
6416 class CPU_State {
6417  public:
6418   FPU_State _fpu_state;
6419   IU_State  _iu_state;
6420 
6421   void print() const {
6422     printf("--------------------------------------------------\n");
6423     _iu_state .print();
6424     printf("\n");
6425     _fpu_state.print();
6426     printf("--------------------------------------------------\n");
6427   }
6428 
6429 };
6430 
6431 
6432 static void _print_CPU_state(CPU_State* state) {
6433   state->print();
6434 };
6435 
6436 
6437 void MacroAssembler::print_CPU_state() {
6438   push_CPU_state();
6439   push(rsp);                // pass CPU state
6440   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
6441   addptr(rsp, wordSize);       // discard argument
6442   pop_CPU_state();
6443 }
6444 
6445 
6446 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
6447   static int counter = 0;
6448   FPU_State* fs = &state->_fpu_state;
6449   counter++;
6450   // For leaf calls, only verify that the top few elements remain empty.
6451   // We only need 1 empty at the top for C2 code.
6452   if( stack_depth < 0 ) {
6453     if( fs->tag_for_st(7) != 3 ) {
6454       printf("FPR7 not empty\n");
6455       state->print();
6456       assert(false, "error");
6457       return false;
6458     }
6459     return true;                // All other stack states do not matter
6460   }
6461 
6462   assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std,
6463          "bad FPU control word");
6464 
6465   // compute stack depth
6466   int i = 0;
6467   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
6468   int d = i;
6469   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
6470   // verify findings
6471   if (i != FPU_State::number_of_registers) {
6472     // stack not contiguous
6473     printf("%s: stack not contiguous at ST%d\n", s, i);
6474     state->print();
6475     assert(false, "error");
6476     return false;
6477   }
6478   // check if computed stack depth corresponds to expected stack depth
6479   if (stack_depth < 0) {
6480     // expected stack depth is -stack_depth or less
6481     if (d > -stack_depth) {
6482       // too many elements on the stack
6483       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
6484       state->print();
6485       assert(false, "error");
6486       return false;
6487     }
6488   } else {
6489     // expected stack depth is stack_depth
6490     if (d != stack_depth) {
6491       // wrong stack depth
6492       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
6493       state->print();
6494       assert(false, "error");
6495       return false;
6496     }
6497   }
6498   // everything is cool
6499   return true;
6500 }
6501 
6502 
6503 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
6504   if (!VerifyFPU) return;
6505   push_CPU_state();
6506   push(rsp);                // pass CPU state
6507   ExternalAddress msg((address) s);
6508   // pass message string s
6509   pushptr(msg.addr());
6510   push(stack_depth);        // pass stack depth
6511   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
6512   addptr(rsp, 3 * wordSize);   // discard arguments
6513   // check for error
6514   { Label L;
6515     testl(rax, rax);
6516     jcc(Assembler::notZero, L);
6517     int3();                  // break if error condition
6518     bind(L);
6519   }
6520   pop_CPU_state();
6521 }
6522 
6523 void MacroAssembler::restore_cpu_control_state_after_jni() {
6524   // Either restore the MXCSR register after returning from the JNI Call
6525   // or verify that it wasn't changed (with -Xcheck:jni flag).
6526   if (VM_Version::supports_sse()) {
6527     if (RestoreMXCSROnJNICalls) {
6528       ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std()));
6529     } else if (CheckJNICalls) {
6530       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
6531     }
6532   }
6533   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
6534   vzeroupper();
6535   // Reset k1 to 0xffff.
6536   if (VM_Version::supports_evex()) {
6537     push(rcx);
6538     movl(rcx, 0xffff);
6539     kmovwl(k1, rcx);
6540     pop(rcx);
6541   }
6542 
6543 #ifndef _LP64
6544   // Either restore the x87 floating pointer control word after returning
6545   // from the JNI call or verify that it wasn't changed.
6546   if (CheckJNICalls) {
6547     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
6548   }
6549 #endif // _LP64
6550 }
6551 
6552 // ((OopHandle)result).resolve();
6553 void MacroAssembler::resolve_oop_handle(Register result) {
6554   // OopHandle::resolve is an indirection.
6555   movptr(result, Address(result, 0));
6556 }
6557 
6558 void MacroAssembler::load_mirror(Register mirror, Register method) {
6559   // get mirror
6560   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
6561   movptr(mirror, Address(method, Method::const_offset()));
6562   movptr(mirror, Address(mirror, ConstMethod::constants_offset()));
6563   movptr(mirror, Address(mirror, ConstantPool::pool_holder_offset_in_bytes()));
6564   movptr(mirror, Address(mirror, mirror_offset));
6565   resolve_oop_handle(mirror);
6566 }
6567 
6568 void MacroAssembler::load_klass(Register dst, Register src) {
6569 #ifdef _LP64
6570   if (UseCompressedClassPointers) {
6571     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
6572     decode_klass_not_null(dst);
6573   } else
6574 #endif
6575     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
6576 }
6577 
6578 void MacroAssembler::load_prototype_header(Register dst, Register src) {
6579   load_klass(dst, src);
6580   movptr(dst, Address(dst, Klass::prototype_header_offset()));
6581 }
6582 
6583 void MacroAssembler::store_klass(Register dst, Register src) {
6584 #ifdef _LP64
6585   if (UseCompressedClassPointers) {
6586     encode_klass_not_null(src);
6587     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
6588   } else
6589 #endif
6590     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
6591 }
6592 
6593 void MacroAssembler::load_heap_oop(Register dst, Address src) {
6594 #ifdef _LP64
6595   // FIXME: Must change all places where we try to load the klass.
6596   if (UseCompressedOops) {
6597     movl(dst, src);
6598     decode_heap_oop(dst);
6599   } else
6600 #endif
6601     movptr(dst, src);
6602 }
6603 
6604 // Doesn't do verfication, generates fixed size code
6605 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src) {
6606 #ifdef _LP64
6607   if (UseCompressedOops) {
6608     movl(dst, src);
6609     decode_heap_oop_not_null(dst);
6610   } else
6611 #endif
6612     movptr(dst, src);
6613 }
6614 
6615 void MacroAssembler::store_heap_oop(Address dst, Register src) {
6616 #ifdef _LP64
6617   if (UseCompressedOops) {
6618     assert(!dst.uses(src), "not enough registers");
6619     encode_heap_oop(src);
6620     movl(dst, src);
6621   } else
6622 #endif
6623     movptr(dst, src);
6624 }
6625 
6626 // Used for storing NULLs.
6627 void MacroAssembler::store_heap_oop_null(Address dst) {
6628 #ifdef _LP64
6629   if (UseCompressedOops) {
6630     movl(dst, (int32_t)NULL_WORD);
6631   } else {
6632     movslq(dst, (int32_t)NULL_WORD);
6633   }
6634 #else
6635   movl(dst, (int32_t)NULL_WORD);
6636 #endif
6637 }
6638 
6639 #ifdef _LP64
6640 void MacroAssembler::store_klass_gap(Register dst, Register src) {
6641   if (UseCompressedClassPointers) {
6642     // Store to klass gap in destination
6643     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
6644   }
6645 }
6646 
6647 #ifdef ASSERT
6648 void MacroAssembler::verify_heapbase(const char* msg) {
6649   assert (UseCompressedOops, "should be compressed");
6650   assert (Universe::heap() != NULL, "java heap should be initialized");
6651   if (CheckCompressedOops) {
6652     Label ok;
6653     push(rscratch1); // cmpptr trashes rscratch1
6654     cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6655     jcc(Assembler::equal, ok);
6656     STOP(msg);
6657     bind(ok);
6658     pop(rscratch1);
6659   }
6660 }
6661 #endif
6662 
6663 // Algorithm must match oop.inline.hpp encode_heap_oop.
6664 void MacroAssembler::encode_heap_oop(Register r) {
6665 #ifdef ASSERT
6666   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
6667 #endif
6668   verify_oop(r, "broken oop in encode_heap_oop");
6669   if (Universe::narrow_oop_base() == NULL) {
6670     if (Universe::narrow_oop_shift() != 0) {
6671       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6672       shrq(r, LogMinObjAlignmentInBytes);
6673     }
6674     return;
6675   }
6676   testq(r, r);
6677   cmovq(Assembler::equal, r, r12_heapbase);
6678   subq(r, r12_heapbase);
6679   shrq(r, LogMinObjAlignmentInBytes);
6680 }
6681 
6682 void MacroAssembler::encode_heap_oop_not_null(Register r) {
6683 #ifdef ASSERT
6684   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
6685   if (CheckCompressedOops) {
6686     Label ok;
6687     testq(r, r);
6688     jcc(Assembler::notEqual, ok);
6689     STOP("null oop passed to encode_heap_oop_not_null");
6690     bind(ok);
6691   }
6692 #endif
6693   verify_oop(r, "broken oop in encode_heap_oop_not_null");
6694   if (Universe::narrow_oop_base() != NULL) {
6695     subq(r, r12_heapbase);
6696   }
6697   if (Universe::narrow_oop_shift() != 0) {
6698     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6699     shrq(r, LogMinObjAlignmentInBytes);
6700   }
6701 }
6702 
6703 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
6704 #ifdef ASSERT
6705   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
6706   if (CheckCompressedOops) {
6707     Label ok;
6708     testq(src, src);
6709     jcc(Assembler::notEqual, ok);
6710     STOP("null oop passed to encode_heap_oop_not_null2");
6711     bind(ok);
6712   }
6713 #endif
6714   verify_oop(src, "broken oop in encode_heap_oop_not_null2");
6715   if (dst != src) {
6716     movq(dst, src);
6717   }
6718   if (Universe::narrow_oop_base() != NULL) {
6719     subq(dst, r12_heapbase);
6720   }
6721   if (Universe::narrow_oop_shift() != 0) {
6722     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6723     shrq(dst, LogMinObjAlignmentInBytes);
6724   }
6725 }
6726 
6727 void  MacroAssembler::decode_heap_oop(Register r) {
6728 #ifdef ASSERT
6729   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
6730 #endif
6731   if (Universe::narrow_oop_base() == NULL) {
6732     if (Universe::narrow_oop_shift() != 0) {
6733       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6734       shlq(r, LogMinObjAlignmentInBytes);
6735     }
6736   } else {
6737     Label done;
6738     shlq(r, LogMinObjAlignmentInBytes);
6739     jccb(Assembler::equal, done);
6740     addq(r, r12_heapbase);
6741     bind(done);
6742   }
6743   verify_oop(r, "broken oop in decode_heap_oop");
6744 }
6745 
6746 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
6747   // Note: it will change flags
6748   assert (UseCompressedOops, "should only be used for compressed headers");
6749   assert (Universe::heap() != NULL, "java heap should be initialized");
6750   // Cannot assert, unverified entry point counts instructions (see .ad file)
6751   // vtableStubs also counts instructions in pd_code_size_limit.
6752   // Also do not verify_oop as this is called by verify_oop.
6753   if (Universe::narrow_oop_shift() != 0) {
6754     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6755     shlq(r, LogMinObjAlignmentInBytes);
6756     if (Universe::narrow_oop_base() != NULL) {
6757       addq(r, r12_heapbase);
6758     }
6759   } else {
6760     assert (Universe::narrow_oop_base() == NULL, "sanity");
6761   }
6762 }
6763 
6764 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
6765   // Note: it will change flags
6766   assert (UseCompressedOops, "should only be used for compressed headers");
6767   assert (Universe::heap() != NULL, "java heap should be initialized");
6768   // Cannot assert, unverified entry point counts instructions (see .ad file)
6769   // vtableStubs also counts instructions in pd_code_size_limit.
6770   // Also do not verify_oop as this is called by verify_oop.
6771   if (Universe::narrow_oop_shift() != 0) {
6772     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6773     if (LogMinObjAlignmentInBytes == Address::times_8) {
6774       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
6775     } else {
6776       if (dst != src) {
6777         movq(dst, src);
6778       }
6779       shlq(dst, LogMinObjAlignmentInBytes);
6780       if (Universe::narrow_oop_base() != NULL) {
6781         addq(dst, r12_heapbase);
6782       }
6783     }
6784   } else {
6785     assert (Universe::narrow_oop_base() == NULL, "sanity");
6786     if (dst != src) {
6787       movq(dst, src);
6788     }
6789   }
6790 }
6791 
6792 void MacroAssembler::encode_klass_not_null(Register r) {
6793   if (Universe::narrow_klass_base() != NULL) {
6794     // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6795     assert(r != r12_heapbase, "Encoding a klass in r12");
6796     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6797     subq(r, r12_heapbase);
6798   }
6799   if (Universe::narrow_klass_shift() != 0) {
6800     assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6801     shrq(r, LogKlassAlignmentInBytes);
6802   }
6803   if (Universe::narrow_klass_base() != NULL) {
6804     reinit_heapbase();
6805   }
6806 }
6807 
6808 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
6809   if (dst == src) {
6810     encode_klass_not_null(src);
6811   } else {
6812     if (Universe::narrow_klass_base() != NULL) {
6813       mov64(dst, (int64_t)Universe::narrow_klass_base());
6814       negq(dst);
6815       addq(dst, src);
6816     } else {
6817       movptr(dst, src);
6818     }
6819     if (Universe::narrow_klass_shift() != 0) {
6820       assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6821       shrq(dst, LogKlassAlignmentInBytes);
6822     }
6823   }
6824 }
6825 
6826 // Function instr_size_for_decode_klass_not_null() counts the instructions
6827 // generated by decode_klass_not_null(register r) and reinit_heapbase(),
6828 // when (Universe::heap() != NULL).  Hence, if the instructions they
6829 // generate change, then this method needs to be updated.
6830 int MacroAssembler::instr_size_for_decode_klass_not_null() {
6831   assert (UseCompressedClassPointers, "only for compressed klass ptrs");
6832   if (Universe::narrow_klass_base() != NULL) {
6833     // mov64 + addq + shlq? + mov64  (for reinit_heapbase()).
6834     return (Universe::narrow_klass_shift() == 0 ? 20 : 24);
6835   } else {
6836     // longest load decode klass function, mov64, leaq
6837     return 16;
6838   }
6839 }
6840 
6841 // !!! If the instructions that get generated here change then function
6842 // instr_size_for_decode_klass_not_null() needs to get updated.
6843 void  MacroAssembler::decode_klass_not_null(Register r) {
6844   // Note: it will change flags
6845   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6846   assert(r != r12_heapbase, "Decoding a klass in r12");
6847   // Cannot assert, unverified entry point counts instructions (see .ad file)
6848   // vtableStubs also counts instructions in pd_code_size_limit.
6849   // Also do not verify_oop as this is called by verify_oop.
6850   if (Universe::narrow_klass_shift() != 0) {
6851     assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6852     shlq(r, LogKlassAlignmentInBytes);
6853   }
6854   // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6855   if (Universe::narrow_klass_base() != NULL) {
6856     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6857     addq(r, r12_heapbase);
6858     reinit_heapbase();
6859   }
6860 }
6861 
6862 void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
6863   // Note: it will change flags
6864   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6865   if (dst == src) {
6866     decode_klass_not_null(dst);
6867   } else {
6868     // Cannot assert, unverified entry point counts instructions (see .ad file)
6869     // vtableStubs also counts instructions in pd_code_size_limit.
6870     // Also do not verify_oop as this is called by verify_oop.
6871     mov64(dst, (int64_t)Universe::narrow_klass_base());
6872     if (Universe::narrow_klass_shift() != 0) {
6873       assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6874       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
6875       leaq(dst, Address(dst, src, Address::times_8, 0));
6876     } else {
6877       addq(dst, src);
6878     }
6879   }
6880 }
6881 
6882 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
6883   assert (UseCompressedOops, "should only be used for compressed headers");
6884   assert (Universe::heap() != NULL, "java heap should be initialized");
6885   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6886   int oop_index = oop_recorder()->find_index(obj);
6887   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6888   mov_narrow_oop(dst, oop_index, rspec);
6889 }
6890 
6891 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
6892   assert (UseCompressedOops, "should only be used for compressed headers");
6893   assert (Universe::heap() != NULL, "java heap should be initialized");
6894   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6895   int oop_index = oop_recorder()->find_index(obj);
6896   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6897   mov_narrow_oop(dst, oop_index, rspec);
6898 }
6899 
6900 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
6901   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6902   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6903   int klass_index = oop_recorder()->find_index(k);
6904   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6905   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6906 }
6907 
6908 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
6909   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6910   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6911   int klass_index = oop_recorder()->find_index(k);
6912   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6913   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6914 }
6915 
6916 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
6917   assert (UseCompressedOops, "should only be used for compressed headers");
6918   assert (Universe::heap() != NULL, "java heap should be initialized");
6919   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6920   int oop_index = oop_recorder()->find_index(obj);
6921   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6922   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6923 }
6924 
6925 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
6926   assert (UseCompressedOops, "should only be used for compressed headers");
6927   assert (Universe::heap() != NULL, "java heap should be initialized");
6928   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6929   int oop_index = oop_recorder()->find_index(obj);
6930   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6931   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6932 }
6933 
6934 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
6935   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6936   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6937   int klass_index = oop_recorder()->find_index(k);
6938   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6939   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6940 }
6941 
6942 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
6943   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6944   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6945   int klass_index = oop_recorder()->find_index(k);
6946   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6947   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6948 }
6949 
6950 void MacroAssembler::reinit_heapbase() {
6951   if (UseCompressedOops || UseCompressedClassPointers) {
6952     if (Universe::heap() != NULL) {
6953       if (Universe::narrow_oop_base() == NULL) {
6954         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
6955       } else {
6956         mov64(r12_heapbase, (int64_t)Universe::narrow_ptrs_base());
6957       }
6958     } else {
6959       movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6960     }
6961   }
6962 }
6963 
6964 #endif // _LP64
6965 
6966 // C2 compiled method's prolog code.
6967 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b) {
6968 
6969   // WARNING: Initial instruction MUST be 5 bytes or longer so that
6970   // NativeJump::patch_verified_entry will be able to patch out the entry
6971   // code safely. The push to verify stack depth is ok at 5 bytes,
6972   // the frame allocation can be either 3 or 6 bytes. So if we don't do
6973   // stack bang then we must use the 6 byte frame allocation even if
6974   // we have no frame. :-(
6975   assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
6976 
6977   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
6978   // Remove word for return addr
6979   framesize -= wordSize;
6980   stack_bang_size -= wordSize;
6981 
6982   // Calls to C2R adapters often do not accept exceptional returns.
6983   // We require that their callers must bang for them.  But be careful, because
6984   // some VM calls (such as call site linkage) can use several kilobytes of
6985   // stack.  But the stack safety zone should account for that.
6986   // See bugs 4446381, 4468289, 4497237.
6987   if (stack_bang_size > 0) {
6988     generate_stack_overflow_check(stack_bang_size);
6989 
6990     // We always push rbp, so that on return to interpreter rbp, will be
6991     // restored correctly and we can correct the stack.
6992     push(rbp);
6993     // Save caller's stack pointer into RBP if the frame pointer is preserved.
6994     if (PreserveFramePointer) {
6995       mov(rbp, rsp);
6996     }
6997     // Remove word for ebp
6998     framesize -= wordSize;
6999 
7000     // Create frame
7001     if (framesize) {
7002       subptr(rsp, framesize);
7003     }
7004   } else {
7005     // Create frame (force generation of a 4 byte immediate value)
7006     subptr_imm32(rsp, framesize);
7007 
7008     // Save RBP register now.
7009     framesize -= wordSize;
7010     movptr(Address(rsp, framesize), rbp);
7011     // Save caller's stack pointer into RBP if the frame pointer is preserved.
7012     if (PreserveFramePointer) {
7013       movptr(rbp, rsp);
7014       if (framesize > 0) {
7015         addptr(rbp, framesize);
7016       }
7017     }
7018   }
7019 
7020   if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
7021     framesize -= wordSize;
7022     movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
7023   }
7024 
7025 #ifndef _LP64
7026   // If method sets FPU control word do it now
7027   if (fp_mode_24b) {
7028     fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
7029   }
7030   if (UseSSE >= 2 && VerifyFPU) {
7031     verify_FPU(0, "FPU stack must be clean on entry");
7032   }
7033 #endif
7034 
7035 #ifdef ASSERT
7036   if (VerifyStackAtCalls) {
7037     Label L;
7038     push(rax);
7039     mov(rax, rsp);
7040     andptr(rax, StackAlignmentInBytes-1);
7041     cmpptr(rax, StackAlignmentInBytes-wordSize);
7042     pop(rax);
7043     jcc(Assembler::equal, L);
7044     STOP("Stack is not properly aligned!");
7045     bind(L);
7046   }
7047 #endif
7048 
7049 }
7050 
7051 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, bool is_large) {
7052   // cnt - number of qwords (8-byte words).
7053   // base - start address, qword aligned.
7054   // is_large - if optimizers know cnt is larger than InitArrayShortSize
7055   assert(base==rdi, "base register must be edi for rep stos");
7056   assert(tmp==rax,   "tmp register must be eax for rep stos");
7057   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
7058   assert(InitArrayShortSize % BytesPerLong == 0,
7059     "InitArrayShortSize should be the multiple of BytesPerLong");
7060 
7061   Label DONE;
7062 
7063   xorptr(tmp, tmp);
7064 
7065   if (!is_large) {
7066     Label LOOP, LONG;
7067     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
7068     jccb(Assembler::greater, LONG);
7069 
7070     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
7071 
7072     decrement(cnt);
7073     jccb(Assembler::negative, DONE); // Zero length
7074 
7075     // Use individual pointer-sized stores for small counts:
7076     BIND(LOOP);
7077     movptr(Address(base, cnt, Address::times_ptr), tmp);
7078     decrement(cnt);
7079     jccb(Assembler::greaterEqual, LOOP);
7080     jmpb(DONE);
7081 
7082     BIND(LONG);
7083   }
7084 
7085   // Use longer rep-prefixed ops for non-small counts:
7086   if (UseFastStosb) {
7087     shlptr(cnt, 3); // convert to number of bytes
7088     rep_stosb();
7089   } else {
7090     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
7091     rep_stos();
7092   }
7093 
7094   BIND(DONE);
7095 }
7096 
7097 #ifdef COMPILER2
7098 
7099 // IndexOf for constant substrings with size >= 8 chars
7100 // which don't need to be loaded through stack.
7101 void MacroAssembler::string_indexofC8(Register str1, Register str2,
7102                                       Register cnt1, Register cnt2,
7103                                       int int_cnt2,  Register result,
7104                                       XMMRegister vec, Register tmp,
7105                                       int ae) {
7106   ShortBranchVerifier sbv(this);
7107   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7108   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
7109 
7110   // This method uses the pcmpestri instruction with bound registers
7111   //   inputs:
7112   //     xmm - substring
7113   //     rax - substring length (elements count)
7114   //     mem - scanned string
7115   //     rdx - string length (elements count)
7116   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
7117   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
7118   //   outputs:
7119   //     rcx - matched index in string
7120   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
7121   int mode   = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
7122   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
7123   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
7124   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
7125 
7126   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR,
7127         RET_FOUND, RET_NOT_FOUND, EXIT, FOUND_SUBSTR,
7128         MATCH_SUBSTR_HEAD, RELOAD_STR, FOUND_CANDIDATE;
7129 
7130   // Note, inline_string_indexOf() generates checks:
7131   // if (substr.count > string.count) return -1;
7132   // if (substr.count == 0) return 0;
7133   assert(int_cnt2 >= stride, "this code is used only for cnt2 >= 8 chars");
7134 
7135   // Load substring.
7136   if (ae == StrIntrinsicNode::UL) {
7137     pmovzxbw(vec, Address(str2, 0));
7138   } else {
7139     movdqu(vec, Address(str2, 0));
7140   }
7141   movl(cnt2, int_cnt2);
7142   movptr(result, str1); // string addr
7143 
7144   if (int_cnt2 > stride) {
7145     jmpb(SCAN_TO_SUBSTR);
7146 
7147     // Reload substr for rescan, this code
7148     // is executed only for large substrings (> 8 chars)
7149     bind(RELOAD_SUBSTR);
7150     if (ae == StrIntrinsicNode::UL) {
7151       pmovzxbw(vec, Address(str2, 0));
7152     } else {
7153       movdqu(vec, Address(str2, 0));
7154     }
7155     negptr(cnt2); // Jumped here with negative cnt2, convert to positive
7156 
7157     bind(RELOAD_STR);
7158     // We came here after the beginning of the substring was
7159     // matched but the rest of it was not so we need to search
7160     // again. Start from the next element after the previous match.
7161 
7162     // cnt2 is number of substring reminding elements and
7163     // cnt1 is number of string reminding elements when cmp failed.
7164     // Restored cnt1 = cnt1 - cnt2 + int_cnt2
7165     subl(cnt1, cnt2);
7166     addl(cnt1, int_cnt2);
7167     movl(cnt2, int_cnt2); // Now restore cnt2
7168 
7169     decrementl(cnt1);     // Shift to next element
7170     cmpl(cnt1, cnt2);
7171     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7172 
7173     addptr(result, (1<<scale1));
7174 
7175   } // (int_cnt2 > 8)
7176 
7177   // Scan string for start of substr in 16-byte vectors
7178   bind(SCAN_TO_SUBSTR);
7179   pcmpestri(vec, Address(result, 0), mode);
7180   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
7181   subl(cnt1, stride);
7182   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
7183   cmpl(cnt1, cnt2);
7184   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7185   addptr(result, 16);
7186   jmpb(SCAN_TO_SUBSTR);
7187 
7188   // Found a potential substr
7189   bind(FOUND_CANDIDATE);
7190   // Matched whole vector if first element matched (tmp(rcx) == 0).
7191   if (int_cnt2 == stride) {
7192     jccb(Assembler::overflow, RET_FOUND);    // OF == 1
7193   } else { // int_cnt2 > 8
7194     jccb(Assembler::overflow, FOUND_SUBSTR);
7195   }
7196   // After pcmpestri tmp(rcx) contains matched element index
7197   // Compute start addr of substr
7198   lea(result, Address(result, tmp, scale1));
7199 
7200   // Make sure string is still long enough
7201   subl(cnt1, tmp);
7202   cmpl(cnt1, cnt2);
7203   if (int_cnt2 == stride) {
7204     jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
7205   } else { // int_cnt2 > 8
7206     jccb(Assembler::greaterEqual, MATCH_SUBSTR_HEAD);
7207   }
7208   // Left less then substring.
7209 
7210   bind(RET_NOT_FOUND);
7211   movl(result, -1);
7212   jmp(EXIT);
7213 
7214   if (int_cnt2 > stride) {
7215     // This code is optimized for the case when whole substring
7216     // is matched if its head is matched.
7217     bind(MATCH_SUBSTR_HEAD);
7218     pcmpestri(vec, Address(result, 0), mode);
7219     // Reload only string if does not match
7220     jcc(Assembler::noOverflow, RELOAD_STR); // OF == 0
7221 
7222     Label CONT_SCAN_SUBSTR;
7223     // Compare the rest of substring (> 8 chars).
7224     bind(FOUND_SUBSTR);
7225     // First 8 chars are already matched.
7226     negptr(cnt2);
7227     addptr(cnt2, stride);
7228 
7229     bind(SCAN_SUBSTR);
7230     subl(cnt1, stride);
7231     cmpl(cnt2, -stride); // Do not read beyond substring
7232     jccb(Assembler::lessEqual, CONT_SCAN_SUBSTR);
7233     // Back-up strings to avoid reading beyond substring:
7234     // cnt1 = cnt1 - cnt2 + 8
7235     addl(cnt1, cnt2); // cnt2 is negative
7236     addl(cnt1, stride);
7237     movl(cnt2, stride); negptr(cnt2);
7238     bind(CONT_SCAN_SUBSTR);
7239     if (int_cnt2 < (int)G) {
7240       int tail_off1 = int_cnt2<<scale1;
7241       int tail_off2 = int_cnt2<<scale2;
7242       if (ae == StrIntrinsicNode::UL) {
7243         pmovzxbw(vec, Address(str2, cnt2, scale2, tail_off2));
7244       } else {
7245         movdqu(vec, Address(str2, cnt2, scale2, tail_off2));
7246       }
7247       pcmpestri(vec, Address(result, cnt2, scale1, tail_off1), mode);
7248     } else {
7249       // calculate index in register to avoid integer overflow (int_cnt2*2)
7250       movl(tmp, int_cnt2);
7251       addptr(tmp, cnt2);
7252       if (ae == StrIntrinsicNode::UL) {
7253         pmovzxbw(vec, Address(str2, tmp, scale2, 0));
7254       } else {
7255         movdqu(vec, Address(str2, tmp, scale2, 0));
7256       }
7257       pcmpestri(vec, Address(result, tmp, scale1, 0), mode);
7258     }
7259     // Need to reload strings pointers if not matched whole vector
7260     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
7261     addptr(cnt2, stride);
7262     jcc(Assembler::negative, SCAN_SUBSTR);
7263     // Fall through if found full substring
7264 
7265   } // (int_cnt2 > 8)
7266 
7267   bind(RET_FOUND);
7268   // Found result if we matched full small substring.
7269   // Compute substr offset
7270   subptr(result, str1);
7271   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7272     shrl(result, 1); // index
7273   }
7274   bind(EXIT);
7275 
7276 } // string_indexofC8
7277 
7278 // Small strings are loaded through stack if they cross page boundary.
7279 void MacroAssembler::string_indexof(Register str1, Register str2,
7280                                     Register cnt1, Register cnt2,
7281                                     int int_cnt2,  Register result,
7282                                     XMMRegister vec, Register tmp,
7283                                     int ae) {
7284   ShortBranchVerifier sbv(this);
7285   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7286   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
7287 
7288   //
7289   // int_cnt2 is length of small (< 8 chars) constant substring
7290   // or (-1) for non constant substring in which case its length
7291   // is in cnt2 register.
7292   //
7293   // Note, inline_string_indexOf() generates checks:
7294   // if (substr.count > string.count) return -1;
7295   // if (substr.count == 0) return 0;
7296   //
7297   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
7298   assert(int_cnt2 == -1 || (0 < int_cnt2 && int_cnt2 < stride), "should be != 0");
7299   // This method uses the pcmpestri instruction with bound registers
7300   //   inputs:
7301   //     xmm - substring
7302   //     rax - substring length (elements count)
7303   //     mem - scanned string
7304   //     rdx - string length (elements count)
7305   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
7306   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
7307   //   outputs:
7308   //     rcx - matched index in string
7309   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
7310   int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
7311   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
7312   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
7313 
7314   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, ADJUST_STR,
7315         RET_FOUND, RET_NOT_FOUND, CLEANUP, FOUND_SUBSTR,
7316         FOUND_CANDIDATE;
7317 
7318   { //========================================================
7319     // We don't know where these strings are located
7320     // and we can't read beyond them. Load them through stack.
7321     Label BIG_STRINGS, CHECK_STR, COPY_SUBSTR, COPY_STR;
7322 
7323     movptr(tmp, rsp); // save old SP
7324 
7325     if (int_cnt2 > 0) {     // small (< 8 chars) constant substring
7326       if (int_cnt2 == (1>>scale2)) { // One byte
7327         assert((ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL), "Only possible for latin1 encoding");
7328         load_unsigned_byte(result, Address(str2, 0));
7329         movdl(vec, result); // move 32 bits
7330       } else if (ae == StrIntrinsicNode::LL && int_cnt2 == 3) {  // Three bytes
7331         // Not enough header space in 32-bit VM: 12+3 = 15.
7332         movl(result, Address(str2, -1));
7333         shrl(result, 8);
7334         movdl(vec, result); // move 32 bits
7335       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (2>>scale2)) {  // One char
7336         load_unsigned_short(result, Address(str2, 0));
7337         movdl(vec, result); // move 32 bits
7338       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (4>>scale2)) { // Two chars
7339         movdl(vec, Address(str2, 0)); // move 32 bits
7340       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (8>>scale2)) { // Four chars
7341         movq(vec, Address(str2, 0));  // move 64 bits
7342       } else { // cnt2 = { 3, 5, 6, 7 } || (ae == StrIntrinsicNode::UL && cnt2 ={2, ..., 7})
7343         // Array header size is 12 bytes in 32-bit VM
7344         // + 6 bytes for 3 chars == 18 bytes,
7345         // enough space to load vec and shift.
7346         assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity");
7347         if (ae == StrIntrinsicNode::UL) {
7348           int tail_off = int_cnt2-8;
7349           pmovzxbw(vec, Address(str2, tail_off));
7350           psrldq(vec, -2*tail_off);
7351         }
7352         else {
7353           int tail_off = int_cnt2*(1<<scale2);
7354           movdqu(vec, Address(str2, tail_off-16));
7355           psrldq(vec, 16-tail_off);
7356         }
7357       }
7358     } else { // not constant substring
7359       cmpl(cnt2, stride);
7360       jccb(Assembler::aboveEqual, BIG_STRINGS); // Both strings are big enough
7361 
7362       // We can read beyond string if srt+16 does not cross page boundary
7363       // since heaps are aligned and mapped by pages.
7364       assert(os::vm_page_size() < (int)G, "default page should be small");
7365       movl(result, str2); // We need only low 32 bits
7366       andl(result, (os::vm_page_size()-1));
7367       cmpl(result, (os::vm_page_size()-16));
7368       jccb(Assembler::belowEqual, CHECK_STR);
7369 
7370       // Move small strings to stack to allow load 16 bytes into vec.
7371       subptr(rsp, 16);
7372       int stk_offset = wordSize-(1<<scale2);
7373       push(cnt2);
7374 
7375       bind(COPY_SUBSTR);
7376       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL) {
7377         load_unsigned_byte(result, Address(str2, cnt2, scale2, -1));
7378         movb(Address(rsp, cnt2, scale2, stk_offset), result);
7379       } else if (ae == StrIntrinsicNode::UU) {
7380         load_unsigned_short(result, Address(str2, cnt2, scale2, -2));
7381         movw(Address(rsp, cnt2, scale2, stk_offset), result);
7382       }
7383       decrement(cnt2);
7384       jccb(Assembler::notZero, COPY_SUBSTR);
7385 
7386       pop(cnt2);
7387       movptr(str2, rsp);  // New substring address
7388     } // non constant
7389 
7390     bind(CHECK_STR);
7391     cmpl(cnt1, stride);
7392     jccb(Assembler::aboveEqual, BIG_STRINGS);
7393 
7394     // Check cross page boundary.
7395     movl(result, str1); // We need only low 32 bits
7396     andl(result, (os::vm_page_size()-1));
7397     cmpl(result, (os::vm_page_size()-16));
7398     jccb(Assembler::belowEqual, BIG_STRINGS);
7399 
7400     subptr(rsp, 16);
7401     int stk_offset = -(1<<scale1);
7402     if (int_cnt2 < 0) { // not constant
7403       push(cnt2);
7404       stk_offset += wordSize;
7405     }
7406     movl(cnt2, cnt1);
7407 
7408     bind(COPY_STR);
7409     if (ae == StrIntrinsicNode::LL) {
7410       load_unsigned_byte(result, Address(str1, cnt2, scale1, -1));
7411       movb(Address(rsp, cnt2, scale1, stk_offset), result);
7412     } else {
7413       load_unsigned_short(result, Address(str1, cnt2, scale1, -2));
7414       movw(Address(rsp, cnt2, scale1, stk_offset), result);
7415     }
7416     decrement(cnt2);
7417     jccb(Assembler::notZero, COPY_STR);
7418 
7419     if (int_cnt2 < 0) { // not constant
7420       pop(cnt2);
7421     }
7422     movptr(str1, rsp);  // New string address
7423 
7424     bind(BIG_STRINGS);
7425     // Load substring.
7426     if (int_cnt2 < 0) { // -1
7427       if (ae == StrIntrinsicNode::UL) {
7428         pmovzxbw(vec, Address(str2, 0));
7429       } else {
7430         movdqu(vec, Address(str2, 0));
7431       }
7432       push(cnt2);       // substr count
7433       push(str2);       // substr addr
7434       push(str1);       // string addr
7435     } else {
7436       // Small (< 8 chars) constant substrings are loaded already.
7437       movl(cnt2, int_cnt2);
7438     }
7439     push(tmp);  // original SP
7440 
7441   } // Finished loading
7442 
7443   //========================================================
7444   // Start search
7445   //
7446 
7447   movptr(result, str1); // string addr
7448 
7449   if (int_cnt2  < 0) {  // Only for non constant substring
7450     jmpb(SCAN_TO_SUBSTR);
7451 
7452     // SP saved at sp+0
7453     // String saved at sp+1*wordSize
7454     // Substr saved at sp+2*wordSize
7455     // Substr count saved at sp+3*wordSize
7456 
7457     // Reload substr for rescan, this code
7458     // is executed only for large substrings (> 8 chars)
7459     bind(RELOAD_SUBSTR);
7460     movptr(str2, Address(rsp, 2*wordSize));
7461     movl(cnt2, Address(rsp, 3*wordSize));
7462     if (ae == StrIntrinsicNode::UL) {
7463       pmovzxbw(vec, Address(str2, 0));
7464     } else {
7465       movdqu(vec, Address(str2, 0));
7466     }
7467     // We came here after the beginning of the substring was
7468     // matched but the rest of it was not so we need to search
7469     // again. Start from the next element after the previous match.
7470     subptr(str1, result); // Restore counter
7471     if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7472       shrl(str1, 1);
7473     }
7474     addl(cnt1, str1);
7475     decrementl(cnt1);   // Shift to next element
7476     cmpl(cnt1, cnt2);
7477     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7478 
7479     addptr(result, (1<<scale1));
7480   } // non constant
7481 
7482   // Scan string for start of substr in 16-byte vectors
7483   bind(SCAN_TO_SUBSTR);
7484   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
7485   pcmpestri(vec, Address(result, 0), mode);
7486   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
7487   subl(cnt1, stride);
7488   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
7489   cmpl(cnt1, cnt2);
7490   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7491   addptr(result, 16);
7492 
7493   bind(ADJUST_STR);
7494   cmpl(cnt1, stride); // Do not read beyond string
7495   jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
7496   // Back-up string to avoid reading beyond string.
7497   lea(result, Address(result, cnt1, scale1, -16));
7498   movl(cnt1, stride);
7499   jmpb(SCAN_TO_SUBSTR);
7500 
7501   // Found a potential substr
7502   bind(FOUND_CANDIDATE);
7503   // After pcmpestri tmp(rcx) contains matched element index
7504 
7505   // Make sure string is still long enough
7506   subl(cnt1, tmp);
7507   cmpl(cnt1, cnt2);
7508   jccb(Assembler::greaterEqual, FOUND_SUBSTR);
7509   // Left less then substring.
7510 
7511   bind(RET_NOT_FOUND);
7512   movl(result, -1);
7513   jmpb(CLEANUP);
7514 
7515   bind(FOUND_SUBSTR);
7516   // Compute start addr of substr
7517   lea(result, Address(result, tmp, scale1));
7518   if (int_cnt2 > 0) { // Constant substring
7519     // Repeat search for small substring (< 8 chars)
7520     // from new point without reloading substring.
7521     // Have to check that we don't read beyond string.
7522     cmpl(tmp, stride-int_cnt2);
7523     jccb(Assembler::greater, ADJUST_STR);
7524     // Fall through if matched whole substring.
7525   } else { // non constant
7526     assert(int_cnt2 == -1, "should be != 0");
7527 
7528     addl(tmp, cnt2);
7529     // Found result if we matched whole substring.
7530     cmpl(tmp, stride);
7531     jccb(Assembler::lessEqual, RET_FOUND);
7532 
7533     // Repeat search for small substring (<= 8 chars)
7534     // from new point 'str1' without reloading substring.
7535     cmpl(cnt2, stride);
7536     // Have to check that we don't read beyond string.
7537     jccb(Assembler::lessEqual, ADJUST_STR);
7538 
7539     Label CHECK_NEXT, CONT_SCAN_SUBSTR, RET_FOUND_LONG;
7540     // Compare the rest of substring (> 8 chars).
7541     movptr(str1, result);
7542 
7543     cmpl(tmp, cnt2);
7544     // First 8 chars are already matched.
7545     jccb(Assembler::equal, CHECK_NEXT);
7546 
7547     bind(SCAN_SUBSTR);
7548     pcmpestri(vec, Address(str1, 0), mode);
7549     // Need to reload strings pointers if not matched whole vector
7550     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
7551 
7552     bind(CHECK_NEXT);
7553     subl(cnt2, stride);
7554     jccb(Assembler::lessEqual, RET_FOUND_LONG); // Found full substring
7555     addptr(str1, 16);
7556     if (ae == StrIntrinsicNode::UL) {
7557       addptr(str2, 8);
7558     } else {
7559       addptr(str2, 16);
7560     }
7561     subl(cnt1, stride);
7562     cmpl(cnt2, stride); // Do not read beyond substring
7563     jccb(Assembler::greaterEqual, CONT_SCAN_SUBSTR);
7564     // Back-up strings to avoid reading beyond substring.
7565 
7566     if (ae == StrIntrinsicNode::UL) {
7567       lea(str2, Address(str2, cnt2, scale2, -8));
7568       lea(str1, Address(str1, cnt2, scale1, -16));
7569     } else {
7570       lea(str2, Address(str2, cnt2, scale2, -16));
7571       lea(str1, Address(str1, cnt2, scale1, -16));
7572     }
7573     subl(cnt1, cnt2);
7574     movl(cnt2, stride);
7575     addl(cnt1, stride);
7576     bind(CONT_SCAN_SUBSTR);
7577     if (ae == StrIntrinsicNode::UL) {
7578       pmovzxbw(vec, Address(str2, 0));
7579     } else {
7580       movdqu(vec, Address(str2, 0));
7581     }
7582     jmp(SCAN_SUBSTR);
7583 
7584     bind(RET_FOUND_LONG);
7585     movptr(str1, Address(rsp, wordSize));
7586   } // non constant
7587 
7588   bind(RET_FOUND);
7589   // Compute substr offset
7590   subptr(result, str1);
7591   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7592     shrl(result, 1); // index
7593   }
7594   bind(CLEANUP);
7595   pop(rsp); // restore SP
7596 
7597 } // string_indexof
7598 
7599 void MacroAssembler::string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
7600                                          XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp) {
7601   ShortBranchVerifier sbv(this);
7602   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7603 
7604   int stride = 8;
7605 
7606   Label FOUND_CHAR, SCAN_TO_CHAR, SCAN_TO_CHAR_LOOP,
7607         SCAN_TO_8_CHAR, SCAN_TO_8_CHAR_LOOP, SCAN_TO_16_CHAR_LOOP,
7608         RET_NOT_FOUND, SCAN_TO_8_CHAR_INIT,
7609         FOUND_SEQ_CHAR, DONE_LABEL;
7610 
7611   movptr(result, str1);
7612   if (UseAVX >= 2) {
7613     cmpl(cnt1, stride);
7614     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7615     cmpl(cnt1, 2*stride);
7616     jcc(Assembler::less, SCAN_TO_8_CHAR_INIT);
7617     movdl(vec1, ch);
7618     vpbroadcastw(vec1, vec1);
7619     vpxor(vec2, vec2);
7620     movl(tmp, cnt1);
7621     andl(tmp, 0xFFFFFFF0);  //vector count (in chars)
7622     andl(cnt1,0x0000000F);  //tail count (in chars)
7623 
7624     bind(SCAN_TO_16_CHAR_LOOP);
7625     vmovdqu(vec3, Address(result, 0));
7626     vpcmpeqw(vec3, vec3, vec1, 1);
7627     vptest(vec2, vec3);
7628     jcc(Assembler::carryClear, FOUND_CHAR);
7629     addptr(result, 32);
7630     subl(tmp, 2*stride);
7631     jccb(Assembler::notZero, SCAN_TO_16_CHAR_LOOP);
7632     jmp(SCAN_TO_8_CHAR);
7633     bind(SCAN_TO_8_CHAR_INIT);
7634     movdl(vec1, ch);
7635     pshuflw(vec1, vec1, 0x00);
7636     pshufd(vec1, vec1, 0);
7637     pxor(vec2, vec2);
7638   }
7639   bind(SCAN_TO_8_CHAR);
7640   cmpl(cnt1, stride);
7641   if (UseAVX >= 2) {
7642     jcc(Assembler::less, SCAN_TO_CHAR);
7643   } else {
7644     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7645     movdl(vec1, ch);
7646     pshuflw(vec1, vec1, 0x00);
7647     pshufd(vec1, vec1, 0);
7648     pxor(vec2, vec2);
7649   }
7650   movl(tmp, cnt1);
7651   andl(tmp, 0xFFFFFFF8);  //vector count (in chars)
7652   andl(cnt1,0x00000007);  //tail count (in chars)
7653 
7654   bind(SCAN_TO_8_CHAR_LOOP);
7655   movdqu(vec3, Address(result, 0));
7656   pcmpeqw(vec3, vec1);
7657   ptest(vec2, vec3);
7658   jcc(Assembler::carryClear, FOUND_CHAR);
7659   addptr(result, 16);
7660   subl(tmp, stride);
7661   jccb(Assembler::notZero, SCAN_TO_8_CHAR_LOOP);
7662   bind(SCAN_TO_CHAR);
7663   testl(cnt1, cnt1);
7664   jcc(Assembler::zero, RET_NOT_FOUND);
7665   bind(SCAN_TO_CHAR_LOOP);
7666   load_unsigned_short(tmp, Address(result, 0));
7667   cmpl(ch, tmp);
7668   jccb(Assembler::equal, FOUND_SEQ_CHAR);
7669   addptr(result, 2);
7670   subl(cnt1, 1);
7671   jccb(Assembler::zero, RET_NOT_FOUND);
7672   jmp(SCAN_TO_CHAR_LOOP);
7673 
7674   bind(RET_NOT_FOUND);
7675   movl(result, -1);
7676   jmpb(DONE_LABEL);
7677 
7678   bind(FOUND_CHAR);
7679   if (UseAVX >= 2) {
7680     vpmovmskb(tmp, vec3);
7681   } else {
7682     pmovmskb(tmp, vec3);
7683   }
7684   bsfl(ch, tmp);
7685   addl(result, ch);
7686 
7687   bind(FOUND_SEQ_CHAR);
7688   subptr(result, str1);
7689   shrl(result, 1);
7690 
7691   bind(DONE_LABEL);
7692 } // string_indexof_char
7693 
7694 // helper function for string_compare
7695 void MacroAssembler::load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
7696                                         Address::ScaleFactor scale, Address::ScaleFactor scale1,
7697                                         Address::ScaleFactor scale2, Register index, int ae) {
7698   if (ae == StrIntrinsicNode::LL) {
7699     load_unsigned_byte(elem1, Address(str1, index, scale, 0));
7700     load_unsigned_byte(elem2, Address(str2, index, scale, 0));
7701   } else if (ae == StrIntrinsicNode::UU) {
7702     load_unsigned_short(elem1, Address(str1, index, scale, 0));
7703     load_unsigned_short(elem2, Address(str2, index, scale, 0));
7704   } else {
7705     load_unsigned_byte(elem1, Address(str1, index, scale1, 0));
7706     load_unsigned_short(elem2, Address(str2, index, scale2, 0));
7707   }
7708 }
7709 
7710 // Compare strings, used for char[] and byte[].
7711 void MacroAssembler::string_compare(Register str1, Register str2,
7712                                     Register cnt1, Register cnt2, Register result,
7713                                     XMMRegister vec1, int ae) {
7714   ShortBranchVerifier sbv(this);
7715   Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
7716   Label COMPARE_WIDE_VECTORS_LOOP_FAILED;  // used only _LP64 && AVX3
7717   int stride, stride2, adr_stride, adr_stride1, adr_stride2;
7718   int stride2x2 = 0x40;
7719   Address::ScaleFactor scale = Address::no_scale;
7720   Address::ScaleFactor scale1 = Address::no_scale;
7721   Address::ScaleFactor scale2 = Address::no_scale;
7722 
7723   if (ae != StrIntrinsicNode::LL) {
7724     stride2x2 = 0x20;
7725   }
7726 
7727   if (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL) {
7728     shrl(cnt2, 1);
7729   }
7730   // Compute the minimum of the string lengths and the
7731   // difference of the string lengths (stack).
7732   // Do the conditional move stuff
7733   movl(result, cnt1);
7734   subl(cnt1, cnt2);
7735   push(cnt1);
7736   cmov32(Assembler::lessEqual, cnt2, result);    // cnt2 = min(cnt1, cnt2)
7737 
7738   // Is the minimum length zero?
7739   testl(cnt2, cnt2);
7740   jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7741   if (ae == StrIntrinsicNode::LL) {
7742     // Load first bytes
7743     load_unsigned_byte(result, Address(str1, 0));  // result = str1[0]
7744     load_unsigned_byte(cnt1, Address(str2, 0));    // cnt1   = str2[0]
7745   } else if (ae == StrIntrinsicNode::UU) {
7746     // Load first characters
7747     load_unsigned_short(result, Address(str1, 0));
7748     load_unsigned_short(cnt1, Address(str2, 0));
7749   } else {
7750     load_unsigned_byte(result, Address(str1, 0));
7751     load_unsigned_short(cnt1, Address(str2, 0));
7752   }
7753   subl(result, cnt1);
7754   jcc(Assembler::notZero,  POP_LABEL);
7755 
7756   if (ae == StrIntrinsicNode::UU) {
7757     // Divide length by 2 to get number of chars
7758     shrl(cnt2, 1);
7759   }
7760   cmpl(cnt2, 1);
7761   jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7762 
7763   // Check if the strings start at the same location and setup scale and stride
7764   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7765     cmpptr(str1, str2);
7766     jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7767     if (ae == StrIntrinsicNode::LL) {
7768       scale = Address::times_1;
7769       stride = 16;
7770     } else {
7771       scale = Address::times_2;
7772       stride = 8;
7773     }
7774   } else {
7775     scale1 = Address::times_1;
7776     scale2 = Address::times_2;
7777     // scale not used
7778     stride = 8;
7779   }
7780 
7781   if (UseAVX >= 2 && UseSSE42Intrinsics) {
7782     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_WIDE_TAIL, COMPARE_SMALL_STR;
7783     Label COMPARE_WIDE_VECTORS_LOOP, COMPARE_16_CHARS, COMPARE_INDEX_CHAR;
7784     Label COMPARE_WIDE_VECTORS_LOOP_AVX2;
7785     Label COMPARE_TAIL_LONG;
7786     Label COMPARE_WIDE_VECTORS_LOOP_AVX3;  // used only _LP64 && AVX3
7787 
7788     int pcmpmask = 0x19;
7789     if (ae == StrIntrinsicNode::LL) {
7790       pcmpmask &= ~0x01;
7791     }
7792 
7793     // Setup to compare 16-chars (32-bytes) vectors,
7794     // start from first character again because it has aligned address.
7795     if (ae == StrIntrinsicNode::LL) {
7796       stride2 = 32;
7797     } else {
7798       stride2 = 16;
7799     }
7800     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7801       adr_stride = stride << scale;
7802     } else {
7803       adr_stride1 = 8;  //stride << scale1;
7804       adr_stride2 = 16; //stride << scale2;
7805     }
7806 
7807     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
7808     // rax and rdx are used by pcmpestri as elements counters
7809     movl(result, cnt2);
7810     andl(cnt2, ~(stride2-1));   // cnt2 holds the vector count
7811     jcc(Assembler::zero, COMPARE_TAIL_LONG);
7812 
7813     // fast path : compare first 2 8-char vectors.
7814     bind(COMPARE_16_CHARS);
7815     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7816       movdqu(vec1, Address(str1, 0));
7817     } else {
7818       pmovzxbw(vec1, Address(str1, 0));
7819     }
7820     pcmpestri(vec1, Address(str2, 0), pcmpmask);
7821     jccb(Assembler::below, COMPARE_INDEX_CHAR);
7822 
7823     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7824       movdqu(vec1, Address(str1, adr_stride));
7825       pcmpestri(vec1, Address(str2, adr_stride), pcmpmask);
7826     } else {
7827       pmovzxbw(vec1, Address(str1, adr_stride1));
7828       pcmpestri(vec1, Address(str2, adr_stride2), pcmpmask);
7829     }
7830     jccb(Assembler::aboveEqual, COMPARE_WIDE_VECTORS);
7831     addl(cnt1, stride);
7832 
7833     // Compare the characters at index in cnt1
7834     bind(COMPARE_INDEX_CHAR); // cnt1 has the offset of the mismatching character
7835     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
7836     subl(result, cnt2);
7837     jmp(POP_LABEL);
7838 
7839     // Setup the registers to start vector comparison loop
7840     bind(COMPARE_WIDE_VECTORS);
7841     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7842       lea(str1, Address(str1, result, scale));
7843       lea(str2, Address(str2, result, scale));
7844     } else {
7845       lea(str1, Address(str1, result, scale1));
7846       lea(str2, Address(str2, result, scale2));
7847     }
7848     subl(result, stride2);
7849     subl(cnt2, stride2);
7850     jcc(Assembler::zero, COMPARE_WIDE_TAIL);
7851     negptr(result);
7852 
7853     //  In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest)
7854     bind(COMPARE_WIDE_VECTORS_LOOP);
7855 
7856 #ifdef _LP64
7857     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
7858       cmpl(cnt2, stride2x2);
7859       jccb(Assembler::below, COMPARE_WIDE_VECTORS_LOOP_AVX2);
7860       testl(cnt2, stride2x2-1);   // cnt2 holds the vector count
7861       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX2);   // means we cannot subtract by 0x40
7862 
7863       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
7864       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7865         evmovdquq(vec1, Address(str1, result, scale), Assembler::AVX_512bit);
7866         evpcmpeqb(k7, vec1, Address(str2, result, scale), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7867       } else {
7868         vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_512bit);
7869         evpcmpeqb(k7, vec1, Address(str2, result, scale2), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7870       }
7871       kortestql(k7, k7);
7872       jcc(Assembler::aboveEqual, COMPARE_WIDE_VECTORS_LOOP_FAILED);     // miscompare
7873       addptr(result, stride2x2);  // update since we already compared at this addr
7874       subl(cnt2, stride2x2);      // and sub the size too
7875       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX3);
7876 
7877       vpxor(vec1, vec1);
7878       jmpb(COMPARE_WIDE_TAIL);
7879     }//if (VM_Version::supports_avx512vlbw())
7880 #endif // _LP64
7881 
7882 
7883     bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7884     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7885       vmovdqu(vec1, Address(str1, result, scale));
7886       vpxor(vec1, Address(str2, result, scale));
7887     } else {
7888       vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_256bit);
7889       vpxor(vec1, Address(str2, result, scale2));
7890     }
7891     vptest(vec1, vec1);
7892     jcc(Assembler::notZero, VECTOR_NOT_EQUAL);
7893     addptr(result, stride2);
7894     subl(cnt2, stride2);
7895     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP);
7896     // clean upper bits of YMM registers
7897     vpxor(vec1, vec1);
7898 
7899     // compare wide vectors tail
7900     bind(COMPARE_WIDE_TAIL);
7901     testptr(result, result);
7902     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7903 
7904     movl(result, stride2);
7905     movl(cnt2, result);
7906     negptr(result);
7907     jmp(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7908 
7909     // Identifies the mismatching (higher or lower)16-bytes in the 32-byte vectors.
7910     bind(VECTOR_NOT_EQUAL);
7911     // clean upper bits of YMM registers
7912     vpxor(vec1, vec1);
7913     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7914       lea(str1, Address(str1, result, scale));
7915       lea(str2, Address(str2, result, scale));
7916     } else {
7917       lea(str1, Address(str1, result, scale1));
7918       lea(str2, Address(str2, result, scale2));
7919     }
7920     jmp(COMPARE_16_CHARS);
7921 
7922     // Compare tail chars, length between 1 to 15 chars
7923     bind(COMPARE_TAIL_LONG);
7924     movl(cnt2, result);
7925     cmpl(cnt2, stride);
7926     jcc(Assembler::less, COMPARE_SMALL_STR);
7927 
7928     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7929       movdqu(vec1, Address(str1, 0));
7930     } else {
7931       pmovzxbw(vec1, Address(str1, 0));
7932     }
7933     pcmpestri(vec1, Address(str2, 0), pcmpmask);
7934     jcc(Assembler::below, COMPARE_INDEX_CHAR);
7935     subptr(cnt2, stride);
7936     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7937     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7938       lea(str1, Address(str1, result, scale));
7939       lea(str2, Address(str2, result, scale));
7940     } else {
7941       lea(str1, Address(str1, result, scale1));
7942       lea(str2, Address(str2, result, scale2));
7943     }
7944     negptr(cnt2);
7945     jmpb(WHILE_HEAD_LABEL);
7946 
7947     bind(COMPARE_SMALL_STR);
7948   } else if (UseSSE42Intrinsics) {
7949     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
7950     int pcmpmask = 0x19;
7951     // Setup to compare 8-char (16-byte) vectors,
7952     // start from first character again because it has aligned address.
7953     movl(result, cnt2);
7954     andl(cnt2, ~(stride - 1));   // cnt2 holds the vector count
7955     if (ae == StrIntrinsicNode::LL) {
7956       pcmpmask &= ~0x01;
7957     }
7958     jcc(Assembler::zero, COMPARE_TAIL);
7959     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7960       lea(str1, Address(str1, result, scale));
7961       lea(str2, Address(str2, result, scale));
7962     } else {
7963       lea(str1, Address(str1, result, scale1));
7964       lea(str2, Address(str2, result, scale2));
7965     }
7966     negptr(result);
7967 
7968     // pcmpestri
7969     //   inputs:
7970     //     vec1- substring
7971     //     rax - negative string length (elements count)
7972     //     mem - scanned string
7973     //     rdx - string length (elements count)
7974     //     pcmpmask - cmp mode: 11000 (string compare with negated result)
7975     //               + 00 (unsigned bytes) or  + 01 (unsigned shorts)
7976     //   outputs:
7977     //     rcx - first mismatched element index
7978     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
7979 
7980     bind(COMPARE_WIDE_VECTORS);
7981     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7982       movdqu(vec1, Address(str1, result, scale));
7983       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
7984     } else {
7985       pmovzxbw(vec1, Address(str1, result, scale1));
7986       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
7987     }
7988     // After pcmpestri cnt1(rcx) contains mismatched element index
7989 
7990     jccb(Assembler::below, VECTOR_NOT_EQUAL);  // CF==1
7991     addptr(result, stride);
7992     subptr(cnt2, stride);
7993     jccb(Assembler::notZero, COMPARE_WIDE_VECTORS);
7994 
7995     // compare wide vectors tail
7996     testptr(result, result);
7997     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7998 
7999     movl(cnt2, stride);
8000     movl(result, stride);
8001     negptr(result);
8002     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
8003       movdqu(vec1, Address(str1, result, scale));
8004       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
8005     } else {
8006       pmovzxbw(vec1, Address(str1, result, scale1));
8007       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
8008     }
8009     jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL);
8010 
8011     // Mismatched characters in the vectors
8012     bind(VECTOR_NOT_EQUAL);
8013     addptr(cnt1, result);
8014     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
8015     subl(result, cnt2);
8016     jmpb(POP_LABEL);
8017 
8018     bind(COMPARE_TAIL); // limit is zero
8019     movl(cnt2, result);
8020     // Fallthru to tail compare
8021   }
8022   // Shift str2 and str1 to the end of the arrays, negate min
8023   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
8024     lea(str1, Address(str1, cnt2, scale));
8025     lea(str2, Address(str2, cnt2, scale));
8026   } else {
8027     lea(str1, Address(str1, cnt2, scale1));
8028     lea(str2, Address(str2, cnt2, scale2));
8029   }
8030   decrementl(cnt2);  // first character was compared already
8031   negptr(cnt2);
8032 
8033   // Compare the rest of the elements
8034   bind(WHILE_HEAD_LABEL);
8035   load_next_elements(result, cnt1, str1, str2, scale, scale1, scale2, cnt2, ae);
8036   subl(result, cnt1);
8037   jccb(Assembler::notZero, POP_LABEL);
8038   increment(cnt2);
8039   jccb(Assembler::notZero, WHILE_HEAD_LABEL);
8040 
8041   // Strings are equal up to min length.  Return the length difference.
8042   bind(LENGTH_DIFF_LABEL);
8043   pop(result);
8044   if (ae == StrIntrinsicNode::UU) {
8045     // Divide diff by 2 to get number of chars
8046     sarl(result, 1);
8047   }
8048   jmpb(DONE_LABEL);
8049 
8050 #ifdef _LP64
8051   if (VM_Version::supports_avx512vlbw()) {
8052 
8053     bind(COMPARE_WIDE_VECTORS_LOOP_FAILED);
8054 
8055     kmovql(cnt1, k7);
8056     notq(cnt1);
8057     bsfq(cnt2, cnt1);
8058     if (ae != StrIntrinsicNode::LL) {
8059       // Divide diff by 2 to get number of chars
8060       sarl(cnt2, 1);
8061     }
8062     addq(result, cnt2);
8063     if (ae == StrIntrinsicNode::LL) {
8064       load_unsigned_byte(cnt1, Address(str2, result));
8065       load_unsigned_byte(result, Address(str1, result));
8066     } else if (ae == StrIntrinsicNode::UU) {
8067       load_unsigned_short(cnt1, Address(str2, result, scale));
8068       load_unsigned_short(result, Address(str1, result, scale));
8069     } else {
8070       load_unsigned_short(cnt1, Address(str2, result, scale2));
8071       load_unsigned_byte(result, Address(str1, result, scale1));
8072     }
8073     subl(result, cnt1);
8074     jmpb(POP_LABEL);
8075   }//if (VM_Version::supports_avx512vlbw())
8076 #endif // _LP64
8077 
8078   // Discard the stored length difference
8079   bind(POP_LABEL);
8080   pop(cnt1);
8081 
8082   // That's it
8083   bind(DONE_LABEL);
8084   if(ae == StrIntrinsicNode::UL) {
8085     negl(result);
8086   }
8087 
8088 }
8089 
8090 // Search for Non-ASCII character (Negative byte value) in a byte array,
8091 // return true if it has any and false otherwise.
8092 //   ..\jdk\src\java.base\share\classes\java\lang\StringCoding.java
8093 //   @HotSpotIntrinsicCandidate
8094 //   private static boolean hasNegatives(byte[] ba, int off, int len) {
8095 //     for (int i = off; i < off + len; i++) {
8096 //       if (ba[i] < 0) {
8097 //         return true;
8098 //       }
8099 //     }
8100 //     return false;
8101 //   }
8102 void MacroAssembler::has_negatives(Register ary1, Register len,
8103   Register result, Register tmp1,
8104   XMMRegister vec1, XMMRegister vec2) {
8105   // rsi: byte array
8106   // rcx: len
8107   // rax: result
8108   ShortBranchVerifier sbv(this);
8109   assert_different_registers(ary1, len, result, tmp1);
8110   assert_different_registers(vec1, vec2);
8111   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_CHAR, COMPARE_VECTORS, COMPARE_BYTE;
8112 
8113   // len == 0
8114   testl(len, len);
8115   jcc(Assembler::zero, FALSE_LABEL);
8116 
8117   if ((UseAVX > 2) && // AVX512
8118     VM_Version::supports_avx512vlbw() &&
8119     VM_Version::supports_bmi2()) {
8120 
8121     set_vector_masking();  // opening of the stub context for programming mask registers
8122 
8123     Label test_64_loop, test_tail;
8124     Register tmp3_aliased = len;
8125 
8126     movl(tmp1, len);
8127     vpxor(vec2, vec2, vec2, Assembler::AVX_512bit);
8128 
8129     andl(tmp1, 64 - 1);   // tail count (in chars) 0x3F
8130     andl(len, ~(64 - 1));    // vector count (in chars)
8131     jccb(Assembler::zero, test_tail);
8132 
8133     lea(ary1, Address(ary1, len, Address::times_1));
8134     negptr(len);
8135 
8136     bind(test_64_loop);
8137     // Check whether our 64 elements of size byte contain negatives
8138     evpcmpgtb(k2, vec2, Address(ary1, len, Address::times_1), Assembler::AVX_512bit);
8139     kortestql(k2, k2);
8140     jcc(Assembler::notZero, TRUE_LABEL);
8141 
8142     addptr(len, 64);
8143     jccb(Assembler::notZero, test_64_loop);
8144 
8145 
8146     bind(test_tail);
8147     // bail out when there is nothing to be done
8148     testl(tmp1, -1);
8149     jcc(Assembler::zero, FALSE_LABEL);
8150 
8151     // Save k1
8152     kmovql(k3, k1);
8153 
8154     // ~(~0 << len) applied up to two times (for 32-bit scenario)
8155 #ifdef _LP64
8156     mov64(tmp3_aliased, 0xFFFFFFFFFFFFFFFF);
8157     shlxq(tmp3_aliased, tmp3_aliased, tmp1);
8158     notq(tmp3_aliased);
8159     kmovql(k1, tmp3_aliased);
8160 #else
8161     Label k_init;
8162     jmp(k_init);
8163 
8164     // We could not read 64-bits from a general purpose register thus we move
8165     // data required to compose 64 1's to the instruction stream
8166     // We emit 64 byte wide series of elements from 0..63 which later on would
8167     // be used as a compare targets with tail count contained in tmp1 register.
8168     // Result would be a k1 register having tmp1 consecutive number or 1
8169     // counting from least significant bit.
8170     address tmp = pc();
8171     emit_int64(0x0706050403020100);
8172     emit_int64(0x0F0E0D0C0B0A0908);
8173     emit_int64(0x1716151413121110);
8174     emit_int64(0x1F1E1D1C1B1A1918);
8175     emit_int64(0x2726252423222120);
8176     emit_int64(0x2F2E2D2C2B2A2928);
8177     emit_int64(0x3736353433323130);
8178     emit_int64(0x3F3E3D3C3B3A3938);
8179 
8180     bind(k_init);
8181     lea(len, InternalAddress(tmp));
8182     // create mask to test for negative byte inside a vector
8183     evpbroadcastb(vec1, tmp1, Assembler::AVX_512bit);
8184     evpcmpgtb(k1, vec1, Address(len, 0), Assembler::AVX_512bit);
8185 
8186 #endif
8187     evpcmpgtb(k2, k1, vec2, Address(ary1, 0), Assembler::AVX_512bit);
8188     ktestq(k2, k1);
8189     // Restore k1
8190     kmovql(k1, k3);
8191     jcc(Assembler::notZero, TRUE_LABEL);
8192 
8193     jmp(FALSE_LABEL);
8194 
8195     clear_vector_masking();   // closing of the stub context for programming mask registers
8196   } else {
8197     movl(result, len); // copy
8198 
8199     if (UseAVX == 2 && UseSSE >= 2) {
8200       // With AVX2, use 32-byte vector compare
8201       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8202 
8203       // Compare 32-byte vectors
8204       andl(result, 0x0000001f);  //   tail count (in bytes)
8205       andl(len, 0xffffffe0);   // vector count (in bytes)
8206       jccb(Assembler::zero, COMPARE_TAIL);
8207 
8208       lea(ary1, Address(ary1, len, Address::times_1));
8209       negptr(len);
8210 
8211       movl(tmp1, 0x80808080);   // create mask to test for Unicode chars in vector
8212       movdl(vec2, tmp1);
8213       vpbroadcastd(vec2, vec2);
8214 
8215       bind(COMPARE_WIDE_VECTORS);
8216       vmovdqu(vec1, Address(ary1, len, Address::times_1));
8217       vptest(vec1, vec2);
8218       jccb(Assembler::notZero, TRUE_LABEL);
8219       addptr(len, 32);
8220       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8221 
8222       testl(result, result);
8223       jccb(Assembler::zero, FALSE_LABEL);
8224 
8225       vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
8226       vptest(vec1, vec2);
8227       jccb(Assembler::notZero, TRUE_LABEL);
8228       jmpb(FALSE_LABEL);
8229 
8230       bind(COMPARE_TAIL); // len is zero
8231       movl(len, result);
8232       // Fallthru to tail compare
8233     } else if (UseSSE42Intrinsics) {
8234       // With SSE4.2, use double quad vector compare
8235       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8236 
8237       // Compare 16-byte vectors
8238       andl(result, 0x0000000f);  //   tail count (in bytes)
8239       andl(len, 0xfffffff0);   // vector count (in bytes)
8240       jccb(Assembler::zero, COMPARE_TAIL);
8241 
8242       lea(ary1, Address(ary1, len, Address::times_1));
8243       negptr(len);
8244 
8245       movl(tmp1, 0x80808080);
8246       movdl(vec2, tmp1);
8247       pshufd(vec2, vec2, 0);
8248 
8249       bind(COMPARE_WIDE_VECTORS);
8250       movdqu(vec1, Address(ary1, len, Address::times_1));
8251       ptest(vec1, vec2);
8252       jccb(Assembler::notZero, TRUE_LABEL);
8253       addptr(len, 16);
8254       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8255 
8256       testl(result, result);
8257       jccb(Assembler::zero, FALSE_LABEL);
8258 
8259       movdqu(vec1, Address(ary1, result, Address::times_1, -16));
8260       ptest(vec1, vec2);
8261       jccb(Assembler::notZero, TRUE_LABEL);
8262       jmpb(FALSE_LABEL);
8263 
8264       bind(COMPARE_TAIL); // len is zero
8265       movl(len, result);
8266       // Fallthru to tail compare
8267     }
8268   }
8269   // Compare 4-byte vectors
8270   andl(len, 0xfffffffc); // vector count (in bytes)
8271   jccb(Assembler::zero, COMPARE_CHAR);
8272 
8273   lea(ary1, Address(ary1, len, Address::times_1));
8274   negptr(len);
8275 
8276   bind(COMPARE_VECTORS);
8277   movl(tmp1, Address(ary1, len, Address::times_1));
8278   andl(tmp1, 0x80808080);
8279   jccb(Assembler::notZero, TRUE_LABEL);
8280   addptr(len, 4);
8281   jcc(Assembler::notZero, COMPARE_VECTORS);
8282 
8283   // Compare trailing char (final 2 bytes), if any
8284   bind(COMPARE_CHAR);
8285   testl(result, 0x2);   // tail  char
8286   jccb(Assembler::zero, COMPARE_BYTE);
8287   load_unsigned_short(tmp1, Address(ary1, 0));
8288   andl(tmp1, 0x00008080);
8289   jccb(Assembler::notZero, TRUE_LABEL);
8290   subptr(result, 2);
8291   lea(ary1, Address(ary1, 2));
8292 
8293   bind(COMPARE_BYTE);
8294   testl(result, 0x1);   // tail  byte
8295   jccb(Assembler::zero, FALSE_LABEL);
8296   load_unsigned_byte(tmp1, Address(ary1, 0));
8297   andl(tmp1, 0x00000080);
8298   jccb(Assembler::notEqual, TRUE_LABEL);
8299   jmpb(FALSE_LABEL);
8300 
8301   bind(TRUE_LABEL);
8302   movl(result, 1);   // return true
8303   jmpb(DONE);
8304 
8305   bind(FALSE_LABEL);
8306   xorl(result, result); // return false
8307 
8308   // That's it
8309   bind(DONE);
8310   if (UseAVX >= 2 && UseSSE >= 2) {
8311     // clean upper bits of YMM registers
8312     vpxor(vec1, vec1);
8313     vpxor(vec2, vec2);
8314   }
8315 }
8316 // Compare char[] or byte[] arrays aligned to 4 bytes or substrings.
8317 void MacroAssembler::arrays_equals(bool is_array_equ, Register ary1, Register ary2,
8318                                    Register limit, Register result, Register chr,
8319                                    XMMRegister vec1, XMMRegister vec2, bool is_char) {
8320   ShortBranchVerifier sbv(this);
8321   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR, COMPARE_BYTE;
8322 
8323   int length_offset  = arrayOopDesc::length_offset_in_bytes();
8324   int base_offset    = arrayOopDesc::base_offset_in_bytes(is_char ? T_CHAR : T_BYTE);
8325 
8326   if (is_array_equ) {
8327     // Check the input args
8328     cmpoop(ary1, ary2);
8329     jcc(Assembler::equal, TRUE_LABEL);
8330 
8331     // Need additional checks for arrays_equals.
8332     testptr(ary1, ary1);
8333     jcc(Assembler::zero, FALSE_LABEL);
8334     testptr(ary2, ary2);
8335     jcc(Assembler::zero, FALSE_LABEL);
8336 
8337     // Check the lengths
8338     movl(limit, Address(ary1, length_offset));
8339     cmpl(limit, Address(ary2, length_offset));
8340     jcc(Assembler::notEqual, FALSE_LABEL);
8341   }
8342 
8343   // count == 0
8344   testl(limit, limit);
8345   jcc(Assembler::zero, TRUE_LABEL);
8346 
8347   if (is_array_equ) {
8348     // Load array address
8349     lea(ary1, Address(ary1, base_offset));
8350     lea(ary2, Address(ary2, base_offset));
8351   }
8352 
8353   if (is_array_equ && is_char) {
8354     // arrays_equals when used for char[].
8355     shll(limit, 1);      // byte count != 0
8356   }
8357   movl(result, limit); // copy
8358 
8359   if (UseAVX >= 2) {
8360     // With AVX2, use 32-byte vector compare
8361     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8362 
8363     // Compare 32-byte vectors
8364     andl(result, 0x0000001f);  //   tail count (in bytes)
8365     andl(limit, 0xffffffe0);   // vector count (in bytes)
8366     jcc(Assembler::zero, COMPARE_TAIL);
8367 
8368     lea(ary1, Address(ary1, limit, Address::times_1));
8369     lea(ary2, Address(ary2, limit, Address::times_1));
8370     negptr(limit);
8371 
8372     bind(COMPARE_WIDE_VECTORS);
8373 
8374 #ifdef _LP64
8375     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
8376       Label COMPARE_WIDE_VECTORS_LOOP_AVX2, COMPARE_WIDE_VECTORS_LOOP_AVX3;
8377 
8378       cmpl(limit, -64);
8379       jccb(Assembler::greater, COMPARE_WIDE_VECTORS_LOOP_AVX2);
8380 
8381       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
8382 
8383       evmovdquq(vec1, Address(ary1, limit, Address::times_1), Assembler::AVX_512bit);
8384       evpcmpeqb(k7, vec1, Address(ary2, limit, Address::times_1), Assembler::AVX_512bit);
8385       kortestql(k7, k7);
8386       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
8387       addptr(limit, 64);  // update since we already compared at this addr
8388       cmpl(limit, -64);
8389       jccb(Assembler::lessEqual, COMPARE_WIDE_VECTORS_LOOP_AVX3);
8390 
8391       // At this point we may still need to compare -limit+result bytes.
8392       // We could execute the next two instruction and just continue via non-wide path:
8393       //  cmpl(limit, 0);
8394       //  jcc(Assembler::equal, COMPARE_TAIL);  // true
8395       // But since we stopped at the points ary{1,2}+limit which are
8396       // not farther than 64 bytes from the ends of arrays ary{1,2}+result
8397       // (|limit| <= 32 and result < 32),
8398       // we may just compare the last 64 bytes.
8399       //
8400       addptr(result, -64);   // it is safe, bc we just came from this area
8401       evmovdquq(vec1, Address(ary1, result, Address::times_1), Assembler::AVX_512bit);
8402       evpcmpeqb(k7, vec1, Address(ary2, result, Address::times_1), Assembler::AVX_512bit);
8403       kortestql(k7, k7);
8404       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
8405 
8406       jmp(TRUE_LABEL);
8407 
8408       bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
8409 
8410     }//if (VM_Version::supports_avx512vlbw())
8411 #endif //_LP64
8412 
8413     vmovdqu(vec1, Address(ary1, limit, Address::times_1));
8414     vmovdqu(vec2, Address(ary2, limit, Address::times_1));
8415     vpxor(vec1, vec2);
8416 
8417     vptest(vec1, vec1);
8418     jcc(Assembler::notZero, FALSE_LABEL);
8419     addptr(limit, 32);
8420     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8421 
8422     testl(result, result);
8423     jcc(Assembler::zero, TRUE_LABEL);
8424 
8425     vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
8426     vmovdqu(vec2, Address(ary2, result, Address::times_1, -32));
8427     vpxor(vec1, vec2);
8428 
8429     vptest(vec1, vec1);
8430     jccb(Assembler::notZero, FALSE_LABEL);
8431     jmpb(TRUE_LABEL);
8432 
8433     bind(COMPARE_TAIL); // limit is zero
8434     movl(limit, result);
8435     // Fallthru to tail compare
8436   } else if (UseSSE42Intrinsics) {
8437     // With SSE4.2, use double quad vector compare
8438     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8439 
8440     // Compare 16-byte vectors
8441     andl(result, 0x0000000f);  //   tail count (in bytes)
8442     andl(limit, 0xfffffff0);   // vector count (in bytes)
8443     jcc(Assembler::zero, COMPARE_TAIL);
8444 
8445     lea(ary1, Address(ary1, limit, Address::times_1));
8446     lea(ary2, Address(ary2, limit, Address::times_1));
8447     negptr(limit);
8448 
8449     bind(COMPARE_WIDE_VECTORS);
8450     movdqu(vec1, Address(ary1, limit, Address::times_1));
8451     movdqu(vec2, Address(ary2, limit, Address::times_1));
8452     pxor(vec1, vec2);
8453 
8454     ptest(vec1, vec1);
8455     jcc(Assembler::notZero, FALSE_LABEL);
8456     addptr(limit, 16);
8457     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8458 
8459     testl(result, result);
8460     jcc(Assembler::zero, TRUE_LABEL);
8461 
8462     movdqu(vec1, Address(ary1, result, Address::times_1, -16));
8463     movdqu(vec2, Address(ary2, result, Address::times_1, -16));
8464     pxor(vec1, vec2);
8465 
8466     ptest(vec1, vec1);
8467     jccb(Assembler::notZero, FALSE_LABEL);
8468     jmpb(TRUE_LABEL);
8469 
8470     bind(COMPARE_TAIL); // limit is zero
8471     movl(limit, result);
8472     // Fallthru to tail compare
8473   }
8474 
8475   // Compare 4-byte vectors
8476   andl(limit, 0xfffffffc); // vector count (in bytes)
8477   jccb(Assembler::zero, COMPARE_CHAR);
8478 
8479   lea(ary1, Address(ary1, limit, Address::times_1));
8480   lea(ary2, Address(ary2, limit, Address::times_1));
8481   negptr(limit);
8482 
8483   bind(COMPARE_VECTORS);
8484   movl(chr, Address(ary1, limit, Address::times_1));
8485   cmpl(chr, Address(ary2, limit, Address::times_1));
8486   jccb(Assembler::notEqual, FALSE_LABEL);
8487   addptr(limit, 4);
8488   jcc(Assembler::notZero, COMPARE_VECTORS);
8489 
8490   // Compare trailing char (final 2 bytes), if any
8491   bind(COMPARE_CHAR);
8492   testl(result, 0x2);   // tail  char
8493   jccb(Assembler::zero, COMPARE_BYTE);
8494   load_unsigned_short(chr, Address(ary1, 0));
8495   load_unsigned_short(limit, Address(ary2, 0));
8496   cmpl(chr, limit);
8497   jccb(Assembler::notEqual, FALSE_LABEL);
8498 
8499   if (is_array_equ && is_char) {
8500     bind(COMPARE_BYTE);
8501   } else {
8502     lea(ary1, Address(ary1, 2));
8503     lea(ary2, Address(ary2, 2));
8504 
8505     bind(COMPARE_BYTE);
8506     testl(result, 0x1);   // tail  byte
8507     jccb(Assembler::zero, TRUE_LABEL);
8508     load_unsigned_byte(chr, Address(ary1, 0));
8509     load_unsigned_byte(limit, Address(ary2, 0));
8510     cmpl(chr, limit);
8511     jccb(Assembler::notEqual, FALSE_LABEL);
8512   }
8513   bind(TRUE_LABEL);
8514   movl(result, 1);   // return true
8515   jmpb(DONE);
8516 
8517   bind(FALSE_LABEL);
8518   xorl(result, result); // return false
8519 
8520   // That's it
8521   bind(DONE);
8522   if (UseAVX >= 2) {
8523     // clean upper bits of YMM registers
8524     vpxor(vec1, vec1);
8525     vpxor(vec2, vec2);
8526   }
8527 }
8528 
8529 #endif
8530 
8531 void MacroAssembler::generate_fill(BasicType t, bool aligned,
8532                                    Register to, Register value, Register count,
8533                                    Register rtmp, XMMRegister xtmp) {
8534   ShortBranchVerifier sbv(this);
8535   assert_different_registers(to, value, count, rtmp);
8536   Label L_exit, L_skip_align1, L_skip_align2, L_fill_byte;
8537   Label L_fill_2_bytes, L_fill_4_bytes;
8538 
8539   int shift = -1;
8540   switch (t) {
8541     case T_BYTE:
8542       shift = 2;
8543       break;
8544     case T_SHORT:
8545       shift = 1;
8546       break;
8547     case T_INT:
8548       shift = 0;
8549       break;
8550     default: ShouldNotReachHere();
8551   }
8552 
8553   if (t == T_BYTE) {
8554     andl(value, 0xff);
8555     movl(rtmp, value);
8556     shll(rtmp, 8);
8557     orl(value, rtmp);
8558   }
8559   if (t == T_SHORT) {
8560     andl(value, 0xffff);
8561   }
8562   if (t == T_BYTE || t == T_SHORT) {
8563     movl(rtmp, value);
8564     shll(rtmp, 16);
8565     orl(value, rtmp);
8566   }
8567 
8568   cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
8569   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
8570   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
8571     // align source address at 4 bytes address boundary
8572     if (t == T_BYTE) {
8573       // One byte misalignment happens only for byte arrays
8574       testptr(to, 1);
8575       jccb(Assembler::zero, L_skip_align1);
8576       movb(Address(to, 0), value);
8577       increment(to);
8578       decrement(count);
8579       BIND(L_skip_align1);
8580     }
8581     // Two bytes misalignment happens only for byte and short (char) arrays
8582     testptr(to, 2);
8583     jccb(Assembler::zero, L_skip_align2);
8584     movw(Address(to, 0), value);
8585     addptr(to, 2);
8586     subl(count, 1<<(shift-1));
8587     BIND(L_skip_align2);
8588   }
8589   if (UseSSE < 2) {
8590     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8591     // Fill 32-byte chunks
8592     subl(count, 8 << shift);
8593     jcc(Assembler::less, L_check_fill_8_bytes);
8594     align(16);
8595 
8596     BIND(L_fill_32_bytes_loop);
8597 
8598     for (int i = 0; i < 32; i += 4) {
8599       movl(Address(to, i), value);
8600     }
8601 
8602     addptr(to, 32);
8603     subl(count, 8 << shift);
8604     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8605     BIND(L_check_fill_8_bytes);
8606     addl(count, 8 << shift);
8607     jccb(Assembler::zero, L_exit);
8608     jmpb(L_fill_8_bytes);
8609 
8610     //
8611     // length is too short, just fill qwords
8612     //
8613     BIND(L_fill_8_bytes_loop);
8614     movl(Address(to, 0), value);
8615     movl(Address(to, 4), value);
8616     addptr(to, 8);
8617     BIND(L_fill_8_bytes);
8618     subl(count, 1 << (shift + 1));
8619     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8620     // fall through to fill 4 bytes
8621   } else {
8622     Label L_fill_32_bytes;
8623     if (!UseUnalignedLoadStores) {
8624       // align to 8 bytes, we know we are 4 byte aligned to start
8625       testptr(to, 4);
8626       jccb(Assembler::zero, L_fill_32_bytes);
8627       movl(Address(to, 0), value);
8628       addptr(to, 4);
8629       subl(count, 1<<shift);
8630     }
8631     BIND(L_fill_32_bytes);
8632     {
8633       assert( UseSSE >= 2, "supported cpu only" );
8634       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8635       if (UseAVX > 2) {
8636         movl(rtmp, 0xffff);
8637         kmovwl(k1, rtmp);
8638       }
8639       movdl(xtmp, value);
8640       if (UseAVX > 2 && UseUnalignedLoadStores) {
8641         // Fill 64-byte chunks
8642         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8643         evpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
8644 
8645         subl(count, 16 << shift);
8646         jcc(Assembler::less, L_check_fill_32_bytes);
8647         align(16);
8648 
8649         BIND(L_fill_64_bytes_loop);
8650         evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
8651         addptr(to, 64);
8652         subl(count, 16 << shift);
8653         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8654 
8655         BIND(L_check_fill_32_bytes);
8656         addl(count, 8 << shift);
8657         jccb(Assembler::less, L_check_fill_8_bytes);
8658         vmovdqu(Address(to, 0), xtmp);
8659         addptr(to, 32);
8660         subl(count, 8 << shift);
8661 
8662         BIND(L_check_fill_8_bytes);
8663       } else if (UseAVX == 2 && UseUnalignedLoadStores) {
8664         // Fill 64-byte chunks
8665         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8666         vpbroadcastd(xtmp, xtmp);
8667 
8668         subl(count, 16 << shift);
8669         jcc(Assembler::less, L_check_fill_32_bytes);
8670         align(16);
8671 
8672         BIND(L_fill_64_bytes_loop);
8673         vmovdqu(Address(to, 0), xtmp);
8674         vmovdqu(Address(to, 32), xtmp);
8675         addptr(to, 64);
8676         subl(count, 16 << shift);
8677         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8678 
8679         BIND(L_check_fill_32_bytes);
8680         addl(count, 8 << shift);
8681         jccb(Assembler::less, L_check_fill_8_bytes);
8682         vmovdqu(Address(to, 0), xtmp);
8683         addptr(to, 32);
8684         subl(count, 8 << shift);
8685 
8686         BIND(L_check_fill_8_bytes);
8687         // clean upper bits of YMM registers
8688         movdl(xtmp, value);
8689         pshufd(xtmp, xtmp, 0);
8690       } else {
8691         // Fill 32-byte chunks
8692         pshufd(xtmp, xtmp, 0);
8693 
8694         subl(count, 8 << shift);
8695         jcc(Assembler::less, L_check_fill_8_bytes);
8696         align(16);
8697 
8698         BIND(L_fill_32_bytes_loop);
8699 
8700         if (UseUnalignedLoadStores) {
8701           movdqu(Address(to, 0), xtmp);
8702           movdqu(Address(to, 16), xtmp);
8703         } else {
8704           movq(Address(to, 0), xtmp);
8705           movq(Address(to, 8), xtmp);
8706           movq(Address(to, 16), xtmp);
8707           movq(Address(to, 24), xtmp);
8708         }
8709 
8710         addptr(to, 32);
8711         subl(count, 8 << shift);
8712         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8713 
8714         BIND(L_check_fill_8_bytes);
8715       }
8716       addl(count, 8 << shift);
8717       jccb(Assembler::zero, L_exit);
8718       jmpb(L_fill_8_bytes);
8719 
8720       //
8721       // length is too short, just fill qwords
8722       //
8723       BIND(L_fill_8_bytes_loop);
8724       movq(Address(to, 0), xtmp);
8725       addptr(to, 8);
8726       BIND(L_fill_8_bytes);
8727       subl(count, 1 << (shift + 1));
8728       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8729     }
8730   }
8731   // fill trailing 4 bytes
8732   BIND(L_fill_4_bytes);
8733   testl(count, 1<<shift);
8734   jccb(Assembler::zero, L_fill_2_bytes);
8735   movl(Address(to, 0), value);
8736   if (t == T_BYTE || t == T_SHORT) {
8737     addptr(to, 4);
8738     BIND(L_fill_2_bytes);
8739     // fill trailing 2 bytes
8740     testl(count, 1<<(shift-1));
8741     jccb(Assembler::zero, L_fill_byte);
8742     movw(Address(to, 0), value);
8743     if (t == T_BYTE) {
8744       addptr(to, 2);
8745       BIND(L_fill_byte);
8746       // fill trailing byte
8747       testl(count, 1);
8748       jccb(Assembler::zero, L_exit);
8749       movb(Address(to, 0), value);
8750     } else {
8751       BIND(L_fill_byte);
8752     }
8753   } else {
8754     BIND(L_fill_2_bytes);
8755   }
8756   BIND(L_exit);
8757 }
8758 
8759 // encode char[] to byte[] in ISO_8859_1
8760    //@HotSpotIntrinsicCandidate
8761    //private static int implEncodeISOArray(byte[] sa, int sp,
8762    //byte[] da, int dp, int len) {
8763    //  int i = 0;
8764    //  for (; i < len; i++) {
8765    //    char c = StringUTF16.getChar(sa, sp++);
8766    //    if (c > '\u00FF')
8767    //      break;
8768    //    da[dp++] = (byte)c;
8769    //  }
8770    //  return i;
8771    //}
8772 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
8773   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
8774   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
8775   Register tmp5, Register result) {
8776 
8777   // rsi: src
8778   // rdi: dst
8779   // rdx: len
8780   // rcx: tmp5
8781   // rax: result
8782   ShortBranchVerifier sbv(this);
8783   assert_different_registers(src, dst, len, tmp5, result);
8784   Label L_done, L_copy_1_char, L_copy_1_char_exit;
8785 
8786   // set result
8787   xorl(result, result);
8788   // check for zero length
8789   testl(len, len);
8790   jcc(Assembler::zero, L_done);
8791 
8792   movl(result, len);
8793 
8794   // Setup pointers
8795   lea(src, Address(src, len, Address::times_2)); // char[]
8796   lea(dst, Address(dst, len, Address::times_1)); // byte[]
8797   negptr(len);
8798 
8799   if (UseSSE42Intrinsics || UseAVX >= 2) {
8800     Label L_chars_8_check, L_copy_8_chars, L_copy_8_chars_exit;
8801     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
8802 
8803     if (UseAVX >= 2) {
8804       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
8805       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8806       movdl(tmp1Reg, tmp5);
8807       vpbroadcastd(tmp1Reg, tmp1Reg);
8808       jmp(L_chars_32_check);
8809 
8810       bind(L_copy_32_chars);
8811       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
8812       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
8813       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8814       vptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8815       jccb(Assembler::notZero, L_copy_32_chars_exit);
8816       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8817       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
8818       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
8819 
8820       bind(L_chars_32_check);
8821       addptr(len, 32);
8822       jcc(Assembler::lessEqual, L_copy_32_chars);
8823 
8824       bind(L_copy_32_chars_exit);
8825       subptr(len, 16);
8826       jccb(Assembler::greater, L_copy_16_chars_exit);
8827 
8828     } else if (UseSSE42Intrinsics) {
8829       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8830       movdl(tmp1Reg, tmp5);
8831       pshufd(tmp1Reg, tmp1Reg, 0);
8832       jmpb(L_chars_16_check);
8833     }
8834 
8835     bind(L_copy_16_chars);
8836     if (UseAVX >= 2) {
8837       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
8838       vptest(tmp2Reg, tmp1Reg);
8839       jcc(Assembler::notZero, L_copy_16_chars_exit);
8840       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
8841       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
8842     } else {
8843       if (UseAVX > 0) {
8844         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8845         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8846         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
8847       } else {
8848         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8849         por(tmp2Reg, tmp3Reg);
8850         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8851         por(tmp2Reg, tmp4Reg);
8852       }
8853       ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8854       jccb(Assembler::notZero, L_copy_16_chars_exit);
8855       packuswb(tmp3Reg, tmp4Reg);
8856     }
8857     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
8858 
8859     bind(L_chars_16_check);
8860     addptr(len, 16);
8861     jcc(Assembler::lessEqual, L_copy_16_chars);
8862 
8863     bind(L_copy_16_chars_exit);
8864     if (UseAVX >= 2) {
8865       // clean upper bits of YMM registers
8866       vpxor(tmp2Reg, tmp2Reg);
8867       vpxor(tmp3Reg, tmp3Reg);
8868       vpxor(tmp4Reg, tmp4Reg);
8869       movdl(tmp1Reg, tmp5);
8870       pshufd(tmp1Reg, tmp1Reg, 0);
8871     }
8872     subptr(len, 8);
8873     jccb(Assembler::greater, L_copy_8_chars_exit);
8874 
8875     bind(L_copy_8_chars);
8876     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
8877     ptest(tmp3Reg, tmp1Reg);
8878     jccb(Assembler::notZero, L_copy_8_chars_exit);
8879     packuswb(tmp3Reg, tmp1Reg);
8880     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
8881     addptr(len, 8);
8882     jccb(Assembler::lessEqual, L_copy_8_chars);
8883 
8884     bind(L_copy_8_chars_exit);
8885     subptr(len, 8);
8886     jccb(Assembler::zero, L_done);
8887   }
8888 
8889   bind(L_copy_1_char);
8890   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
8891   testl(tmp5, 0xff00);      // check if Unicode char
8892   jccb(Assembler::notZero, L_copy_1_char_exit);
8893   movb(Address(dst, len, Address::times_1, 0), tmp5);
8894   addptr(len, 1);
8895   jccb(Assembler::less, L_copy_1_char);
8896 
8897   bind(L_copy_1_char_exit);
8898   addptr(result, len); // len is negative count of not processed elements
8899 
8900   bind(L_done);
8901 }
8902 
8903 #ifdef _LP64
8904 /**
8905  * Helper for multiply_to_len().
8906  */
8907 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
8908   addq(dest_lo, src1);
8909   adcq(dest_hi, 0);
8910   addq(dest_lo, src2);
8911   adcq(dest_hi, 0);
8912 }
8913 
8914 /**
8915  * Multiply 64 bit by 64 bit first loop.
8916  */
8917 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
8918                                            Register y, Register y_idx, Register z,
8919                                            Register carry, Register product,
8920                                            Register idx, Register kdx) {
8921   //
8922   //  jlong carry, x[], y[], z[];
8923   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
8924   //    huge_128 product = y[idx] * x[xstart] + carry;
8925   //    z[kdx] = (jlong)product;
8926   //    carry  = (jlong)(product >>> 64);
8927   //  }
8928   //  z[xstart] = carry;
8929   //
8930 
8931   Label L_first_loop, L_first_loop_exit;
8932   Label L_one_x, L_one_y, L_multiply;
8933 
8934   decrementl(xstart);
8935   jcc(Assembler::negative, L_one_x);
8936 
8937   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
8938   rorq(x_xstart, 32); // convert big-endian to little-endian
8939 
8940   bind(L_first_loop);
8941   decrementl(idx);
8942   jcc(Assembler::negative, L_first_loop_exit);
8943   decrementl(idx);
8944   jcc(Assembler::negative, L_one_y);
8945   movq(y_idx, Address(y, idx, Address::times_4,  0));
8946   rorq(y_idx, 32); // convert big-endian to little-endian
8947   bind(L_multiply);
8948   movq(product, x_xstart);
8949   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
8950   addq(product, carry);
8951   adcq(rdx, 0);
8952   subl(kdx, 2);
8953   movl(Address(z, kdx, Address::times_4,  4), product);
8954   shrq(product, 32);
8955   movl(Address(z, kdx, Address::times_4,  0), product);
8956   movq(carry, rdx);
8957   jmp(L_first_loop);
8958 
8959   bind(L_one_y);
8960   movl(y_idx, Address(y,  0));
8961   jmp(L_multiply);
8962 
8963   bind(L_one_x);
8964   movl(x_xstart, Address(x,  0));
8965   jmp(L_first_loop);
8966 
8967   bind(L_first_loop_exit);
8968 }
8969 
8970 /**
8971  * Multiply 64 bit by 64 bit and add 128 bit.
8972  */
8973 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
8974                                             Register yz_idx, Register idx,
8975                                             Register carry, Register product, int offset) {
8976   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
8977   //     z[kdx] = (jlong)product;
8978 
8979   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
8980   rorq(yz_idx, 32); // convert big-endian to little-endian
8981   movq(product, x_xstart);
8982   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
8983   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
8984   rorq(yz_idx, 32); // convert big-endian to little-endian
8985 
8986   add2_with_carry(rdx, product, carry, yz_idx);
8987 
8988   movl(Address(z, idx, Address::times_4,  offset+4), product);
8989   shrq(product, 32);
8990   movl(Address(z, idx, Address::times_4,  offset), product);
8991 
8992 }
8993 
8994 /**
8995  * Multiply 128 bit by 128 bit. Unrolled inner loop.
8996  */
8997 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
8998                                              Register yz_idx, Register idx, Register jdx,
8999                                              Register carry, Register product,
9000                                              Register carry2) {
9001   //   jlong carry, x[], y[], z[];
9002   //   int kdx = ystart+1;
9003   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
9004   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
9005   //     z[kdx+idx+1] = (jlong)product;
9006   //     jlong carry2  = (jlong)(product >>> 64);
9007   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
9008   //     z[kdx+idx] = (jlong)product;
9009   //     carry  = (jlong)(product >>> 64);
9010   //   }
9011   //   idx += 2;
9012   //   if (idx > 0) {
9013   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
9014   //     z[kdx+idx] = (jlong)product;
9015   //     carry  = (jlong)(product >>> 64);
9016   //   }
9017   //
9018 
9019   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
9020 
9021   movl(jdx, idx);
9022   andl(jdx, 0xFFFFFFFC);
9023   shrl(jdx, 2);
9024 
9025   bind(L_third_loop);
9026   subl(jdx, 1);
9027   jcc(Assembler::negative, L_third_loop_exit);
9028   subl(idx, 4);
9029 
9030   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
9031   movq(carry2, rdx);
9032 
9033   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
9034   movq(carry, rdx);
9035   jmp(L_third_loop);
9036 
9037   bind (L_third_loop_exit);
9038 
9039   andl (idx, 0x3);
9040   jcc(Assembler::zero, L_post_third_loop_done);
9041 
9042   Label L_check_1;
9043   subl(idx, 2);
9044   jcc(Assembler::negative, L_check_1);
9045 
9046   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
9047   movq(carry, rdx);
9048 
9049   bind (L_check_1);
9050   addl (idx, 0x2);
9051   andl (idx, 0x1);
9052   subl(idx, 1);
9053   jcc(Assembler::negative, L_post_third_loop_done);
9054 
9055   movl(yz_idx, Address(y, idx, Address::times_4,  0));
9056   movq(product, x_xstart);
9057   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
9058   movl(yz_idx, Address(z, idx, Address::times_4,  0));
9059 
9060   add2_with_carry(rdx, product, yz_idx, carry);
9061 
9062   movl(Address(z, idx, Address::times_4,  0), product);
9063   shrq(product, 32);
9064 
9065   shlq(rdx, 32);
9066   orq(product, rdx);
9067   movq(carry, product);
9068 
9069   bind(L_post_third_loop_done);
9070 }
9071 
9072 /**
9073  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
9074  *
9075  */
9076 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
9077                                                   Register carry, Register carry2,
9078                                                   Register idx, Register jdx,
9079                                                   Register yz_idx1, Register yz_idx2,
9080                                                   Register tmp, Register tmp3, Register tmp4) {
9081   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
9082 
9083   //   jlong carry, x[], y[], z[];
9084   //   int kdx = ystart+1;
9085   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
9086   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
9087   //     jlong carry2  = (jlong)(tmp3 >>> 64);
9088   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
9089   //     carry  = (jlong)(tmp4 >>> 64);
9090   //     z[kdx+idx+1] = (jlong)tmp3;
9091   //     z[kdx+idx] = (jlong)tmp4;
9092   //   }
9093   //   idx += 2;
9094   //   if (idx > 0) {
9095   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
9096   //     z[kdx+idx] = (jlong)yz_idx1;
9097   //     carry  = (jlong)(yz_idx1 >>> 64);
9098   //   }
9099   //
9100 
9101   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
9102 
9103   movl(jdx, idx);
9104   andl(jdx, 0xFFFFFFFC);
9105   shrl(jdx, 2);
9106 
9107   bind(L_third_loop);
9108   subl(jdx, 1);
9109   jcc(Assembler::negative, L_third_loop_exit);
9110   subl(idx, 4);
9111 
9112   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
9113   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
9114   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
9115   rorxq(yz_idx2, yz_idx2, 32);
9116 
9117   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
9118   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
9119 
9120   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
9121   rorxq(yz_idx1, yz_idx1, 32);
9122   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
9123   rorxq(yz_idx2, yz_idx2, 32);
9124 
9125   if (VM_Version::supports_adx()) {
9126     adcxq(tmp3, carry);
9127     adoxq(tmp3, yz_idx1);
9128 
9129     adcxq(tmp4, tmp);
9130     adoxq(tmp4, yz_idx2);
9131 
9132     movl(carry, 0); // does not affect flags
9133     adcxq(carry2, carry);
9134     adoxq(carry2, carry);
9135   } else {
9136     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
9137     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
9138   }
9139   movq(carry, carry2);
9140 
9141   movl(Address(z, idx, Address::times_4, 12), tmp3);
9142   shrq(tmp3, 32);
9143   movl(Address(z, idx, Address::times_4,  8), tmp3);
9144 
9145   movl(Address(z, idx, Address::times_4,  4), tmp4);
9146   shrq(tmp4, 32);
9147   movl(Address(z, idx, Address::times_4,  0), tmp4);
9148 
9149   jmp(L_third_loop);
9150 
9151   bind (L_third_loop_exit);
9152 
9153   andl (idx, 0x3);
9154   jcc(Assembler::zero, L_post_third_loop_done);
9155 
9156   Label L_check_1;
9157   subl(idx, 2);
9158   jcc(Assembler::negative, L_check_1);
9159 
9160   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
9161   rorxq(yz_idx1, yz_idx1, 32);
9162   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
9163   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
9164   rorxq(yz_idx2, yz_idx2, 32);
9165 
9166   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
9167 
9168   movl(Address(z, idx, Address::times_4,  4), tmp3);
9169   shrq(tmp3, 32);
9170   movl(Address(z, idx, Address::times_4,  0), tmp3);
9171   movq(carry, tmp4);
9172 
9173   bind (L_check_1);
9174   addl (idx, 0x2);
9175   andl (idx, 0x1);
9176   subl(idx, 1);
9177   jcc(Assembler::negative, L_post_third_loop_done);
9178   movl(tmp4, Address(y, idx, Address::times_4,  0));
9179   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
9180   movl(tmp4, Address(z, idx, Address::times_4,  0));
9181 
9182   add2_with_carry(carry2, tmp3, tmp4, carry);
9183 
9184   movl(Address(z, idx, Address::times_4,  0), tmp3);
9185   shrq(tmp3, 32);
9186 
9187   shlq(carry2, 32);
9188   orq(tmp3, carry2);
9189   movq(carry, tmp3);
9190 
9191   bind(L_post_third_loop_done);
9192 }
9193 
9194 /**
9195  * Code for BigInteger::multiplyToLen() instrinsic.
9196  *
9197  * rdi: x
9198  * rax: xlen
9199  * rsi: y
9200  * rcx: ylen
9201  * r8:  z
9202  * r11: zlen
9203  * r12: tmp1
9204  * r13: tmp2
9205  * r14: tmp3
9206  * r15: tmp4
9207  * rbx: tmp5
9208  *
9209  */
9210 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
9211                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
9212   ShortBranchVerifier sbv(this);
9213   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
9214 
9215   push(tmp1);
9216   push(tmp2);
9217   push(tmp3);
9218   push(tmp4);
9219   push(tmp5);
9220 
9221   push(xlen);
9222   push(zlen);
9223 
9224   const Register idx = tmp1;
9225   const Register kdx = tmp2;
9226   const Register xstart = tmp3;
9227 
9228   const Register y_idx = tmp4;
9229   const Register carry = tmp5;
9230   const Register product  = xlen;
9231   const Register x_xstart = zlen;  // reuse register
9232 
9233   // First Loop.
9234   //
9235   //  final static long LONG_MASK = 0xffffffffL;
9236   //  int xstart = xlen - 1;
9237   //  int ystart = ylen - 1;
9238   //  long carry = 0;
9239   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
9240   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
9241   //    z[kdx] = (int)product;
9242   //    carry = product >>> 32;
9243   //  }
9244   //  z[xstart] = (int)carry;
9245   //
9246 
9247   movl(idx, ylen);      // idx = ylen;
9248   movl(kdx, zlen);      // kdx = xlen+ylen;
9249   xorq(carry, carry);   // carry = 0;
9250 
9251   Label L_done;
9252 
9253   movl(xstart, xlen);
9254   decrementl(xstart);
9255   jcc(Assembler::negative, L_done);
9256 
9257   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
9258 
9259   Label L_second_loop;
9260   testl(kdx, kdx);
9261   jcc(Assembler::zero, L_second_loop);
9262 
9263   Label L_carry;
9264   subl(kdx, 1);
9265   jcc(Assembler::zero, L_carry);
9266 
9267   movl(Address(z, kdx, Address::times_4,  0), carry);
9268   shrq(carry, 32);
9269   subl(kdx, 1);
9270 
9271   bind(L_carry);
9272   movl(Address(z, kdx, Address::times_4,  0), carry);
9273 
9274   // Second and third (nested) loops.
9275   //
9276   // for (int i = xstart-1; i >= 0; i--) { // Second loop
9277   //   carry = 0;
9278   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
9279   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
9280   //                    (z[k] & LONG_MASK) + carry;
9281   //     z[k] = (int)product;
9282   //     carry = product >>> 32;
9283   //   }
9284   //   z[i] = (int)carry;
9285   // }
9286   //
9287   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
9288 
9289   const Register jdx = tmp1;
9290 
9291   bind(L_second_loop);
9292   xorl(carry, carry);    // carry = 0;
9293   movl(jdx, ylen);       // j = ystart+1
9294 
9295   subl(xstart, 1);       // i = xstart-1;
9296   jcc(Assembler::negative, L_done);
9297 
9298   push (z);
9299 
9300   Label L_last_x;
9301   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
9302   subl(xstart, 1);       // i = xstart-1;
9303   jcc(Assembler::negative, L_last_x);
9304 
9305   if (UseBMI2Instructions) {
9306     movq(rdx,  Address(x, xstart, Address::times_4,  0));
9307     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
9308   } else {
9309     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
9310     rorq(x_xstart, 32);  // convert big-endian to little-endian
9311   }
9312 
9313   Label L_third_loop_prologue;
9314   bind(L_third_loop_prologue);
9315 
9316   push (x);
9317   push (xstart);
9318   push (ylen);
9319 
9320 
9321   if (UseBMI2Instructions) {
9322     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
9323   } else { // !UseBMI2Instructions
9324     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
9325   }
9326 
9327   pop(ylen);
9328   pop(xlen);
9329   pop(x);
9330   pop(z);
9331 
9332   movl(tmp3, xlen);
9333   addl(tmp3, 1);
9334   movl(Address(z, tmp3, Address::times_4,  0), carry);
9335   subl(tmp3, 1);
9336   jccb(Assembler::negative, L_done);
9337 
9338   shrq(carry, 32);
9339   movl(Address(z, tmp3, Address::times_4,  0), carry);
9340   jmp(L_second_loop);
9341 
9342   // Next infrequent code is moved outside loops.
9343   bind(L_last_x);
9344   if (UseBMI2Instructions) {
9345     movl(rdx, Address(x,  0));
9346   } else {
9347     movl(x_xstart, Address(x,  0));
9348   }
9349   jmp(L_third_loop_prologue);
9350 
9351   bind(L_done);
9352 
9353   pop(zlen);
9354   pop(xlen);
9355 
9356   pop(tmp5);
9357   pop(tmp4);
9358   pop(tmp3);
9359   pop(tmp2);
9360   pop(tmp1);
9361 }
9362 
9363 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
9364   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
9365   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
9366   Label VECTOR64_LOOP, VECTOR64_TAIL, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
9367   Label VECTOR32_LOOP, VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
9368   Label VECTOR16_TAIL, VECTOR8_TAIL, VECTOR4_TAIL;
9369   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
9370   Label SAME_TILL_END, DONE;
9371   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
9372 
9373   //scale is in rcx in both Win64 and Unix
9374   ShortBranchVerifier sbv(this);
9375 
9376   shlq(length);
9377   xorq(result, result);
9378 
9379   if ((UseAVX > 2) &&
9380       VM_Version::supports_avx512vlbw()) {
9381     set_vector_masking();  // opening of the stub context for programming mask registers
9382     cmpq(length, 64);
9383     jcc(Assembler::less, VECTOR32_TAIL);
9384     movq(tmp1, length);
9385     andq(tmp1, 0x3F);      // tail count
9386     andq(length, ~(0x3F)); //vector count
9387 
9388     bind(VECTOR64_LOOP);
9389     // AVX512 code to compare 64 byte vectors.
9390     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
9391     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
9392     kortestql(k7, k7);
9393     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
9394     addq(result, 64);
9395     subq(length, 64);
9396     jccb(Assembler::notZero, VECTOR64_LOOP);
9397 
9398     //bind(VECTOR64_TAIL);
9399     testq(tmp1, tmp1);
9400     jcc(Assembler::zero, SAME_TILL_END);
9401 
9402     bind(VECTOR64_TAIL);
9403     // AVX512 code to compare upto 63 byte vectors.
9404     // Save k1
9405     kmovql(k3, k1);
9406     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
9407     shlxq(tmp2, tmp2, tmp1);
9408     notq(tmp2);
9409     kmovql(k1, tmp2);
9410 
9411     evmovdqub(rymm0, k1, Address(obja, result), Assembler::AVX_512bit);
9412     evpcmpeqb(k7, k1, rymm0, Address(objb, result), Assembler::AVX_512bit);
9413 
9414     ktestql(k7, k1);
9415     // Restore k1
9416     kmovql(k1, k3);
9417     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
9418 
9419     bind(VECTOR64_NOT_EQUAL);
9420     kmovql(tmp1, k7);
9421     notq(tmp1);
9422     tzcntq(tmp1, tmp1);
9423     addq(result, tmp1);
9424     shrq(result);
9425     jmp(DONE);
9426     bind(VECTOR32_TAIL);
9427     clear_vector_masking();   // closing of the stub context for programming mask registers
9428   }
9429 
9430   cmpq(length, 8);
9431   jcc(Assembler::equal, VECTOR8_LOOP);
9432   jcc(Assembler::less, VECTOR4_TAIL);
9433 
9434   if (UseAVX >= 2) {
9435 
9436     cmpq(length, 16);
9437     jcc(Assembler::equal, VECTOR16_LOOP);
9438     jcc(Assembler::less, VECTOR8_LOOP);
9439 
9440     cmpq(length, 32);
9441     jccb(Assembler::less, VECTOR16_TAIL);
9442 
9443     subq(length, 32);
9444     bind(VECTOR32_LOOP);
9445     vmovdqu(rymm0, Address(obja, result));
9446     vmovdqu(rymm1, Address(objb, result));
9447     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
9448     vptest(rymm2, rymm2);
9449     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
9450     addq(result, 32);
9451     subq(length, 32);
9452     jccb(Assembler::greaterEqual, VECTOR32_LOOP);
9453     addq(length, 32);
9454     jcc(Assembler::equal, SAME_TILL_END);
9455     //falling through if less than 32 bytes left //close the branch here.
9456 
9457     bind(VECTOR16_TAIL);
9458     cmpq(length, 16);
9459     jccb(Assembler::less, VECTOR8_TAIL);
9460     bind(VECTOR16_LOOP);
9461     movdqu(rymm0, Address(obja, result));
9462     movdqu(rymm1, Address(objb, result));
9463     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
9464     ptest(rymm2, rymm2);
9465     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
9466     addq(result, 16);
9467     subq(length, 16);
9468     jcc(Assembler::equal, SAME_TILL_END);
9469     //falling through if less than 16 bytes left
9470   } else {//regular intrinsics
9471 
9472     cmpq(length, 16);
9473     jccb(Assembler::less, VECTOR8_TAIL);
9474 
9475     subq(length, 16);
9476     bind(VECTOR16_LOOP);
9477     movdqu(rymm0, Address(obja, result));
9478     movdqu(rymm1, Address(objb, result));
9479     pxor(rymm0, rymm1);
9480     ptest(rymm0, rymm0);
9481     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
9482     addq(result, 16);
9483     subq(length, 16);
9484     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
9485     addq(length, 16);
9486     jcc(Assembler::equal, SAME_TILL_END);
9487     //falling through if less than 16 bytes left
9488   }
9489 
9490   bind(VECTOR8_TAIL);
9491   cmpq(length, 8);
9492   jccb(Assembler::less, VECTOR4_TAIL);
9493   bind(VECTOR8_LOOP);
9494   movq(tmp1, Address(obja, result));
9495   movq(tmp2, Address(objb, result));
9496   xorq(tmp1, tmp2);
9497   testq(tmp1, tmp1);
9498   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
9499   addq(result, 8);
9500   subq(length, 8);
9501   jcc(Assembler::equal, SAME_TILL_END);
9502   //falling through if less than 8 bytes left
9503 
9504   bind(VECTOR4_TAIL);
9505   cmpq(length, 4);
9506   jccb(Assembler::less, BYTES_TAIL);
9507   bind(VECTOR4_LOOP);
9508   movl(tmp1, Address(obja, result));
9509   xorl(tmp1, Address(objb, result));
9510   testl(tmp1, tmp1);
9511   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
9512   addq(result, 4);
9513   subq(length, 4);
9514   jcc(Assembler::equal, SAME_TILL_END);
9515   //falling through if less than 4 bytes left
9516 
9517   bind(BYTES_TAIL);
9518   bind(BYTES_LOOP);
9519   load_unsigned_byte(tmp1, Address(obja, result));
9520   load_unsigned_byte(tmp2, Address(objb, result));
9521   xorl(tmp1, tmp2);
9522   testl(tmp1, tmp1);
9523   jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9524   decq(length);
9525   jccb(Assembler::zero, SAME_TILL_END);
9526   incq(result);
9527   load_unsigned_byte(tmp1, Address(obja, result));
9528   load_unsigned_byte(tmp2, Address(objb, result));
9529   xorl(tmp1, tmp2);
9530   testl(tmp1, tmp1);
9531   jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9532   decq(length);
9533   jccb(Assembler::zero, SAME_TILL_END);
9534   incq(result);
9535   load_unsigned_byte(tmp1, Address(obja, result));
9536   load_unsigned_byte(tmp2, Address(objb, result));
9537   xorl(tmp1, tmp2);
9538   testl(tmp1, tmp1);
9539   jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9540   jmpb(SAME_TILL_END);
9541 
9542   if (UseAVX >= 2) {
9543     bind(VECTOR32_NOT_EQUAL);
9544     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
9545     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
9546     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
9547     vpmovmskb(tmp1, rymm0);
9548     bsfq(tmp1, tmp1);
9549     addq(result, tmp1);
9550     shrq(result);
9551     jmpb(DONE);
9552   }
9553 
9554   bind(VECTOR16_NOT_EQUAL);
9555   if (UseAVX >= 2) {
9556     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
9557     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
9558     pxor(rymm0, rymm2);
9559   } else {
9560     pcmpeqb(rymm2, rymm2);
9561     pxor(rymm0, rymm1);
9562     pcmpeqb(rymm0, rymm1);
9563     pxor(rymm0, rymm2);
9564   }
9565   pmovmskb(tmp1, rymm0);
9566   bsfq(tmp1, tmp1);
9567   addq(result, tmp1);
9568   shrq(result);
9569   jmpb(DONE);
9570 
9571   bind(VECTOR8_NOT_EQUAL);
9572   bind(VECTOR4_NOT_EQUAL);
9573   bsfq(tmp1, tmp1);
9574   shrq(tmp1, 3);
9575   addq(result, tmp1);
9576   bind(BYTES_NOT_EQUAL);
9577   shrq(result);
9578   jmpb(DONE);
9579 
9580   bind(SAME_TILL_END);
9581   mov64(result, -1);
9582 
9583   bind(DONE);
9584 }
9585 
9586 //Helper functions for square_to_len()
9587 
9588 /**
9589  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
9590  * Preserves x and z and modifies rest of the registers.
9591  */
9592 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9593   // Perform square and right shift by 1
9594   // Handle odd xlen case first, then for even xlen do the following
9595   // jlong carry = 0;
9596   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
9597   //     huge_128 product = x[j:j+1] * x[j:j+1];
9598   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
9599   //     z[i+2:i+3] = (jlong)(product >>> 1);
9600   //     carry = (jlong)product;
9601   // }
9602 
9603   xorq(tmp5, tmp5);     // carry
9604   xorq(rdxReg, rdxReg);
9605   xorl(tmp1, tmp1);     // index for x
9606   xorl(tmp4, tmp4);     // index for z
9607 
9608   Label L_first_loop, L_first_loop_exit;
9609 
9610   testl(xlen, 1);
9611   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
9612 
9613   // Square and right shift by 1 the odd element using 32 bit multiply
9614   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
9615   imulq(raxReg, raxReg);
9616   shrq(raxReg, 1);
9617   adcq(tmp5, 0);
9618   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
9619   incrementl(tmp1);
9620   addl(tmp4, 2);
9621 
9622   // Square and  right shift by 1 the rest using 64 bit multiply
9623   bind(L_first_loop);
9624   cmpptr(tmp1, xlen);
9625   jccb(Assembler::equal, L_first_loop_exit);
9626 
9627   // Square
9628   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
9629   rorq(raxReg, 32);    // convert big-endian to little-endian
9630   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
9631 
9632   // Right shift by 1 and save carry
9633   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
9634   rcrq(rdxReg, 1);
9635   rcrq(raxReg, 1);
9636   adcq(tmp5, 0);
9637 
9638   // Store result in z
9639   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
9640   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
9641 
9642   // Update indices for x and z
9643   addl(tmp1, 2);
9644   addl(tmp4, 4);
9645   jmp(L_first_loop);
9646 
9647   bind(L_first_loop_exit);
9648 }
9649 
9650 
9651 /**
9652  * Perform the following multiply add operation using BMI2 instructions
9653  * carry:sum = sum + op1*op2 + carry
9654  * op2 should be in rdx
9655  * op2 is preserved, all other registers are modified
9656  */
9657 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
9658   // assert op2 is rdx
9659   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
9660   addq(sum, carry);
9661   adcq(tmp2, 0);
9662   addq(sum, op1);
9663   adcq(tmp2, 0);
9664   movq(carry, tmp2);
9665 }
9666 
9667 /**
9668  * Perform the following multiply add operation:
9669  * carry:sum = sum + op1*op2 + carry
9670  * Preserves op1, op2 and modifies rest of registers
9671  */
9672 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
9673   // rdx:rax = op1 * op2
9674   movq(raxReg, op2);
9675   mulq(op1);
9676 
9677   //  rdx:rax = sum + carry + rdx:rax
9678   addq(sum, carry);
9679   adcq(rdxReg, 0);
9680   addq(sum, raxReg);
9681   adcq(rdxReg, 0);
9682 
9683   // carry:sum = rdx:sum
9684   movq(carry, rdxReg);
9685 }
9686 
9687 /**
9688  * Add 64 bit long carry into z[] with carry propogation.
9689  * Preserves z and carry register values and modifies rest of registers.
9690  *
9691  */
9692 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
9693   Label L_fourth_loop, L_fourth_loop_exit;
9694 
9695   movl(tmp1, 1);
9696   subl(zlen, 2);
9697   addq(Address(z, zlen, Address::times_4, 0), carry);
9698 
9699   bind(L_fourth_loop);
9700   jccb(Assembler::carryClear, L_fourth_loop_exit);
9701   subl(zlen, 2);
9702   jccb(Assembler::negative, L_fourth_loop_exit);
9703   addq(Address(z, zlen, Address::times_4, 0), tmp1);
9704   jmp(L_fourth_loop);
9705   bind(L_fourth_loop_exit);
9706 }
9707 
9708 /**
9709  * Shift z[] left by 1 bit.
9710  * Preserves x, len, z and zlen registers and modifies rest of the registers.
9711  *
9712  */
9713 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
9714 
9715   Label L_fifth_loop, L_fifth_loop_exit;
9716 
9717   // Fifth loop
9718   // Perform primitiveLeftShift(z, zlen, 1)
9719 
9720   const Register prev_carry = tmp1;
9721   const Register new_carry = tmp4;
9722   const Register value = tmp2;
9723   const Register zidx = tmp3;
9724 
9725   // int zidx, carry;
9726   // long value;
9727   // carry = 0;
9728   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
9729   //    (carry:value)  = (z[i] << 1) | carry ;
9730   //    z[i] = value;
9731   // }
9732 
9733   movl(zidx, zlen);
9734   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
9735 
9736   bind(L_fifth_loop);
9737   decl(zidx);  // Use decl to preserve carry flag
9738   decl(zidx);
9739   jccb(Assembler::negative, L_fifth_loop_exit);
9740 
9741   if (UseBMI2Instructions) {
9742      movq(value, Address(z, zidx, Address::times_4, 0));
9743      rclq(value, 1);
9744      rorxq(value, value, 32);
9745      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9746   }
9747   else {
9748     // clear new_carry
9749     xorl(new_carry, new_carry);
9750 
9751     // Shift z[i] by 1, or in previous carry and save new carry
9752     movq(value, Address(z, zidx, Address::times_4, 0));
9753     shlq(value, 1);
9754     adcl(new_carry, 0);
9755 
9756     orq(value, prev_carry);
9757     rorq(value, 0x20);
9758     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9759 
9760     // Set previous carry = new carry
9761     movl(prev_carry, new_carry);
9762   }
9763   jmp(L_fifth_loop);
9764 
9765   bind(L_fifth_loop_exit);
9766 }
9767 
9768 
9769 /**
9770  * Code for BigInteger::squareToLen() intrinsic
9771  *
9772  * rdi: x
9773  * rsi: len
9774  * r8:  z
9775  * rcx: zlen
9776  * r12: tmp1
9777  * r13: tmp2
9778  * r14: tmp3
9779  * r15: tmp4
9780  * rbx: tmp5
9781  *
9782  */
9783 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9784 
9785   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, fifth_loop, fifth_loop_exit, L_last_x, L_multiply;
9786   push(tmp1);
9787   push(tmp2);
9788   push(tmp3);
9789   push(tmp4);
9790   push(tmp5);
9791 
9792   // First loop
9793   // Store the squares, right shifted one bit (i.e., divided by 2).
9794   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
9795 
9796   // Add in off-diagonal sums.
9797   //
9798   // Second, third (nested) and fourth loops.
9799   // zlen +=2;
9800   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
9801   //    carry = 0;
9802   //    long op2 = x[xidx:xidx+1];
9803   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
9804   //       k -= 2;
9805   //       long op1 = x[j:j+1];
9806   //       long sum = z[k:k+1];
9807   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
9808   //       z[k:k+1] = sum;
9809   //    }
9810   //    add_one_64(z, k, carry, tmp_regs);
9811   // }
9812 
9813   const Register carry = tmp5;
9814   const Register sum = tmp3;
9815   const Register op1 = tmp4;
9816   Register op2 = tmp2;
9817 
9818   push(zlen);
9819   push(len);
9820   addl(zlen,2);
9821   bind(L_second_loop);
9822   xorq(carry, carry);
9823   subl(zlen, 4);
9824   subl(len, 2);
9825   push(zlen);
9826   push(len);
9827   cmpl(len, 0);
9828   jccb(Assembler::lessEqual, L_second_loop_exit);
9829 
9830   // Multiply an array by one 64 bit long.
9831   if (UseBMI2Instructions) {
9832     op2 = rdxReg;
9833     movq(op2, Address(x, len, Address::times_4,  0));
9834     rorxq(op2, op2, 32);
9835   }
9836   else {
9837     movq(op2, Address(x, len, Address::times_4,  0));
9838     rorq(op2, 32);
9839   }
9840 
9841   bind(L_third_loop);
9842   decrementl(len);
9843   jccb(Assembler::negative, L_third_loop_exit);
9844   decrementl(len);
9845   jccb(Assembler::negative, L_last_x);
9846 
9847   movq(op1, Address(x, len, Address::times_4,  0));
9848   rorq(op1, 32);
9849 
9850   bind(L_multiply);
9851   subl(zlen, 2);
9852   movq(sum, Address(z, zlen, Address::times_4,  0));
9853 
9854   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
9855   if (UseBMI2Instructions) {
9856     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
9857   }
9858   else {
9859     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9860   }
9861 
9862   movq(Address(z, zlen, Address::times_4, 0), sum);
9863 
9864   jmp(L_third_loop);
9865   bind(L_third_loop_exit);
9866 
9867   // Fourth loop
9868   // Add 64 bit long carry into z with carry propogation.
9869   // Uses offsetted zlen.
9870   add_one_64(z, zlen, carry, tmp1);
9871 
9872   pop(len);
9873   pop(zlen);
9874   jmp(L_second_loop);
9875 
9876   // Next infrequent code is moved outside loops.
9877   bind(L_last_x);
9878   movl(op1, Address(x, 0));
9879   jmp(L_multiply);
9880 
9881   bind(L_second_loop_exit);
9882   pop(len);
9883   pop(zlen);
9884   pop(len);
9885   pop(zlen);
9886 
9887   // Fifth loop
9888   // Shift z left 1 bit.
9889   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
9890 
9891   // z[zlen-1] |= x[len-1] & 1;
9892   movl(tmp3, Address(x, len, Address::times_4, -4));
9893   andl(tmp3, 1);
9894   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
9895 
9896   pop(tmp5);
9897   pop(tmp4);
9898   pop(tmp3);
9899   pop(tmp2);
9900   pop(tmp1);
9901 }
9902 
9903 /**
9904  * Helper function for mul_add()
9905  * Multiply the in[] by int k and add to out[] starting at offset offs using
9906  * 128 bit by 32 bit multiply and return the carry in tmp5.
9907  * Only quad int aligned length of in[] is operated on in this function.
9908  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
9909  * This function preserves out, in and k registers.
9910  * len and offset point to the appropriate index in "in" & "out" correspondingly
9911  * tmp5 has the carry.
9912  * other registers are temporary and are modified.
9913  *
9914  */
9915 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
9916   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
9917   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9918 
9919   Label L_first_loop, L_first_loop_exit;
9920 
9921   movl(tmp1, len);
9922   shrl(tmp1, 2);
9923 
9924   bind(L_first_loop);
9925   subl(tmp1, 1);
9926   jccb(Assembler::negative, L_first_loop_exit);
9927 
9928   subl(len, 4);
9929   subl(offset, 4);
9930 
9931   Register op2 = tmp2;
9932   const Register sum = tmp3;
9933   const Register op1 = tmp4;
9934   const Register carry = tmp5;
9935 
9936   if (UseBMI2Instructions) {
9937     op2 = rdxReg;
9938   }
9939 
9940   movq(op1, Address(in, len, Address::times_4,  8));
9941   rorq(op1, 32);
9942   movq(sum, Address(out, offset, Address::times_4,  8));
9943   rorq(sum, 32);
9944   if (UseBMI2Instructions) {
9945     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9946   }
9947   else {
9948     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9949   }
9950   // Store back in big endian from little endian
9951   rorq(sum, 0x20);
9952   movq(Address(out, offset, Address::times_4,  8), sum);
9953 
9954   movq(op1, Address(in, len, Address::times_4,  0));
9955   rorq(op1, 32);
9956   movq(sum, Address(out, offset, Address::times_4,  0));
9957   rorq(sum, 32);
9958   if (UseBMI2Instructions) {
9959     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9960   }
9961   else {
9962     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9963   }
9964   // Store back in big endian from little endian
9965   rorq(sum, 0x20);
9966   movq(Address(out, offset, Address::times_4,  0), sum);
9967 
9968   jmp(L_first_loop);
9969   bind(L_first_loop_exit);
9970 }
9971 
9972 /**
9973  * Code for BigInteger::mulAdd() intrinsic
9974  *
9975  * rdi: out
9976  * rsi: in
9977  * r11: offs (out.length - offset)
9978  * rcx: len
9979  * r8:  k
9980  * r12: tmp1
9981  * r13: tmp2
9982  * r14: tmp3
9983  * r15: tmp4
9984  * rbx: tmp5
9985  * Multiply the in[] by word k and add to out[], return the carry in rax
9986  */
9987 void MacroAssembler::mul_add(Register out, Register in, Register offs,
9988    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
9989    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9990 
9991   Label L_carry, L_last_in, L_done;
9992 
9993 // carry = 0;
9994 // for (int j=len-1; j >= 0; j--) {
9995 //    long product = (in[j] & LONG_MASK) * kLong +
9996 //                   (out[offs] & LONG_MASK) + carry;
9997 //    out[offs--] = (int)product;
9998 //    carry = product >>> 32;
9999 // }
10000 //
10001   push(tmp1);
10002   push(tmp2);
10003   push(tmp3);
10004   push(tmp4);
10005   push(tmp5);
10006 
10007   Register op2 = tmp2;
10008   const Register sum = tmp3;
10009   const Register op1 = tmp4;
10010   const Register carry =  tmp5;
10011 
10012   if (UseBMI2Instructions) {
10013     op2 = rdxReg;
10014     movl(op2, k);
10015   }
10016   else {
10017     movl(op2, k);
10018   }
10019 
10020   xorq(carry, carry);
10021 
10022   //First loop
10023 
10024   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
10025   //The carry is in tmp5
10026   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
10027 
10028   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
10029   decrementl(len);
10030   jccb(Assembler::negative, L_carry);
10031   decrementl(len);
10032   jccb(Assembler::negative, L_last_in);
10033 
10034   movq(op1, Address(in, len, Address::times_4,  0));
10035   rorq(op1, 32);
10036 
10037   subl(offs, 2);
10038   movq(sum, Address(out, offs, Address::times_4,  0));
10039   rorq(sum, 32);
10040 
10041   if (UseBMI2Instructions) {
10042     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
10043   }
10044   else {
10045     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
10046   }
10047 
10048   // Store back in big endian from little endian
10049   rorq(sum, 0x20);
10050   movq(Address(out, offs, Address::times_4,  0), sum);
10051 
10052   testl(len, len);
10053   jccb(Assembler::zero, L_carry);
10054 
10055   //Multiply the last in[] entry, if any
10056   bind(L_last_in);
10057   movl(op1, Address(in, 0));
10058   movl(sum, Address(out, offs, Address::times_4,  -4));
10059 
10060   movl(raxReg, k);
10061   mull(op1); //tmp4 * eax -> edx:eax
10062   addl(sum, carry);
10063   adcl(rdxReg, 0);
10064   addl(sum, raxReg);
10065   adcl(rdxReg, 0);
10066   movl(carry, rdxReg);
10067 
10068   movl(Address(out, offs, Address::times_4,  -4), sum);
10069 
10070   bind(L_carry);
10071   //return tmp5/carry as carry in rax
10072   movl(rax, carry);
10073 
10074   bind(L_done);
10075   pop(tmp5);
10076   pop(tmp4);
10077   pop(tmp3);
10078   pop(tmp2);
10079   pop(tmp1);
10080 }
10081 #endif
10082 
10083 /**
10084  * Emits code to update CRC-32 with a byte value according to constants in table
10085  *
10086  * @param [in,out]crc   Register containing the crc.
10087  * @param [in]val       Register containing the byte to fold into the CRC.
10088  * @param [in]table     Register containing the table of crc constants.
10089  *
10090  * uint32_t crc;
10091  * val = crc_table[(val ^ crc) & 0xFF];
10092  * crc = val ^ (crc >> 8);
10093  *
10094  */
10095 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
10096   xorl(val, crc);
10097   andl(val, 0xFF);
10098   shrl(crc, 8); // unsigned shift
10099   xorl(crc, Address(table, val, Address::times_4, 0));
10100 }
10101 
10102 /**
10103 * Fold four 128-bit data chunks
10104 */
10105 void MacroAssembler::fold_128bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
10106   evpclmulhdq(xtmp, xK, xcrc, Assembler::AVX_512bit); // [123:64]
10107   evpclmulldq(xcrc, xK, xcrc, Assembler::AVX_512bit); // [63:0]
10108   evpxorq(xcrc, xcrc, Address(buf, offset), Assembler::AVX_512bit /* vector_len */);
10109   evpxorq(xcrc, xcrc, xtmp, Assembler::AVX_512bit /* vector_len */);
10110 }
10111 
10112 /**
10113  * Fold 128-bit data chunk
10114  */
10115 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
10116   if (UseAVX > 0) {
10117     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
10118     vpclmulldq(xcrc, xK, xcrc); // [63:0]
10119     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
10120     pxor(xcrc, xtmp);
10121   } else {
10122     movdqa(xtmp, xcrc);
10123     pclmulhdq(xtmp, xK);   // [123:64]
10124     pclmulldq(xcrc, xK);   // [63:0]
10125     pxor(xcrc, xtmp);
10126     movdqu(xtmp, Address(buf, offset));
10127     pxor(xcrc, xtmp);
10128   }
10129 }
10130 
10131 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
10132   if (UseAVX > 0) {
10133     vpclmulhdq(xtmp, xK, xcrc);
10134     vpclmulldq(xcrc, xK, xcrc);
10135     pxor(xcrc, xbuf);
10136     pxor(xcrc, xtmp);
10137   } else {
10138     movdqa(xtmp, xcrc);
10139     pclmulhdq(xtmp, xK);
10140     pclmulldq(xcrc, xK);
10141     pxor(xcrc, xbuf);
10142     pxor(xcrc, xtmp);
10143   }
10144 }
10145 
10146 /**
10147  * 8-bit folds to compute 32-bit CRC
10148  *
10149  * uint64_t xcrc;
10150  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
10151  */
10152 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
10153   movdl(tmp, xcrc);
10154   andl(tmp, 0xFF);
10155   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
10156   psrldq(xcrc, 1); // unsigned shift one byte
10157   pxor(xcrc, xtmp);
10158 }
10159 
10160 /**
10161  * uint32_t crc;
10162  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
10163  */
10164 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
10165   movl(tmp, crc);
10166   andl(tmp, 0xFF);
10167   shrl(crc, 8);
10168   xorl(crc, Address(table, tmp, Address::times_4, 0));
10169 }
10170 
10171 /**
10172  * @param crc   register containing existing CRC (32-bit)
10173  * @param buf   register pointing to input byte buffer (byte*)
10174  * @param len   register containing number of bytes
10175  * @param table register that will contain address of CRC table
10176  * @param tmp   scratch register
10177  */
10178 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
10179   assert_different_registers(crc, buf, len, table, tmp, rax);
10180 
10181   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
10182   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
10183 
10184   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
10185   // context for the registers used, where all instructions below are using 128-bit mode
10186   // On EVEX without VL and BW, these instructions will all be AVX.
10187   if (VM_Version::supports_avx512vlbw()) {
10188     movl(tmp, 0xffff);
10189     kmovwl(k1, tmp);
10190   }
10191 
10192   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
10193   notl(crc); // ~crc
10194   cmpl(len, 16);
10195   jcc(Assembler::less, L_tail);
10196 
10197   // Align buffer to 16 bytes
10198   movl(tmp, buf);
10199   andl(tmp, 0xF);
10200   jccb(Assembler::zero, L_aligned);
10201   subl(tmp,  16);
10202   addl(len, tmp);
10203 
10204   align(4);
10205   BIND(L_align_loop);
10206   movsbl(rax, Address(buf, 0)); // load byte with sign extension
10207   update_byte_crc32(crc, rax, table);
10208   increment(buf);
10209   incrementl(tmp);
10210   jccb(Assembler::less, L_align_loop);
10211 
10212   BIND(L_aligned);
10213   movl(tmp, len); // save
10214   shrl(len, 4);
10215   jcc(Assembler::zero, L_tail_restore);
10216 
10217   // Fold total 512 bits of polynomial on each iteration
10218   if (VM_Version::supports_vpclmulqdq()) {
10219     Label Parallel_loop, L_No_Parallel;
10220 
10221     cmpl(len, 8);
10222     jccb(Assembler::less, L_No_Parallel);
10223 
10224     movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
10225     evmovdquq(xmm1, Address(buf, 0), Assembler::AVX_512bit);
10226     movdl(xmm5, crc);
10227     evpxorq(xmm1, xmm1, xmm5, Assembler::AVX_512bit);
10228     addptr(buf, 64);
10229     subl(len, 7);
10230     evshufi64x2(xmm0, xmm0, xmm0, 0x00, Assembler::AVX_512bit); //propagate the mask from 128 bits to 512 bits
10231 
10232     BIND(Parallel_loop);
10233     fold_128bit_crc32_avx512(xmm1, xmm0, xmm5, buf, 0);
10234     addptr(buf, 64);
10235     subl(len, 4);
10236     jcc(Assembler::greater, Parallel_loop);
10237 
10238     vextracti64x2(xmm2, xmm1, 0x01);
10239     vextracti64x2(xmm3, xmm1, 0x02);
10240     vextracti64x2(xmm4, xmm1, 0x03);
10241     jmp(L_fold_512b);
10242 
10243     BIND(L_No_Parallel);
10244   }
10245   // Fold crc into first bytes of vector
10246   movdqa(xmm1, Address(buf, 0));
10247   movdl(rax, xmm1);
10248   xorl(crc, rax);
10249   if (VM_Version::supports_sse4_1()) {
10250     pinsrd(xmm1, crc, 0);
10251   } else {
10252     pinsrw(xmm1, crc, 0);
10253     shrl(crc, 16);
10254     pinsrw(xmm1, crc, 1);
10255   }
10256   addptr(buf, 16);
10257   subl(len, 4); // len > 0
10258   jcc(Assembler::less, L_fold_tail);
10259 
10260   movdqa(xmm2, Address(buf,  0));
10261   movdqa(xmm3, Address(buf, 16));
10262   movdqa(xmm4, Address(buf, 32));
10263   addptr(buf, 48);
10264   subl(len, 3);
10265   jcc(Assembler::lessEqual, L_fold_512b);
10266 
10267   // Fold total 512 bits of polynomial on each iteration,
10268   // 128 bits per each of 4 parallel streams.
10269   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
10270 
10271   align(32);
10272   BIND(L_fold_512b_loop);
10273   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
10274   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
10275   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
10276   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
10277   addptr(buf, 64);
10278   subl(len, 4);
10279   jcc(Assembler::greater, L_fold_512b_loop);
10280 
10281   // Fold 512 bits to 128 bits.
10282   BIND(L_fold_512b);
10283   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
10284   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
10285   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
10286   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
10287 
10288   // Fold the rest of 128 bits data chunks
10289   BIND(L_fold_tail);
10290   addl(len, 3);
10291   jccb(Assembler::lessEqual, L_fold_128b);
10292   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
10293 
10294   BIND(L_fold_tail_loop);
10295   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
10296   addptr(buf, 16);
10297   decrementl(len);
10298   jccb(Assembler::greater, L_fold_tail_loop);
10299 
10300   // Fold 128 bits in xmm1 down into 32 bits in crc register.
10301   BIND(L_fold_128b);
10302   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()));
10303   if (UseAVX > 0) {
10304     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
10305     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
10306     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
10307   } else {
10308     movdqa(xmm2, xmm0);
10309     pclmulqdq(xmm2, xmm1, 0x1);
10310     movdqa(xmm3, xmm0);
10311     pand(xmm3, xmm2);
10312     pclmulqdq(xmm0, xmm3, 0x1);
10313   }
10314   psrldq(xmm1, 8);
10315   psrldq(xmm2, 4);
10316   pxor(xmm0, xmm1);
10317   pxor(xmm0, xmm2);
10318 
10319   // 8 8-bit folds to compute 32-bit CRC.
10320   for (int j = 0; j < 4; j++) {
10321     fold_8bit_crc32(xmm0, table, xmm1, rax);
10322   }
10323   movdl(crc, xmm0); // mov 32 bits to general register
10324   for (int j = 0; j < 4; j++) {
10325     fold_8bit_crc32(crc, table, rax);
10326   }
10327 
10328   BIND(L_tail_restore);
10329   movl(len, tmp); // restore
10330   BIND(L_tail);
10331   andl(len, 0xf);
10332   jccb(Assembler::zero, L_exit);
10333 
10334   // Fold the rest of bytes
10335   align(4);
10336   BIND(L_tail_loop);
10337   movsbl(rax, Address(buf, 0)); // load byte with sign extension
10338   update_byte_crc32(crc, rax, table);
10339   increment(buf);
10340   decrementl(len);
10341   jccb(Assembler::greater, L_tail_loop);
10342 
10343   BIND(L_exit);
10344   notl(crc); // ~c
10345 }
10346 
10347 #ifdef _LP64
10348 // S. Gueron / Information Processing Letters 112 (2012) 184
10349 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
10350 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
10351 // Output: the 64-bit carry-less product of B * CONST
10352 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
10353                                      Register tmp1, Register tmp2, Register tmp3) {
10354   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
10355   if (n > 0) {
10356     addq(tmp3, n * 256 * 8);
10357   }
10358   //    Q1 = TABLEExt[n][B & 0xFF];
10359   movl(tmp1, in);
10360   andl(tmp1, 0x000000FF);
10361   shll(tmp1, 3);
10362   addq(tmp1, tmp3);
10363   movq(tmp1, Address(tmp1, 0));
10364 
10365   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
10366   movl(tmp2, in);
10367   shrl(tmp2, 8);
10368   andl(tmp2, 0x000000FF);
10369   shll(tmp2, 3);
10370   addq(tmp2, tmp3);
10371   movq(tmp2, Address(tmp2, 0));
10372 
10373   shlq(tmp2, 8);
10374   xorq(tmp1, tmp2);
10375 
10376   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
10377   movl(tmp2, in);
10378   shrl(tmp2, 16);
10379   andl(tmp2, 0x000000FF);
10380   shll(tmp2, 3);
10381   addq(tmp2, tmp3);
10382   movq(tmp2, Address(tmp2, 0));
10383 
10384   shlq(tmp2, 16);
10385   xorq(tmp1, tmp2);
10386 
10387   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
10388   shrl(in, 24);
10389   andl(in, 0x000000FF);
10390   shll(in, 3);
10391   addq(in, tmp3);
10392   movq(in, Address(in, 0));
10393 
10394   shlq(in, 24);
10395   xorq(in, tmp1);
10396   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
10397 }
10398 
10399 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
10400                                       Register in_out,
10401                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
10402                                       XMMRegister w_xtmp2,
10403                                       Register tmp1,
10404                                       Register n_tmp2, Register n_tmp3) {
10405   if (is_pclmulqdq_supported) {
10406     movdl(w_xtmp1, in_out); // modified blindly
10407 
10408     movl(tmp1, const_or_pre_comp_const_index);
10409     movdl(w_xtmp2, tmp1);
10410     pclmulqdq(w_xtmp1, w_xtmp2, 0);
10411 
10412     movdq(in_out, w_xtmp1);
10413   } else {
10414     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
10415   }
10416 }
10417 
10418 // Recombination Alternative 2: No bit-reflections
10419 // T1 = (CRC_A * U1) << 1
10420 // T2 = (CRC_B * U2) << 1
10421 // C1 = T1 >> 32
10422 // C2 = T2 >> 32
10423 // T1 = T1 & 0xFFFFFFFF
10424 // T2 = T2 & 0xFFFFFFFF
10425 // T1 = CRC32(0, T1)
10426 // T2 = CRC32(0, T2)
10427 // C1 = C1 ^ T1
10428 // C2 = C2 ^ T2
10429 // CRC = C1 ^ C2 ^ CRC_C
10430 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
10431                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10432                                      Register tmp1, Register tmp2,
10433                                      Register n_tmp3) {
10434   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10435   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10436   shlq(in_out, 1);
10437   movl(tmp1, in_out);
10438   shrq(in_out, 32);
10439   xorl(tmp2, tmp2);
10440   crc32(tmp2, tmp1, 4);
10441   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
10442   shlq(in1, 1);
10443   movl(tmp1, in1);
10444   shrq(in1, 32);
10445   xorl(tmp2, tmp2);
10446   crc32(tmp2, tmp1, 4);
10447   xorl(in1, tmp2);
10448   xorl(in_out, in1);
10449   xorl(in_out, in2);
10450 }
10451 
10452 // Set N to predefined value
10453 // Subtract from a lenght of a buffer
10454 // execute in a loop:
10455 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
10456 // for i = 1 to N do
10457 //  CRC_A = CRC32(CRC_A, A[i])
10458 //  CRC_B = CRC32(CRC_B, B[i])
10459 //  CRC_C = CRC32(CRC_C, C[i])
10460 // end for
10461 // Recombine
10462 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
10463                                        Register in_out1, Register in_out2, Register in_out3,
10464                                        Register tmp1, Register tmp2, Register tmp3,
10465                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10466                                        Register tmp4, Register tmp5,
10467                                        Register n_tmp6) {
10468   Label L_processPartitions;
10469   Label L_processPartition;
10470   Label L_exit;
10471 
10472   bind(L_processPartitions);
10473   cmpl(in_out1, 3 * size);
10474   jcc(Assembler::less, L_exit);
10475     xorl(tmp1, tmp1);
10476     xorl(tmp2, tmp2);
10477     movq(tmp3, in_out2);
10478     addq(tmp3, size);
10479 
10480     bind(L_processPartition);
10481       crc32(in_out3, Address(in_out2, 0), 8);
10482       crc32(tmp1, Address(in_out2, size), 8);
10483       crc32(tmp2, Address(in_out2, size * 2), 8);
10484       addq(in_out2, 8);
10485       cmpq(in_out2, tmp3);
10486       jcc(Assembler::less, L_processPartition);
10487     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
10488             w_xtmp1, w_xtmp2, w_xtmp3,
10489             tmp4, tmp5,
10490             n_tmp6);
10491     addq(in_out2, 2 * size);
10492     subl(in_out1, 3 * size);
10493     jmp(L_processPartitions);
10494 
10495   bind(L_exit);
10496 }
10497 #else
10498 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
10499                                      Register tmp1, Register tmp2, Register tmp3,
10500                                      XMMRegister xtmp1, XMMRegister xtmp2) {
10501   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
10502   if (n > 0) {
10503     addl(tmp3, n * 256 * 8);
10504   }
10505   //    Q1 = TABLEExt[n][B & 0xFF];
10506   movl(tmp1, in_out);
10507   andl(tmp1, 0x000000FF);
10508   shll(tmp1, 3);
10509   addl(tmp1, tmp3);
10510   movq(xtmp1, Address(tmp1, 0));
10511 
10512   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
10513   movl(tmp2, in_out);
10514   shrl(tmp2, 8);
10515   andl(tmp2, 0x000000FF);
10516   shll(tmp2, 3);
10517   addl(tmp2, tmp3);
10518   movq(xtmp2, Address(tmp2, 0));
10519 
10520   psllq(xtmp2, 8);
10521   pxor(xtmp1, xtmp2);
10522 
10523   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
10524   movl(tmp2, in_out);
10525   shrl(tmp2, 16);
10526   andl(tmp2, 0x000000FF);
10527   shll(tmp2, 3);
10528   addl(tmp2, tmp3);
10529   movq(xtmp2, Address(tmp2, 0));
10530 
10531   psllq(xtmp2, 16);
10532   pxor(xtmp1, xtmp2);
10533 
10534   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
10535   shrl(in_out, 24);
10536   andl(in_out, 0x000000FF);
10537   shll(in_out, 3);
10538   addl(in_out, tmp3);
10539   movq(xtmp2, Address(in_out, 0));
10540 
10541   psllq(xtmp2, 24);
10542   pxor(xtmp1, xtmp2); // Result in CXMM
10543   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
10544 }
10545 
10546 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
10547                                       Register in_out,
10548                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
10549                                       XMMRegister w_xtmp2,
10550                                       Register tmp1,
10551                                       Register n_tmp2, Register n_tmp3) {
10552   if (is_pclmulqdq_supported) {
10553     movdl(w_xtmp1, in_out);
10554 
10555     movl(tmp1, const_or_pre_comp_const_index);
10556     movdl(w_xtmp2, tmp1);
10557     pclmulqdq(w_xtmp1, w_xtmp2, 0);
10558     // Keep result in XMM since GPR is 32 bit in length
10559   } else {
10560     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
10561   }
10562 }
10563 
10564 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
10565                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10566                                      Register tmp1, Register tmp2,
10567                                      Register n_tmp3) {
10568   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10569   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10570 
10571   psllq(w_xtmp1, 1);
10572   movdl(tmp1, w_xtmp1);
10573   psrlq(w_xtmp1, 32);
10574   movdl(in_out, w_xtmp1);
10575 
10576   xorl(tmp2, tmp2);
10577   crc32(tmp2, tmp1, 4);
10578   xorl(in_out, tmp2);
10579 
10580   psllq(w_xtmp2, 1);
10581   movdl(tmp1, w_xtmp2);
10582   psrlq(w_xtmp2, 32);
10583   movdl(in1, w_xtmp2);
10584 
10585   xorl(tmp2, tmp2);
10586   crc32(tmp2, tmp1, 4);
10587   xorl(in1, tmp2);
10588   xorl(in_out, in1);
10589   xorl(in_out, in2);
10590 }
10591 
10592 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
10593                                        Register in_out1, Register in_out2, Register in_out3,
10594                                        Register tmp1, Register tmp2, Register tmp3,
10595                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10596                                        Register tmp4, Register tmp5,
10597                                        Register n_tmp6) {
10598   Label L_processPartitions;
10599   Label L_processPartition;
10600   Label L_exit;
10601 
10602   bind(L_processPartitions);
10603   cmpl(in_out1, 3 * size);
10604   jcc(Assembler::less, L_exit);
10605     xorl(tmp1, tmp1);
10606     xorl(tmp2, tmp2);
10607     movl(tmp3, in_out2);
10608     addl(tmp3, size);
10609 
10610     bind(L_processPartition);
10611       crc32(in_out3, Address(in_out2, 0), 4);
10612       crc32(tmp1, Address(in_out2, size), 4);
10613       crc32(tmp2, Address(in_out2, size*2), 4);
10614       crc32(in_out3, Address(in_out2, 0+4), 4);
10615       crc32(tmp1, Address(in_out2, size+4), 4);
10616       crc32(tmp2, Address(in_out2, size*2+4), 4);
10617       addl(in_out2, 8);
10618       cmpl(in_out2, tmp3);
10619       jcc(Assembler::less, L_processPartition);
10620 
10621         push(tmp3);
10622         push(in_out1);
10623         push(in_out2);
10624         tmp4 = tmp3;
10625         tmp5 = in_out1;
10626         n_tmp6 = in_out2;
10627 
10628       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
10629             w_xtmp1, w_xtmp2, w_xtmp3,
10630             tmp4, tmp5,
10631             n_tmp6);
10632 
10633         pop(in_out2);
10634         pop(in_out1);
10635         pop(tmp3);
10636 
10637     addl(in_out2, 2 * size);
10638     subl(in_out1, 3 * size);
10639     jmp(L_processPartitions);
10640 
10641   bind(L_exit);
10642 }
10643 #endif //LP64
10644 
10645 #ifdef _LP64
10646 // Algorithm 2: Pipelined usage of the CRC32 instruction.
10647 // Input: A buffer I of L bytes.
10648 // Output: the CRC32C value of the buffer.
10649 // Notations:
10650 // Write L = 24N + r, with N = floor (L/24).
10651 // r = L mod 24 (0 <= r < 24).
10652 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
10653 // N quadwords, and R consists of r bytes.
10654 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
10655 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
10656 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
10657 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
10658 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10659                                           Register tmp1, Register tmp2, Register tmp3,
10660                                           Register tmp4, Register tmp5, Register tmp6,
10661                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10662                                           bool is_pclmulqdq_supported) {
10663   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10664   Label L_wordByWord;
10665   Label L_byteByByteProlog;
10666   Label L_byteByByte;
10667   Label L_exit;
10668 
10669   if (is_pclmulqdq_supported ) {
10670     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10671     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
10672 
10673     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10674     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10675 
10676     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10677     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10678     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
10679   } else {
10680     const_or_pre_comp_const_index[0] = 1;
10681     const_or_pre_comp_const_index[1] = 0;
10682 
10683     const_or_pre_comp_const_index[2] = 3;
10684     const_or_pre_comp_const_index[3] = 2;
10685 
10686     const_or_pre_comp_const_index[4] = 5;
10687     const_or_pre_comp_const_index[5] = 4;
10688    }
10689   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10690                     in2, in1, in_out,
10691                     tmp1, tmp2, tmp3,
10692                     w_xtmp1, w_xtmp2, w_xtmp3,
10693                     tmp4, tmp5,
10694                     tmp6);
10695   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10696                     in2, in1, in_out,
10697                     tmp1, tmp2, tmp3,
10698                     w_xtmp1, w_xtmp2, w_xtmp3,
10699                     tmp4, tmp5,
10700                     tmp6);
10701   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10702                     in2, in1, in_out,
10703                     tmp1, tmp2, tmp3,
10704                     w_xtmp1, w_xtmp2, w_xtmp3,
10705                     tmp4, tmp5,
10706                     tmp6);
10707   movl(tmp1, in2);
10708   andl(tmp1, 0x00000007);
10709   negl(tmp1);
10710   addl(tmp1, in2);
10711   addq(tmp1, in1);
10712 
10713   BIND(L_wordByWord);
10714   cmpq(in1, tmp1);
10715   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10716     crc32(in_out, Address(in1, 0), 4);
10717     addq(in1, 4);
10718     jmp(L_wordByWord);
10719 
10720   BIND(L_byteByByteProlog);
10721   andl(in2, 0x00000007);
10722   movl(tmp2, 1);
10723 
10724   BIND(L_byteByByte);
10725   cmpl(tmp2, in2);
10726   jccb(Assembler::greater, L_exit);
10727     crc32(in_out, Address(in1, 0), 1);
10728     incq(in1);
10729     incl(tmp2);
10730     jmp(L_byteByByte);
10731 
10732   BIND(L_exit);
10733 }
10734 #else
10735 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10736                                           Register tmp1, Register  tmp2, Register tmp3,
10737                                           Register tmp4, Register  tmp5, Register tmp6,
10738                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10739                                           bool is_pclmulqdq_supported) {
10740   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10741   Label L_wordByWord;
10742   Label L_byteByByteProlog;
10743   Label L_byteByByte;
10744   Label L_exit;
10745 
10746   if (is_pclmulqdq_supported) {
10747     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10748     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
10749 
10750     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10751     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10752 
10753     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10754     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10755   } else {
10756     const_or_pre_comp_const_index[0] = 1;
10757     const_or_pre_comp_const_index[1] = 0;
10758 
10759     const_or_pre_comp_const_index[2] = 3;
10760     const_or_pre_comp_const_index[3] = 2;
10761 
10762     const_or_pre_comp_const_index[4] = 5;
10763     const_or_pre_comp_const_index[5] = 4;
10764   }
10765   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10766                     in2, in1, in_out,
10767                     tmp1, tmp2, tmp3,
10768                     w_xtmp1, w_xtmp2, w_xtmp3,
10769                     tmp4, tmp5,
10770                     tmp6);
10771   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10772                     in2, in1, in_out,
10773                     tmp1, tmp2, tmp3,
10774                     w_xtmp1, w_xtmp2, w_xtmp3,
10775                     tmp4, tmp5,
10776                     tmp6);
10777   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10778                     in2, in1, in_out,
10779                     tmp1, tmp2, tmp3,
10780                     w_xtmp1, w_xtmp2, w_xtmp3,
10781                     tmp4, tmp5,
10782                     tmp6);
10783   movl(tmp1, in2);
10784   andl(tmp1, 0x00000007);
10785   negl(tmp1);
10786   addl(tmp1, in2);
10787   addl(tmp1, in1);
10788 
10789   BIND(L_wordByWord);
10790   cmpl(in1, tmp1);
10791   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10792     crc32(in_out, Address(in1,0), 4);
10793     addl(in1, 4);
10794     jmp(L_wordByWord);
10795 
10796   BIND(L_byteByByteProlog);
10797   andl(in2, 0x00000007);
10798   movl(tmp2, 1);
10799 
10800   BIND(L_byteByByte);
10801   cmpl(tmp2, in2);
10802   jccb(Assembler::greater, L_exit);
10803     movb(tmp1, Address(in1, 0));
10804     crc32(in_out, tmp1, 1);
10805     incl(in1);
10806     incl(tmp2);
10807     jmp(L_byteByByte);
10808 
10809   BIND(L_exit);
10810 }
10811 #endif // LP64
10812 #undef BIND
10813 #undef BLOCK_COMMENT
10814 
10815 // Compress char[] array to byte[].
10816 //   ..\jdk\src\java.base\share\classes\java\lang\StringUTF16.java
10817 //   @HotSpotIntrinsicCandidate
10818 //   private static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
10819 //     for (int i = 0; i < len; i++) {
10820 //       int c = src[srcOff++];
10821 //       if (c >>> 8 != 0) {
10822 //         return 0;
10823 //       }
10824 //       dst[dstOff++] = (byte)c;
10825 //     }
10826 //     return len;
10827 //   }
10828 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
10829   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
10830   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
10831   Register tmp5, Register result) {
10832   Label copy_chars_loop, return_length, return_zero, done, below_threshold;
10833 
10834   // rsi: src
10835   // rdi: dst
10836   // rdx: len
10837   // rcx: tmp5
10838   // rax: result
10839 
10840   // rsi holds start addr of source char[] to be compressed
10841   // rdi holds start addr of destination byte[]
10842   // rdx holds length
10843 
10844   assert(len != result, "");
10845 
10846   // save length for return
10847   push(len);
10848 
10849   if ((UseAVX > 2) && // AVX512
10850     VM_Version::supports_avx512vlbw() &&
10851     VM_Version::supports_bmi2()) {
10852 
10853     set_vector_masking();  // opening of the stub context for programming mask registers
10854 
10855     Label copy_32_loop, copy_loop_tail, restore_k1_return_zero;
10856 
10857     // alignement
10858     Label post_alignement;
10859 
10860     // if length of the string is less than 16, handle it in an old fashioned
10861     // way
10862     testl(len, -32);
10863     jcc(Assembler::zero, below_threshold);
10864 
10865     // First check whether a character is compressable ( <= 0xFF).
10866     // Create mask to test for Unicode chars inside zmm vector
10867     movl(result, 0x00FF);
10868     evpbroadcastw(tmp2Reg, result, Assembler::AVX_512bit);
10869 
10870     // Save k1
10871     kmovql(k3, k1);
10872 
10873     testl(len, -64);
10874     jcc(Assembler::zero, post_alignement);
10875 
10876     movl(tmp5, dst);
10877     andl(tmp5, (32 - 1));
10878     negl(tmp5);
10879     andl(tmp5, (32 - 1));
10880 
10881     // bail out when there is nothing to be done
10882     testl(tmp5, 0xFFFFFFFF);
10883     jcc(Assembler::zero, post_alignement);
10884 
10885     // ~(~0 << len), where len is the # of remaining elements to process
10886     movl(result, 0xFFFFFFFF);
10887     shlxl(result, result, tmp5);
10888     notl(result);
10889     kmovdl(k1, result);
10890 
10891     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
10892     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10893     ktestd(k2, k1);
10894     jcc(Assembler::carryClear, restore_k1_return_zero);
10895 
10896     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
10897 
10898     addptr(src, tmp5);
10899     addptr(src, tmp5);
10900     addptr(dst, tmp5);
10901     subl(len, tmp5);
10902 
10903     bind(post_alignement);
10904     // end of alignement
10905 
10906     movl(tmp5, len);
10907     andl(tmp5, (32 - 1));    // tail count (in chars)
10908     andl(len, ~(32 - 1));    // vector count (in chars)
10909     jcc(Assembler::zero, copy_loop_tail);
10910 
10911     lea(src, Address(src, len, Address::times_2));
10912     lea(dst, Address(dst, len, Address::times_1));
10913     negptr(len);
10914 
10915     bind(copy_32_loop);
10916     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
10917     evpcmpuw(k2, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10918     kortestdl(k2, k2);
10919     jcc(Assembler::carryClear, restore_k1_return_zero);
10920 
10921     // All elements in current processed chunk are valid candidates for
10922     // compression. Write a truncated byte elements to the memory.
10923     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
10924     addptr(len, 32);
10925     jcc(Assembler::notZero, copy_32_loop);
10926 
10927     bind(copy_loop_tail);
10928     // bail out when there is nothing to be done
10929     testl(tmp5, 0xFFFFFFFF);
10930     // Restore k1
10931     kmovql(k1, k3);
10932     jcc(Assembler::zero, return_length);
10933 
10934     movl(len, tmp5);
10935 
10936     // ~(~0 << len), where len is the # of remaining elements to process
10937     movl(result, 0xFFFFFFFF);
10938     shlxl(result, result, len);
10939     notl(result);
10940 
10941     kmovdl(k1, result);
10942 
10943     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
10944     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10945     ktestd(k2, k1);
10946     jcc(Assembler::carryClear, restore_k1_return_zero);
10947 
10948     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
10949     // Restore k1
10950     kmovql(k1, k3);
10951     jmp(return_length);
10952 
10953     bind(restore_k1_return_zero);
10954     // Restore k1
10955     kmovql(k1, k3);
10956     jmp(return_zero);
10957 
10958     clear_vector_masking();   // closing of the stub context for programming mask registers
10959   }
10960   if (UseSSE42Intrinsics) {
10961     Label copy_32_loop, copy_16, copy_tail;
10962 
10963     bind(below_threshold);
10964 
10965     movl(result, len);
10966 
10967     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
10968 
10969     // vectored compression
10970     andl(len, 0xfffffff0);    // vector count (in chars)
10971     andl(result, 0x0000000f);    // tail count (in chars)
10972     testl(len, len);
10973     jccb(Assembler::zero, copy_16);
10974 
10975     // compress 16 chars per iter
10976     movdl(tmp1Reg, tmp5);
10977     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
10978     pxor(tmp4Reg, tmp4Reg);
10979 
10980     lea(src, Address(src, len, Address::times_2));
10981     lea(dst, Address(dst, len, Address::times_1));
10982     negptr(len);
10983 
10984     bind(copy_32_loop);
10985     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
10986     por(tmp4Reg, tmp2Reg);
10987     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
10988     por(tmp4Reg, tmp3Reg);
10989     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
10990     jcc(Assembler::notZero, return_zero);
10991     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
10992     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
10993     addptr(len, 16);
10994     jcc(Assembler::notZero, copy_32_loop);
10995 
10996     // compress next vector of 8 chars (if any)
10997     bind(copy_16);
10998     movl(len, result);
10999     andl(len, 0xfffffff8);    // vector count (in chars)
11000     andl(result, 0x00000007);    // tail count (in chars)
11001     testl(len, len);
11002     jccb(Assembler::zero, copy_tail);
11003 
11004     movdl(tmp1Reg, tmp5);
11005     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
11006     pxor(tmp3Reg, tmp3Reg);
11007 
11008     movdqu(tmp2Reg, Address(src, 0));
11009     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
11010     jccb(Assembler::notZero, return_zero);
11011     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
11012     movq(Address(dst, 0), tmp2Reg);
11013     addptr(src, 16);
11014     addptr(dst, 8);
11015 
11016     bind(copy_tail);
11017     movl(len, result);
11018   }
11019   // compress 1 char per iter
11020   testl(len, len);
11021   jccb(Assembler::zero, return_length);
11022   lea(src, Address(src, len, Address::times_2));
11023   lea(dst, Address(dst, len, Address::times_1));
11024   negptr(len);
11025 
11026   bind(copy_chars_loop);
11027   load_unsigned_short(result, Address(src, len, Address::times_2));
11028   testl(result, 0xff00);      // check if Unicode char
11029   jccb(Assembler::notZero, return_zero);
11030   movb(Address(dst, len, Address::times_1), result);  // ASCII char; compress to 1 byte
11031   increment(len);
11032   jcc(Assembler::notZero, copy_chars_loop);
11033 
11034   // if compression succeeded, return length
11035   bind(return_length);
11036   pop(result);
11037   jmpb(done);
11038 
11039   // if compression failed, return 0
11040   bind(return_zero);
11041   xorl(result, result);
11042   addptr(rsp, wordSize);
11043 
11044   bind(done);
11045 }
11046 
11047 // Inflate byte[] array to char[].
11048 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
11049 //   @HotSpotIntrinsicCandidate
11050 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
11051 //     for (int i = 0; i < len; i++) {
11052 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
11053 //     }
11054 //   }
11055 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
11056   XMMRegister tmp1, Register tmp2) {
11057   Label copy_chars_loop, done, below_threshold;
11058   // rsi: src
11059   // rdi: dst
11060   // rdx: len
11061   // rcx: tmp2
11062 
11063   // rsi holds start addr of source byte[] to be inflated
11064   // rdi holds start addr of destination char[]
11065   // rdx holds length
11066   assert_different_registers(src, dst, len, tmp2);
11067 
11068   if ((UseAVX > 2) && // AVX512
11069     VM_Version::supports_avx512vlbw() &&
11070     VM_Version::supports_bmi2()) {
11071 
11072     set_vector_masking();  // opening of the stub context for programming mask registers
11073 
11074     Label copy_32_loop, copy_tail;
11075     Register tmp3_aliased = len;
11076 
11077     // if length of the string is less than 16, handle it in an old fashioned
11078     // way
11079     testl(len, -16);
11080     jcc(Assembler::zero, below_threshold);
11081 
11082     // In order to use only one arithmetic operation for the main loop we use
11083     // this pre-calculation
11084     movl(tmp2, len);
11085     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
11086     andl(len, -32);     // vector count
11087     jccb(Assembler::zero, copy_tail);
11088 
11089     lea(src, Address(src, len, Address::times_1));
11090     lea(dst, Address(dst, len, Address::times_2));
11091     negptr(len);
11092 
11093 
11094     // inflate 32 chars per iter
11095     bind(copy_32_loop);
11096     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
11097     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
11098     addptr(len, 32);
11099     jcc(Assembler::notZero, copy_32_loop);
11100 
11101     bind(copy_tail);
11102     // bail out when there is nothing to be done
11103     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
11104     jcc(Assembler::zero, done);
11105 
11106     // Save k1
11107     kmovql(k2, k1);
11108 
11109     // ~(~0 << length), where length is the # of remaining elements to process
11110     movl(tmp3_aliased, -1);
11111     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
11112     notl(tmp3_aliased);
11113     kmovdl(k1, tmp3_aliased);
11114     evpmovzxbw(tmp1, k1, Address(src, 0), Assembler::AVX_512bit);
11115     evmovdquw(Address(dst, 0), k1, tmp1, Assembler::AVX_512bit);
11116 
11117     // Restore k1
11118     kmovql(k1, k2);
11119     jmp(done);
11120 
11121     clear_vector_masking();   // closing of the stub context for programming mask registers
11122   }
11123   if (UseSSE42Intrinsics) {
11124     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
11125 
11126     movl(tmp2, len);
11127 
11128     if (UseAVX > 1) {
11129       andl(tmp2, (16 - 1));
11130       andl(len, -16);
11131       jccb(Assembler::zero, copy_new_tail);
11132     } else {
11133       andl(tmp2, 0x00000007);   // tail count (in chars)
11134       andl(len, 0xfffffff8);    // vector count (in chars)
11135       jccb(Assembler::zero, copy_tail);
11136     }
11137 
11138     // vectored inflation
11139     lea(src, Address(src, len, Address::times_1));
11140     lea(dst, Address(dst, len, Address::times_2));
11141     negptr(len);
11142 
11143     if (UseAVX > 1) {
11144       bind(copy_16_loop);
11145       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
11146       vmovdqu(Address(dst, len, Address::times_2), tmp1);
11147       addptr(len, 16);
11148       jcc(Assembler::notZero, copy_16_loop);
11149 
11150       bind(below_threshold);
11151       bind(copy_new_tail);
11152       if ((UseAVX > 2) &&
11153         VM_Version::supports_avx512vlbw() &&
11154         VM_Version::supports_bmi2()) {
11155         movl(tmp2, len);
11156       } else {
11157         movl(len, tmp2);
11158       }
11159       andl(tmp2, 0x00000007);
11160       andl(len, 0xFFFFFFF8);
11161       jccb(Assembler::zero, copy_tail);
11162 
11163       pmovzxbw(tmp1, Address(src, 0));
11164       movdqu(Address(dst, 0), tmp1);
11165       addptr(src, 8);
11166       addptr(dst, 2 * 8);
11167 
11168       jmp(copy_tail, true);
11169     }
11170 
11171     // inflate 8 chars per iter
11172     bind(copy_8_loop);
11173     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
11174     movdqu(Address(dst, len, Address::times_2), tmp1);
11175     addptr(len, 8);
11176     jcc(Assembler::notZero, copy_8_loop);
11177 
11178     bind(copy_tail);
11179     movl(len, tmp2);
11180 
11181     cmpl(len, 4);
11182     jccb(Assembler::less, copy_bytes);
11183 
11184     movdl(tmp1, Address(src, 0));  // load 4 byte chars
11185     pmovzxbw(tmp1, tmp1);
11186     movq(Address(dst, 0), tmp1);
11187     subptr(len, 4);
11188     addptr(src, 4);
11189     addptr(dst, 8);
11190 
11191     bind(copy_bytes);
11192   }
11193   testl(len, len);
11194   jccb(Assembler::zero, done);
11195   lea(src, Address(src, len, Address::times_1));
11196   lea(dst, Address(dst, len, Address::times_2));
11197   negptr(len);
11198 
11199   // inflate 1 char per iter
11200   bind(copy_chars_loop);
11201   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
11202   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
11203   increment(len);
11204   jcc(Assembler::notZero, copy_chars_loop);
11205 
11206   bind(done);
11207 }
11208 
11209 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
11210   switch (cond) {
11211     // Note some conditions are synonyms for others
11212     case Assembler::zero:         return Assembler::notZero;
11213     case Assembler::notZero:      return Assembler::zero;
11214     case Assembler::less:         return Assembler::greaterEqual;
11215     case Assembler::lessEqual:    return Assembler::greater;
11216     case Assembler::greater:      return Assembler::lessEqual;
11217     case Assembler::greaterEqual: return Assembler::less;
11218     case Assembler::below:        return Assembler::aboveEqual;
11219     case Assembler::belowEqual:   return Assembler::above;
11220     case Assembler::above:        return Assembler::belowEqual;
11221     case Assembler::aboveEqual:   return Assembler::below;
11222     case Assembler::overflow:     return Assembler::noOverflow;
11223     case Assembler::noOverflow:   return Assembler::overflow;
11224     case Assembler::negative:     return Assembler::positive;
11225     case Assembler::positive:     return Assembler::negative;
11226     case Assembler::parity:       return Assembler::noParity;
11227     case Assembler::noParity:     return Assembler::parity;
11228   }
11229   ShouldNotReachHere(); return Assembler::overflow;
11230 }
11231 
11232 SkipIfEqual::SkipIfEqual(
11233     MacroAssembler* masm, const bool* flag_addr, bool value) {
11234   _masm = masm;
11235   _masm->cmp8(ExternalAddress((address)flag_addr), value);
11236   _masm->jcc(Assembler::equal, _label);
11237 }
11238 
11239 SkipIfEqual::~SkipIfEqual() {
11240   _masm->bind(_label);
11241 }
11242 
11243 // 32-bit Windows has its own fast-path implementation
11244 // of get_thread
11245 #if !defined(WIN32) || defined(_LP64)
11246 
11247 // This is simply a call to Thread::current()
11248 void MacroAssembler::get_thread(Register thread) {
11249   if (thread != rax) {
11250     push(rax);
11251   }
11252   LP64_ONLY(push(rdi);)
11253   LP64_ONLY(push(rsi);)
11254   push(rdx);
11255   push(rcx);
11256 #ifdef _LP64
11257   push(r8);
11258   push(r9);
11259   push(r10);
11260   push(r11);
11261 #endif
11262 
11263   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
11264 
11265 #ifdef _LP64
11266   pop(r11);
11267   pop(r10);
11268   pop(r9);
11269   pop(r8);
11270 #endif
11271   pop(rcx);
11272   pop(rdx);
11273   LP64_ONLY(pop(rsi);)
11274   LP64_ONLY(pop(rdi);)
11275   if (thread != rax) {
11276     mov(thread, rax);
11277     pop(rax);
11278   }
11279 }
11280 
11281 #endif