1 /*
   2  * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "asm/assembler.hpp"
  28 #include "asm/assembler.inline.hpp"
  29 #include "compiler/disassembler.hpp"
  30 #include "gc/shared/barrierSet.hpp"
  31 #include "gc/shared/barrierSetAssembler.hpp"
  32 #include "gc/shared/collectedHeap.inline.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "memory/universe.hpp"
  36 #include "oops/accessDecorators.hpp"
  37 #include "oops/klass.inline.hpp"
  38 #include "prims/methodHandles.hpp"
  39 #include "runtime/biasedLocking.hpp"
  40 #include "runtime/flags/flagSetting.hpp"
  41 #include "runtime/interfaceSupport.inline.hpp"
  42 #include "runtime/objectMonitor.hpp"
  43 #include "runtime/os.hpp"
  44 #include "runtime/safepoint.hpp"
  45 #include "runtime/safepointMechanism.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/stubRoutines.hpp"
  48 #include "runtime/thread.hpp"
  49 #include "utilities/macros.hpp"
  50 #include "crc32c.h"
  51 #ifdef COMPILER2
  52 #include "opto/intrinsicnode.hpp"
  53 #endif
  54 
  55 #ifdef PRODUCT
  56 #define BLOCK_COMMENT(str) /* nothing */
  57 #define STOP(error) stop(error)
  58 #else
  59 #define BLOCK_COMMENT(str) block_comment(str)
  60 #define STOP(error) block_comment(error); stop(error)
  61 #endif
  62 
  63 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  64 
  65 #ifdef ASSERT
  66 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  67 #endif
  68 
  69 static Assembler::Condition reverse[] = {
  70     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  71     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  72     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  73     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  74     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
  75     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
  76     Assembler::above          /* belowEqual    = 0x6 */ ,
  77     Assembler::belowEqual     /* above         = 0x7 */ ,
  78     Assembler::positive       /* negative      = 0x8 */ ,
  79     Assembler::negative       /* positive      = 0x9 */ ,
  80     Assembler::noParity       /* parity        = 0xa */ ,
  81     Assembler::parity         /* noParity      = 0xb */ ,
  82     Assembler::greaterEqual   /* less          = 0xc */ ,
  83     Assembler::less           /* greaterEqual  = 0xd */ ,
  84     Assembler::greater        /* lessEqual     = 0xe */ ,
  85     Assembler::lessEqual      /* greater       = 0xf, */
  86 
  87 };
  88 
  89 
  90 // Implementation of MacroAssembler
  91 
  92 // First all the versions that have distinct versions depending on 32/64 bit
  93 // Unless the difference is trivial (1 line or so).
  94 
  95 #ifndef _LP64
  96 
  97 // 32bit versions
  98 
  99 Address MacroAssembler::as_Address(AddressLiteral adr) {
 100   return Address(adr.target(), adr.rspec());
 101 }
 102 
 103 Address MacroAssembler::as_Address(ArrayAddress adr) {
 104   return Address::make_array(adr);
 105 }
 106 
 107 void MacroAssembler::call_VM_leaf_base(address entry_point,
 108                                        int number_of_arguments) {
 109   call(RuntimeAddress(entry_point));
 110   increment(rsp, number_of_arguments * wordSize);
 111 }
 112 
 113 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
 114   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 115 }
 116 
 117 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
 118   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 119 }
 120 
 121 void MacroAssembler::cmpoop_raw(Address src1, jobject obj) {
 122   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 123 }
 124 
 125 void MacroAssembler::cmpoop_raw(Register src1, jobject obj) {
 126   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 127 }
 128 
 129 void MacroAssembler::cmpoop(Address src1, jobject obj) {
 130   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 131   bs->obj_equals(this, src1, obj);
 132 }
 133 
 134 void MacroAssembler::cmpoop(Register src1, jobject obj) {
 135   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 136   bs->obj_equals(this, src1, obj);
 137 }
 138 
 139 void MacroAssembler::extend_sign(Register hi, Register lo) {
 140   // According to Intel Doc. AP-526, "Integer Divide", p.18.
 141   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
 142     cdql();
 143   } else {
 144     movl(hi, lo);
 145     sarl(hi, 31);
 146   }
 147 }
 148 
 149 void MacroAssembler::jC2(Register tmp, Label& L) {
 150   // set parity bit if FPU flag C2 is set (via rax)
 151   save_rax(tmp);
 152   fwait(); fnstsw_ax();
 153   sahf();
 154   restore_rax(tmp);
 155   // branch
 156   jcc(Assembler::parity, L);
 157 }
 158 
 159 void MacroAssembler::jnC2(Register tmp, Label& L) {
 160   // set parity bit if FPU flag C2 is set (via rax)
 161   save_rax(tmp);
 162   fwait(); fnstsw_ax();
 163   sahf();
 164   restore_rax(tmp);
 165   // branch
 166   jcc(Assembler::noParity, L);
 167 }
 168 
 169 // 32bit can do a case table jump in one instruction but we no longer allow the base
 170 // to be installed in the Address class
 171 void MacroAssembler::jump(ArrayAddress entry) {
 172   jmp(as_Address(entry));
 173 }
 174 
 175 // Note: y_lo will be destroyed
 176 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 177   // Long compare for Java (semantics as described in JVM spec.)
 178   Label high, low, done;
 179 
 180   cmpl(x_hi, y_hi);
 181   jcc(Assembler::less, low);
 182   jcc(Assembler::greater, high);
 183   // x_hi is the return register
 184   xorl(x_hi, x_hi);
 185   cmpl(x_lo, y_lo);
 186   jcc(Assembler::below, low);
 187   jcc(Assembler::equal, done);
 188 
 189   bind(high);
 190   xorl(x_hi, x_hi);
 191   increment(x_hi);
 192   jmp(done);
 193 
 194   bind(low);
 195   xorl(x_hi, x_hi);
 196   decrementl(x_hi);
 197 
 198   bind(done);
 199 }
 200 
 201 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 202     mov_literal32(dst, (int32_t)src.target(), src.rspec());
 203 }
 204 
 205 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 206   // leal(dst, as_Address(adr));
 207   // see note in movl as to why we must use a move
 208   mov_literal32(dst, (int32_t) adr.target(), adr.rspec());
 209 }
 210 
 211 void MacroAssembler::leave() {
 212   mov(rsp, rbp);
 213   pop(rbp);
 214 }
 215 
 216 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
 217   // Multiplication of two Java long values stored on the stack
 218   // as illustrated below. Result is in rdx:rax.
 219   //
 220   // rsp ---> [  ??  ] \               \
 221   //            ....    | y_rsp_offset  |
 222   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
 223   //          [ y_hi ]                  | (in bytes)
 224   //            ....                    |
 225   //          [ x_lo ]                 /
 226   //          [ x_hi ]
 227   //            ....
 228   //
 229   // Basic idea: lo(result) = lo(x_lo * y_lo)
 230   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
 231   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
 232   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
 233   Label quick;
 234   // load x_hi, y_hi and check if quick
 235   // multiplication is possible
 236   movl(rbx, x_hi);
 237   movl(rcx, y_hi);
 238   movl(rax, rbx);
 239   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
 240   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
 241   // do full multiplication
 242   // 1st step
 243   mull(y_lo);                                    // x_hi * y_lo
 244   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
 245   // 2nd step
 246   movl(rax, x_lo);
 247   mull(rcx);                                     // x_lo * y_hi
 248   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
 249   // 3rd step
 250   bind(quick);                                   // note: rbx, = 0 if quick multiply!
 251   movl(rax, x_lo);
 252   mull(y_lo);                                    // x_lo * y_lo
 253   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
 254 }
 255 
 256 void MacroAssembler::lneg(Register hi, Register lo) {
 257   negl(lo);
 258   adcl(hi, 0);
 259   negl(hi);
 260 }
 261 
 262 void MacroAssembler::lshl(Register hi, Register lo) {
 263   // Java shift left long support (semantics as described in JVM spec., p.305)
 264   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
 265   // shift value is in rcx !
 266   assert(hi != rcx, "must not use rcx");
 267   assert(lo != rcx, "must not use rcx");
 268   const Register s = rcx;                        // shift count
 269   const int      n = BitsPerWord;
 270   Label L;
 271   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 272   cmpl(s, n);                                    // if (s < n)
 273   jcc(Assembler::less, L);                       // else (s >= n)
 274   movl(hi, lo);                                  // x := x << n
 275   xorl(lo, lo);
 276   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 277   bind(L);                                       // s (mod n) < n
 278   shldl(hi, lo);                                 // x := x << s
 279   shll(lo);
 280 }
 281 
 282 
 283 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
 284   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
 285   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
 286   assert(hi != rcx, "must not use rcx");
 287   assert(lo != rcx, "must not use rcx");
 288   const Register s = rcx;                        // shift count
 289   const int      n = BitsPerWord;
 290   Label L;
 291   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 292   cmpl(s, n);                                    // if (s < n)
 293   jcc(Assembler::less, L);                       // else (s >= n)
 294   movl(lo, hi);                                  // x := x >> n
 295   if (sign_extension) sarl(hi, 31);
 296   else                xorl(hi, hi);
 297   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 298   bind(L);                                       // s (mod n) < n
 299   shrdl(lo, hi);                                 // x := x >> s
 300   if (sign_extension) sarl(hi);
 301   else                shrl(hi);
 302 }
 303 
 304 void MacroAssembler::movoop(Register dst, jobject obj) {
 305   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 306 }
 307 
 308 void MacroAssembler::movoop(Address dst, jobject obj) {
 309   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 310 }
 311 
 312 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 313   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 314 }
 315 
 316 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 317   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 318 }
 319 
 320 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 321   // scratch register is not used,
 322   // it is defined to match parameters of 64-bit version of this method.
 323   if (src.is_lval()) {
 324     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
 325   } else {
 326     movl(dst, as_Address(src));
 327   }
 328 }
 329 
 330 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 331   movl(as_Address(dst), src);
 332 }
 333 
 334 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 335   movl(dst, as_Address(src));
 336 }
 337 
 338 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 339 void MacroAssembler::movptr(Address dst, intptr_t src) {
 340   movl(dst, src);
 341 }
 342 
 343 
 344 void MacroAssembler::pop_callee_saved_registers() {
 345   pop(rcx);
 346   pop(rdx);
 347   pop(rdi);
 348   pop(rsi);
 349 }
 350 
 351 void MacroAssembler::pop_fTOS() {
 352   fld_d(Address(rsp, 0));
 353   addl(rsp, 2 * wordSize);
 354 }
 355 
 356 void MacroAssembler::push_callee_saved_registers() {
 357   push(rsi);
 358   push(rdi);
 359   push(rdx);
 360   push(rcx);
 361 }
 362 
 363 void MacroAssembler::push_fTOS() {
 364   subl(rsp, 2 * wordSize);
 365   fstp_d(Address(rsp, 0));
 366 }
 367 
 368 
 369 void MacroAssembler::pushoop(jobject obj) {
 370   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
 371 }
 372 
 373 void MacroAssembler::pushklass(Metadata* obj) {
 374   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
 375 }
 376 
 377 void MacroAssembler::pushptr(AddressLiteral src) {
 378   if (src.is_lval()) {
 379     push_literal32((int32_t)src.target(), src.rspec());
 380   } else {
 381     pushl(as_Address(src));
 382   }
 383 }
 384 
 385 void MacroAssembler::set_word_if_not_zero(Register dst) {
 386   xorl(dst, dst);
 387   set_byte_if_not_zero(dst);
 388 }
 389 
 390 static void pass_arg0(MacroAssembler* masm, Register arg) {
 391   masm->push(arg);
 392 }
 393 
 394 static void pass_arg1(MacroAssembler* masm, Register arg) {
 395   masm->push(arg);
 396 }
 397 
 398 static void pass_arg2(MacroAssembler* masm, Register arg) {
 399   masm->push(arg);
 400 }
 401 
 402 static void pass_arg3(MacroAssembler* masm, Register arg) {
 403   masm->push(arg);
 404 }
 405 
 406 #ifndef PRODUCT
 407 extern "C" void findpc(intptr_t x);
 408 #endif
 409 
 410 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
 411   // In order to get locks to work, we need to fake a in_VM state
 412   JavaThread* thread = JavaThread::current();
 413   JavaThreadState saved_state = thread->thread_state();
 414   thread->set_thread_state(_thread_in_vm);
 415   if (ShowMessageBoxOnError) {
 416     JavaThread* thread = JavaThread::current();
 417     JavaThreadState saved_state = thread->thread_state();
 418     thread->set_thread_state(_thread_in_vm);
 419     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 420       ttyLocker ttyl;
 421       BytecodeCounter::print();
 422     }
 423     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 424     // This is the value of eip which points to where verify_oop will return.
 425     if (os::message_box(msg, "Execution stopped, print registers?")) {
 426       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
 427       BREAKPOINT;
 428     }
 429   } else {
 430     ttyLocker ttyl;
 431     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
 432   }
 433   // Don't assert holding the ttyLock
 434     assert(false, "DEBUG MESSAGE: %s", msg);
 435   ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 436 }
 437 
 438 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
 439   ttyLocker ttyl;
 440   FlagSetting fs(Debugging, true);
 441   tty->print_cr("eip = 0x%08x", eip);
 442 #ifndef PRODUCT
 443   if ((WizardMode || Verbose) && PrintMiscellaneous) {
 444     tty->cr();
 445     findpc(eip);
 446     tty->cr();
 447   }
 448 #endif
 449 #define PRINT_REG(rax) \
 450   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
 451   PRINT_REG(rax);
 452   PRINT_REG(rbx);
 453   PRINT_REG(rcx);
 454   PRINT_REG(rdx);
 455   PRINT_REG(rdi);
 456   PRINT_REG(rsi);
 457   PRINT_REG(rbp);
 458   PRINT_REG(rsp);
 459 #undef PRINT_REG
 460   // Print some words near top of staack.
 461   int* dump_sp = (int*) rsp;
 462   for (int col1 = 0; col1 < 8; col1++) {
 463     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 464     os::print_location(tty, *dump_sp++);
 465   }
 466   for (int row = 0; row < 16; row++) {
 467     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 468     for (int col = 0; col < 8; col++) {
 469       tty->print(" 0x%08x", *dump_sp++);
 470     }
 471     tty->cr();
 472   }
 473   // Print some instructions around pc:
 474   Disassembler::decode((address)eip-64, (address)eip);
 475   tty->print_cr("--------");
 476   Disassembler::decode((address)eip, (address)eip+32);
 477 }
 478 
 479 void MacroAssembler::stop(const char* msg) {
 480   ExternalAddress message((address)msg);
 481   // push address of message
 482   pushptr(message.addr());
 483   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 484   pusha();                                            // push registers
 485   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 486   hlt();
 487 }
 488 
 489 void MacroAssembler::warn(const char* msg) {
 490   push_CPU_state();
 491 
 492   ExternalAddress message((address) msg);
 493   // push address of message
 494   pushptr(message.addr());
 495 
 496   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
 497   addl(rsp, wordSize);       // discard argument
 498   pop_CPU_state();
 499 }
 500 
 501 void MacroAssembler::print_state() {
 502   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 503   pusha();                                            // push registers
 504 
 505   push_CPU_state();
 506   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
 507   pop_CPU_state();
 508 
 509   popa();
 510   addl(rsp, wordSize);
 511 }
 512 
 513 #else // _LP64
 514 
 515 // 64 bit versions
 516 
 517 Address MacroAssembler::as_Address(AddressLiteral adr) {
 518   // amd64 always does this as a pc-rel
 519   // we can be absolute or disp based on the instruction type
 520   // jmp/call are displacements others are absolute
 521   assert(!adr.is_lval(), "must be rval");
 522   assert(reachable(adr), "must be");
 523   return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
 524 
 525 }
 526 
 527 Address MacroAssembler::as_Address(ArrayAddress adr) {
 528   AddressLiteral base = adr.base();
 529   lea(rscratch1, base);
 530   Address index = adr.index();
 531   assert(index._disp == 0, "must not have disp"); // maybe it can?
 532   Address array(rscratch1, index._index, index._scale, index._disp);
 533   return array;
 534 }
 535 
 536 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
 537   Label L, E;
 538 
 539 #ifdef _WIN64
 540   // Windows always allocates space for it's register args
 541   assert(num_args <= 4, "only register arguments supported");
 542   subq(rsp,  frame::arg_reg_save_area_bytes);
 543 #endif
 544 
 545   // Align stack if necessary
 546   testl(rsp, 15);
 547   jcc(Assembler::zero, L);
 548 
 549   subq(rsp, 8);
 550   {
 551     call(RuntimeAddress(entry_point));
 552   }
 553   addq(rsp, 8);
 554   jmp(E);
 555 
 556   bind(L);
 557   {
 558     call(RuntimeAddress(entry_point));
 559   }
 560 
 561   bind(E);
 562 
 563 #ifdef _WIN64
 564   // restore stack pointer
 565   addq(rsp, frame::arg_reg_save_area_bytes);
 566 #endif
 567 
 568 }
 569 
 570 void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
 571   assert(!src2.is_lval(), "should use cmpptr");
 572 
 573   if (reachable(src2)) {
 574     cmpq(src1, as_Address(src2));
 575   } else {
 576     lea(rscratch1, src2);
 577     Assembler::cmpq(src1, Address(rscratch1, 0));
 578   }
 579 }
 580 
 581 int MacroAssembler::corrected_idivq(Register reg) {
 582   // Full implementation of Java ldiv and lrem; checks for special
 583   // case as described in JVM spec., p.243 & p.271.  The function
 584   // returns the (pc) offset of the idivl instruction - may be needed
 585   // for implicit exceptions.
 586   //
 587   //         normal case                           special case
 588   //
 589   // input : rax: dividend                         min_long
 590   //         reg: divisor   (may not be eax/edx)   -1
 591   //
 592   // output: rax: quotient  (= rax idiv reg)       min_long
 593   //         rdx: remainder (= rax irem reg)       0
 594   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
 595   static const int64_t min_long = 0x8000000000000000;
 596   Label normal_case, special_case;
 597 
 598   // check for special case
 599   cmp64(rax, ExternalAddress((address) &min_long));
 600   jcc(Assembler::notEqual, normal_case);
 601   xorl(rdx, rdx); // prepare rdx for possible special case (where
 602                   // remainder = 0)
 603   cmpq(reg, -1);
 604   jcc(Assembler::equal, special_case);
 605 
 606   // handle normal case
 607   bind(normal_case);
 608   cdqq();
 609   int idivq_offset = offset();
 610   idivq(reg);
 611 
 612   // normal and special case exit
 613   bind(special_case);
 614 
 615   return idivq_offset;
 616 }
 617 
 618 void MacroAssembler::decrementq(Register reg, int value) {
 619   if (value == min_jint) { subq(reg, value); return; }
 620   if (value <  0) { incrementq(reg, -value); return; }
 621   if (value == 0) {                        ; return; }
 622   if (value == 1 && UseIncDec) { decq(reg) ; return; }
 623   /* else */      { subq(reg, value)       ; return; }
 624 }
 625 
 626 void MacroAssembler::decrementq(Address dst, int value) {
 627   if (value == min_jint) { subq(dst, value); return; }
 628   if (value <  0) { incrementq(dst, -value); return; }
 629   if (value == 0) {                        ; return; }
 630   if (value == 1 && UseIncDec) { decq(dst) ; return; }
 631   /* else */      { subq(dst, value)       ; return; }
 632 }
 633 
 634 void MacroAssembler::incrementq(AddressLiteral dst) {
 635   if (reachable(dst)) {
 636     incrementq(as_Address(dst));
 637   } else {
 638     lea(rscratch1, dst);
 639     incrementq(Address(rscratch1, 0));
 640   }
 641 }
 642 
 643 void MacroAssembler::incrementq(Register reg, int value) {
 644   if (value == min_jint) { addq(reg, value); return; }
 645   if (value <  0) { decrementq(reg, -value); return; }
 646   if (value == 0) {                        ; return; }
 647   if (value == 1 && UseIncDec) { incq(reg) ; return; }
 648   /* else */      { addq(reg, value)       ; return; }
 649 }
 650 
 651 void MacroAssembler::incrementq(Address dst, int value) {
 652   if (value == min_jint) { addq(dst, value); return; }
 653   if (value <  0) { decrementq(dst, -value); return; }
 654   if (value == 0) {                        ; return; }
 655   if (value == 1 && UseIncDec) { incq(dst) ; return; }
 656   /* else */      { addq(dst, value)       ; return; }
 657 }
 658 
 659 // 32bit can do a case table jump in one instruction but we no longer allow the base
 660 // to be installed in the Address class
 661 void MacroAssembler::jump(ArrayAddress entry) {
 662   lea(rscratch1, entry.base());
 663   Address dispatch = entry.index();
 664   assert(dispatch._base == noreg, "must be");
 665   dispatch._base = rscratch1;
 666   jmp(dispatch);
 667 }
 668 
 669 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 670   ShouldNotReachHere(); // 64bit doesn't use two regs
 671   cmpq(x_lo, y_lo);
 672 }
 673 
 674 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 675     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 676 }
 677 
 678 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 679   mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());
 680   movptr(dst, rscratch1);
 681 }
 682 
 683 void MacroAssembler::leave() {
 684   // %%% is this really better? Why not on 32bit too?
 685   emit_int8((unsigned char)0xC9); // LEAVE
 686 }
 687 
 688 void MacroAssembler::lneg(Register hi, Register lo) {
 689   ShouldNotReachHere(); // 64bit doesn't use two regs
 690   negq(lo);
 691 }
 692 
 693 void MacroAssembler::movoop(Register dst, jobject obj) {
 694   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 695 }
 696 
 697 void MacroAssembler::movoop(Address dst, jobject obj) {
 698   mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 699   movq(dst, rscratch1);
 700 }
 701 
 702 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 703   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 704 }
 705 
 706 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 707   mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 708   movq(dst, rscratch1);
 709 }
 710 
 711 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 712   if (src.is_lval()) {
 713     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 714   } else {
 715     if (reachable(src)) {
 716       movq(dst, as_Address(src));
 717     } else {
 718       lea(scratch, src);
 719       movq(dst, Address(scratch, 0));
 720     }
 721   }
 722 }
 723 
 724 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 725   movq(as_Address(dst), src);
 726 }
 727 
 728 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 729   movq(dst, as_Address(src));
 730 }
 731 
 732 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 733 void MacroAssembler::movptr(Address dst, intptr_t src) {
 734   mov64(rscratch1, src);
 735   movq(dst, rscratch1);
 736 }
 737 
 738 // These are mostly for initializing NULL
 739 void MacroAssembler::movptr(Address dst, int32_t src) {
 740   movslq(dst, src);
 741 }
 742 
 743 void MacroAssembler::movptr(Register dst, int32_t src) {
 744   mov64(dst, (intptr_t)src);
 745 }
 746 
 747 void MacroAssembler::pushoop(jobject obj) {
 748   movoop(rscratch1, obj);
 749   push(rscratch1);
 750 }
 751 
 752 void MacroAssembler::pushklass(Metadata* obj) {
 753   mov_metadata(rscratch1, obj);
 754   push(rscratch1);
 755 }
 756 
 757 void MacroAssembler::pushptr(AddressLiteral src) {
 758   lea(rscratch1, src);
 759   if (src.is_lval()) {
 760     push(rscratch1);
 761   } else {
 762     pushq(Address(rscratch1, 0));
 763   }
 764 }
 765 
 766 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
 767   // we must set sp to zero to clear frame
 768   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 769   // must clear fp, so that compiled frames are not confused; it is
 770   // possible that we need it only for debugging
 771   if (clear_fp) {
 772     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 773   }
 774 
 775   // Always clear the pc because it could have been set by make_walkable()
 776   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 777   vzeroupper();
 778 }
 779 
 780 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 781                                          Register last_java_fp,
 782                                          address  last_java_pc) {
 783   vzeroupper();
 784   // determine last_java_sp register
 785   if (!last_java_sp->is_valid()) {
 786     last_java_sp = rsp;
 787   }
 788 
 789   // last_java_fp is optional
 790   if (last_java_fp->is_valid()) {
 791     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),
 792            last_java_fp);
 793   }
 794 
 795   // last_java_pc is optional
 796   if (last_java_pc != NULL) {
 797     Address java_pc(r15_thread,
 798                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 799     lea(rscratch1, InternalAddress(last_java_pc));
 800     movptr(java_pc, rscratch1);
 801   }
 802 
 803   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 804 }
 805 
 806 static void pass_arg0(MacroAssembler* masm, Register arg) {
 807   if (c_rarg0 != arg ) {
 808     masm->mov(c_rarg0, arg);
 809   }
 810 }
 811 
 812 static void pass_arg1(MacroAssembler* masm, Register arg) {
 813   if (c_rarg1 != arg ) {
 814     masm->mov(c_rarg1, arg);
 815   }
 816 }
 817 
 818 static void pass_arg2(MacroAssembler* masm, Register arg) {
 819   if (c_rarg2 != arg ) {
 820     masm->mov(c_rarg2, arg);
 821   }
 822 }
 823 
 824 static void pass_arg3(MacroAssembler* masm, Register arg) {
 825   if (c_rarg3 != arg ) {
 826     masm->mov(c_rarg3, arg);
 827   }
 828 }
 829 
 830 void MacroAssembler::stop(const char* msg) {
 831   address rip = pc();
 832   pusha(); // get regs on stack
 833   lea(c_rarg0, ExternalAddress((address) msg));
 834   lea(c_rarg1, InternalAddress(rip));
 835   movq(c_rarg2, rsp); // pass pointer to regs array
 836   andq(rsp, -16); // align stack as required by ABI
 837   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
 838   hlt();
 839 }
 840 
 841 void MacroAssembler::warn(const char* msg) {
 842   push(rbp);
 843   movq(rbp, rsp);
 844   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 845   push_CPU_state();   // keeps alignment at 16 bytes
 846   lea(c_rarg0, ExternalAddress((address) msg));
 847   lea(rax, ExternalAddress(CAST_FROM_FN_PTR(address, warning)));
 848   call(rax);
 849   pop_CPU_state();
 850   mov(rsp, rbp);
 851   pop(rbp);
 852 }
 853 
 854 void MacroAssembler::print_state() {
 855   address rip = pc();
 856   pusha();            // get regs on stack
 857   push(rbp);
 858   movq(rbp, rsp);
 859   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 860   push_CPU_state();   // keeps alignment at 16 bytes
 861 
 862   lea(c_rarg0, InternalAddress(rip));
 863   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
 864   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
 865 
 866   pop_CPU_state();
 867   mov(rsp, rbp);
 868   pop(rbp);
 869   popa();
 870 }
 871 
 872 #ifndef PRODUCT
 873 extern "C" void findpc(intptr_t x);
 874 #endif
 875 
 876 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
 877   // In order to get locks to work, we need to fake a in_VM state
 878   if (ShowMessageBoxOnError) {
 879     JavaThread* thread = JavaThread::current();
 880     JavaThreadState saved_state = thread->thread_state();
 881     thread->set_thread_state(_thread_in_vm);
 882 #ifndef PRODUCT
 883     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 884       ttyLocker ttyl;
 885       BytecodeCounter::print();
 886     }
 887 #endif
 888     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 889     // XXX correct this offset for amd64
 890     // This is the value of eip which points to where verify_oop will return.
 891     if (os::message_box(msg, "Execution stopped, print registers?")) {
 892       print_state64(pc, regs);
 893       BREAKPOINT;
 894       assert(false, "start up GDB");
 895     }
 896     ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 897   } else {
 898     ttyLocker ttyl;
 899     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
 900                     msg);
 901     assert(false, "DEBUG MESSAGE: %s", msg);
 902   }
 903 }
 904 
 905 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
 906   ttyLocker ttyl;
 907   FlagSetting fs(Debugging, true);
 908   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
 909 #ifndef PRODUCT
 910   tty->cr();
 911   findpc(pc);
 912   tty->cr();
 913 #endif
 914 #define PRINT_REG(rax, value) \
 915   { tty->print("%s = ", #rax); os::print_location(tty, value); }
 916   PRINT_REG(rax, regs[15]);
 917   PRINT_REG(rbx, regs[12]);
 918   PRINT_REG(rcx, regs[14]);
 919   PRINT_REG(rdx, regs[13]);
 920   PRINT_REG(rdi, regs[8]);
 921   PRINT_REG(rsi, regs[9]);
 922   PRINT_REG(rbp, regs[10]);
 923   PRINT_REG(rsp, regs[11]);
 924   PRINT_REG(r8 , regs[7]);
 925   PRINT_REG(r9 , regs[6]);
 926   PRINT_REG(r10, regs[5]);
 927   PRINT_REG(r11, regs[4]);
 928   PRINT_REG(r12, regs[3]);
 929   PRINT_REG(r13, regs[2]);
 930   PRINT_REG(r14, regs[1]);
 931   PRINT_REG(r15, regs[0]);
 932 #undef PRINT_REG
 933   // Print some words near top of staack.
 934   int64_t* rsp = (int64_t*) regs[11];
 935   int64_t* dump_sp = rsp;
 936   for (int col1 = 0; col1 < 8; col1++) {
 937     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 938     os::print_location(tty, *dump_sp++);
 939   }
 940   for (int row = 0; row < 25; row++) {
 941     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 942     for (int col = 0; col < 4; col++) {
 943       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
 944     }
 945     tty->cr();
 946   }
 947   // Print some instructions around pc:
 948   Disassembler::decode((address)pc-64, (address)pc);
 949   tty->print_cr("--------");
 950   Disassembler::decode((address)pc, (address)pc+32);
 951 }
 952 
 953 #endif // _LP64
 954 
 955 // Now versions that are common to 32/64 bit
 956 
 957 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 958   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 959 }
 960 
 961 void MacroAssembler::addptr(Register dst, Register src) {
 962   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 963 }
 964 
 965 void MacroAssembler::addptr(Address dst, Register src) {
 966   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 967 }
 968 
 969 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {
 970   if (reachable(src)) {
 971     Assembler::addsd(dst, as_Address(src));
 972   } else {
 973     lea(rscratch1, src);
 974     Assembler::addsd(dst, Address(rscratch1, 0));
 975   }
 976 }
 977 
 978 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {
 979   if (reachable(src)) {
 980     addss(dst, as_Address(src));
 981   } else {
 982     lea(rscratch1, src);
 983     addss(dst, Address(rscratch1, 0));
 984   }
 985 }
 986 
 987 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) {
 988   if (reachable(src)) {
 989     Assembler::addpd(dst, as_Address(src));
 990   } else {
 991     lea(rscratch1, src);
 992     Assembler::addpd(dst, Address(rscratch1, 0));
 993   }
 994 }
 995 
 996 void MacroAssembler::align(int modulus) {
 997   align(modulus, offset());
 998 }
 999 
1000 void MacroAssembler::align(int modulus, int target) {
1001   if (target % modulus != 0) {
1002     nop(modulus - (target % modulus));
1003   }
1004 }
1005 
1006 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {
1007   // Used in sign-masking with aligned address.
1008   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1009   if (reachable(src)) {
1010     Assembler::andpd(dst, as_Address(src));
1011   } else {
1012     lea(rscratch1, src);
1013     Assembler::andpd(dst, Address(rscratch1, 0));
1014   }
1015 }
1016 
1017 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {
1018   // Used in sign-masking with aligned address.
1019   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1020   if (reachable(src)) {
1021     Assembler::andps(dst, as_Address(src));
1022   } else {
1023     lea(rscratch1, src);
1024     Assembler::andps(dst, Address(rscratch1, 0));
1025   }
1026 }
1027 
1028 void MacroAssembler::andptr(Register dst, int32_t imm32) {
1029   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
1030 }
1031 
1032 void MacroAssembler::atomic_incl(Address counter_addr) {
1033   if (os::is_MP())
1034     lock();
1035   incrementl(counter_addr);
1036 }
1037 
1038 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
1039   if (reachable(counter_addr)) {
1040     atomic_incl(as_Address(counter_addr));
1041   } else {
1042     lea(scr, counter_addr);
1043     atomic_incl(Address(scr, 0));
1044   }
1045 }
1046 
1047 #ifdef _LP64
1048 void MacroAssembler::atomic_incq(Address counter_addr) {
1049   if (os::is_MP())
1050     lock();
1051   incrementq(counter_addr);
1052 }
1053 
1054 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) {
1055   if (reachable(counter_addr)) {
1056     atomic_incq(as_Address(counter_addr));
1057   } else {
1058     lea(scr, counter_addr);
1059     atomic_incq(Address(scr, 0));
1060   }
1061 }
1062 #endif
1063 
1064 // Writes to stack successive pages until offset reached to check for
1065 // stack overflow + shadow pages.  This clobbers tmp.
1066 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
1067   movptr(tmp, rsp);
1068   // Bang stack for total size given plus shadow page size.
1069   // Bang one page at a time because large size can bang beyond yellow and
1070   // red zones.
1071   Label loop;
1072   bind(loop);
1073   movl(Address(tmp, (-os::vm_page_size())), size );
1074   subptr(tmp, os::vm_page_size());
1075   subl(size, os::vm_page_size());
1076   jcc(Assembler::greater, loop);
1077 
1078   // Bang down shadow pages too.
1079   // At this point, (tmp-0) is the last address touched, so don't
1080   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
1081   // was post-decremented.)  Skip this address by starting at i=1, and
1082   // touch a few more pages below.  N.B.  It is important to touch all
1083   // the way down including all pages in the shadow zone.
1084   for (int i = 1; i < ((int)JavaThread::stack_shadow_zone_size() / os::vm_page_size()); i++) {
1085     // this could be any sized move but this is can be a debugging crumb
1086     // so the bigger the better.
1087     movptr(Address(tmp, (-i*os::vm_page_size())), size );
1088   }
1089 }
1090 
1091 void MacroAssembler::reserved_stack_check() {
1092     // testing if reserved zone needs to be enabled
1093     Label no_reserved_zone_enabling;
1094     Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
1095     NOT_LP64(get_thread(rsi);)
1096 
1097     cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
1098     jcc(Assembler::below, no_reserved_zone_enabling);
1099 
1100     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
1101     jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
1102     should_not_reach_here();
1103 
1104     bind(no_reserved_zone_enabling);
1105 }
1106 
1107 int MacroAssembler::biased_locking_enter(Register lock_reg,
1108                                          Register obj_reg,
1109                                          Register swap_reg,
1110                                          Register tmp_reg,
1111                                          bool swap_reg_contains_mark,
1112                                          Label& done,
1113                                          Label* slow_case,
1114                                          BiasedLockingCounters* counters) {
1115   assert(UseBiasedLocking, "why call this otherwise?");
1116   assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
1117   assert(tmp_reg != noreg, "tmp_reg must be supplied");
1118   assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
1119   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
1120   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
1121   NOT_LP64( Address saved_mark_addr(lock_reg, 0); )
1122 
1123   if (PrintBiasedLockingStatistics && counters == NULL) {
1124     counters = BiasedLocking::counters();
1125   }
1126   // Biased locking
1127   // See whether the lock is currently biased toward our thread and
1128   // whether the epoch is still valid
1129   // Note that the runtime guarantees sufficient alignment of JavaThread
1130   // pointers to allow age to be placed into low bits
1131   // First check to see whether biasing is even enabled for this object
1132   Label cas_label;
1133   int null_check_offset = -1;
1134   if (!swap_reg_contains_mark) {
1135     null_check_offset = offset();
1136     movptr(swap_reg, mark_addr);
1137   }
1138   movptr(tmp_reg, swap_reg);
1139   andptr(tmp_reg, markOopDesc::biased_lock_mask_in_place);
1140   cmpptr(tmp_reg, markOopDesc::biased_lock_pattern);
1141   jcc(Assembler::notEqual, cas_label);
1142   // The bias pattern is present in the object's header. Need to check
1143   // whether the bias owner and the epoch are both still current.
1144 #ifndef _LP64
1145   // Note that because there is no current thread register on x86_32 we
1146   // need to store off the mark word we read out of the object to
1147   // avoid reloading it and needing to recheck invariants below. This
1148   // store is unfortunate but it makes the overall code shorter and
1149   // simpler.
1150   movptr(saved_mark_addr, swap_reg);
1151 #endif
1152   if (swap_reg_contains_mark) {
1153     null_check_offset = offset();
1154   }
1155   load_prototype_header(tmp_reg, obj_reg);
1156 #ifdef _LP64
1157   orptr(tmp_reg, r15_thread);
1158   xorptr(tmp_reg, swap_reg);
1159   Register header_reg = tmp_reg;
1160 #else
1161   xorptr(tmp_reg, swap_reg);
1162   get_thread(swap_reg);
1163   xorptr(swap_reg, tmp_reg);
1164   Register header_reg = swap_reg;
1165 #endif
1166   andptr(header_reg, ~((int) markOopDesc::age_mask_in_place));
1167   if (counters != NULL) {
1168     cond_inc32(Assembler::zero,
1169                ExternalAddress((address) counters->biased_lock_entry_count_addr()));
1170   }
1171   jcc(Assembler::equal, done);
1172 
1173   Label try_revoke_bias;
1174   Label try_rebias;
1175 
1176   // At this point we know that the header has the bias pattern and
1177   // that we are not the bias owner in the current epoch. We need to
1178   // figure out more details about the state of the header in order to
1179   // know what operations can be legally performed on the object's
1180   // header.
1181 
1182   // If the low three bits in the xor result aren't clear, that means
1183   // the prototype header is no longer biased and we have to revoke
1184   // the bias on this object.
1185   testptr(header_reg, markOopDesc::biased_lock_mask_in_place);
1186   jccb(Assembler::notZero, try_revoke_bias);
1187 
1188   // Biasing is still enabled for this data type. See whether the
1189   // epoch of the current bias is still valid, meaning that the epoch
1190   // bits of the mark word are equal to the epoch bits of the
1191   // prototype header. (Note that the prototype header's epoch bits
1192   // only change at a safepoint.) If not, attempt to rebias the object
1193   // toward the current thread. Note that we must be absolutely sure
1194   // that the current epoch is invalid in order to do this because
1195   // otherwise the manipulations it performs on the mark word are
1196   // illegal.
1197   testptr(header_reg, markOopDesc::epoch_mask_in_place);
1198   jccb(Assembler::notZero, try_rebias);
1199 
1200   // The epoch of the current bias is still valid but we know nothing
1201   // about the owner; it might be set or it might be clear. Try to
1202   // acquire the bias of the object using an atomic operation. If this
1203   // fails we will go in to the runtime to revoke the object's bias.
1204   // Note that we first construct the presumed unbiased header so we
1205   // don't accidentally blow away another thread's valid bias.
1206   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1207   andptr(swap_reg,
1208          markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
1209 #ifdef _LP64
1210   movptr(tmp_reg, swap_reg);
1211   orptr(tmp_reg, r15_thread);
1212 #else
1213   get_thread(tmp_reg);
1214   orptr(tmp_reg, swap_reg);
1215 #endif
1216   if (os::is_MP()) {
1217     lock();
1218   }
1219   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1220   // If the biasing toward our thread failed, this means that
1221   // another thread succeeded in biasing it toward itself and we
1222   // need to revoke that bias. The revocation will occur in the
1223   // interpreter runtime in the slow case.
1224   if (counters != NULL) {
1225     cond_inc32(Assembler::zero,
1226                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
1227   }
1228   if (slow_case != NULL) {
1229     jcc(Assembler::notZero, *slow_case);
1230   }
1231   jmp(done);
1232 
1233   bind(try_rebias);
1234   // At this point we know the epoch has expired, meaning that the
1235   // current "bias owner", if any, is actually invalid. Under these
1236   // circumstances _only_, we are allowed to use the current header's
1237   // value as the comparison value when doing the cas to acquire the
1238   // bias in the current epoch. In other words, we allow transfer of
1239   // the bias from one thread to another directly in this situation.
1240   //
1241   // FIXME: due to a lack of registers we currently blow away the age
1242   // bits in this situation. Should attempt to preserve them.
1243   load_prototype_header(tmp_reg, obj_reg);
1244 #ifdef _LP64
1245   orptr(tmp_reg, r15_thread);
1246 #else
1247   get_thread(swap_reg);
1248   orptr(tmp_reg, swap_reg);
1249   movptr(swap_reg, saved_mark_addr);
1250 #endif
1251   if (os::is_MP()) {
1252     lock();
1253   }
1254   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1255   // If the biasing toward our thread failed, then another thread
1256   // succeeded in biasing it toward itself and we need to revoke that
1257   // bias. The revocation will occur in the runtime in the slow case.
1258   if (counters != NULL) {
1259     cond_inc32(Assembler::zero,
1260                ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
1261   }
1262   if (slow_case != NULL) {
1263     jcc(Assembler::notZero, *slow_case);
1264   }
1265   jmp(done);
1266 
1267   bind(try_revoke_bias);
1268   // The prototype mark in the klass doesn't have the bias bit set any
1269   // more, indicating that objects of this data type are not supposed
1270   // to be biased any more. We are going to try to reset the mark of
1271   // this object to the prototype value and fall through to the
1272   // CAS-based locking scheme. Note that if our CAS fails, it means
1273   // that another thread raced us for the privilege of revoking the
1274   // bias of this particular object, so it's okay to continue in the
1275   // normal locking code.
1276   //
1277   // FIXME: due to a lack of registers we currently blow away the age
1278   // bits in this situation. Should attempt to preserve them.
1279   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1280   load_prototype_header(tmp_reg, obj_reg);
1281   if (os::is_MP()) {
1282     lock();
1283   }
1284   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1285   // Fall through to the normal CAS-based lock, because no matter what
1286   // the result of the above CAS, some thread must have succeeded in
1287   // removing the bias bit from the object's header.
1288   if (counters != NULL) {
1289     cond_inc32(Assembler::zero,
1290                ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
1291   }
1292 
1293   bind(cas_label);
1294 
1295   return null_check_offset;
1296 }
1297 
1298 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
1299   assert(UseBiasedLocking, "why call this otherwise?");
1300 
1301   // Check for biased locking unlock case, which is a no-op
1302   // Note: we do not have to check the thread ID for two reasons.
1303   // First, the interpreter checks for IllegalMonitorStateException at
1304   // a higher level. Second, if the bias was revoked while we held the
1305   // lock, the object could not be rebiased toward another thread, so
1306   // the bias bit would be clear.
1307   movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1308   andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);
1309   cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
1310   jcc(Assembler::equal, done);
1311 }
1312 
1313 #ifdef COMPILER2
1314 
1315 #if INCLUDE_RTM_OPT
1316 
1317 // Update rtm_counters based on abort status
1318 // input: abort_status
1319 //        rtm_counters (RTMLockingCounters*)
1320 // flags are killed
1321 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) {
1322 
1323   atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset()));
1324   if (PrintPreciseRTMLockingStatistics) {
1325     for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
1326       Label check_abort;
1327       testl(abort_status, (1<<i));
1328       jccb(Assembler::equal, check_abort);
1329       atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx))));
1330       bind(check_abort);
1331     }
1332   }
1333 }
1334 
1335 // Branch if (random & (count-1) != 0), count is 2^n
1336 // tmp, scr and flags are killed
1337 void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) {
1338   assert(tmp == rax, "");
1339   assert(scr == rdx, "");
1340   rdtsc(); // modifies EDX:EAX
1341   andptr(tmp, count-1);
1342   jccb(Assembler::notZero, brLabel);
1343 }
1344 
1345 // Perform abort ratio calculation, set no_rtm bit if high ratio
1346 // input:  rtm_counters_Reg (RTMLockingCounters* address)
1347 // tmpReg, rtm_counters_Reg and flags are killed
1348 void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg,
1349                                                  Register rtm_counters_Reg,
1350                                                  RTMLockingCounters* rtm_counters,
1351                                                  Metadata* method_data) {
1352   Label L_done, L_check_always_rtm1, L_check_always_rtm2;
1353 
1354   if (RTMLockingCalculationDelay > 0) {
1355     // Delay calculation
1356     movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg);
1357     testptr(tmpReg, tmpReg);
1358     jccb(Assembler::equal, L_done);
1359   }
1360   // Abort ratio calculation only if abort_count > RTMAbortThreshold
1361   //   Aborted transactions = abort_count * 100
1362   //   All transactions = total_count *  RTMTotalCountIncrRate
1363   //   Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio)
1364 
1365   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset()));
1366   cmpptr(tmpReg, RTMAbortThreshold);
1367   jccb(Assembler::below, L_check_always_rtm2);
1368   imulptr(tmpReg, tmpReg, 100);
1369 
1370   Register scrReg = rtm_counters_Reg;
1371   movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1372   imulptr(scrReg, scrReg, RTMTotalCountIncrRate);
1373   imulptr(scrReg, scrReg, RTMAbortRatio);
1374   cmpptr(tmpReg, scrReg);
1375   jccb(Assembler::below, L_check_always_rtm1);
1376   if (method_data != NULL) {
1377     // set rtm_state to "no rtm" in MDO
1378     mov_metadata(tmpReg, method_data);
1379     if (os::is_MP()) {
1380       lock();
1381     }
1382     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM);
1383   }
1384   jmpb(L_done);
1385   bind(L_check_always_rtm1);
1386   // Reload RTMLockingCounters* address
1387   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1388   bind(L_check_always_rtm2);
1389   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1390   cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate);
1391   jccb(Assembler::below, L_done);
1392   if (method_data != NULL) {
1393     // set rtm_state to "always rtm" in MDO
1394     mov_metadata(tmpReg, method_data);
1395     if (os::is_MP()) {
1396       lock();
1397     }
1398     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM);
1399   }
1400   bind(L_done);
1401 }
1402 
1403 // Update counters and perform abort ratio calculation
1404 // input:  abort_status_Reg
1405 // rtm_counters_Reg, flags are killed
1406 void MacroAssembler::rtm_profiling(Register abort_status_Reg,
1407                                    Register rtm_counters_Reg,
1408                                    RTMLockingCounters* rtm_counters,
1409                                    Metadata* method_data,
1410                                    bool profile_rtm) {
1411 
1412   assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1413   // update rtm counters based on rax value at abort
1414   // reads abort_status_Reg, updates flags
1415   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1416   rtm_counters_update(abort_status_Reg, rtm_counters_Reg);
1417   if (profile_rtm) {
1418     // Save abort status because abort_status_Reg is used by following code.
1419     if (RTMRetryCount > 0) {
1420       push(abort_status_Reg);
1421     }
1422     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1423     rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data);
1424     // restore abort status
1425     if (RTMRetryCount > 0) {
1426       pop(abort_status_Reg);
1427     }
1428   }
1429 }
1430 
1431 // Retry on abort if abort's status is 0x6: can retry (0x2) | memory conflict (0x4)
1432 // inputs: retry_count_Reg
1433 //       : abort_status_Reg
1434 // output: retry_count_Reg decremented by 1
1435 // flags are killed
1436 void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) {
1437   Label doneRetry;
1438   assert(abort_status_Reg == rax, "");
1439   // The abort reason bits are in eax (see all states in rtmLocking.hpp)
1440   // 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4)
1441   // if reason is in 0x6 and retry count != 0 then retry
1442   andptr(abort_status_Reg, 0x6);
1443   jccb(Assembler::zero, doneRetry);
1444   testl(retry_count_Reg, retry_count_Reg);
1445   jccb(Assembler::zero, doneRetry);
1446   pause();
1447   decrementl(retry_count_Reg);
1448   jmp(retryLabel);
1449   bind(doneRetry);
1450 }
1451 
1452 // Spin and retry if lock is busy,
1453 // inputs: box_Reg (monitor address)
1454 //       : retry_count_Reg
1455 // output: retry_count_Reg decremented by 1
1456 //       : clear z flag if retry count exceeded
1457 // tmp_Reg, scr_Reg, flags are killed
1458 void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg,
1459                                             Register tmp_Reg, Register scr_Reg, Label& retryLabel) {
1460   Label SpinLoop, SpinExit, doneRetry;
1461   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1462 
1463   testl(retry_count_Reg, retry_count_Reg);
1464   jccb(Assembler::zero, doneRetry);
1465   decrementl(retry_count_Reg);
1466   movptr(scr_Reg, RTMSpinLoopCount);
1467 
1468   bind(SpinLoop);
1469   pause();
1470   decrementl(scr_Reg);
1471   jccb(Assembler::lessEqual, SpinExit);
1472   movptr(tmp_Reg, Address(box_Reg, owner_offset));
1473   testptr(tmp_Reg, tmp_Reg);
1474   jccb(Assembler::notZero, SpinLoop);
1475 
1476   bind(SpinExit);
1477   jmp(retryLabel);
1478   bind(doneRetry);
1479   incrementl(retry_count_Reg); // clear z flag
1480 }
1481 
1482 // Use RTM for normal stack locks
1483 // Input: objReg (object to lock)
1484 void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg,
1485                                        Register retry_on_abort_count_Reg,
1486                                        RTMLockingCounters* stack_rtm_counters,
1487                                        Metadata* method_data, bool profile_rtm,
1488                                        Label& DONE_LABEL, Label& IsInflated) {
1489   assert(UseRTMForStackLocks, "why call this otherwise?");
1490   assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1491   assert(tmpReg == rax, "");
1492   assert(scrReg == rdx, "");
1493   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1494 
1495   if (RTMRetryCount > 0) {
1496     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1497     bind(L_rtm_retry);
1498   }
1499   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1500   testptr(tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased
1501   jcc(Assembler::notZero, IsInflated);
1502 
1503   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1504     Label L_noincrement;
1505     if (RTMTotalCountIncrRate > 1) {
1506       // tmpReg, scrReg and flags are killed
1507       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1508     }
1509     assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM");
1510     atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg);
1511     bind(L_noincrement);
1512   }
1513   xbegin(L_on_abort);
1514   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));       // fetch markword
1515   andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
1516   cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
1517   jcc(Assembler::equal, DONE_LABEL);        // all done if unlocked
1518 
1519   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1520   if (UseRTMXendForLockBusy) {
1521     xend();
1522     movptr(abort_status_Reg, 0x2);   // Set the abort status to 2 (so we can retry)
1523     jmp(L_decrement_retry);
1524   }
1525   else {
1526     xabort(0);
1527   }
1528   bind(L_on_abort);
1529   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1530     rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm);
1531   }
1532   bind(L_decrement_retry);
1533   if (RTMRetryCount > 0) {
1534     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1535     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1536   }
1537 }
1538 
1539 // Use RTM for inflating locks
1540 // inputs: objReg (object to lock)
1541 //         boxReg (on-stack box address (displaced header location) - KILLED)
1542 //         tmpReg (ObjectMonitor address + markOopDesc::monitor_value)
1543 void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg,
1544                                           Register scrReg, Register retry_on_busy_count_Reg,
1545                                           Register retry_on_abort_count_Reg,
1546                                           RTMLockingCounters* rtm_counters,
1547                                           Metadata* method_data, bool profile_rtm,
1548                                           Label& DONE_LABEL) {
1549   assert(UseRTMLocking, "why call this otherwise?");
1550   assert(tmpReg == rax, "");
1551   assert(scrReg == rdx, "");
1552   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1553   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1554 
1555   // Without cast to int32_t a movptr will destroy r10 which is typically obj
1556   movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1557   movptr(boxReg, tmpReg); // Save ObjectMonitor address
1558 
1559   if (RTMRetryCount > 0) {
1560     movl(retry_on_busy_count_Reg, RTMRetryCount);  // Retry on lock busy
1561     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1562     bind(L_rtm_retry);
1563   }
1564   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1565     Label L_noincrement;
1566     if (RTMTotalCountIncrRate > 1) {
1567       // tmpReg, scrReg and flags are killed
1568       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1569     }
1570     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1571     atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);
1572     bind(L_noincrement);
1573   }
1574   xbegin(L_on_abort);
1575   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1576   movptr(tmpReg, Address(tmpReg, owner_offset));
1577   testptr(tmpReg, tmpReg);
1578   jcc(Assembler::zero, DONE_LABEL);
1579   if (UseRTMXendForLockBusy) {
1580     xend();
1581     jmp(L_decrement_retry);
1582   }
1583   else {
1584     xabort(0);
1585   }
1586   bind(L_on_abort);
1587   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1588   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1589     rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm);
1590   }
1591   if (RTMRetryCount > 0) {
1592     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1593     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1594   }
1595 
1596   movptr(tmpReg, Address(boxReg, owner_offset)) ;
1597   testptr(tmpReg, tmpReg) ;
1598   jccb(Assembler::notZero, L_decrement_retry) ;
1599 
1600   // Appears unlocked - try to swing _owner from null to non-null.
1601   // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1602 #ifdef _LP64
1603   Register threadReg = r15_thread;
1604 #else
1605   get_thread(scrReg);
1606   Register threadReg = scrReg;
1607 #endif
1608   if (os::is_MP()) {
1609     lock();
1610   }
1611   cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg
1612 
1613   if (RTMRetryCount > 0) {
1614     // success done else retry
1615     jccb(Assembler::equal, DONE_LABEL) ;
1616     bind(L_decrement_retry);
1617     // Spin and retry if lock is busy.
1618     rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
1619   }
1620   else {
1621     bind(L_decrement_retry);
1622   }
1623 }
1624 
1625 #endif //  INCLUDE_RTM_OPT
1626 
1627 // Fast_Lock and Fast_Unlock used by C2
1628 
1629 // Because the transitions from emitted code to the runtime
1630 // monitorenter/exit helper stubs are so slow it's critical that
1631 // we inline both the stack-locking fast-path and the inflated fast path.
1632 //
1633 // See also: cmpFastLock and cmpFastUnlock.
1634 //
1635 // What follows is a specialized inline transliteration of the code
1636 // in slow_enter() and slow_exit().  If we're concerned about I$ bloat
1637 // another option would be to emit TrySlowEnter and TrySlowExit methods
1638 // at startup-time.  These methods would accept arguments as
1639 // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure
1640 // indications in the icc.ZFlag.  Fast_Lock and Fast_Unlock would simply
1641 // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.
1642 // In practice, however, the # of lock sites is bounded and is usually small.
1643 // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer
1644 // if the processor uses simple bimodal branch predictors keyed by EIP
1645 // Since the helper routines would be called from multiple synchronization
1646 // sites.
1647 //
1648 // An even better approach would be write "MonitorEnter()" and "MonitorExit()"
1649 // in java - using j.u.c and unsafe - and just bind the lock and unlock sites
1650 // to those specialized methods.  That'd give us a mostly platform-independent
1651 // implementation that the JITs could optimize and inline at their pleasure.
1652 // Done correctly, the only time we'd need to cross to native could would be
1653 // to park() or unpark() threads.  We'd also need a few more unsafe operators
1654 // to (a) prevent compiler-JIT reordering of non-volatile accesses, and
1655 // (b) explicit barriers or fence operations.
1656 //
1657 // TODO:
1658 //
1659 // *  Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr).
1660 //    This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals.
1661 //    Given TLAB allocation, Self is usually manifested in a register, so passing it into
1662 //    the lock operators would typically be faster than reifying Self.
1663 //
1664 // *  Ideally I'd define the primitives as:
1665 //       fast_lock   (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.
1666 //       fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED
1667 //    Unfortunately ADLC bugs prevent us from expressing the ideal form.
1668 //    Instead, we're stuck with a rather awkward and brittle register assignments below.
1669 //    Furthermore the register assignments are overconstrained, possibly resulting in
1670 //    sub-optimal code near the synchronization site.
1671 //
1672 // *  Eliminate the sp-proximity tests and just use "== Self" tests instead.
1673 //    Alternately, use a better sp-proximity test.
1674 //
1675 // *  Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.
1676 //    Either one is sufficient to uniquely identify a thread.
1677 //    TODO: eliminate use of sp in _owner and use get_thread(tr) instead.
1678 //
1679 // *  Intrinsify notify() and notifyAll() for the common cases where the
1680 //    object is locked by the calling thread but the waitlist is empty.
1681 //    avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().
1682 //
1683 // *  use jccb and jmpb instead of jcc and jmp to improve code density.
1684 //    But beware of excessive branch density on AMD Opterons.
1685 //
1686 // *  Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success
1687 //    or failure of the fast-path.  If the fast-path fails then we pass
1688 //    control to the slow-path, typically in C.  In Fast_Lock and
1689 //    Fast_Unlock we often branch to DONE_LABEL, just to find that C2
1690 //    will emit a conditional branch immediately after the node.
1691 //    So we have branches to branches and lots of ICC.ZF games.
1692 //    Instead, it might be better to have C2 pass a "FailureLabel"
1693 //    into Fast_Lock and Fast_Unlock.  In the case of success, control
1694 //    will drop through the node.  ICC.ZF is undefined at exit.
1695 //    In the case of failure, the node will branch directly to the
1696 //    FailureLabel
1697 
1698 
1699 // obj: object to lock
1700 // box: on-stack box address (displaced header location) - KILLED
1701 // rax,: tmp -- KILLED
1702 // scr: tmp -- KILLED
1703 void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
1704                                Register scrReg, Register cx1Reg, Register cx2Reg,
1705                                BiasedLockingCounters* counters,
1706                                RTMLockingCounters* rtm_counters,
1707                                RTMLockingCounters* stack_rtm_counters,
1708                                Metadata* method_data,
1709                                bool use_rtm, bool profile_rtm) {
1710   // Ensure the register assignments are disjoint
1711   assert(tmpReg == rax, "");
1712 
1713   if (use_rtm) {
1714     assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg);
1715   } else {
1716     assert(cx1Reg == noreg, "");
1717     assert(cx2Reg == noreg, "");
1718     assert_different_registers(objReg, boxReg, tmpReg, scrReg);
1719   }
1720 
1721   if (counters != NULL) {
1722     atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);
1723   }
1724   if (EmitSync & 1) {
1725       // set box->dhw = markOopDesc::unused_mark()
1726       // Force all sync thru slow-path: slow_enter() and slow_exit()
1727       movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1728       cmpptr (rsp, (int32_t)NULL_WORD);
1729   } else {
1730     // Possible cases that we'll encounter in fast_lock
1731     // ------------------------------------------------
1732     // * Inflated
1733     //    -- unlocked
1734     //    -- Locked
1735     //       = by self
1736     //       = by other
1737     // * biased
1738     //    -- by Self
1739     //    -- by other
1740     // * neutral
1741     // * stack-locked
1742     //    -- by self
1743     //       = sp-proximity test hits
1744     //       = sp-proximity test generates false-negative
1745     //    -- by other
1746     //
1747 
1748     Label IsInflated, DONE_LABEL;
1749 
1750     // it's stack-locked, biased or neutral
1751     // TODO: optimize away redundant LDs of obj->mark and improve the markword triage
1752     // order to reduce the number of conditional branches in the most common cases.
1753     // Beware -- there's a subtle invariant that fetch of the markword
1754     // at [FETCH], below, will never observe a biased encoding (*101b).
1755     // If this invariant is not held we risk exclusion (safety) failure.
1756     if (UseBiasedLocking && !UseOptoBiasInlining) {
1757       biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters);
1758     }
1759 
1760 #if INCLUDE_RTM_OPT
1761     if (UseRTMForStackLocks && use_rtm) {
1762       rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
1763                         stack_rtm_counters, method_data, profile_rtm,
1764                         DONE_LABEL, IsInflated);
1765     }
1766 #endif // INCLUDE_RTM_OPT
1767 
1768     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));          // [FETCH]
1769     testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
1770     jccb(Assembler::notZero, IsInflated);
1771 
1772     // Attempt stack-locking ...
1773     orptr (tmpReg, markOopDesc::unlocked_value);
1774     movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
1775     if (os::is_MP()) {
1776       lock();
1777     }
1778     cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes()));      // Updates tmpReg
1779     if (counters != NULL) {
1780       cond_inc32(Assembler::equal,
1781                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
1782     }
1783     jcc(Assembler::equal, DONE_LABEL);           // Success
1784 
1785     // Recursive locking.
1786     // The object is stack-locked: markword contains stack pointer to BasicLock.
1787     // Locked by current thread if difference with current SP is less than one page.
1788     subptr(tmpReg, rsp);
1789     // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
1790     andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
1791     movptr(Address(boxReg, 0), tmpReg);
1792     if (counters != NULL) {
1793       cond_inc32(Assembler::equal,
1794                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
1795     }
1796     jmp(DONE_LABEL);
1797 
1798     bind(IsInflated);
1799     // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markOopDesc::monitor_value
1800 
1801 #if INCLUDE_RTM_OPT
1802     // Use the same RTM locking code in 32- and 64-bit VM.
1803     if (use_rtm) {
1804       rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg,
1805                            rtm_counters, method_data, profile_rtm, DONE_LABEL);
1806     } else {
1807 #endif // INCLUDE_RTM_OPT
1808 
1809 #ifndef _LP64
1810     // The object is inflated.
1811 
1812     // boxReg refers to the on-stack BasicLock in the current frame.
1813     // We'd like to write:
1814     //   set box->_displaced_header = markOopDesc::unused_mark().  Any non-0 value suffices.
1815     // This is convenient but results a ST-before-CAS penalty.  The following CAS suffers
1816     // additional latency as we have another ST in the store buffer that must drain.
1817 
1818     if (EmitSync & 8192) {
1819        movptr(Address(boxReg, 0), 3);            // results in ST-before-CAS penalty
1820        get_thread (scrReg);
1821        movptr(boxReg, tmpReg);                    // consider: LEA box, [tmp-2]
1822        movptr(tmpReg, NULL_WORD);                 // consider: xor vs mov
1823        if (os::is_MP()) {
1824          lock();
1825        }
1826        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1827     } else
1828     if ((EmitSync & 128) == 0) {                      // avoid ST-before-CAS
1829        // register juggle because we need tmpReg for cmpxchgptr below
1830        movptr(scrReg, boxReg);
1831        movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2]
1832 
1833        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
1834        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
1835           // prefetchw [eax + Offset(_owner)-2]
1836           prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1837        }
1838 
1839        if ((EmitSync & 64) == 0) {
1840          // Optimistic form: consider XORL tmpReg,tmpReg
1841          movptr(tmpReg, NULL_WORD);
1842        } else {
1843          // Can suffer RTS->RTO upgrades on shared or cold $ lines
1844          // Test-And-CAS instead of CAS
1845          movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));   // rax, = m->_owner
1846          testptr(tmpReg, tmpReg);                   // Locked ?
1847          jccb  (Assembler::notZero, DONE_LABEL);
1848        }
1849 
1850        // Appears unlocked - try to swing _owner from null to non-null.
1851        // Ideally, I'd manifest "Self" with get_thread and then attempt
1852        // to CAS the register containing Self into m->Owner.
1853        // But we don't have enough registers, so instead we can either try to CAS
1854        // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
1855        // we later store "Self" into m->Owner.  Transiently storing a stack address
1856        // (rsp or the address of the box) into  m->owner is harmless.
1857        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1858        if (os::is_MP()) {
1859          lock();
1860        }
1861        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1862        movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3
1863        // If we weren't able to swing _owner from NULL to the BasicLock
1864        // then take the slow path.
1865        jccb  (Assembler::notZero, DONE_LABEL);
1866        // update _owner from BasicLock to thread
1867        get_thread (scrReg);                    // beware: clobbers ICCs
1868        movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
1869        xorptr(boxReg, boxReg);                 // set icc.ZFlag = 1 to indicate success
1870 
1871        // If the CAS fails we can either retry or pass control to the slow-path.
1872        // We use the latter tactic.
1873        // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1874        // If the CAS was successful ...
1875        //   Self has acquired the lock
1876        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1877        // Intentional fall-through into DONE_LABEL ...
1878     } else {
1879        movptr(Address(boxReg, 0), intptr_t(markOopDesc::unused_mark()));  // results in ST-before-CAS penalty
1880        movptr(boxReg, tmpReg);
1881 
1882        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
1883        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
1884           // prefetchw [eax + Offset(_owner)-2]
1885           prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1886        }
1887 
1888        if ((EmitSync & 64) == 0) {
1889          // Optimistic form
1890          xorptr  (tmpReg, tmpReg);
1891        } else {
1892          // Can suffer RTS->RTO upgrades on shared or cold $ lines
1893          movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));   // rax, = m->_owner
1894          testptr(tmpReg, tmpReg);                   // Locked ?
1895          jccb  (Assembler::notZero, DONE_LABEL);
1896        }
1897 
1898        // Appears unlocked - try to swing _owner from null to non-null.
1899        // Use either "Self" (in scr) or rsp as thread identity in _owner.
1900        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1901        get_thread (scrReg);
1902        if (os::is_MP()) {
1903          lock();
1904        }
1905        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1906 
1907        // If the CAS fails we can either retry or pass control to the slow-path.
1908        // We use the latter tactic.
1909        // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1910        // If the CAS was successful ...
1911        //   Self has acquired the lock
1912        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1913        // Intentional fall-through into DONE_LABEL ...
1914     }
1915 #else // _LP64
1916     // It's inflated
1917     movq(scrReg, tmpReg);
1918     xorq(tmpReg, tmpReg);
1919 
1920     if (os::is_MP()) {
1921       lock();
1922     }
1923     cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1924     // Unconditionally set box->_displaced_header = markOopDesc::unused_mark().
1925     // Without cast to int32_t movptr will destroy r10 which is typically obj.
1926     movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1927     // Intentional fall-through into DONE_LABEL ...
1928     // Propagate ICC.ZF from CAS above into DONE_LABEL.
1929 #endif // _LP64
1930 #if INCLUDE_RTM_OPT
1931     } // use_rtm()
1932 #endif
1933     // DONE_LABEL is a hot target - we'd really like to place it at the
1934     // start of cache line by padding with NOPs.
1935     // See the AMD and Intel software optimization manuals for the
1936     // most efficient "long" NOP encodings.
1937     // Unfortunately none of our alignment mechanisms suffice.
1938     bind(DONE_LABEL);
1939 
1940     // At DONE_LABEL the icc ZFlag is set as follows ...
1941     // Fast_Unlock uses the same protocol.
1942     // ZFlag == 1 -> Success
1943     // ZFlag == 0 -> Failure - force control through the slow-path
1944   }
1945 }
1946 
1947 // obj: object to unlock
1948 // box: box address (displaced header location), killed.  Must be EAX.
1949 // tmp: killed, cannot be obj nor box.
1950 //
1951 // Some commentary on balanced locking:
1952 //
1953 // Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites.
1954 // Methods that don't have provably balanced locking are forced to run in the
1955 // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
1956 // The interpreter provides two properties:
1957 // I1:  At return-time the interpreter automatically and quietly unlocks any
1958 //      objects acquired the current activation (frame).  Recall that the
1959 //      interpreter maintains an on-stack list of locks currently held by
1960 //      a frame.
1961 // I2:  If a method attempts to unlock an object that is not held by the
1962 //      the frame the interpreter throws IMSX.
1963 //
1964 // Lets say A(), which has provably balanced locking, acquires O and then calls B().
1965 // B() doesn't have provably balanced locking so it runs in the interpreter.
1966 // Control returns to A() and A() unlocks O.  By I1 and I2, above, we know that O
1967 // is still locked by A().
1968 //
1969 // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:
1970 // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
1971 // should not be unlocked by "normal" java-level locking and vice-versa.  The specification
1972 // doesn't specify what will occur if a program engages in such mixed-mode locking, however.
1973 // Arguably given that the spec legislates the JNI case as undefined our implementation
1974 // could reasonably *avoid* checking owner in Fast_Unlock().
1975 // In the interest of performance we elide m->Owner==Self check in unlock.
1976 // A perfectly viable alternative is to elide the owner check except when
1977 // Xcheck:jni is enabled.
1978 
1979 void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
1980   assert(boxReg == rax, "");
1981   assert_different_registers(objReg, boxReg, tmpReg);
1982 
1983   if (EmitSync & 4) {
1984     // Disable - inhibit all inlining.  Force control through the slow-path
1985     cmpptr (rsp, 0);
1986   } else {
1987     Label DONE_LABEL, Stacked, CheckSucc;
1988 
1989     // Critically, the biased locking test must have precedence over
1990     // and appear before the (box->dhw == 0) recursive stack-lock test.
1991     if (UseBiasedLocking && !UseOptoBiasInlining) {
1992        biased_locking_exit(objReg, tmpReg, DONE_LABEL);
1993     }
1994 
1995 #if INCLUDE_RTM_OPT
1996     if (UseRTMForStackLocks && use_rtm) {
1997       assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1998       Label L_regular_unlock;
1999       movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));           // fetch markword
2000       andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
2001       cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
2002       jccb(Assembler::notEqual, L_regular_unlock);  // if !HLE RegularLock
2003       xend();                                       // otherwise end...
2004       jmp(DONE_LABEL);                              // ... and we're done
2005       bind(L_regular_unlock);
2006     }
2007 #endif
2008 
2009     cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
2010     jcc   (Assembler::zero, DONE_LABEL);            // 0 indicates recursive stack-lock
2011     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));             // Examine the object's markword
2012     testptr(tmpReg, markOopDesc::monitor_value);    // Inflated?
2013     jccb  (Assembler::zero, Stacked);
2014 
2015     // It's inflated.
2016 #if INCLUDE_RTM_OPT
2017     if (use_rtm) {
2018       Label L_regular_inflated_unlock;
2019       int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
2020       movptr(boxReg, Address(tmpReg, owner_offset));
2021       testptr(boxReg, boxReg);
2022       jccb(Assembler::notZero, L_regular_inflated_unlock);
2023       xend();
2024       jmpb(DONE_LABEL);
2025       bind(L_regular_inflated_unlock);
2026     }
2027 #endif
2028 
2029     // Despite our balanced locking property we still check that m->_owner == Self
2030     // as java routines or native JNI code called by this thread might
2031     // have released the lock.
2032     // Refer to the comments in synchronizer.cpp for how we might encode extra
2033     // state in _succ so we can avoid fetching EntryList|cxq.
2034     //
2035     // I'd like to add more cases in fast_lock() and fast_unlock() --
2036     // such as recursive enter and exit -- but we have to be wary of
2037     // I$ bloat, T$ effects and BP$ effects.
2038     //
2039     // If there's no contention try a 1-0 exit.  That is, exit without
2040     // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how
2041     // we detect and recover from the race that the 1-0 exit admits.
2042     //
2043     // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier
2044     // before it STs null into _owner, releasing the lock.  Updates
2045     // to data protected by the critical section must be visible before
2046     // we drop the lock (and thus before any other thread could acquire
2047     // the lock and observe the fields protected by the lock).
2048     // IA32's memory-model is SPO, so STs are ordered with respect to
2049     // each other and there's no need for an explicit barrier (fence).
2050     // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
2051 #ifndef _LP64
2052     get_thread (boxReg);
2053     if ((EmitSync & 4096) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
2054       // prefetchw [ebx + Offset(_owner)-2]
2055       prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2056     }
2057 
2058     // Note that we could employ various encoding schemes to reduce
2059     // the number of loads below (currently 4) to just 2 or 3.
2060     // Refer to the comments in synchronizer.cpp.
2061     // In practice the chain of fetches doesn't seem to impact performance, however.
2062     xorptr(boxReg, boxReg);
2063     if ((EmitSync & 65536) == 0 && (EmitSync & 256)) {
2064        // Attempt to reduce branch density - AMD's branch predictor.
2065        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2066        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2067        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2068        jccb  (Assembler::notZero, DONE_LABEL);
2069        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2070        jmpb  (DONE_LABEL);
2071     } else {
2072        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2073        jccb  (Assembler::notZero, DONE_LABEL);
2074        movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2075        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2076        jccb  (Assembler::notZero, CheckSucc);
2077        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2078        jmpb  (DONE_LABEL);
2079     }
2080 
2081     // The Following code fragment (EmitSync & 65536) improves the performance of
2082     // contended applications and contended synchronization microbenchmarks.
2083     // Unfortunately the emission of the code - even though not executed - causes regressions
2084     // in scimark and jetstream, evidently because of $ effects.  Replacing the code
2085     // with an equal number of never-executed NOPs results in the same regression.
2086     // We leave it off by default.
2087 
2088     if ((EmitSync & 65536) != 0) {
2089        Label LSuccess, LGoSlowPath ;
2090 
2091        bind  (CheckSucc);
2092 
2093        // Optional pre-test ... it's safe to elide this
2094        cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2095        jccb(Assembler::zero, LGoSlowPath);
2096 
2097        // We have a classic Dekker-style idiom:
2098        //    ST m->_owner = 0 ; MEMBAR; LD m->_succ
2099        // There are a number of ways to implement the barrier:
2100        // (1) lock:andl &m->_owner, 0
2101        //     is fast, but mask doesn't currently support the "ANDL M,IMM32" form.
2102        //     LOCK: ANDL [ebx+Offset(_Owner)-2], 0
2103        //     Encodes as 81 31 OFF32 IMM32 or 83 63 OFF8 IMM8
2104        // (2) If supported, an explicit MFENCE is appealing.
2105        //     In older IA32 processors MFENCE is slower than lock:add or xchg
2106        //     particularly if the write-buffer is full as might be the case if
2107        //     if stores closely precede the fence or fence-equivalent instruction.
2108        //     See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences
2109        //     as the situation has changed with Nehalem and Shanghai.
2110        // (3) In lieu of an explicit fence, use lock:addl to the top-of-stack
2111        //     The $lines underlying the top-of-stack should be in M-state.
2112        //     The locked add instruction is serializing, of course.
2113        // (4) Use xchg, which is serializing
2114        //     mov boxReg, 0; xchgl boxReg, [tmpReg + Offset(_owner)-2] also works
2115        // (5) ST m->_owner = 0 and then execute lock:orl &m->_succ, 0.
2116        //     The integer condition codes will tell us if succ was 0.
2117        //     Since _succ and _owner should reside in the same $line and
2118        //     we just stored into _owner, it's likely that the $line
2119        //     remains in M-state for the lock:orl.
2120        //
2121        // We currently use (3), although it's likely that switching to (2)
2122        // is correct for the future.
2123 
2124        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2125        if (os::is_MP()) {
2126          lock(); addptr(Address(rsp, 0), 0);
2127        }
2128        // Ratify _succ remains non-null
2129        cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), 0);
2130        jccb  (Assembler::notZero, LSuccess);
2131 
2132        xorptr(boxReg, boxReg);                  // box is really EAX
2133        if (os::is_MP()) { lock(); }
2134        cmpxchgptr(rsp, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2135        // There's no successor so we tried to regrab the lock with the
2136        // placeholder value. If that didn't work, then another thread
2137        // grabbed the lock so we're done (and exit was a success).
2138        jccb  (Assembler::notEqual, LSuccess);
2139        // Since we're low on registers we installed rsp as a placeholding in _owner.
2140        // Now install Self over rsp.  This is safe as we're transitioning from
2141        // non-null to non=null
2142        get_thread (boxReg);
2143        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), boxReg);
2144        // Intentional fall-through into LGoSlowPath ...
2145 
2146        bind  (LGoSlowPath);
2147        orptr(boxReg, 1);                      // set ICC.ZF=0 to indicate failure
2148        jmpb  (DONE_LABEL);
2149 
2150        bind  (LSuccess);
2151        xorptr(boxReg, boxReg);                 // set ICC.ZF=1 to indicate success
2152        jmpb  (DONE_LABEL);
2153     }
2154 
2155     bind (Stacked);
2156     // It's not inflated and it's not recursively stack-locked and it's not biased.
2157     // It must be stack-locked.
2158     // Try to reset the header to displaced header.
2159     // The "box" value on the stack is stable, so we can reload
2160     // and be assured we observe the same value as above.
2161     movptr(tmpReg, Address(boxReg, 0));
2162     if (os::is_MP()) {
2163       lock();
2164     }
2165     cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
2166     // Intention fall-thru into DONE_LABEL
2167 
2168     // DONE_LABEL is a hot target - we'd really like to place it at the
2169     // start of cache line by padding with NOPs.
2170     // See the AMD and Intel software optimization manuals for the
2171     // most efficient "long" NOP encodings.
2172     // Unfortunately none of our alignment mechanisms suffice.
2173     if ((EmitSync & 65536) == 0) {
2174        bind (CheckSucc);
2175     }
2176 #else // _LP64
2177     // It's inflated
2178     if (EmitSync & 1024) {
2179       // Emit code to check that _owner == Self
2180       // We could fold the _owner test into subsequent code more efficiently
2181       // than using a stand-alone check, but since _owner checking is off by
2182       // default we don't bother. We also might consider predicating the
2183       // _owner==Self check on Xcheck:jni or running on a debug build.
2184       movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2185       xorptr(boxReg, r15_thread);
2186     } else {
2187       xorptr(boxReg, boxReg);
2188     }
2189     orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2190     jccb  (Assembler::notZero, DONE_LABEL);
2191     movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2192     orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2193     jccb  (Assembler::notZero, CheckSucc);
2194     movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2195     jmpb  (DONE_LABEL);
2196 
2197     if ((EmitSync & 65536) == 0) {
2198       // Try to avoid passing control into the slow_path ...
2199       Label LSuccess, LGoSlowPath ;
2200       bind  (CheckSucc);
2201 
2202       // The following optional optimization can be elided if necessary
2203       // Effectively: if (succ == null) goto SlowPath
2204       // The code reduces the window for a race, however,
2205       // and thus benefits performance.
2206       cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2207       jccb  (Assembler::zero, LGoSlowPath);
2208 
2209       xorptr(boxReg, boxReg);
2210       if ((EmitSync & 16) && os::is_MP()) {
2211         xchgptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2212       } else {
2213         movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2214         if (os::is_MP()) {
2215           // Memory barrier/fence
2216           // Dekker pivot point -- fulcrum : ST Owner; MEMBAR; LD Succ
2217           // Instead of MFENCE we use a dummy locked add of 0 to the top-of-stack.
2218           // This is faster on Nehalem and AMD Shanghai/Barcelona.
2219           // See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences
2220           // We might also restructure (ST Owner=0;barrier;LD _Succ) to
2221           // (mov box,0; xchgq box, &m->Owner; LD _succ) .
2222           lock(); addl(Address(rsp, 0), 0);
2223         }
2224       }
2225       cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2226       jccb  (Assembler::notZero, LSuccess);
2227 
2228       // Rare inopportune interleaving - race.
2229       // The successor vanished in the small window above.
2230       // The lock is contended -- (cxq|EntryList) != null -- and there's no apparent successor.
2231       // We need to ensure progress and succession.
2232       // Try to reacquire the lock.
2233       // If that fails then the new owner is responsible for succession and this
2234       // thread needs to take no further action and can exit via the fast path (success).
2235       // If the re-acquire succeeds then pass control into the slow path.
2236       // As implemented, this latter mode is horrible because we generated more
2237       // coherence traffic on the lock *and* artifically extended the critical section
2238       // length while by virtue of passing control into the slow path.
2239 
2240       // box is really RAX -- the following CMPXCHG depends on that binding
2241       // cmpxchg R,[M] is equivalent to rax = CAS(M,rax,R)
2242       if (os::is_MP()) { lock(); }
2243       cmpxchgptr(r15_thread, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2244       // There's no successor so we tried to regrab the lock.
2245       // If that didn't work, then another thread grabbed the
2246       // lock so we're done (and exit was a success).
2247       jccb  (Assembler::notEqual, LSuccess);
2248       // Intentional fall-through into slow-path
2249 
2250       bind  (LGoSlowPath);
2251       orl   (boxReg, 1);                      // set ICC.ZF=0 to indicate failure
2252       jmpb  (DONE_LABEL);
2253 
2254       bind  (LSuccess);
2255       testl (boxReg, 0);                      // set ICC.ZF=1 to indicate success
2256       jmpb  (DONE_LABEL);
2257     }
2258 
2259     bind  (Stacked);
2260     movptr(tmpReg, Address (boxReg, 0));      // re-fetch
2261     if (os::is_MP()) { lock(); }
2262     cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
2263 
2264     if (EmitSync & 65536) {
2265        bind (CheckSucc);
2266     }
2267 #endif
2268     bind(DONE_LABEL);
2269   }
2270 }
2271 #endif // COMPILER2
2272 
2273 void MacroAssembler::c2bool(Register x) {
2274   // implements x == 0 ? 0 : 1
2275   // note: must only look at least-significant byte of x
2276   //       since C-style booleans are stored in one byte
2277   //       only! (was bug)
2278   andl(x, 0xFF);
2279   setb(Assembler::notZero, x);
2280 }
2281 
2282 // Wouldn't need if AddressLiteral version had new name
2283 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
2284   Assembler::call(L, rtype);
2285 }
2286 
2287 void MacroAssembler::call(Register entry) {
2288   Assembler::call(entry);
2289 }
2290 
2291 void MacroAssembler::call(AddressLiteral entry) {
2292   if (reachable(entry)) {
2293     Assembler::call_literal(entry.target(), entry.rspec());
2294   } else {
2295     lea(rscratch1, entry);
2296     Assembler::call(rscratch1);
2297   }
2298 }
2299 
2300 void MacroAssembler::ic_call(address entry, jint method_index) {
2301   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
2302   movptr(rax, (intptr_t)Universe::non_oop_word());
2303   call(AddressLiteral(entry, rh));
2304 }
2305 
2306 // Implementation of call_VM versions
2307 
2308 void MacroAssembler::call_VM(Register oop_result,
2309                              address entry_point,
2310                              bool check_exceptions) {
2311   Label C, E;
2312   call(C, relocInfo::none);
2313   jmp(E);
2314 
2315   bind(C);
2316   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
2317   ret(0);
2318 
2319   bind(E);
2320 }
2321 
2322 void MacroAssembler::call_VM(Register oop_result,
2323                              address entry_point,
2324                              Register arg_1,
2325                              bool check_exceptions) {
2326   Label C, E;
2327   call(C, relocInfo::none);
2328   jmp(E);
2329 
2330   bind(C);
2331   pass_arg1(this, arg_1);
2332   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
2333   ret(0);
2334 
2335   bind(E);
2336 }
2337 
2338 void MacroAssembler::call_VM(Register oop_result,
2339                              address entry_point,
2340                              Register arg_1,
2341                              Register arg_2,
2342                              bool check_exceptions) {
2343   Label C, E;
2344   call(C, relocInfo::none);
2345   jmp(E);
2346 
2347   bind(C);
2348 
2349   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2350 
2351   pass_arg2(this, arg_2);
2352   pass_arg1(this, arg_1);
2353   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
2354   ret(0);
2355 
2356   bind(E);
2357 }
2358 
2359 void MacroAssembler::call_VM(Register oop_result,
2360                              address entry_point,
2361                              Register arg_1,
2362                              Register arg_2,
2363                              Register arg_3,
2364                              bool check_exceptions) {
2365   Label C, E;
2366   call(C, relocInfo::none);
2367   jmp(E);
2368 
2369   bind(C);
2370 
2371   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2372   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2373   pass_arg3(this, arg_3);
2374 
2375   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2376   pass_arg2(this, arg_2);
2377 
2378   pass_arg1(this, arg_1);
2379   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
2380   ret(0);
2381 
2382   bind(E);
2383 }
2384 
2385 void MacroAssembler::call_VM(Register oop_result,
2386                              Register last_java_sp,
2387                              address entry_point,
2388                              int number_of_arguments,
2389                              bool check_exceptions) {
2390   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2391   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2392 }
2393 
2394 void MacroAssembler::call_VM(Register oop_result,
2395                              Register last_java_sp,
2396                              address entry_point,
2397                              Register arg_1,
2398                              bool check_exceptions) {
2399   pass_arg1(this, arg_1);
2400   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2401 }
2402 
2403 void MacroAssembler::call_VM(Register oop_result,
2404                              Register last_java_sp,
2405                              address entry_point,
2406                              Register arg_1,
2407                              Register arg_2,
2408                              bool check_exceptions) {
2409 
2410   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2411   pass_arg2(this, arg_2);
2412   pass_arg1(this, arg_1);
2413   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2414 }
2415 
2416 void MacroAssembler::call_VM(Register oop_result,
2417                              Register last_java_sp,
2418                              address entry_point,
2419                              Register arg_1,
2420                              Register arg_2,
2421                              Register arg_3,
2422                              bool check_exceptions) {
2423   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2424   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2425   pass_arg3(this, arg_3);
2426   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2427   pass_arg2(this, arg_2);
2428   pass_arg1(this, arg_1);
2429   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2430 }
2431 
2432 void MacroAssembler::super_call_VM(Register oop_result,
2433                                    Register last_java_sp,
2434                                    address entry_point,
2435                                    int number_of_arguments,
2436                                    bool check_exceptions) {
2437   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2438   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2439 }
2440 
2441 void MacroAssembler::super_call_VM(Register oop_result,
2442                                    Register last_java_sp,
2443                                    address entry_point,
2444                                    Register arg_1,
2445                                    bool check_exceptions) {
2446   pass_arg1(this, arg_1);
2447   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2448 }
2449 
2450 void MacroAssembler::super_call_VM(Register oop_result,
2451                                    Register last_java_sp,
2452                                    address entry_point,
2453                                    Register arg_1,
2454                                    Register arg_2,
2455                                    bool check_exceptions) {
2456 
2457   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2458   pass_arg2(this, arg_2);
2459   pass_arg1(this, arg_1);
2460   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2461 }
2462 
2463 void MacroAssembler::super_call_VM(Register oop_result,
2464                                    Register last_java_sp,
2465                                    address entry_point,
2466                                    Register arg_1,
2467                                    Register arg_2,
2468                                    Register arg_3,
2469                                    bool check_exceptions) {
2470   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2471   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2472   pass_arg3(this, arg_3);
2473   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2474   pass_arg2(this, arg_2);
2475   pass_arg1(this, arg_1);
2476   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2477 }
2478 
2479 void MacroAssembler::call_VM_base(Register oop_result,
2480                                   Register java_thread,
2481                                   Register last_java_sp,
2482                                   address  entry_point,
2483                                   int      number_of_arguments,
2484                                   bool     check_exceptions) {
2485   // determine java_thread register
2486   if (!java_thread->is_valid()) {
2487 #ifdef _LP64
2488     java_thread = r15_thread;
2489 #else
2490     java_thread = rdi;
2491     get_thread(java_thread);
2492 #endif // LP64
2493   }
2494   // determine last_java_sp register
2495   if (!last_java_sp->is_valid()) {
2496     last_java_sp = rsp;
2497   }
2498   // debugging support
2499   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
2500   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
2501 #ifdef ASSERT
2502   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
2503   // r12 is the heapbase.
2504   LP64_ONLY(if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
2505 #endif // ASSERT
2506 
2507   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
2508   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
2509 
2510   // push java thread (becomes first argument of C function)
2511 
2512   NOT_LP64(push(java_thread); number_of_arguments++);
2513   LP64_ONLY(mov(c_rarg0, r15_thread));
2514 
2515   // set last Java frame before call
2516   assert(last_java_sp != rbp, "can't use ebp/rbp");
2517 
2518   // Only interpreter should have to set fp
2519   set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);
2520 
2521   // do the call, remove parameters
2522   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
2523 
2524   // restore the thread (cannot use the pushed argument since arguments
2525   // may be overwritten by C code generated by an optimizing compiler);
2526   // however can use the register value directly if it is callee saved.
2527   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
2528     // rdi & rsi (also r15) are callee saved -> nothing to do
2529 #ifdef ASSERT
2530     guarantee(java_thread != rax, "change this code");
2531     push(rax);
2532     { Label L;
2533       get_thread(rax);
2534       cmpptr(java_thread, rax);
2535       jcc(Assembler::equal, L);
2536       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
2537       bind(L);
2538     }
2539     pop(rax);
2540 #endif
2541   } else {
2542     get_thread(java_thread);
2543   }
2544   // reset last Java frame
2545   // Only interpreter should have to clear fp
2546   reset_last_Java_frame(java_thread, true);
2547 
2548    // C++ interp handles this in the interpreter
2549   check_and_handle_popframe(java_thread);
2550   check_and_handle_earlyret(java_thread);
2551 
2552   if (check_exceptions) {
2553     // check for pending exceptions (java_thread is set upon return)
2554     cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
2555 #ifndef _LP64
2556     jump_cc(Assembler::notEqual,
2557             RuntimeAddress(StubRoutines::forward_exception_entry()));
2558 #else
2559     // This used to conditionally jump to forward_exception however it is
2560     // possible if we relocate that the branch will not reach. So we must jump
2561     // around so we can always reach
2562 
2563     Label ok;
2564     jcc(Assembler::equal, ok);
2565     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2566     bind(ok);
2567 #endif // LP64
2568   }
2569 
2570   // get oop result if there is one and reset the value in the thread
2571   if (oop_result->is_valid()) {
2572     get_vm_result(oop_result, java_thread);
2573   }
2574 }
2575 
2576 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
2577 
2578   // Calculate the value for last_Java_sp
2579   // somewhat subtle. call_VM does an intermediate call
2580   // which places a return address on the stack just under the
2581   // stack pointer as the user finsihed with it. This allows
2582   // use to retrieve last_Java_pc from last_Java_sp[-1].
2583   // On 32bit we then have to push additional args on the stack to accomplish
2584   // the actual requested call. On 64bit call_VM only can use register args
2585   // so the only extra space is the return address that call_VM created.
2586   // This hopefully explains the calculations here.
2587 
2588 #ifdef _LP64
2589   // We've pushed one address, correct last_Java_sp
2590   lea(rax, Address(rsp, wordSize));
2591 #else
2592   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
2593 #endif // LP64
2594 
2595   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
2596 
2597 }
2598 
2599 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
2600 void MacroAssembler::call_VM_leaf0(address entry_point) {
2601   MacroAssembler::call_VM_leaf_base(entry_point, 0);
2602 }
2603 
2604 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
2605   call_VM_leaf_base(entry_point, number_of_arguments);
2606 }
2607 
2608 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
2609   pass_arg0(this, arg_0);
2610   call_VM_leaf(entry_point, 1);
2611 }
2612 
2613 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2614 
2615   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2616   pass_arg1(this, arg_1);
2617   pass_arg0(this, arg_0);
2618   call_VM_leaf(entry_point, 2);
2619 }
2620 
2621 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2622   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2623   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2624   pass_arg2(this, arg_2);
2625   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2626   pass_arg1(this, arg_1);
2627   pass_arg0(this, arg_0);
2628   call_VM_leaf(entry_point, 3);
2629 }
2630 
2631 void MacroAssembler::super_call_VM_leaf(address entry_point) {
2632   MacroAssembler::call_VM_leaf_base(entry_point, 1);
2633 }
2634 
2635 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
2636   pass_arg0(this, arg_0);
2637   MacroAssembler::call_VM_leaf_base(entry_point, 1);
2638 }
2639 
2640 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2641 
2642   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2643   pass_arg1(this, arg_1);
2644   pass_arg0(this, arg_0);
2645   MacroAssembler::call_VM_leaf_base(entry_point, 2);
2646 }
2647 
2648 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2649   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2650   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2651   pass_arg2(this, arg_2);
2652   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2653   pass_arg1(this, arg_1);
2654   pass_arg0(this, arg_0);
2655   MacroAssembler::call_VM_leaf_base(entry_point, 3);
2656 }
2657 
2658 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
2659   LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg"));
2660   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2661   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2662   pass_arg3(this, arg_3);
2663   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2664   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2665   pass_arg2(this, arg_2);
2666   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2667   pass_arg1(this, arg_1);
2668   pass_arg0(this, arg_0);
2669   MacroAssembler::call_VM_leaf_base(entry_point, 4);
2670 }
2671 
2672 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
2673   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
2674   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
2675   verify_oop(oop_result, "broken oop in call_VM_base");
2676 }
2677 
2678 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
2679   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
2680   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
2681 }
2682 
2683 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
2684 }
2685 
2686 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
2687 }
2688 
2689 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) {
2690   if (reachable(src1)) {
2691     cmpl(as_Address(src1), imm);
2692   } else {
2693     lea(rscratch1, src1);
2694     cmpl(Address(rscratch1, 0), imm);
2695   }
2696 }
2697 
2698 void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
2699   assert(!src2.is_lval(), "use cmpptr");
2700   if (reachable(src2)) {
2701     cmpl(src1, as_Address(src2));
2702   } else {
2703     lea(rscratch1, src2);
2704     cmpl(src1, Address(rscratch1, 0));
2705   }
2706 }
2707 
2708 void MacroAssembler::cmp32(Register src1, int32_t imm) {
2709   Assembler::cmpl(src1, imm);
2710 }
2711 
2712 void MacroAssembler::cmp32(Register src1, Address src2) {
2713   Assembler::cmpl(src1, src2);
2714 }
2715 
2716 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2717   ucomisd(opr1, opr2);
2718 
2719   Label L;
2720   if (unordered_is_less) {
2721     movl(dst, -1);
2722     jcc(Assembler::parity, L);
2723     jcc(Assembler::below , L);
2724     movl(dst, 0);
2725     jcc(Assembler::equal , L);
2726     increment(dst);
2727   } else { // unordered is greater
2728     movl(dst, 1);
2729     jcc(Assembler::parity, L);
2730     jcc(Assembler::above , L);
2731     movl(dst, 0);
2732     jcc(Assembler::equal , L);
2733     decrementl(dst);
2734   }
2735   bind(L);
2736 }
2737 
2738 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2739   ucomiss(opr1, opr2);
2740 
2741   Label L;
2742   if (unordered_is_less) {
2743     movl(dst, -1);
2744     jcc(Assembler::parity, L);
2745     jcc(Assembler::below , L);
2746     movl(dst, 0);
2747     jcc(Assembler::equal , L);
2748     increment(dst);
2749   } else { // unordered is greater
2750     movl(dst, 1);
2751     jcc(Assembler::parity, L);
2752     jcc(Assembler::above , L);
2753     movl(dst, 0);
2754     jcc(Assembler::equal , L);
2755     decrementl(dst);
2756   }
2757   bind(L);
2758 }
2759 
2760 
2761 void MacroAssembler::cmp8(AddressLiteral src1, int imm) {
2762   if (reachable(src1)) {
2763     cmpb(as_Address(src1), imm);
2764   } else {
2765     lea(rscratch1, src1);
2766     cmpb(Address(rscratch1, 0), imm);
2767   }
2768 }
2769 
2770 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
2771 #ifdef _LP64
2772   if (src2.is_lval()) {
2773     movptr(rscratch1, src2);
2774     Assembler::cmpq(src1, rscratch1);
2775   } else if (reachable(src2)) {
2776     cmpq(src1, as_Address(src2));
2777   } else {
2778     lea(rscratch1, src2);
2779     Assembler::cmpq(src1, Address(rscratch1, 0));
2780   }
2781 #else
2782   if (src2.is_lval()) {
2783     cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2784   } else {
2785     cmpl(src1, as_Address(src2));
2786   }
2787 #endif // _LP64
2788 }
2789 
2790 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
2791   assert(src2.is_lval(), "not a mem-mem compare");
2792 #ifdef _LP64
2793   // moves src2's literal address
2794   movptr(rscratch1, src2);
2795   Assembler::cmpq(src1, rscratch1);
2796 #else
2797   cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2798 #endif // _LP64
2799 }
2800 
2801 void MacroAssembler::cmpoop(Register src1, Register src2) {
2802   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2803   bs->obj_equals(this, src1, src2);
2804 }
2805 
2806 void MacroAssembler::cmpoop(Register src1, Address src2) {
2807   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2808   bs->obj_equals(this, src1, src2);
2809 }
2810 
2811 #ifdef _LP64
2812 void MacroAssembler::cmpoop(Register src1, jobject src2) {
2813   movoop(rscratch1, src2);
2814   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2815   bs->obj_equals(this, src1, rscratch1);
2816 }
2817 #endif
2818 
2819 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) {
2820   if (reachable(adr)) {
2821     if (os::is_MP())
2822       lock();
2823     cmpxchgptr(reg, as_Address(adr));
2824   } else {
2825     lea(rscratch1, adr);
2826     if (os::is_MP())
2827       lock();
2828     cmpxchgptr(reg, Address(rscratch1, 0));
2829   }
2830 }
2831 
2832 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
2833   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
2834 }
2835 
2836 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {
2837   if (reachable(src)) {
2838     Assembler::comisd(dst, as_Address(src));
2839   } else {
2840     lea(rscratch1, src);
2841     Assembler::comisd(dst, Address(rscratch1, 0));
2842   }
2843 }
2844 
2845 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {
2846   if (reachable(src)) {
2847     Assembler::comiss(dst, as_Address(src));
2848   } else {
2849     lea(rscratch1, src);
2850     Assembler::comiss(dst, Address(rscratch1, 0));
2851   }
2852 }
2853 
2854 
2855 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
2856   Condition negated_cond = negate_condition(cond);
2857   Label L;
2858   jcc(negated_cond, L);
2859   pushf(); // Preserve flags
2860   atomic_incl(counter_addr);
2861   popf();
2862   bind(L);
2863 }
2864 
2865 int MacroAssembler::corrected_idivl(Register reg) {
2866   // Full implementation of Java idiv and irem; checks for
2867   // special case as described in JVM spec., p.243 & p.271.
2868   // The function returns the (pc) offset of the idivl
2869   // instruction - may be needed for implicit exceptions.
2870   //
2871   //         normal case                           special case
2872   //
2873   // input : rax,: dividend                         min_int
2874   //         reg: divisor   (may not be rax,/rdx)   -1
2875   //
2876   // output: rax,: quotient  (= rax, idiv reg)       min_int
2877   //         rdx: remainder (= rax, irem reg)       0
2878   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
2879   const int min_int = 0x80000000;
2880   Label normal_case, special_case;
2881 
2882   // check for special case
2883   cmpl(rax, min_int);
2884   jcc(Assembler::notEqual, normal_case);
2885   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
2886   cmpl(reg, -1);
2887   jcc(Assembler::equal, special_case);
2888 
2889   // handle normal case
2890   bind(normal_case);
2891   cdql();
2892   int idivl_offset = offset();
2893   idivl(reg);
2894 
2895   // normal and special case exit
2896   bind(special_case);
2897 
2898   return idivl_offset;
2899 }
2900 
2901 
2902 
2903 void MacroAssembler::decrementl(Register reg, int value) {
2904   if (value == min_jint) {subl(reg, value) ; return; }
2905   if (value <  0) { incrementl(reg, -value); return; }
2906   if (value == 0) {                        ; return; }
2907   if (value == 1 && UseIncDec) { decl(reg) ; return; }
2908   /* else */      { subl(reg, value)       ; return; }
2909 }
2910 
2911 void MacroAssembler::decrementl(Address dst, int value) {
2912   if (value == min_jint) {subl(dst, value) ; return; }
2913   if (value <  0) { incrementl(dst, -value); return; }
2914   if (value == 0) {                        ; return; }
2915   if (value == 1 && UseIncDec) { decl(dst) ; return; }
2916   /* else */      { subl(dst, value)       ; return; }
2917 }
2918 
2919 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
2920   assert (shift_value > 0, "illegal shift value");
2921   Label _is_positive;
2922   testl (reg, reg);
2923   jcc (Assembler::positive, _is_positive);
2924   int offset = (1 << shift_value) - 1 ;
2925 
2926   if (offset == 1) {
2927     incrementl(reg);
2928   } else {
2929     addl(reg, offset);
2930   }
2931 
2932   bind (_is_positive);
2933   sarl(reg, shift_value);
2934 }
2935 
2936 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
2937   if (reachable(src)) {
2938     Assembler::divsd(dst, as_Address(src));
2939   } else {
2940     lea(rscratch1, src);
2941     Assembler::divsd(dst, Address(rscratch1, 0));
2942   }
2943 }
2944 
2945 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
2946   if (reachable(src)) {
2947     Assembler::divss(dst, as_Address(src));
2948   } else {
2949     lea(rscratch1, src);
2950     Assembler::divss(dst, Address(rscratch1, 0));
2951   }
2952 }
2953 
2954 // !defined(COMPILER2) is because of stupid core builds
2955 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) || INCLUDE_JVMCI
2956 void MacroAssembler::empty_FPU_stack() {
2957   if (VM_Version::supports_mmx()) {
2958     emms();
2959   } else {
2960     for (int i = 8; i-- > 0; ) ffree(i);
2961   }
2962 }
2963 #endif // !LP64 || C1 || !C2 || INCLUDE_JVMCI
2964 
2965 
2966 void MacroAssembler::enter() {
2967   push(rbp);
2968   mov(rbp, rsp);
2969 }
2970 
2971 // A 5 byte nop that is safe for patching (see patch_verified_entry)
2972 void MacroAssembler::fat_nop() {
2973   if (UseAddressNop) {
2974     addr_nop_5();
2975   } else {
2976     emit_int8(0x26); // es:
2977     emit_int8(0x2e); // cs:
2978     emit_int8(0x64); // fs:
2979     emit_int8(0x65); // gs:
2980     emit_int8((unsigned char)0x90);
2981   }
2982 }
2983 
2984 void MacroAssembler::fcmp(Register tmp) {
2985   fcmp(tmp, 1, true, true);
2986 }
2987 
2988 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
2989   assert(!pop_right || pop_left, "usage error");
2990   if (VM_Version::supports_cmov()) {
2991     assert(tmp == noreg, "unneeded temp");
2992     if (pop_left) {
2993       fucomip(index);
2994     } else {
2995       fucomi(index);
2996     }
2997     if (pop_right) {
2998       fpop();
2999     }
3000   } else {
3001     assert(tmp != noreg, "need temp");
3002     if (pop_left) {
3003       if (pop_right) {
3004         fcompp();
3005       } else {
3006         fcomp(index);
3007       }
3008     } else {
3009       fcom(index);
3010     }
3011     // convert FPU condition into eflags condition via rax,
3012     save_rax(tmp);
3013     fwait(); fnstsw_ax();
3014     sahf();
3015     restore_rax(tmp);
3016   }
3017   // condition codes set as follows:
3018   //
3019   // CF (corresponds to C0) if x < y
3020   // PF (corresponds to C2) if unordered
3021   // ZF (corresponds to C3) if x = y
3022 }
3023 
3024 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
3025   fcmp2int(dst, unordered_is_less, 1, true, true);
3026 }
3027 
3028 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
3029   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
3030   Label L;
3031   if (unordered_is_less) {
3032     movl(dst, -1);
3033     jcc(Assembler::parity, L);
3034     jcc(Assembler::below , L);
3035     movl(dst, 0);
3036     jcc(Assembler::equal , L);
3037     increment(dst);
3038   } else { // unordered is greater
3039     movl(dst, 1);
3040     jcc(Assembler::parity, L);
3041     jcc(Assembler::above , L);
3042     movl(dst, 0);
3043     jcc(Assembler::equal , L);
3044     decrementl(dst);
3045   }
3046   bind(L);
3047 }
3048 
3049 void MacroAssembler::fld_d(AddressLiteral src) {
3050   fld_d(as_Address(src));
3051 }
3052 
3053 void MacroAssembler::fld_s(AddressLiteral src) {
3054   fld_s(as_Address(src));
3055 }
3056 
3057 void MacroAssembler::fld_x(AddressLiteral src) {
3058   Assembler::fld_x(as_Address(src));
3059 }
3060 
3061 void MacroAssembler::fldcw(AddressLiteral src) {
3062   Assembler::fldcw(as_Address(src));
3063 }
3064 
3065 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src) {
3066   if (reachable(src)) {
3067     Assembler::mulpd(dst, as_Address(src));
3068   } else {
3069     lea(rscratch1, src);
3070     Assembler::mulpd(dst, Address(rscratch1, 0));
3071   }
3072 }
3073 
3074 void MacroAssembler::increase_precision() {
3075   subptr(rsp, BytesPerWord);
3076   fnstcw(Address(rsp, 0));
3077   movl(rax, Address(rsp, 0));
3078   orl(rax, 0x300);
3079   push(rax);
3080   fldcw(Address(rsp, 0));
3081   pop(rax);
3082 }
3083 
3084 void MacroAssembler::restore_precision() {
3085   fldcw(Address(rsp, 0));
3086   addptr(rsp, BytesPerWord);
3087 }
3088 
3089 void MacroAssembler::fpop() {
3090   ffree();
3091   fincstp();
3092 }
3093 
3094 void MacroAssembler::load_float(Address src) {
3095   if (UseSSE >= 1) {
3096     movflt(xmm0, src);
3097   } else {
3098     LP64_ONLY(ShouldNotReachHere());
3099     NOT_LP64(fld_s(src));
3100   }
3101 }
3102 
3103 void MacroAssembler::store_float(Address dst) {
3104   if (UseSSE >= 1) {
3105     movflt(dst, xmm0);
3106   } else {
3107     LP64_ONLY(ShouldNotReachHere());
3108     NOT_LP64(fstp_s(dst));
3109   }
3110 }
3111 
3112 void MacroAssembler::load_double(Address src) {
3113   if (UseSSE >= 2) {
3114     movdbl(xmm0, src);
3115   } else {
3116     LP64_ONLY(ShouldNotReachHere());
3117     NOT_LP64(fld_d(src));
3118   }
3119 }
3120 
3121 void MacroAssembler::store_double(Address dst) {
3122   if (UseSSE >= 2) {
3123     movdbl(dst, xmm0);
3124   } else {
3125     LP64_ONLY(ShouldNotReachHere());
3126     NOT_LP64(fstp_d(dst));
3127   }
3128 }
3129 
3130 void MacroAssembler::push_zmm(XMMRegister reg) {
3131   lea(rsp, Address(rsp, -64)); // Use lea to not affect flags
3132   evmovdqul(Address(rsp, 0), reg, Assembler::AVX_512bit);
3133 }
3134 
3135 void MacroAssembler::pop_zmm(XMMRegister reg) {
3136   evmovdqul(reg, Address(rsp, 0), Assembler::AVX_512bit);
3137   lea(rsp, Address(rsp, 64)); // Use lea to not affect flags
3138 }
3139 
3140 void MacroAssembler::fremr(Register tmp) {
3141   save_rax(tmp);
3142   { Label L;
3143     bind(L);
3144     fprem();
3145     fwait(); fnstsw_ax();
3146 #ifdef _LP64
3147     testl(rax, 0x400);
3148     jcc(Assembler::notEqual, L);
3149 #else
3150     sahf();
3151     jcc(Assembler::parity, L);
3152 #endif // _LP64
3153   }
3154   restore_rax(tmp);
3155   // Result is in ST0.
3156   // Note: fxch & fpop to get rid of ST1
3157   // (otherwise FPU stack could overflow eventually)
3158   fxch(1);
3159   fpop();
3160 }
3161 
3162 // dst = c = a * b + c
3163 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
3164   Assembler::vfmadd231sd(c, a, b);
3165   if (dst != c) {
3166     movdbl(dst, c);
3167   }
3168 }
3169 
3170 // dst = c = a * b + c
3171 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
3172   Assembler::vfmadd231ss(c, a, b);
3173   if (dst != c) {
3174     movflt(dst, c);
3175   }
3176 }
3177 
3178 // dst = c = a * b + c
3179 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
3180   Assembler::vfmadd231pd(c, a, b, vector_len);
3181   if (dst != c) {
3182     vmovdqu(dst, c);
3183   }
3184 }
3185 
3186 // dst = c = a * b + c
3187 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
3188   Assembler::vfmadd231ps(c, a, b, vector_len);
3189   if (dst != c) {
3190     vmovdqu(dst, c);
3191   }
3192 }
3193 
3194 // dst = c = a * b + c
3195 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
3196   Assembler::vfmadd231pd(c, a, b, vector_len);
3197   if (dst != c) {
3198     vmovdqu(dst, c);
3199   }
3200 }
3201 
3202 // dst = c = a * b + c
3203 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
3204   Assembler::vfmadd231ps(c, a, b, vector_len);
3205   if (dst != c) {
3206     vmovdqu(dst, c);
3207   }
3208 }
3209 
3210 void MacroAssembler::incrementl(AddressLiteral dst) {
3211   if (reachable(dst)) {
3212     incrementl(as_Address(dst));
3213   } else {
3214     lea(rscratch1, dst);
3215     incrementl(Address(rscratch1, 0));
3216   }
3217 }
3218 
3219 void MacroAssembler::incrementl(ArrayAddress dst) {
3220   incrementl(as_Address(dst));
3221 }
3222 
3223 void MacroAssembler::incrementl(Register reg, int value) {
3224   if (value == min_jint) {addl(reg, value) ; return; }
3225   if (value <  0) { decrementl(reg, -value); return; }
3226   if (value == 0) {                        ; return; }
3227   if (value == 1 && UseIncDec) { incl(reg) ; return; }
3228   /* else */      { addl(reg, value)       ; return; }
3229 }
3230 
3231 void MacroAssembler::incrementl(Address dst, int value) {
3232   if (value == min_jint) {addl(dst, value) ; return; }
3233   if (value <  0) { decrementl(dst, -value); return; }
3234   if (value == 0) {                        ; return; }
3235   if (value == 1 && UseIncDec) { incl(dst) ; return; }
3236   /* else */      { addl(dst, value)       ; return; }
3237 }
3238 
3239 void MacroAssembler::jump(AddressLiteral dst) {
3240   if (reachable(dst)) {
3241     jmp_literal(dst.target(), dst.rspec());
3242   } else {
3243     lea(rscratch1, dst);
3244     jmp(rscratch1);
3245   }
3246 }
3247 
3248 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
3249   if (reachable(dst)) {
3250     InstructionMark im(this);
3251     relocate(dst.reloc());
3252     const int short_size = 2;
3253     const int long_size = 6;
3254     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
3255     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
3256       // 0111 tttn #8-bit disp
3257       emit_int8(0x70 | cc);
3258       emit_int8((offs - short_size) & 0xFF);
3259     } else {
3260       // 0000 1111 1000 tttn #32-bit disp
3261       emit_int8(0x0F);
3262       emit_int8((unsigned char)(0x80 | cc));
3263       emit_int32(offs - long_size);
3264     }
3265   } else {
3266 #ifdef ASSERT
3267     warning("reversing conditional branch");
3268 #endif /* ASSERT */
3269     Label skip;
3270     jccb(reverse[cc], skip);
3271     lea(rscratch1, dst);
3272     Assembler::jmp(rscratch1);
3273     bind(skip);
3274   }
3275 }
3276 
3277 void MacroAssembler::ldmxcsr(AddressLiteral src) {
3278   if (reachable(src)) {
3279     Assembler::ldmxcsr(as_Address(src));
3280   } else {
3281     lea(rscratch1, src);
3282     Assembler::ldmxcsr(Address(rscratch1, 0));
3283   }
3284 }
3285 
3286 int MacroAssembler::load_signed_byte(Register dst, Address src) {
3287   int off;
3288   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3289     off = offset();
3290     movsbl(dst, src); // movsxb
3291   } else {
3292     off = load_unsigned_byte(dst, src);
3293     shll(dst, 24);
3294     sarl(dst, 24);
3295   }
3296   return off;
3297 }
3298 
3299 // Note: load_signed_short used to be called load_signed_word.
3300 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
3301 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
3302 // The term "word" in HotSpot means a 32- or 64-bit machine word.
3303 int MacroAssembler::load_signed_short(Register dst, Address src) {
3304   int off;
3305   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3306     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
3307     // version but this is what 64bit has always done. This seems to imply
3308     // that users are only using 32bits worth.
3309     off = offset();
3310     movswl(dst, src); // movsxw
3311   } else {
3312     off = load_unsigned_short(dst, src);
3313     shll(dst, 16);
3314     sarl(dst, 16);
3315   }
3316   return off;
3317 }
3318 
3319 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
3320   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3321   // and "3.9 Partial Register Penalties", p. 22).
3322   int off;
3323   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
3324     off = offset();
3325     movzbl(dst, src); // movzxb
3326   } else {
3327     xorl(dst, dst);
3328     off = offset();
3329     movb(dst, src);
3330   }
3331   return off;
3332 }
3333 
3334 // Note: load_unsigned_short used to be called load_unsigned_word.
3335 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
3336   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3337   // and "3.9 Partial Register Penalties", p. 22).
3338   int off;
3339   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
3340     off = offset();
3341     movzwl(dst, src); // movzxw
3342   } else {
3343     xorl(dst, dst);
3344     off = offset();
3345     movw(dst, src);
3346   }
3347   return off;
3348 }
3349 
3350 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
3351   switch (size_in_bytes) {
3352 #ifndef _LP64
3353   case  8:
3354     assert(dst2 != noreg, "second dest register required");
3355     movl(dst,  src);
3356     movl(dst2, src.plus_disp(BytesPerInt));
3357     break;
3358 #else
3359   case  8:  movq(dst, src); break;
3360 #endif
3361   case  4:  movl(dst, src); break;
3362   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
3363   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
3364   default:  ShouldNotReachHere();
3365   }
3366 }
3367 
3368 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
3369   switch (size_in_bytes) {
3370 #ifndef _LP64
3371   case  8:
3372     assert(src2 != noreg, "second source register required");
3373     movl(dst,                        src);
3374     movl(dst.plus_disp(BytesPerInt), src2);
3375     break;
3376 #else
3377   case  8:  movq(dst, src); break;
3378 #endif
3379   case  4:  movl(dst, src); break;
3380   case  2:  movw(dst, src); break;
3381   case  1:  movb(dst, src); break;
3382   default:  ShouldNotReachHere();
3383   }
3384 }
3385 
3386 void MacroAssembler::mov32(AddressLiteral dst, Register src) {
3387   if (reachable(dst)) {
3388     movl(as_Address(dst), src);
3389   } else {
3390     lea(rscratch1, dst);
3391     movl(Address(rscratch1, 0), src);
3392   }
3393 }
3394 
3395 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
3396   if (reachable(src)) {
3397     movl(dst, as_Address(src));
3398   } else {
3399     lea(rscratch1, src);
3400     movl(dst, Address(rscratch1, 0));
3401   }
3402 }
3403 
3404 // C++ bool manipulation
3405 
3406 void MacroAssembler::movbool(Register dst, Address src) {
3407   if(sizeof(bool) == 1)
3408     movb(dst, src);
3409   else if(sizeof(bool) == 2)
3410     movw(dst, src);
3411   else if(sizeof(bool) == 4)
3412     movl(dst, src);
3413   else
3414     // unsupported
3415     ShouldNotReachHere();
3416 }
3417 
3418 void MacroAssembler::movbool(Address dst, bool boolconst) {
3419   if(sizeof(bool) == 1)
3420     movb(dst, (int) boolconst);
3421   else if(sizeof(bool) == 2)
3422     movw(dst, (int) boolconst);
3423   else if(sizeof(bool) == 4)
3424     movl(dst, (int) boolconst);
3425   else
3426     // unsupported
3427     ShouldNotReachHere();
3428 }
3429 
3430 void MacroAssembler::movbool(Address dst, Register src) {
3431   if(sizeof(bool) == 1)
3432     movb(dst, src);
3433   else if(sizeof(bool) == 2)
3434     movw(dst, src);
3435   else if(sizeof(bool) == 4)
3436     movl(dst, src);
3437   else
3438     // unsupported
3439     ShouldNotReachHere();
3440 }
3441 
3442 void MacroAssembler::movbyte(ArrayAddress dst, int src) {
3443   movb(as_Address(dst), src);
3444 }
3445 
3446 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src) {
3447   if (reachable(src)) {
3448     movdl(dst, as_Address(src));
3449   } else {
3450     lea(rscratch1, src);
3451     movdl(dst, Address(rscratch1, 0));
3452   }
3453 }
3454 
3455 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src) {
3456   if (reachable(src)) {
3457     movq(dst, as_Address(src));
3458   } else {
3459     lea(rscratch1, src);
3460     movq(dst, Address(rscratch1, 0));
3461   }
3462 }
3463 
3464 void MacroAssembler::setvectmask(Register dst, Register src) {
3465   Assembler::movl(dst, 1);
3466   Assembler::shlxl(dst, dst, src);
3467   Assembler::decl(dst);
3468   Assembler::kmovdl(k1, dst);
3469   Assembler::movl(dst, src);
3470 }
3471 
3472 void MacroAssembler::restorevectmask() {
3473   Assembler::knotwl(k1, k0);
3474 }
3475 
3476 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
3477   if (reachable(src)) {
3478     if (UseXmmLoadAndClearUpper) {
3479       movsd (dst, as_Address(src));
3480     } else {
3481       movlpd(dst, as_Address(src));
3482     }
3483   } else {
3484     lea(rscratch1, src);
3485     if (UseXmmLoadAndClearUpper) {
3486       movsd (dst, Address(rscratch1, 0));
3487     } else {
3488       movlpd(dst, Address(rscratch1, 0));
3489     }
3490   }
3491 }
3492 
3493 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
3494   if (reachable(src)) {
3495     movss(dst, as_Address(src));
3496   } else {
3497     lea(rscratch1, src);
3498     movss(dst, Address(rscratch1, 0));
3499   }
3500 }
3501 
3502 void MacroAssembler::movptr(Register dst, Register src) {
3503   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3504 }
3505 
3506 void MacroAssembler::movptr(Register dst, Address src) {
3507   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3508 }
3509 
3510 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
3511 void MacroAssembler::movptr(Register dst, intptr_t src) {
3512   LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src));
3513 }
3514 
3515 void MacroAssembler::movptr(Address dst, Register src) {
3516   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3517 }
3518 
3519 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
3520   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3521     Assembler::vextractf32x4(dst, src, 0);
3522   } else {
3523     Assembler::movdqu(dst, src);
3524   }
3525 }
3526 
3527 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
3528   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3529     Assembler::vinsertf32x4(dst, dst, src, 0);
3530   } else {
3531     Assembler::movdqu(dst, src);
3532   }
3533 }
3534 
3535 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
3536   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3537     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3538   } else {
3539     Assembler::movdqu(dst, src);
3540   }
3541 }
3542 
3543 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register scratchReg) {
3544   if (reachable(src)) {
3545     movdqu(dst, as_Address(src));
3546   } else {
3547     lea(scratchReg, src);
3548     movdqu(dst, Address(scratchReg, 0));
3549   }
3550 }
3551 
3552 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
3553   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3554     vextractf64x4_low(dst, src);
3555   } else {
3556     Assembler::vmovdqu(dst, src);
3557   }
3558 }
3559 
3560 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
3561   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3562     vinsertf64x4_low(dst, src);
3563   } else {
3564     Assembler::vmovdqu(dst, src);
3565   }
3566 }
3567 
3568 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3569   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3570     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3571   }
3572   else {
3573     Assembler::vmovdqu(dst, src);
3574   }
3575 }
3576 
3577 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src) {
3578   if (reachable(src)) {
3579     vmovdqu(dst, as_Address(src));
3580   }
3581   else {
3582     lea(rscratch1, src);
3583     vmovdqu(dst, Address(rscratch1, 0));
3584   }
3585 }
3586 
3587 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
3588   if (reachable(src)) {
3589     Assembler::evmovdquq(dst, as_Address(src), vector_len);
3590   } else {
3591     lea(rscratch, src);
3592     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
3593   }
3594 }
3595 
3596 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) {
3597   if (reachable(src)) {
3598     Assembler::movdqa(dst, as_Address(src));
3599   } else {
3600     lea(rscratch1, src);
3601     Assembler::movdqa(dst, Address(rscratch1, 0));
3602   }
3603 }
3604 
3605 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
3606   if (reachable(src)) {
3607     Assembler::movsd(dst, as_Address(src));
3608   } else {
3609     lea(rscratch1, src);
3610     Assembler::movsd(dst, Address(rscratch1, 0));
3611   }
3612 }
3613 
3614 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
3615   if (reachable(src)) {
3616     Assembler::movss(dst, as_Address(src));
3617   } else {
3618     lea(rscratch1, src);
3619     Assembler::movss(dst, Address(rscratch1, 0));
3620   }
3621 }
3622 
3623 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) {
3624   if (reachable(src)) {
3625     Assembler::mulsd(dst, as_Address(src));
3626   } else {
3627     lea(rscratch1, src);
3628     Assembler::mulsd(dst, Address(rscratch1, 0));
3629   }
3630 }
3631 
3632 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) {
3633   if (reachable(src)) {
3634     Assembler::mulss(dst, as_Address(src));
3635   } else {
3636     lea(rscratch1, src);
3637     Assembler::mulss(dst, Address(rscratch1, 0));
3638   }
3639 }
3640 
3641 void MacroAssembler::null_check(Register reg, int offset) {
3642   if (needs_explicit_null_check(offset)) {
3643     // provoke OS NULL exception if reg = NULL by
3644     // accessing M[reg] w/o changing any (non-CC) registers
3645     // NOTE: cmpl is plenty here to provoke a segv
3646     cmpptr(rax, Address(reg, 0));
3647     // Note: should probably use testl(rax, Address(reg, 0));
3648     //       may be shorter code (however, this version of
3649     //       testl needs to be implemented first)
3650   } else {
3651     // nothing to do, (later) access of M[reg + offset]
3652     // will provoke OS NULL exception if reg = NULL
3653   }
3654 }
3655 
3656 void MacroAssembler::test_klass_is_value(Register klass, Register temp_reg, Label& is_value) {
3657   movl(temp_reg, Address(klass, Klass::access_flags_offset()));
3658   testl(temp_reg, JVM_ACC_VALUE);
3659   jcc(Assembler::notZero, is_value);
3660 }
3661 
3662 void MacroAssembler::test_field_is_flattenable(Register flags, Register temp_reg, Label& is_flattenable) {
3663   movl(temp_reg, flags);
3664   shrl(temp_reg, ConstantPoolCacheEntry::is_flattenable_field_shift);
3665   andl(temp_reg, 0x1);
3666   testl(temp_reg, temp_reg);
3667   jcc(Assembler::notZero, is_flattenable);
3668 }
3669 
3670 void MacroAssembler::test_field_is_not_flattenable(Register flags, Register temp_reg, Label& notFlattenable) {
3671   movl(temp_reg, flags);
3672   shrl(temp_reg, ConstantPoolCacheEntry::is_flattenable_field_shift);
3673   andl(temp_reg, 0x1);
3674   testl(temp_reg, temp_reg);
3675   jcc(Assembler::zero, notFlattenable);
3676 }
3677 
3678 void MacroAssembler::test_field_is_flattened(Register flags, Register temp_reg, Label& is_flattened) {
3679   movl(temp_reg, flags);
3680   shrl(temp_reg, ConstantPoolCacheEntry::is_flattened_field_shift);
3681   andl(temp_reg, 0x1);
3682   testl(temp_reg, temp_reg);
3683   jcc(Assembler::notZero, is_flattened);
3684 }
3685 
3686 void MacroAssembler::test_flat_array_klass(Register klass, Register temp_reg,
3687                                            Label& is_flat_array) {
3688   movl(temp_reg, Address(klass, Klass::layout_helper_offset()));
3689   sarl(temp_reg, Klass::_lh_array_tag_shift);
3690   cmpl(temp_reg, Klass::_lh_array_tag_vt_value);
3691   jcc(Assembler::equal, is_flat_array);
3692 }
3693 
3694 
3695 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg,
3696                                          Label& is_flat_array) {
3697   load_klass(temp_reg, oop);
3698   test_flat_array_klass(temp_reg, temp_reg, is_flat_array);
3699 }
3700 
3701 void MacroAssembler::os_breakpoint() {
3702   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
3703   // (e.g., MSVC can't call ps() otherwise)
3704   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
3705 }
3706 
3707 void MacroAssembler::unimplemented(const char* what) {
3708   const char* buf = NULL;
3709   {
3710     ResourceMark rm;
3711     stringStream ss;
3712     ss.print("unimplemented: %s", what);
3713     buf = code_string(ss.as_string());
3714   }
3715   stop(buf);
3716 }
3717 
3718 #ifdef _LP64
3719 #define XSTATE_BV 0x200
3720 #endif
3721 
3722 void MacroAssembler::pop_CPU_state() {
3723   pop_FPU_state();
3724   pop_IU_state();
3725 }
3726 
3727 void MacroAssembler::pop_FPU_state() {
3728 #ifndef _LP64
3729   frstor(Address(rsp, 0));
3730 #else
3731   fxrstor(Address(rsp, 0));
3732 #endif
3733   addptr(rsp, FPUStateSizeInWords * wordSize);
3734 }
3735 
3736 void MacroAssembler::pop_IU_state() {
3737   popa();
3738   LP64_ONLY(addq(rsp, 8));
3739   popf();
3740 }
3741 
3742 // Save Integer and Float state
3743 // Warning: Stack must be 16 byte aligned (64bit)
3744 void MacroAssembler::push_CPU_state() {
3745   push_IU_state();
3746   push_FPU_state();
3747 }
3748 
3749 void MacroAssembler::push_FPU_state() {
3750   subptr(rsp, FPUStateSizeInWords * wordSize);
3751 #ifndef _LP64
3752   fnsave(Address(rsp, 0));
3753   fwait();
3754 #else
3755   fxsave(Address(rsp, 0));
3756 #endif // LP64
3757 }
3758 
3759 void MacroAssembler::push_IU_state() {
3760   // Push flags first because pusha kills them
3761   pushf();
3762   // Make sure rsp stays 16-byte aligned
3763   LP64_ONLY(subq(rsp, 8));
3764   pusha();
3765 }
3766 
3767 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
3768   if (!java_thread->is_valid()) {
3769     java_thread = rdi;
3770     get_thread(java_thread);
3771   }
3772   // we must set sp to zero to clear frame
3773   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
3774   if (clear_fp) {
3775     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
3776   }
3777 
3778   // Always clear the pc because it could have been set by make_walkable()
3779   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
3780 
3781   vzeroupper();
3782 }
3783 
3784 void MacroAssembler::restore_rax(Register tmp) {
3785   if (tmp == noreg) pop(rax);
3786   else if (tmp != rax) mov(rax, tmp);
3787 }
3788 
3789 void MacroAssembler::round_to(Register reg, int modulus) {
3790   addptr(reg, modulus - 1);
3791   andptr(reg, -modulus);
3792 }
3793 
3794 void MacroAssembler::save_rax(Register tmp) {
3795   if (tmp == noreg) push(rax);
3796   else if (tmp != rax) mov(tmp, rax);
3797 }
3798 
3799 // Write serialization page so VM thread can do a pseudo remote membar.
3800 // We use the current thread pointer to calculate a thread specific
3801 // offset to write to within the page. This minimizes bus traffic
3802 // due to cache line collision.
3803 void MacroAssembler::serialize_memory(Register thread, Register tmp) {
3804   movl(tmp, thread);
3805   shrl(tmp, os::get_serialize_page_shift_count());
3806   andl(tmp, (os::vm_page_size() - sizeof(int)));
3807 
3808   Address index(noreg, tmp, Address::times_1);
3809   ExternalAddress page(os::get_memory_serialize_page());
3810 
3811   // Size of store must match masking code above
3812   movl(as_Address(ArrayAddress(page, index)), tmp);
3813 }
3814 
3815 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, Register temp_reg) {
3816   if (SafepointMechanism::uses_thread_local_poll()) {
3817 #ifdef _LP64
3818     assert(thread_reg == r15_thread, "should be");
3819 #else
3820     if (thread_reg == noreg) {
3821       thread_reg = temp_reg;
3822       get_thread(thread_reg);
3823     }
3824 #endif
3825     testb(Address(thread_reg, Thread::polling_page_offset()), SafepointMechanism::poll_bit());
3826     jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
3827   } else {
3828     cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
3829         SafepointSynchronize::_not_synchronized);
3830     jcc(Assembler::notEqual, slow_path);
3831   }
3832 }
3833 
3834 // Calls to C land
3835 //
3836 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
3837 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
3838 // has to be reset to 0. This is required to allow proper stack traversal.
3839 void MacroAssembler::set_last_Java_frame(Register java_thread,
3840                                          Register last_java_sp,
3841                                          Register last_java_fp,
3842                                          address  last_java_pc) {
3843   vzeroupper();
3844   // determine java_thread register
3845   if (!java_thread->is_valid()) {
3846     java_thread = rdi;
3847     get_thread(java_thread);
3848   }
3849   // determine last_java_sp register
3850   if (!last_java_sp->is_valid()) {
3851     last_java_sp = rsp;
3852   }
3853 
3854   // last_java_fp is optional
3855 
3856   if (last_java_fp->is_valid()) {
3857     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
3858   }
3859 
3860   // last_java_pc is optional
3861 
3862   if (last_java_pc != NULL) {
3863     lea(Address(java_thread,
3864                  JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()),
3865         InternalAddress(last_java_pc));
3866 
3867   }
3868   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
3869 }
3870 
3871 void MacroAssembler::shlptr(Register dst, int imm8) {
3872   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
3873 }
3874 
3875 void MacroAssembler::shrptr(Register dst, int imm8) {
3876   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
3877 }
3878 
3879 void MacroAssembler::sign_extend_byte(Register reg) {
3880   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
3881     movsbl(reg, reg); // movsxb
3882   } else {
3883     shll(reg, 24);
3884     sarl(reg, 24);
3885   }
3886 }
3887 
3888 void MacroAssembler::sign_extend_short(Register reg) {
3889   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3890     movswl(reg, reg); // movsxw
3891   } else {
3892     shll(reg, 16);
3893     sarl(reg, 16);
3894   }
3895 }
3896 
3897 void MacroAssembler::testl(Register dst, AddressLiteral src) {
3898   assert(reachable(src), "Address should be reachable");
3899   testl(dst, as_Address(src));
3900 }
3901 
3902 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3903   int dst_enc = dst->encoding();
3904   int src_enc = src->encoding();
3905   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3906     Assembler::pcmpeqb(dst, src);
3907   } else if ((dst_enc < 16) && (src_enc < 16)) {
3908     Assembler::pcmpeqb(dst, src);
3909   } else if (src_enc < 16) {
3910     push_zmm(xmm0);
3911     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3912     Assembler::pcmpeqb(xmm0, src);
3913     movdqu(dst, xmm0);
3914     pop_zmm(xmm0);
3915   } else if (dst_enc < 16) {
3916     push_zmm(xmm0);
3917     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3918     Assembler::pcmpeqb(dst, xmm0);
3919     pop_zmm(xmm0);
3920   } else {
3921     push_zmm(xmm0);
3922     push_zmm(xmm1);
3923     movdqu(xmm0, src);
3924     movdqu(xmm1, dst);
3925     Assembler::pcmpeqb(xmm1, xmm0);
3926     movdqu(dst, xmm1);
3927     pop_zmm(xmm1);
3928     pop_zmm(xmm0);
3929   }
3930 }
3931 
3932 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3933   int dst_enc = dst->encoding();
3934   int src_enc = src->encoding();
3935   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3936     Assembler::pcmpeqw(dst, src);
3937   } else if ((dst_enc < 16) && (src_enc < 16)) {
3938     Assembler::pcmpeqw(dst, src);
3939   } else if (src_enc < 16) {
3940     push_zmm(xmm0);
3941     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3942     Assembler::pcmpeqw(xmm0, src);
3943     movdqu(dst, xmm0);
3944     pop_zmm(xmm0);
3945   } else if (dst_enc < 16) {
3946     push_zmm(xmm0);
3947     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3948     Assembler::pcmpeqw(dst, xmm0);
3949     pop_zmm(xmm0);
3950   } else {
3951     push_zmm(xmm0);
3952     push_zmm(xmm1);
3953     movdqu(xmm0, src);
3954     movdqu(xmm1, dst);
3955     Assembler::pcmpeqw(xmm1, xmm0);
3956     movdqu(dst, xmm1);
3957     pop_zmm(xmm1);
3958     pop_zmm(xmm0);
3959   }
3960 }
3961 
3962 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3963   int dst_enc = dst->encoding();
3964   if (dst_enc < 16) {
3965     Assembler::pcmpestri(dst, src, imm8);
3966   } else {
3967     push_zmm(xmm0);
3968     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3969     Assembler::pcmpestri(xmm0, src, imm8);
3970     movdqu(dst, xmm0);
3971     pop_zmm(xmm0);
3972   }
3973 }
3974 
3975 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3976   int dst_enc = dst->encoding();
3977   int src_enc = src->encoding();
3978   if ((dst_enc < 16) && (src_enc < 16)) {
3979     Assembler::pcmpestri(dst, src, imm8);
3980   } else if (src_enc < 16) {
3981     push_zmm(xmm0);
3982     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3983     Assembler::pcmpestri(xmm0, src, imm8);
3984     movdqu(dst, xmm0);
3985     pop_zmm(xmm0);
3986   } else if (dst_enc < 16) {
3987     push_zmm(xmm0);
3988     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3989     Assembler::pcmpestri(dst, xmm0, imm8);
3990     pop_zmm(xmm0);
3991   } else {
3992     push_zmm(xmm0);
3993     push_zmm(xmm1);
3994     movdqu(xmm0, src);
3995     movdqu(xmm1, dst);
3996     Assembler::pcmpestri(xmm1, xmm0, imm8);
3997     movdqu(dst, xmm1);
3998     pop_zmm(xmm1);
3999     pop_zmm(xmm0);
4000   }
4001 }
4002 
4003 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
4004   int dst_enc = dst->encoding();
4005   int src_enc = src->encoding();
4006   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4007     Assembler::pmovzxbw(dst, src);
4008   } else if ((dst_enc < 16) && (src_enc < 16)) {
4009     Assembler::pmovzxbw(dst, src);
4010   } else if (src_enc < 16) {
4011     push_zmm(xmm0);
4012     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4013     Assembler::pmovzxbw(xmm0, src);
4014     movdqu(dst, xmm0);
4015     pop_zmm(xmm0);
4016   } else if (dst_enc < 16) {
4017     push_zmm(xmm0);
4018     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4019     Assembler::pmovzxbw(dst, xmm0);
4020     pop_zmm(xmm0);
4021   } else {
4022     push_zmm(xmm0);
4023     push_zmm(xmm1);
4024     movdqu(xmm0, src);
4025     movdqu(xmm1, dst);
4026     Assembler::pmovzxbw(xmm1, xmm0);
4027     movdqu(dst, xmm1);
4028     pop_zmm(xmm1);
4029     pop_zmm(xmm0);
4030   }
4031 }
4032 
4033 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
4034   int dst_enc = dst->encoding();
4035   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4036     Assembler::pmovzxbw(dst, src);
4037   } else if (dst_enc < 16) {
4038     Assembler::pmovzxbw(dst, src);
4039   } else {
4040     push_zmm(xmm0);
4041     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4042     Assembler::pmovzxbw(xmm0, src);
4043     movdqu(dst, xmm0);
4044     pop_zmm(xmm0);
4045   }
4046 }
4047 
4048 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
4049   int src_enc = src->encoding();
4050   if (src_enc < 16) {
4051     Assembler::pmovmskb(dst, src);
4052   } else {
4053     push_zmm(xmm0);
4054     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4055     Assembler::pmovmskb(dst, xmm0);
4056     pop_zmm(xmm0);
4057   }
4058 }
4059 
4060 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
4061   int dst_enc = dst->encoding();
4062   int src_enc = src->encoding();
4063   if ((dst_enc < 16) && (src_enc < 16)) {
4064     Assembler::ptest(dst, src);
4065   } else if (src_enc < 16) {
4066     push_zmm(xmm0);
4067     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4068     Assembler::ptest(xmm0, src);
4069     pop_zmm(xmm0);
4070   } else if (dst_enc < 16) {
4071     push_zmm(xmm0);
4072     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4073     Assembler::ptest(dst, xmm0);
4074     pop_zmm(xmm0);
4075   } else {
4076     push_zmm(xmm0);
4077     push_zmm(xmm1);
4078     movdqu(xmm0, src);
4079     movdqu(xmm1, dst);
4080     Assembler::ptest(xmm1, xmm0);
4081     pop_zmm(xmm1);
4082     pop_zmm(xmm0);
4083   }
4084 }
4085 
4086 void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) {
4087   if (reachable(src)) {
4088     Assembler::sqrtsd(dst, as_Address(src));
4089   } else {
4090     lea(rscratch1, src);
4091     Assembler::sqrtsd(dst, Address(rscratch1, 0));
4092   }
4093 }
4094 
4095 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) {
4096   if (reachable(src)) {
4097     Assembler::sqrtss(dst, as_Address(src));
4098   } else {
4099     lea(rscratch1, src);
4100     Assembler::sqrtss(dst, Address(rscratch1, 0));
4101   }
4102 }
4103 
4104 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) {
4105   if (reachable(src)) {
4106     Assembler::subsd(dst, as_Address(src));
4107   } else {
4108     lea(rscratch1, src);
4109     Assembler::subsd(dst, Address(rscratch1, 0));
4110   }
4111 }
4112 
4113 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) {
4114   if (reachable(src)) {
4115     Assembler::subss(dst, as_Address(src));
4116   } else {
4117     lea(rscratch1, src);
4118     Assembler::subss(dst, Address(rscratch1, 0));
4119   }
4120 }
4121 
4122 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
4123   if (reachable(src)) {
4124     Assembler::ucomisd(dst, as_Address(src));
4125   } else {
4126     lea(rscratch1, src);
4127     Assembler::ucomisd(dst, Address(rscratch1, 0));
4128   }
4129 }
4130 
4131 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
4132   if (reachable(src)) {
4133     Assembler::ucomiss(dst, as_Address(src));
4134   } else {
4135     lea(rscratch1, src);
4136     Assembler::ucomiss(dst, Address(rscratch1, 0));
4137   }
4138 }
4139 
4140 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
4141   // Used in sign-bit flipping with aligned address.
4142   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
4143   if (reachable(src)) {
4144     Assembler::xorpd(dst, as_Address(src));
4145   } else {
4146     lea(rscratch1, src);
4147     Assembler::xorpd(dst, Address(rscratch1, 0));
4148   }
4149 }
4150 
4151 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
4152   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
4153     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
4154   }
4155   else {
4156     Assembler::xorpd(dst, src);
4157   }
4158 }
4159 
4160 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
4161   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
4162     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
4163   } else {
4164     Assembler::xorps(dst, src);
4165   }
4166 }
4167 
4168 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
4169   // Used in sign-bit flipping with aligned address.
4170   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
4171   if (reachable(src)) {
4172     Assembler::xorps(dst, as_Address(src));
4173   } else {
4174     lea(rscratch1, src);
4175     Assembler::xorps(dst, Address(rscratch1, 0));
4176   }
4177 }
4178 
4179 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) {
4180   // Used in sign-bit flipping with aligned address.
4181   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
4182   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
4183   if (reachable(src)) {
4184     Assembler::pshufb(dst, as_Address(src));
4185   } else {
4186     lea(rscratch1, src);
4187     Assembler::pshufb(dst, Address(rscratch1, 0));
4188   }
4189 }
4190 
4191 // AVX 3-operands instructions
4192 
4193 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4194   if (reachable(src)) {
4195     vaddsd(dst, nds, as_Address(src));
4196   } else {
4197     lea(rscratch1, src);
4198     vaddsd(dst, nds, Address(rscratch1, 0));
4199   }
4200 }
4201 
4202 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4203   if (reachable(src)) {
4204     vaddss(dst, nds, as_Address(src));
4205   } else {
4206     lea(rscratch1, src);
4207     vaddss(dst, nds, Address(rscratch1, 0));
4208   }
4209 }
4210 
4211 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
4212   int dst_enc = dst->encoding();
4213   int nds_enc = nds->encoding();
4214   int src_enc = src->encoding();
4215   if ((dst_enc < 16) && (nds_enc < 16)) {
4216     vandps(dst, nds, negate_field, vector_len);
4217   } else if ((src_enc < 16) && (dst_enc < 16)) {
4218     // Use src scratch register
4219     evmovdqul(src, nds, Assembler::AVX_512bit);
4220     vandps(dst, src, negate_field, vector_len);
4221   } else if (dst_enc < 16) {
4222     evmovdqul(dst, nds, Assembler::AVX_512bit);
4223     vandps(dst, dst, negate_field, vector_len);
4224   } else if (nds_enc < 16) {
4225     vandps(nds, nds, negate_field, vector_len);
4226     evmovdqul(dst, nds, Assembler::AVX_512bit);
4227   } else if (src_enc < 16) {
4228     evmovdqul(src, nds, Assembler::AVX_512bit);
4229     vandps(src, src, negate_field, vector_len);
4230     evmovdqul(dst, src, Assembler::AVX_512bit);
4231   } else {
4232     if (src_enc != dst_enc) {
4233       // Use src scratch register
4234       evmovdqul(src, xmm0, Assembler::AVX_512bit);
4235       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4236       vandps(xmm0, xmm0, negate_field, vector_len);
4237       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4238       evmovdqul(xmm0, src, Assembler::AVX_512bit);
4239     } else {
4240       push_zmm(xmm0);
4241       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4242       vandps(xmm0, xmm0, negate_field, vector_len);
4243       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4244       pop_zmm(xmm0);
4245     }
4246   }
4247 }
4248 
4249 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
4250   int dst_enc = dst->encoding();
4251   int nds_enc = nds->encoding();
4252   int src_enc = src->encoding();
4253   if ((dst_enc < 16) && (nds_enc < 16)) {
4254     vandpd(dst, nds, negate_field, vector_len);
4255   } else if ((src_enc < 16) && (dst_enc < 16)) {
4256     // Use src scratch register
4257     evmovdqul(src, nds, Assembler::AVX_512bit);
4258     vandpd(dst, src, negate_field, vector_len);
4259   } else if (dst_enc < 16) {
4260     evmovdqul(dst, nds, Assembler::AVX_512bit);
4261     vandpd(dst, dst, negate_field, vector_len);
4262   } else if (nds_enc < 16) {
4263     vandpd(nds, nds, negate_field, vector_len);
4264     evmovdqul(dst, nds, Assembler::AVX_512bit);
4265   } else if (src_enc < 16) {
4266     evmovdqul(src, nds, Assembler::AVX_512bit);
4267     vandpd(src, src, negate_field, vector_len);
4268     evmovdqul(dst, src, Assembler::AVX_512bit);
4269   } else {
4270     if (src_enc != dst_enc) {
4271       evmovdqul(src, xmm0, Assembler::AVX_512bit);
4272       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4273       vandpd(xmm0, xmm0, negate_field, vector_len);
4274       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4275       evmovdqul(xmm0, src, Assembler::AVX_512bit);
4276     } else {
4277       push_zmm(xmm0);
4278       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4279       vandpd(xmm0, xmm0, negate_field, vector_len);
4280       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4281       pop_zmm(xmm0);
4282     }
4283   }
4284 }
4285 
4286 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4287   int dst_enc = dst->encoding();
4288   int nds_enc = nds->encoding();
4289   int src_enc = src->encoding();
4290   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4291     Assembler::vpaddb(dst, nds, src, vector_len);
4292   } else if ((dst_enc < 16) && (src_enc < 16)) {
4293     Assembler::vpaddb(dst, dst, src, vector_len);
4294   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4295     // use nds as scratch for src
4296     evmovdqul(nds, src, Assembler::AVX_512bit);
4297     Assembler::vpaddb(dst, dst, nds, vector_len);
4298   } else if ((src_enc < 16) && (nds_enc < 16)) {
4299     // use nds as scratch for dst
4300     evmovdqul(nds, dst, Assembler::AVX_512bit);
4301     Assembler::vpaddb(nds, nds, src, vector_len);
4302     evmovdqul(dst, nds, Assembler::AVX_512bit);
4303   } else if (dst_enc < 16) {
4304     // use nds as scatch for xmm0 to hold src
4305     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4306     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4307     Assembler::vpaddb(dst, dst, xmm0, vector_len);
4308     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4309   } else {
4310     // worse case scenario, all regs are in the upper bank
4311     push_zmm(xmm1);
4312     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4313     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4314     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4315     Assembler::vpaddb(xmm0, xmm0, xmm1, vector_len);
4316     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4317     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4318     pop_zmm(xmm1);
4319   }
4320 }
4321 
4322 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4323   int dst_enc = dst->encoding();
4324   int nds_enc = nds->encoding();
4325   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4326     Assembler::vpaddb(dst, nds, src, vector_len);
4327   } else if (dst_enc < 16) {
4328     Assembler::vpaddb(dst, dst, src, vector_len);
4329   } else if (nds_enc < 16) {
4330     // implies dst_enc in upper bank with src as scratch
4331     evmovdqul(nds, dst, Assembler::AVX_512bit);
4332     Assembler::vpaddb(nds, nds, src, vector_len);
4333     evmovdqul(dst, nds, Assembler::AVX_512bit);
4334   } else {
4335     // worse case scenario, all regs in upper bank
4336     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4337     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4338     Assembler::vpaddb(xmm0, xmm0, src, vector_len);
4339     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4340     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4341   }
4342 }
4343 
4344 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4345   int dst_enc = dst->encoding();
4346   int nds_enc = nds->encoding();
4347   int src_enc = src->encoding();
4348   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4349     Assembler::vpaddw(dst, nds, src, vector_len);
4350   } else if ((dst_enc < 16) && (src_enc < 16)) {
4351     Assembler::vpaddw(dst, dst, src, vector_len);
4352   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4353     // use nds as scratch for src
4354     evmovdqul(nds, src, Assembler::AVX_512bit);
4355     Assembler::vpaddw(dst, dst, nds, vector_len);
4356   } else if ((src_enc < 16) && (nds_enc < 16)) {
4357     // use nds as scratch for dst
4358     evmovdqul(nds, dst, Assembler::AVX_512bit);
4359     Assembler::vpaddw(nds, nds, src, vector_len);
4360     evmovdqul(dst, nds, Assembler::AVX_512bit);
4361   } else if (dst_enc < 16) {
4362     // use nds as scatch for xmm0 to hold src
4363     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4364     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4365     Assembler::vpaddw(dst, dst, xmm0, vector_len);
4366     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4367   } else {
4368     // worse case scenario, all regs are in the upper bank
4369     push_zmm(xmm1);
4370     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4371     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4372     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4373     Assembler::vpaddw(xmm0, xmm0, xmm1, vector_len);
4374     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4375     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4376     pop_zmm(xmm1);
4377   }
4378 }
4379 
4380 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4381   int dst_enc = dst->encoding();
4382   int nds_enc = nds->encoding();
4383   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4384     Assembler::vpaddw(dst, nds, src, vector_len);
4385   } else if (dst_enc < 16) {
4386     Assembler::vpaddw(dst, dst, src, vector_len);
4387   } else if (nds_enc < 16) {
4388     // implies dst_enc in upper bank with nds as scratch
4389     evmovdqul(nds, dst, Assembler::AVX_512bit);
4390     Assembler::vpaddw(nds, nds, src, vector_len);
4391     evmovdqul(dst, nds, Assembler::AVX_512bit);
4392   } else {
4393     // worse case scenario, all regs in upper bank
4394     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4395     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4396     Assembler::vpaddw(xmm0, xmm0, src, vector_len);
4397     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4398     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4399   }
4400 }
4401 
4402 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4403   if (reachable(src)) {
4404     Assembler::vpand(dst, nds, as_Address(src), vector_len);
4405   } else {
4406     lea(rscratch1, src);
4407     Assembler::vpand(dst, nds, Address(rscratch1, 0), vector_len);
4408   }
4409 }
4410 
4411 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
4412   int dst_enc = dst->encoding();
4413   int src_enc = src->encoding();
4414   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4415     Assembler::vpbroadcastw(dst, src);
4416   } else if ((dst_enc < 16) && (src_enc < 16)) {
4417     Assembler::vpbroadcastw(dst, src);
4418   } else if (src_enc < 16) {
4419     push_zmm(xmm0);
4420     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4421     Assembler::vpbroadcastw(xmm0, src);
4422     movdqu(dst, xmm0);
4423     pop_zmm(xmm0);
4424   } else if (dst_enc < 16) {
4425     push_zmm(xmm0);
4426     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4427     Assembler::vpbroadcastw(dst, xmm0);
4428     pop_zmm(xmm0);
4429   } else {
4430     push_zmm(xmm0);
4431     push_zmm(xmm1);
4432     movdqu(xmm0, src);
4433     movdqu(xmm1, dst);
4434     Assembler::vpbroadcastw(xmm1, xmm0);
4435     movdqu(dst, xmm1);
4436     pop_zmm(xmm1);
4437     pop_zmm(xmm0);
4438   }
4439 }
4440 
4441 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4442   int dst_enc = dst->encoding();
4443   int nds_enc = nds->encoding();
4444   int src_enc = src->encoding();
4445   assert(dst_enc == nds_enc, "");
4446   if ((dst_enc < 16) && (src_enc < 16)) {
4447     Assembler::vpcmpeqb(dst, nds, src, vector_len);
4448   } else if (src_enc < 16) {
4449     push_zmm(xmm0);
4450     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4451     Assembler::vpcmpeqb(xmm0, xmm0, src, vector_len);
4452     movdqu(dst, xmm0);
4453     pop_zmm(xmm0);
4454   } else if (dst_enc < 16) {
4455     push_zmm(xmm0);
4456     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4457     Assembler::vpcmpeqb(dst, dst, xmm0, vector_len);
4458     pop_zmm(xmm0);
4459   } else {
4460     push_zmm(xmm0);
4461     push_zmm(xmm1);
4462     movdqu(xmm0, src);
4463     movdqu(xmm1, dst);
4464     Assembler::vpcmpeqb(xmm1, xmm1, xmm0, vector_len);
4465     movdqu(dst, xmm1);
4466     pop_zmm(xmm1);
4467     pop_zmm(xmm0);
4468   }
4469 }
4470 
4471 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4472   int dst_enc = dst->encoding();
4473   int nds_enc = nds->encoding();
4474   int src_enc = src->encoding();
4475   assert(dst_enc == nds_enc, "");
4476   if ((dst_enc < 16) && (src_enc < 16)) {
4477     Assembler::vpcmpeqw(dst, nds, src, vector_len);
4478   } else if (src_enc < 16) {
4479     push_zmm(xmm0);
4480     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4481     Assembler::vpcmpeqw(xmm0, xmm0, src, vector_len);
4482     movdqu(dst, xmm0);
4483     pop_zmm(xmm0);
4484   } else if (dst_enc < 16) {
4485     push_zmm(xmm0);
4486     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4487     Assembler::vpcmpeqw(dst, dst, xmm0, vector_len);
4488     pop_zmm(xmm0);
4489   } else {
4490     push_zmm(xmm0);
4491     push_zmm(xmm1);
4492     movdqu(xmm0, src);
4493     movdqu(xmm1, dst);
4494     Assembler::vpcmpeqw(xmm1, xmm1, xmm0, vector_len);
4495     movdqu(dst, xmm1);
4496     pop_zmm(xmm1);
4497     pop_zmm(xmm0);
4498   }
4499 }
4500 
4501 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
4502   int dst_enc = dst->encoding();
4503   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4504     Assembler::vpmovzxbw(dst, src, vector_len);
4505   } else if (dst_enc < 16) {
4506     Assembler::vpmovzxbw(dst, src, vector_len);
4507   } else {
4508     push_zmm(xmm0);
4509     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4510     Assembler::vpmovzxbw(xmm0, src, vector_len);
4511     movdqu(dst, xmm0);
4512     pop_zmm(xmm0);
4513   }
4514 }
4515 
4516 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src) {
4517   int src_enc = src->encoding();
4518   if (src_enc < 16) {
4519     Assembler::vpmovmskb(dst, src);
4520   } else {
4521     push_zmm(xmm0);
4522     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4523     Assembler::vpmovmskb(dst, xmm0);
4524     pop_zmm(xmm0);
4525   }
4526 }
4527 
4528 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4529   int dst_enc = dst->encoding();
4530   int nds_enc = nds->encoding();
4531   int src_enc = src->encoding();
4532   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4533     Assembler::vpmullw(dst, nds, src, vector_len);
4534   } else if ((dst_enc < 16) && (src_enc < 16)) {
4535     Assembler::vpmullw(dst, dst, src, vector_len);
4536   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4537     // use nds as scratch for src
4538     evmovdqul(nds, src, Assembler::AVX_512bit);
4539     Assembler::vpmullw(dst, dst, nds, vector_len);
4540   } else if ((src_enc < 16) && (nds_enc < 16)) {
4541     // use nds as scratch for dst
4542     evmovdqul(nds, dst, Assembler::AVX_512bit);
4543     Assembler::vpmullw(nds, nds, src, vector_len);
4544     evmovdqul(dst, nds, Assembler::AVX_512bit);
4545   } else if (dst_enc < 16) {
4546     // use nds as scatch for xmm0 to hold src
4547     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4548     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4549     Assembler::vpmullw(dst, dst, xmm0, vector_len);
4550     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4551   } else {
4552     // worse case scenario, all regs are in the upper bank
4553     push_zmm(xmm1);
4554     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4555     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4556     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4557     Assembler::vpmullw(xmm0, xmm0, xmm1, vector_len);
4558     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4559     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4560     pop_zmm(xmm1);
4561   }
4562 }
4563 
4564 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4565   int dst_enc = dst->encoding();
4566   int nds_enc = nds->encoding();
4567   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4568     Assembler::vpmullw(dst, nds, src, vector_len);
4569   } else if (dst_enc < 16) {
4570     Assembler::vpmullw(dst, dst, src, vector_len);
4571   } else if (nds_enc < 16) {
4572     // implies dst_enc in upper bank with src as scratch
4573     evmovdqul(nds, dst, Assembler::AVX_512bit);
4574     Assembler::vpmullw(nds, nds, src, vector_len);
4575     evmovdqul(dst, nds, Assembler::AVX_512bit);
4576   } else {
4577     // worse case scenario, all regs in upper bank
4578     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4579     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4580     Assembler::vpmullw(xmm0, xmm0, src, vector_len);
4581     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4582     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4583   }
4584 }
4585 
4586 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4587   int dst_enc = dst->encoding();
4588   int nds_enc = nds->encoding();
4589   int src_enc = src->encoding();
4590   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4591     Assembler::vpsubb(dst, nds, src, vector_len);
4592   } else if ((dst_enc < 16) && (src_enc < 16)) {
4593     Assembler::vpsubb(dst, dst, src, vector_len);
4594   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4595     // use nds as scratch for src
4596     evmovdqul(nds, src, Assembler::AVX_512bit);
4597     Assembler::vpsubb(dst, dst, nds, vector_len);
4598   } else if ((src_enc < 16) && (nds_enc < 16)) {
4599     // use nds as scratch for dst
4600     evmovdqul(nds, dst, Assembler::AVX_512bit);
4601     Assembler::vpsubb(nds, nds, src, vector_len);
4602     evmovdqul(dst, nds, Assembler::AVX_512bit);
4603   } else if (dst_enc < 16) {
4604     // use nds as scatch for xmm0 to hold src
4605     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4606     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4607     Assembler::vpsubb(dst, dst, xmm0, vector_len);
4608     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4609   } else {
4610     // worse case scenario, all regs are in the upper bank
4611     push_zmm(xmm1);
4612     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4613     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4614     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4615     Assembler::vpsubb(xmm0, xmm0, xmm1, vector_len);
4616     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4617     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4618     pop_zmm(xmm1);
4619   }
4620 }
4621 
4622 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4623   int dst_enc = dst->encoding();
4624   int nds_enc = nds->encoding();
4625   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4626     Assembler::vpsubb(dst, nds, src, vector_len);
4627   } else if (dst_enc < 16) {
4628     Assembler::vpsubb(dst, dst, src, vector_len);
4629   } else if (nds_enc < 16) {
4630     // implies dst_enc in upper bank with src as scratch
4631     evmovdqul(nds, dst, Assembler::AVX_512bit);
4632     Assembler::vpsubb(nds, nds, src, vector_len);
4633     evmovdqul(dst, nds, Assembler::AVX_512bit);
4634   } else {
4635     // worse case scenario, all regs in upper bank
4636     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4637     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4638     Assembler::vpsubb(xmm0, xmm0, src, vector_len);
4639     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4640     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4641   }
4642 }
4643 
4644 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4645   int dst_enc = dst->encoding();
4646   int nds_enc = nds->encoding();
4647   int src_enc = src->encoding();
4648   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4649     Assembler::vpsubw(dst, nds, src, vector_len);
4650   } else if ((dst_enc < 16) && (src_enc < 16)) {
4651     Assembler::vpsubw(dst, dst, src, vector_len);
4652   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4653     // use nds as scratch for src
4654     evmovdqul(nds, src, Assembler::AVX_512bit);
4655     Assembler::vpsubw(dst, dst, nds, vector_len);
4656   } else if ((src_enc < 16) && (nds_enc < 16)) {
4657     // use nds as scratch for dst
4658     evmovdqul(nds, dst, Assembler::AVX_512bit);
4659     Assembler::vpsubw(nds, nds, src, vector_len);
4660     evmovdqul(dst, nds, Assembler::AVX_512bit);
4661   } else if (dst_enc < 16) {
4662     // use nds as scatch for xmm0 to hold src
4663     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4664     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4665     Assembler::vpsubw(dst, dst, xmm0, vector_len);
4666     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4667   } else {
4668     // worse case scenario, all regs are in the upper bank
4669     push_zmm(xmm1);
4670     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4671     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4672     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4673     Assembler::vpsubw(xmm0, xmm0, xmm1, vector_len);
4674     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4675     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4676     pop_zmm(xmm1);
4677   }
4678 }
4679 
4680 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4681   int dst_enc = dst->encoding();
4682   int nds_enc = nds->encoding();
4683   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4684     Assembler::vpsubw(dst, nds, src, vector_len);
4685   } else if (dst_enc < 16) {
4686     Assembler::vpsubw(dst, dst, src, vector_len);
4687   } else if (nds_enc < 16) {
4688     // implies dst_enc in upper bank with src as scratch
4689     evmovdqul(nds, dst, Assembler::AVX_512bit);
4690     Assembler::vpsubw(nds, nds, src, vector_len);
4691     evmovdqul(dst, nds, Assembler::AVX_512bit);
4692   } else {
4693     // worse case scenario, all regs in upper bank
4694     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4695     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4696     Assembler::vpsubw(xmm0, xmm0, src, vector_len);
4697     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4698     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4699   }
4700 }
4701 
4702 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4703   int dst_enc = dst->encoding();
4704   int nds_enc = nds->encoding();
4705   int shift_enc = shift->encoding();
4706   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4707     Assembler::vpsraw(dst, nds, shift, vector_len);
4708   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4709     Assembler::vpsraw(dst, dst, shift, vector_len);
4710   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4711     // use nds as scratch with shift
4712     evmovdqul(nds, shift, Assembler::AVX_512bit);
4713     Assembler::vpsraw(dst, dst, nds, vector_len);
4714   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4715     // use nds as scratch with dst
4716     evmovdqul(nds, dst, Assembler::AVX_512bit);
4717     Assembler::vpsraw(nds, nds, shift, vector_len);
4718     evmovdqul(dst, nds, Assembler::AVX_512bit);
4719   } else if (dst_enc < 16) {
4720     // use nds to save a copy of xmm0 and hold shift
4721     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4722     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4723     Assembler::vpsraw(dst, dst, xmm0, vector_len);
4724     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4725   } else if (nds_enc < 16) {
4726     // use nds and dst as temps
4727     evmovdqul(nds, dst, Assembler::AVX_512bit);
4728     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4729     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4730     Assembler::vpsraw(nds, nds, xmm0, vector_len);
4731     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4732     evmovdqul(dst, nds, Assembler::AVX_512bit);
4733   } else {
4734     // worse case scenario, all regs are in the upper bank
4735     push_zmm(xmm1);
4736     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4737     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4738     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4739     Assembler::vpsraw(xmm0, xmm0, xmm1, vector_len);
4740     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4741     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4742     pop_zmm(xmm1);
4743   }
4744 }
4745 
4746 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4747   int dst_enc = dst->encoding();
4748   int nds_enc = nds->encoding();
4749   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4750     Assembler::vpsraw(dst, nds, shift, vector_len);
4751   } else if (dst_enc < 16) {
4752     Assembler::vpsraw(dst, dst, shift, vector_len);
4753   } else if (nds_enc < 16) {
4754     // use nds as scratch
4755     evmovdqul(nds, dst, Assembler::AVX_512bit);
4756     Assembler::vpsraw(nds, nds, shift, vector_len);
4757     evmovdqul(dst, nds, Assembler::AVX_512bit);
4758   } else {
4759     // use nds as scratch for xmm0
4760     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4761     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4762     Assembler::vpsraw(xmm0, xmm0, shift, vector_len);
4763     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4764     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4765   }
4766 }
4767 
4768 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4769   int dst_enc = dst->encoding();
4770   int nds_enc = nds->encoding();
4771   int shift_enc = shift->encoding();
4772   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4773     Assembler::vpsrlw(dst, nds, shift, vector_len);
4774   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4775     Assembler::vpsrlw(dst, dst, shift, vector_len);
4776   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4777     // use nds as scratch with shift
4778     evmovdqul(nds, shift, Assembler::AVX_512bit);
4779     Assembler::vpsrlw(dst, dst, nds, vector_len);
4780   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4781     // use nds as scratch with dst
4782     evmovdqul(nds, dst, Assembler::AVX_512bit);
4783     Assembler::vpsrlw(nds, nds, shift, vector_len);
4784     evmovdqul(dst, nds, Assembler::AVX_512bit);
4785   } else if (dst_enc < 16) {
4786     // use nds to save a copy of xmm0 and hold shift
4787     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4788     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4789     Assembler::vpsrlw(dst, dst, xmm0, vector_len);
4790     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4791   } else if (nds_enc < 16) {
4792     // use nds and dst as temps
4793     evmovdqul(nds, dst, Assembler::AVX_512bit);
4794     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4795     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4796     Assembler::vpsrlw(nds, nds, xmm0, vector_len);
4797     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4798     evmovdqul(dst, nds, Assembler::AVX_512bit);
4799   } else {
4800     // worse case scenario, all regs are in the upper bank
4801     push_zmm(xmm1);
4802     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4803     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4804     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4805     Assembler::vpsrlw(xmm0, xmm0, xmm1, vector_len);
4806     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4807     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4808     pop_zmm(xmm1);
4809   }
4810 }
4811 
4812 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4813   int dst_enc = dst->encoding();
4814   int nds_enc = nds->encoding();
4815   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4816     Assembler::vpsrlw(dst, nds, shift, vector_len);
4817   } else if (dst_enc < 16) {
4818     Assembler::vpsrlw(dst, dst, shift, vector_len);
4819   } else if (nds_enc < 16) {
4820     // use nds as scratch
4821     evmovdqul(nds, dst, Assembler::AVX_512bit);
4822     Assembler::vpsrlw(nds, nds, shift, vector_len);
4823     evmovdqul(dst, nds, Assembler::AVX_512bit);
4824   } else {
4825     // use nds as scratch for xmm0
4826     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4827     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4828     Assembler::vpsrlw(xmm0, xmm0, shift, vector_len);
4829     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4830     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4831   }
4832 }
4833 
4834 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4835   int dst_enc = dst->encoding();
4836   int nds_enc = nds->encoding();
4837   int shift_enc = shift->encoding();
4838   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4839     Assembler::vpsllw(dst, nds, shift, vector_len);
4840   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4841     Assembler::vpsllw(dst, dst, shift, vector_len);
4842   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4843     // use nds as scratch with shift
4844     evmovdqul(nds, shift, Assembler::AVX_512bit);
4845     Assembler::vpsllw(dst, dst, nds, vector_len);
4846   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4847     // use nds as scratch with dst
4848     evmovdqul(nds, dst, Assembler::AVX_512bit);
4849     Assembler::vpsllw(nds, nds, shift, vector_len);
4850     evmovdqul(dst, nds, Assembler::AVX_512bit);
4851   } else if (dst_enc < 16) {
4852     // use nds to save a copy of xmm0 and hold shift
4853     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4854     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4855     Assembler::vpsllw(dst, dst, xmm0, vector_len);
4856     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4857   } else if (nds_enc < 16) {
4858     // use nds and dst as temps
4859     evmovdqul(nds, dst, Assembler::AVX_512bit);
4860     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4861     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4862     Assembler::vpsllw(nds, nds, xmm0, vector_len);
4863     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4864     evmovdqul(dst, nds, Assembler::AVX_512bit);
4865   } else {
4866     // worse case scenario, all regs are in the upper bank
4867     push_zmm(xmm1);
4868     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4869     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4870     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4871     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4872     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4873     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4874     pop_zmm(xmm1);
4875   }
4876 }
4877 
4878 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4879   int dst_enc = dst->encoding();
4880   int nds_enc = nds->encoding();
4881   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4882     Assembler::vpsllw(dst, nds, shift, vector_len);
4883   } else if (dst_enc < 16) {
4884     Assembler::vpsllw(dst, dst, shift, vector_len);
4885   } else if (nds_enc < 16) {
4886     // use nds as scratch
4887     evmovdqul(nds, dst, Assembler::AVX_512bit);
4888     Assembler::vpsllw(nds, nds, shift, vector_len);
4889     evmovdqul(dst, nds, Assembler::AVX_512bit);
4890   } else {
4891     // use nds as scratch for xmm0
4892     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4893     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4894     Assembler::vpsllw(xmm0, xmm0, shift, vector_len);
4895     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4896     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4897   }
4898 }
4899 
4900 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
4901   int dst_enc = dst->encoding();
4902   int src_enc = src->encoding();
4903   if ((dst_enc < 16) && (src_enc < 16)) {
4904     Assembler::vptest(dst, src);
4905   } else if (src_enc < 16) {
4906     push_zmm(xmm0);
4907     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4908     Assembler::vptest(xmm0, src);
4909     pop_zmm(xmm0);
4910   } else if (dst_enc < 16) {
4911     push_zmm(xmm0);
4912     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4913     Assembler::vptest(dst, xmm0);
4914     pop_zmm(xmm0);
4915   } else {
4916     push_zmm(xmm0);
4917     push_zmm(xmm1);
4918     movdqu(xmm0, src);
4919     movdqu(xmm1, dst);
4920     Assembler::vptest(xmm1, xmm0);
4921     pop_zmm(xmm1);
4922     pop_zmm(xmm0);
4923   }
4924 }
4925 
4926 // This instruction exists within macros, ergo we cannot control its input
4927 // when emitted through those patterns.
4928 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4929   if (VM_Version::supports_avx512nobw()) {
4930     int dst_enc = dst->encoding();
4931     int src_enc = src->encoding();
4932     if (dst_enc == src_enc) {
4933       if (dst_enc < 16) {
4934         Assembler::punpcklbw(dst, src);
4935       } else {
4936         push_zmm(xmm0);
4937         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4938         Assembler::punpcklbw(xmm0, xmm0);
4939         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4940         pop_zmm(xmm0);
4941       }
4942     } else {
4943       if ((src_enc < 16) && (dst_enc < 16)) {
4944         Assembler::punpcklbw(dst, src);
4945       } else if (src_enc < 16) {
4946         push_zmm(xmm0);
4947         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4948         Assembler::punpcklbw(xmm0, src);
4949         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4950         pop_zmm(xmm0);
4951       } else if (dst_enc < 16) {
4952         push_zmm(xmm0);
4953         evmovdqul(xmm0, src, Assembler::AVX_512bit);
4954         Assembler::punpcklbw(dst, xmm0);
4955         pop_zmm(xmm0);
4956       } else {
4957         push_zmm(xmm0);
4958         push_zmm(xmm1);
4959         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4960         evmovdqul(xmm1, src, Assembler::AVX_512bit);
4961         Assembler::punpcklbw(xmm0, xmm1);
4962         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4963         pop_zmm(xmm1);
4964         pop_zmm(xmm0);
4965       }
4966     }
4967   } else {
4968     Assembler::punpcklbw(dst, src);
4969   }
4970 }
4971 
4972 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
4973   if (VM_Version::supports_avx512vl()) {
4974     Assembler::pshufd(dst, src, mode);
4975   } else {
4976     int dst_enc = dst->encoding();
4977     if (dst_enc < 16) {
4978       Assembler::pshufd(dst, src, mode);
4979     } else {
4980       push_zmm(xmm0);
4981       Assembler::pshufd(xmm0, src, mode);
4982       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4983       pop_zmm(xmm0);
4984     }
4985   }
4986 }
4987 
4988 // This instruction exists within macros, ergo we cannot control its input
4989 // when emitted through those patterns.
4990 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
4991   if (VM_Version::supports_avx512nobw()) {
4992     int dst_enc = dst->encoding();
4993     int src_enc = src->encoding();
4994     if (dst_enc == src_enc) {
4995       if (dst_enc < 16) {
4996         Assembler::pshuflw(dst, src, mode);
4997       } else {
4998         push_zmm(xmm0);
4999         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5000         Assembler::pshuflw(xmm0, xmm0, mode);
5001         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5002         pop_zmm(xmm0);
5003       }
5004     } else {
5005       if ((src_enc < 16) && (dst_enc < 16)) {
5006         Assembler::pshuflw(dst, src, mode);
5007       } else if (src_enc < 16) {
5008         push_zmm(xmm0);
5009         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5010         Assembler::pshuflw(xmm0, src, mode);
5011         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5012         pop_zmm(xmm0);
5013       } else if (dst_enc < 16) {
5014         push_zmm(xmm0);
5015         evmovdqul(xmm0, src, Assembler::AVX_512bit);
5016         Assembler::pshuflw(dst, xmm0, mode);
5017         pop_zmm(xmm0);
5018       } else {
5019         push_zmm(xmm0);
5020         push_zmm(xmm1);
5021         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5022         evmovdqul(xmm1, src, Assembler::AVX_512bit);
5023         Assembler::pshuflw(xmm0, xmm1, mode);
5024         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5025         pop_zmm(xmm1);
5026         pop_zmm(xmm0);
5027       }
5028     }
5029   } else {
5030     Assembler::pshuflw(dst, src, mode);
5031   }
5032 }
5033 
5034 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5035   if (reachable(src)) {
5036     vandpd(dst, nds, as_Address(src), vector_len);
5037   } else {
5038     lea(rscratch1, src);
5039     vandpd(dst, nds, Address(rscratch1, 0), vector_len);
5040   }
5041 }
5042 
5043 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5044   if (reachable(src)) {
5045     vandps(dst, nds, as_Address(src), vector_len);
5046   } else {
5047     lea(rscratch1, src);
5048     vandps(dst, nds, Address(rscratch1, 0), vector_len);
5049   }
5050 }
5051 
5052 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5053   if (reachable(src)) {
5054     vdivsd(dst, nds, as_Address(src));
5055   } else {
5056     lea(rscratch1, src);
5057     vdivsd(dst, nds, Address(rscratch1, 0));
5058   }
5059 }
5060 
5061 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5062   if (reachable(src)) {
5063     vdivss(dst, nds, as_Address(src));
5064   } else {
5065     lea(rscratch1, src);
5066     vdivss(dst, nds, Address(rscratch1, 0));
5067   }
5068 }
5069 
5070 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5071   if (reachable(src)) {
5072     vmulsd(dst, nds, as_Address(src));
5073   } else {
5074     lea(rscratch1, src);
5075     vmulsd(dst, nds, Address(rscratch1, 0));
5076   }
5077 }
5078 
5079 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5080   if (reachable(src)) {
5081     vmulss(dst, nds, as_Address(src));
5082   } else {
5083     lea(rscratch1, src);
5084     vmulss(dst, nds, Address(rscratch1, 0));
5085   }
5086 }
5087 
5088 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5089   if (reachable(src)) {
5090     vsubsd(dst, nds, as_Address(src));
5091   } else {
5092     lea(rscratch1, src);
5093     vsubsd(dst, nds, Address(rscratch1, 0));
5094   }
5095 }
5096 
5097 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5098   if (reachable(src)) {
5099     vsubss(dst, nds, as_Address(src));
5100   } else {
5101     lea(rscratch1, src);
5102     vsubss(dst, nds, Address(rscratch1, 0));
5103   }
5104 }
5105 
5106 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5107   int nds_enc = nds->encoding();
5108   int dst_enc = dst->encoding();
5109   bool dst_upper_bank = (dst_enc > 15);
5110   bool nds_upper_bank = (nds_enc > 15);
5111   if (VM_Version::supports_avx512novl() &&
5112       (nds_upper_bank || dst_upper_bank)) {
5113     if (dst_upper_bank) {
5114       push_zmm(xmm0);
5115       movflt(xmm0, nds);
5116       vxorps(xmm0, xmm0, src, Assembler::AVX_128bit);
5117       movflt(dst, xmm0);
5118       pop_zmm(xmm0);
5119     } else {
5120       movflt(dst, nds);
5121       vxorps(dst, dst, src, Assembler::AVX_128bit);
5122     }
5123   } else {
5124     vxorps(dst, nds, src, Assembler::AVX_128bit);
5125   }
5126 }
5127 
5128 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5129   int nds_enc = nds->encoding();
5130   int dst_enc = dst->encoding();
5131   bool dst_upper_bank = (dst_enc > 15);
5132   bool nds_upper_bank = (nds_enc > 15);
5133   if (VM_Version::supports_avx512novl() &&
5134       (nds_upper_bank || dst_upper_bank)) {
5135     if (dst_upper_bank) {
5136       push_zmm(xmm0);
5137       movdbl(xmm0, nds);
5138       vxorpd(xmm0, xmm0, src, Assembler::AVX_128bit);
5139       movdbl(dst, xmm0);
5140       pop_zmm(xmm0);
5141     } else {
5142       movdbl(dst, nds);
5143       vxorpd(dst, dst, src, Assembler::AVX_128bit);
5144     }
5145   } else {
5146     vxorpd(dst, nds, src, Assembler::AVX_128bit);
5147   }
5148 }
5149 
5150 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5151   if (reachable(src)) {
5152     vxorpd(dst, nds, as_Address(src), vector_len);
5153   } else {
5154     lea(rscratch1, src);
5155     vxorpd(dst, nds, Address(rscratch1, 0), vector_len);
5156   }
5157 }
5158 
5159 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5160   if (reachable(src)) {
5161     vxorps(dst, nds, as_Address(src), vector_len);
5162   } else {
5163     lea(rscratch1, src);
5164     vxorps(dst, nds, Address(rscratch1, 0), vector_len);
5165   }
5166 }
5167 
5168 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) {
5169   const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask);
5170   STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code
5171   // The inverted mask is sign-extended
5172   andptr(possibly_jweak, inverted_jweak_mask);
5173 }
5174 
5175 void MacroAssembler::resolve_jobject(Register value,
5176                                      Register thread,
5177                                      Register tmp) {
5178   assert_different_registers(value, thread, tmp);
5179   Label done, not_weak;
5180   testptr(value, value);
5181   jcc(Assembler::zero, done);                // Use NULL as-is.
5182   testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag.
5183   jcc(Assembler::zero, not_weak);
5184   // Resolve jweak.
5185   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
5186                  value, Address(value, -JNIHandles::weak_tag_value), tmp, thread);
5187   verify_oop(value);
5188   jmp(done);
5189   bind(not_weak);
5190   // Resolve (untagged) jobject.
5191   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, 0), tmp, thread);
5192   verify_oop(value);
5193   bind(done);
5194 }
5195 
5196 void MacroAssembler::subptr(Register dst, int32_t imm32) {
5197   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
5198 }
5199 
5200 // Force generation of a 4 byte immediate value even if it fits into 8bit
5201 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
5202   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
5203 }
5204 
5205 void MacroAssembler::subptr(Register dst, Register src) {
5206   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
5207 }
5208 
5209 // C++ bool manipulation
5210 void MacroAssembler::testbool(Register dst) {
5211   if(sizeof(bool) == 1)
5212     testb(dst, 0xff);
5213   else if(sizeof(bool) == 2) {
5214     // testw implementation needed for two byte bools
5215     ShouldNotReachHere();
5216   } else if(sizeof(bool) == 4)
5217     testl(dst, dst);
5218   else
5219     // unsupported
5220     ShouldNotReachHere();
5221 }
5222 
5223 void MacroAssembler::testptr(Register dst, Register src) {
5224   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
5225 }
5226 
5227 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
5228 void MacroAssembler::tlab_allocate(Register thread, Register obj,
5229                                    Register var_size_in_bytes,
5230                                    int con_size_in_bytes,
5231                                    Register t1,
5232                                    Register t2,
5233                                    Label& slow_case) {
5234   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5235   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
5236 }
5237 
5238 // Defines obj, preserves var_size_in_bytes
5239 void MacroAssembler::eden_allocate(Register thread, Register obj,
5240                                    Register var_size_in_bytes,
5241                                    int con_size_in_bytes,
5242                                    Register t1,
5243                                    Label& slow_case) {
5244   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5245   bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
5246 }
5247 
5248 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
5249 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
5250   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
5251   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
5252   Label done;
5253 
5254   testptr(length_in_bytes, length_in_bytes);
5255   jcc(Assembler::zero, done);
5256 
5257   // initialize topmost word, divide index by 2, check if odd and test if zero
5258   // note: for the remaining code to work, index must be a multiple of BytesPerWord
5259 #ifdef ASSERT
5260   {
5261     Label L;
5262     testptr(length_in_bytes, BytesPerWord - 1);
5263     jcc(Assembler::zero, L);
5264     stop("length must be a multiple of BytesPerWord");
5265     bind(L);
5266   }
5267 #endif
5268   Register index = length_in_bytes;
5269   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
5270   if (UseIncDec) {
5271     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
5272   } else {
5273     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
5274     shrptr(index, 1);
5275   }
5276 #ifndef _LP64
5277   // index could have not been a multiple of 8 (i.e., bit 2 was set)
5278   {
5279     Label even;
5280     // note: if index was a multiple of 8, then it cannot
5281     //       be 0 now otherwise it must have been 0 before
5282     //       => if it is even, we don't need to check for 0 again
5283     jcc(Assembler::carryClear, even);
5284     // clear topmost word (no jump would be needed if conditional assignment worked here)
5285     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
5286     // index could be 0 now, must check again
5287     jcc(Assembler::zero, done);
5288     bind(even);
5289   }
5290 #endif // !_LP64
5291   // initialize remaining object fields: index is a multiple of 2 now
5292   {
5293     Label loop;
5294     bind(loop);
5295     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
5296     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
5297     decrement(index);
5298     jcc(Assembler::notZero, loop);
5299   }
5300 
5301   bind(done);
5302 }
5303 
5304 // Look up the method for a megamorphic invokeinterface call.
5305 // The target method is determined by <intf_klass, itable_index>.
5306 // The receiver klass is in recv_klass.
5307 // On success, the result will be in method_result, and execution falls through.
5308 // On failure, execution transfers to the given label.
5309 void MacroAssembler::lookup_interface_method(Register recv_klass,
5310                                              Register intf_klass,
5311                                              RegisterOrConstant itable_index,
5312                                              Register method_result,
5313                                              Register scan_temp,
5314                                              Label& L_no_such_interface,
5315                                              bool return_method) {
5316   assert_different_registers(recv_klass, intf_klass, scan_temp);
5317   assert_different_registers(method_result, intf_klass, scan_temp);
5318   assert(recv_klass != method_result || !return_method,
5319          "recv_klass can be destroyed when method isn't needed");
5320 
5321   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
5322          "caller must use same register for non-constant itable index as for method");
5323 
5324   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
5325   int vtable_base = in_bytes(Klass::vtable_start_offset());
5326   int itentry_off = itableMethodEntry::method_offset_in_bytes();
5327   int scan_step   = itableOffsetEntry::size() * wordSize;
5328   int vte_size    = vtableEntry::size_in_bytes();
5329   Address::ScaleFactor times_vte_scale = Address::times_ptr;
5330   assert(vte_size == wordSize, "else adjust times_vte_scale");
5331 
5332   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
5333 
5334   // %%% Could store the aligned, prescaled offset in the klassoop.
5335   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
5336 
5337   if (return_method) {
5338     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
5339     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
5340     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
5341   }
5342 
5343   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
5344   //   if (scan->interface() == intf) {
5345   //     result = (klass + scan->offset() + itable_index);
5346   //   }
5347   // }
5348   Label search, found_method;
5349 
5350   for (int peel = 1; peel >= 0; peel--) {
5351     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
5352     cmpptr(intf_klass, method_result);
5353 
5354     if (peel) {
5355       jccb(Assembler::equal, found_method);
5356     } else {
5357       jccb(Assembler::notEqual, search);
5358       // (invert the test to fall through to found_method...)
5359     }
5360 
5361     if (!peel)  break;
5362 
5363     bind(search);
5364 
5365     // Check that the previous entry is non-null.  A null entry means that
5366     // the receiver class doesn't implement the interface, and wasn't the
5367     // same as when the caller was compiled.
5368     testptr(method_result, method_result);
5369     jcc(Assembler::zero, L_no_such_interface);
5370     addptr(scan_temp, scan_step);
5371   }
5372 
5373   bind(found_method);
5374 
5375   if (return_method) {
5376     // Got a hit.
5377     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
5378     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
5379   }
5380 }
5381 
5382 
5383 // virtual method calling
5384 void MacroAssembler::lookup_virtual_method(Register recv_klass,
5385                                            RegisterOrConstant vtable_index,
5386                                            Register method_result) {
5387   const int base = in_bytes(Klass::vtable_start_offset());
5388   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
5389   Address vtable_entry_addr(recv_klass,
5390                             vtable_index, Address::times_ptr,
5391                             base + vtableEntry::method_offset_in_bytes());
5392   movptr(method_result, vtable_entry_addr);
5393 }
5394 
5395 
5396 void MacroAssembler::check_klass_subtype(Register sub_klass,
5397                            Register super_klass,
5398                            Register temp_reg,
5399                            Label& L_success) {
5400   Label L_failure;
5401   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, NULL);
5402   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
5403   bind(L_failure);
5404 }
5405 
5406 
5407 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
5408                                                    Register super_klass,
5409                                                    Register temp_reg,
5410                                                    Label* L_success,
5411                                                    Label* L_failure,
5412                                                    Label* L_slow_path,
5413                                         RegisterOrConstant super_check_offset) {
5414   assert_different_registers(sub_klass, super_klass, temp_reg);
5415   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
5416   if (super_check_offset.is_register()) {
5417     assert_different_registers(sub_klass, super_klass,
5418                                super_check_offset.as_register());
5419   } else if (must_load_sco) {
5420     assert(temp_reg != noreg, "supply either a temp or a register offset");
5421   }
5422 
5423   Label L_fallthrough;
5424   int label_nulls = 0;
5425   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
5426   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
5427   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
5428   assert(label_nulls <= 1, "at most one NULL in the batch");
5429 
5430   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
5431   int sco_offset = in_bytes(Klass::super_check_offset_offset());
5432   Address super_check_offset_addr(super_klass, sco_offset);
5433 
5434   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
5435   // range of a jccb.  If this routine grows larger, reconsider at
5436   // least some of these.
5437 #define local_jcc(assembler_cond, label)                                \
5438   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
5439   else                             jcc( assembler_cond, label) /*omit semi*/
5440 
5441   // Hacked jmp, which may only be used just before L_fallthrough.
5442 #define final_jmp(label)                                                \
5443   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
5444   else                            jmp(label)                /*omit semi*/
5445 
5446   // If the pointers are equal, we are done (e.g., String[] elements).
5447   // This self-check enables sharing of secondary supertype arrays among
5448   // non-primary types such as array-of-interface.  Otherwise, each such
5449   // type would need its own customized SSA.
5450   // We move this check to the front of the fast path because many
5451   // type checks are in fact trivially successful in this manner,
5452   // so we get a nicely predicted branch right at the start of the check.
5453   cmpptr(sub_klass, super_klass);
5454   local_jcc(Assembler::equal, *L_success);
5455 
5456   // Check the supertype display:
5457   if (must_load_sco) {
5458     // Positive movl does right thing on LP64.
5459     movl(temp_reg, super_check_offset_addr);
5460     super_check_offset = RegisterOrConstant(temp_reg);
5461   }
5462   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
5463   cmpptr(super_klass, super_check_addr); // load displayed supertype
5464 
5465   // This check has worked decisively for primary supers.
5466   // Secondary supers are sought in the super_cache ('super_cache_addr').
5467   // (Secondary supers are interfaces and very deeply nested subtypes.)
5468   // This works in the same check above because of a tricky aliasing
5469   // between the super_cache and the primary super display elements.
5470   // (The 'super_check_addr' can address either, as the case requires.)
5471   // Note that the cache is updated below if it does not help us find
5472   // what we need immediately.
5473   // So if it was a primary super, we can just fail immediately.
5474   // Otherwise, it's the slow path for us (no success at this point).
5475 
5476   if (super_check_offset.is_register()) {
5477     local_jcc(Assembler::equal, *L_success);
5478     cmpl(super_check_offset.as_register(), sc_offset);
5479     if (L_failure == &L_fallthrough) {
5480       local_jcc(Assembler::equal, *L_slow_path);
5481     } else {
5482       local_jcc(Assembler::notEqual, *L_failure);
5483       final_jmp(*L_slow_path);
5484     }
5485   } else if (super_check_offset.as_constant() == sc_offset) {
5486     // Need a slow path; fast failure is impossible.
5487     if (L_slow_path == &L_fallthrough) {
5488       local_jcc(Assembler::equal, *L_success);
5489     } else {
5490       local_jcc(Assembler::notEqual, *L_slow_path);
5491       final_jmp(*L_success);
5492     }
5493   } else {
5494     // No slow path; it's a fast decision.
5495     if (L_failure == &L_fallthrough) {
5496       local_jcc(Assembler::equal, *L_success);
5497     } else {
5498       local_jcc(Assembler::notEqual, *L_failure);
5499       final_jmp(*L_success);
5500     }
5501   }
5502 
5503   bind(L_fallthrough);
5504 
5505 #undef local_jcc
5506 #undef final_jmp
5507 }
5508 
5509 
5510 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
5511                                                    Register super_klass,
5512                                                    Register temp_reg,
5513                                                    Register temp2_reg,
5514                                                    Label* L_success,
5515                                                    Label* L_failure,
5516                                                    bool set_cond_codes) {
5517   assert_different_registers(sub_klass, super_klass, temp_reg);
5518   if (temp2_reg != noreg)
5519     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
5520 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
5521 
5522   Label L_fallthrough;
5523   int label_nulls = 0;
5524   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
5525   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
5526   assert(label_nulls <= 1, "at most one NULL in the batch");
5527 
5528   // a couple of useful fields in sub_klass:
5529   int ss_offset = in_bytes(Klass::secondary_supers_offset());
5530   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
5531   Address secondary_supers_addr(sub_klass, ss_offset);
5532   Address super_cache_addr(     sub_klass, sc_offset);
5533 
5534   // Do a linear scan of the secondary super-klass chain.
5535   // This code is rarely used, so simplicity is a virtue here.
5536   // The repne_scan instruction uses fixed registers, which we must spill.
5537   // Don't worry too much about pre-existing connections with the input regs.
5538 
5539   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
5540   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
5541 
5542   // Get super_klass value into rax (even if it was in rdi or rcx).
5543   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
5544   if (super_klass != rax || UseCompressedOops) {
5545     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
5546     mov(rax, super_klass);
5547   }
5548   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
5549   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
5550 
5551 #ifndef PRODUCT
5552   int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
5553   ExternalAddress pst_counter_addr((address) pst_counter);
5554   NOT_LP64(  incrementl(pst_counter_addr) );
5555   LP64_ONLY( lea(rcx, pst_counter_addr) );
5556   LP64_ONLY( incrementl(Address(rcx, 0)) );
5557 #endif //PRODUCT
5558 
5559   // We will consult the secondary-super array.
5560   movptr(rdi, secondary_supers_addr);
5561   // Load the array length.  (Positive movl does right thing on LP64.)
5562   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
5563   // Skip to start of data.
5564   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
5565 
5566   // Scan RCX words at [RDI] for an occurrence of RAX.
5567   // Set NZ/Z based on last compare.
5568   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
5569   // not change flags (only scas instruction which is repeated sets flags).
5570   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
5571 
5572     testptr(rax,rax); // Set Z = 0
5573     repne_scan();
5574 
5575   // Unspill the temp. registers:
5576   if (pushed_rdi)  pop(rdi);
5577   if (pushed_rcx)  pop(rcx);
5578   if (pushed_rax)  pop(rax);
5579 
5580   if (set_cond_codes) {
5581     // Special hack for the AD files:  rdi is guaranteed non-zero.
5582     assert(!pushed_rdi, "rdi must be left non-NULL");
5583     // Also, the condition codes are properly set Z/NZ on succeed/failure.
5584   }
5585 
5586   if (L_failure == &L_fallthrough)
5587         jccb(Assembler::notEqual, *L_failure);
5588   else  jcc(Assembler::notEqual, *L_failure);
5589 
5590   // Success.  Cache the super we found and proceed in triumph.
5591   movptr(super_cache_addr, super_klass);
5592 
5593   if (L_success != &L_fallthrough) {
5594     jmp(*L_success);
5595   }
5596 
5597 #undef IS_A_TEMP
5598 
5599   bind(L_fallthrough);
5600 }
5601 
5602 
5603 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
5604   if (VM_Version::supports_cmov()) {
5605     cmovl(cc, dst, src);
5606   } else {
5607     Label L;
5608     jccb(negate_condition(cc), L);
5609     movl(dst, src);
5610     bind(L);
5611   }
5612 }
5613 
5614 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
5615   if (VM_Version::supports_cmov()) {
5616     cmovl(cc, dst, src);
5617   } else {
5618     Label L;
5619     jccb(negate_condition(cc), L);
5620     movl(dst, src);
5621     bind(L);
5622   }
5623 }
5624 
5625 void MacroAssembler::verify_oop(Register reg, const char* s) {
5626   if (!VerifyOops || VerifyAdapterSharing) {
5627     // Below address of the code string confuses VerifyAdapterSharing
5628     // because it may differ between otherwise equivalent adapters.
5629     return;
5630   }
5631 
5632   // Pass register number to verify_oop_subroutine
5633   const char* b = NULL;
5634   {
5635     ResourceMark rm;
5636     stringStream ss;
5637     ss.print("verify_oop: %s: %s", reg->name(), s);
5638     b = code_string(ss.as_string());
5639   }
5640   BLOCK_COMMENT("verify_oop {");
5641 #ifdef _LP64
5642   push(rscratch1);                    // save r10, trashed by movptr()
5643 #endif
5644   push(rax);                          // save rax,
5645   push(reg);                          // pass register argument
5646   ExternalAddress buffer((address) b);
5647   // avoid using pushptr, as it modifies scratch registers
5648   // and our contract is not to modify anything
5649   movptr(rax, buffer.addr());
5650   push(rax);
5651   // call indirectly to solve generation ordering problem
5652   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
5653   call(rax);
5654   // Caller pops the arguments (oop, message) and restores rax, r10
5655   BLOCK_COMMENT("} verify_oop");
5656 }
5657 
5658 
5659 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
5660                                                       Register tmp,
5661                                                       int offset) {
5662   intptr_t value = *delayed_value_addr;
5663   if (value != 0)
5664     return RegisterOrConstant(value + offset);
5665 
5666   // load indirectly to solve generation ordering problem
5667   movptr(tmp, ExternalAddress((address) delayed_value_addr));
5668 
5669 #ifdef ASSERT
5670   { Label L;
5671     testptr(tmp, tmp);
5672     if (WizardMode) {
5673       const char* buf = NULL;
5674       {
5675         ResourceMark rm;
5676         stringStream ss;
5677         ss.print("DelayedValue=" INTPTR_FORMAT, delayed_value_addr[1]);
5678         buf = code_string(ss.as_string());
5679       }
5680       jcc(Assembler::notZero, L);
5681       STOP(buf);
5682     } else {
5683       jccb(Assembler::notZero, L);
5684       hlt();
5685     }
5686     bind(L);
5687   }
5688 #endif
5689 
5690   if (offset != 0)
5691     addptr(tmp, offset);
5692 
5693   return RegisterOrConstant(tmp);
5694 }
5695 
5696 
5697 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
5698                                          int extra_slot_offset) {
5699   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
5700   int stackElementSize = Interpreter::stackElementSize;
5701   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
5702 #ifdef ASSERT
5703   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
5704   assert(offset1 - offset == stackElementSize, "correct arithmetic");
5705 #endif
5706   Register             scale_reg    = noreg;
5707   Address::ScaleFactor scale_factor = Address::no_scale;
5708   if (arg_slot.is_constant()) {
5709     offset += arg_slot.as_constant() * stackElementSize;
5710   } else {
5711     scale_reg    = arg_slot.as_register();
5712     scale_factor = Address::times(stackElementSize);
5713   }
5714   offset += wordSize;           // return PC is on stack
5715   return Address(rsp, scale_reg, scale_factor, offset);
5716 }
5717 
5718 
5719 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
5720   if (!VerifyOops || VerifyAdapterSharing) {
5721     // Below address of the code string confuses VerifyAdapterSharing
5722     // because it may differ between otherwise equivalent adapters.
5723     return;
5724   }
5725 
5726   // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
5727   // Pass register number to verify_oop_subroutine
5728   const char* b = NULL;
5729   {
5730     ResourceMark rm;
5731     stringStream ss;
5732     ss.print("verify_oop_addr: %s", s);
5733     b = code_string(ss.as_string());
5734   }
5735 #ifdef _LP64
5736   push(rscratch1);                    // save r10, trashed by movptr()
5737 #endif
5738   push(rax);                          // save rax,
5739   // addr may contain rsp so we will have to adjust it based on the push
5740   // we just did (and on 64 bit we do two pushes)
5741   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
5742   // stores rax into addr which is backwards of what was intended.
5743   if (addr.uses(rsp)) {
5744     lea(rax, addr);
5745     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
5746   } else {
5747     pushptr(addr);
5748   }
5749 
5750   ExternalAddress buffer((address) b);
5751   // pass msg argument
5752   // avoid using pushptr, as it modifies scratch registers
5753   // and our contract is not to modify anything
5754   movptr(rax, buffer.addr());
5755   push(rax);
5756 
5757   // call indirectly to solve generation ordering problem
5758   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
5759   call(rax);
5760   // Caller pops the arguments (addr, message) and restores rax, r10.
5761 }
5762 
5763 void MacroAssembler::verify_tlab() {
5764 #ifdef ASSERT
5765   if (UseTLAB && VerifyOops) {
5766     Label next, ok;
5767     Register t1 = rsi;
5768     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
5769 
5770     push(t1);
5771     NOT_LP64(push(thread_reg));
5772     NOT_LP64(get_thread(thread_reg));
5773 
5774     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5775     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
5776     jcc(Assembler::aboveEqual, next);
5777     STOP("assert(top >= start)");
5778     should_not_reach_here();
5779 
5780     bind(next);
5781     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
5782     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5783     jcc(Assembler::aboveEqual, ok);
5784     STOP("assert(top <= end)");
5785     should_not_reach_here();
5786 
5787     bind(ok);
5788     NOT_LP64(pop(thread_reg));
5789     pop(t1);
5790   }
5791 #endif
5792 }
5793 
5794 class ControlWord {
5795  public:
5796   int32_t _value;
5797 
5798   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
5799   int  precision_control() const       { return  (_value >>  8) & 3      ; }
5800   bool precision() const               { return ((_value >>  5) & 1) != 0; }
5801   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
5802   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
5803   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
5804   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
5805   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
5806 
5807   void print() const {
5808     // rounding control
5809     const char* rc;
5810     switch (rounding_control()) {
5811       case 0: rc = "round near"; break;
5812       case 1: rc = "round down"; break;
5813       case 2: rc = "round up  "; break;
5814       case 3: rc = "chop      "; break;
5815     };
5816     // precision control
5817     const char* pc;
5818     switch (precision_control()) {
5819       case 0: pc = "24 bits "; break;
5820       case 1: pc = "reserved"; break;
5821       case 2: pc = "53 bits "; break;
5822       case 3: pc = "64 bits "; break;
5823     };
5824     // flags
5825     char f[9];
5826     f[0] = ' ';
5827     f[1] = ' ';
5828     f[2] = (precision   ()) ? 'P' : 'p';
5829     f[3] = (underflow   ()) ? 'U' : 'u';
5830     f[4] = (overflow    ()) ? 'O' : 'o';
5831     f[5] = (zero_divide ()) ? 'Z' : 'z';
5832     f[6] = (denormalized()) ? 'D' : 'd';
5833     f[7] = (invalid     ()) ? 'I' : 'i';
5834     f[8] = '\x0';
5835     // output
5836     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
5837   }
5838 
5839 };
5840 
5841 class StatusWord {
5842  public:
5843   int32_t _value;
5844 
5845   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
5846   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
5847   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
5848   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
5849   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
5850   int  top() const                     { return  (_value >> 11) & 7      ; }
5851   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
5852   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
5853   bool precision() const               { return ((_value >>  5) & 1) != 0; }
5854   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
5855   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
5856   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
5857   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
5858   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
5859 
5860   void print() const {
5861     // condition codes
5862     char c[5];
5863     c[0] = (C3()) ? '3' : '-';
5864     c[1] = (C2()) ? '2' : '-';
5865     c[2] = (C1()) ? '1' : '-';
5866     c[3] = (C0()) ? '0' : '-';
5867     c[4] = '\x0';
5868     // flags
5869     char f[9];
5870     f[0] = (error_status()) ? 'E' : '-';
5871     f[1] = (stack_fault ()) ? 'S' : '-';
5872     f[2] = (precision   ()) ? 'P' : '-';
5873     f[3] = (underflow   ()) ? 'U' : '-';
5874     f[4] = (overflow    ()) ? 'O' : '-';
5875     f[5] = (zero_divide ()) ? 'Z' : '-';
5876     f[6] = (denormalized()) ? 'D' : '-';
5877     f[7] = (invalid     ()) ? 'I' : '-';
5878     f[8] = '\x0';
5879     // output
5880     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
5881   }
5882 
5883 };
5884 
5885 class TagWord {
5886  public:
5887   int32_t _value;
5888 
5889   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
5890 
5891   void print() const {
5892     printf("%04x", _value & 0xFFFF);
5893   }
5894 
5895 };
5896 
5897 class FPU_Register {
5898  public:
5899   int32_t _m0;
5900   int32_t _m1;
5901   int16_t _ex;
5902 
5903   bool is_indefinite() const           {
5904     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
5905   }
5906 
5907   void print() const {
5908     char  sign = (_ex < 0) ? '-' : '+';
5909     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
5910     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
5911   };
5912 
5913 };
5914 
5915 class FPU_State {
5916  public:
5917   enum {
5918     register_size       = 10,
5919     number_of_registers =  8,
5920     register_mask       =  7
5921   };
5922 
5923   ControlWord  _control_word;
5924   StatusWord   _status_word;
5925   TagWord      _tag_word;
5926   int32_t      _error_offset;
5927   int32_t      _error_selector;
5928   int32_t      _data_offset;
5929   int32_t      _data_selector;
5930   int8_t       _register[register_size * number_of_registers];
5931 
5932   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
5933   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
5934 
5935   const char* tag_as_string(int tag) const {
5936     switch (tag) {
5937       case 0: return "valid";
5938       case 1: return "zero";
5939       case 2: return "special";
5940       case 3: return "empty";
5941     }
5942     ShouldNotReachHere();
5943     return NULL;
5944   }
5945 
5946   void print() const {
5947     // print computation registers
5948     { int t = _status_word.top();
5949       for (int i = 0; i < number_of_registers; i++) {
5950         int j = (i - t) & register_mask;
5951         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
5952         st(j)->print();
5953         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
5954       }
5955     }
5956     printf("\n");
5957     // print control registers
5958     printf("ctrl = "); _control_word.print(); printf("\n");
5959     printf("stat = "); _status_word .print(); printf("\n");
5960     printf("tags = "); _tag_word    .print(); printf("\n");
5961   }
5962 
5963 };
5964 
5965 class Flag_Register {
5966  public:
5967   int32_t _value;
5968 
5969   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
5970   bool direction() const               { return ((_value >> 10) & 1) != 0; }
5971   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
5972   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
5973   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
5974   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
5975   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
5976 
5977   void print() const {
5978     // flags
5979     char f[8];
5980     f[0] = (overflow       ()) ? 'O' : '-';
5981     f[1] = (direction      ()) ? 'D' : '-';
5982     f[2] = (sign           ()) ? 'S' : '-';
5983     f[3] = (zero           ()) ? 'Z' : '-';
5984     f[4] = (auxiliary_carry()) ? 'A' : '-';
5985     f[5] = (parity         ()) ? 'P' : '-';
5986     f[6] = (carry          ()) ? 'C' : '-';
5987     f[7] = '\x0';
5988     // output
5989     printf("%08x  flags = %s", _value, f);
5990   }
5991 
5992 };
5993 
5994 class IU_Register {
5995  public:
5996   int32_t _value;
5997 
5998   void print() const {
5999     printf("%08x  %11d", _value, _value);
6000   }
6001 
6002 };
6003 
6004 class IU_State {
6005  public:
6006   Flag_Register _eflags;
6007   IU_Register   _rdi;
6008   IU_Register   _rsi;
6009   IU_Register   _rbp;
6010   IU_Register   _rsp;
6011   IU_Register   _rbx;
6012   IU_Register   _rdx;
6013   IU_Register   _rcx;
6014   IU_Register   _rax;
6015 
6016   void print() const {
6017     // computation registers
6018     printf("rax,  = "); _rax.print(); printf("\n");
6019     printf("rbx,  = "); _rbx.print(); printf("\n");
6020     printf("rcx  = "); _rcx.print(); printf("\n");
6021     printf("rdx  = "); _rdx.print(); printf("\n");
6022     printf("rdi  = "); _rdi.print(); printf("\n");
6023     printf("rsi  = "); _rsi.print(); printf("\n");
6024     printf("rbp,  = "); _rbp.print(); printf("\n");
6025     printf("rsp  = "); _rsp.print(); printf("\n");
6026     printf("\n");
6027     // control registers
6028     printf("flgs = "); _eflags.print(); printf("\n");
6029   }
6030 };
6031 
6032 
6033 class CPU_State {
6034  public:
6035   FPU_State _fpu_state;
6036   IU_State  _iu_state;
6037 
6038   void print() const {
6039     printf("--------------------------------------------------\n");
6040     _iu_state .print();
6041     printf("\n");
6042     _fpu_state.print();
6043     printf("--------------------------------------------------\n");
6044   }
6045 
6046 };
6047 
6048 
6049 static void _print_CPU_state(CPU_State* state) {
6050   state->print();
6051 };
6052 
6053 
6054 void MacroAssembler::print_CPU_state() {
6055   push_CPU_state();
6056   push(rsp);                // pass CPU state
6057   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
6058   addptr(rsp, wordSize);       // discard argument
6059   pop_CPU_state();
6060 }
6061 
6062 
6063 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
6064   static int counter = 0;
6065   FPU_State* fs = &state->_fpu_state;
6066   counter++;
6067   // For leaf calls, only verify that the top few elements remain empty.
6068   // We only need 1 empty at the top for C2 code.
6069   if( stack_depth < 0 ) {
6070     if( fs->tag_for_st(7) != 3 ) {
6071       printf("FPR7 not empty\n");
6072       state->print();
6073       assert(false, "error");
6074       return false;
6075     }
6076     return true;                // All other stack states do not matter
6077   }
6078 
6079   assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std,
6080          "bad FPU control word");
6081 
6082   // compute stack depth
6083   int i = 0;
6084   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
6085   int d = i;
6086   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
6087   // verify findings
6088   if (i != FPU_State::number_of_registers) {
6089     // stack not contiguous
6090     printf("%s: stack not contiguous at ST%d\n", s, i);
6091     state->print();
6092     assert(false, "error");
6093     return false;
6094   }
6095   // check if computed stack depth corresponds to expected stack depth
6096   if (stack_depth < 0) {
6097     // expected stack depth is -stack_depth or less
6098     if (d > -stack_depth) {
6099       // too many elements on the stack
6100       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
6101       state->print();
6102       assert(false, "error");
6103       return false;
6104     }
6105   } else {
6106     // expected stack depth is stack_depth
6107     if (d != stack_depth) {
6108       // wrong stack depth
6109       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
6110       state->print();
6111       assert(false, "error");
6112       return false;
6113     }
6114   }
6115   // everything is cool
6116   return true;
6117 }
6118 
6119 
6120 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
6121   if (!VerifyFPU) return;
6122   push_CPU_state();
6123   push(rsp);                // pass CPU state
6124   ExternalAddress msg((address) s);
6125   // pass message string s
6126   pushptr(msg.addr());
6127   push(stack_depth);        // pass stack depth
6128   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
6129   addptr(rsp, 3 * wordSize);   // discard arguments
6130   // check for error
6131   { Label L;
6132     testl(rax, rax);
6133     jcc(Assembler::notZero, L);
6134     int3();                  // break if error condition
6135     bind(L);
6136   }
6137   pop_CPU_state();
6138 }
6139 
6140 void MacroAssembler::restore_cpu_control_state_after_jni() {
6141   // Either restore the MXCSR register after returning from the JNI Call
6142   // or verify that it wasn't changed (with -Xcheck:jni flag).
6143   if (VM_Version::supports_sse()) {
6144     if (RestoreMXCSROnJNICalls) {
6145       ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std()));
6146     } else if (CheckJNICalls) {
6147       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
6148     }
6149   }
6150   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
6151   vzeroupper();
6152   // Reset k1 to 0xffff.
6153   if (VM_Version::supports_evex()) {
6154     push(rcx);
6155     movl(rcx, 0xffff);
6156     kmovwl(k1, rcx);
6157     pop(rcx);
6158   }
6159 
6160 #ifndef _LP64
6161   // Either restore the x87 floating pointer control word after returning
6162   // from the JNI call or verify that it wasn't changed.
6163   if (CheckJNICalls) {
6164     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
6165   }
6166 #endif // _LP64
6167 }
6168 
6169 // ((OopHandle)result).resolve();
6170 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
6171   assert_different_registers(result, tmp);
6172 
6173   // Only 64 bit platforms support GCs that require a tmp register
6174   // Only IN_HEAP loads require a thread_tmp register
6175   // OopHandle::resolve is an indirection like jobject.
6176   access_load_at(T_OBJECT, IN_NATIVE,
6177                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
6178 }
6179 
6180 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
6181   // get mirror
6182   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
6183   movptr(mirror, Address(method, Method::const_offset()));
6184   movptr(mirror, Address(mirror, ConstMethod::constants_offset()));
6185   movptr(mirror, Address(mirror, ConstantPool::pool_holder_offset_in_bytes()));
6186   movptr(mirror, Address(mirror, mirror_offset));
6187   resolve_oop_handle(mirror, tmp);
6188 }
6189 
6190 void MacroAssembler::load_klass(Register dst, Register src) {
6191 #ifdef _LP64
6192   if (UseCompressedClassPointers) {
6193     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
6194     decode_klass_not_null(dst);
6195   } else
6196 #endif
6197     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
6198 }
6199 
6200 void MacroAssembler::load_prototype_header(Register dst, Register src) {
6201   load_klass(dst, src);
6202   movptr(dst, Address(dst, Klass::prototype_header_offset()));
6203 }
6204 
6205 void MacroAssembler::store_klass(Register dst, Register src) {
6206 #ifdef _LP64
6207   if (UseCompressedClassPointers) {
6208     encode_klass_not_null(src);
6209     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
6210   } else
6211 #endif
6212     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
6213 }
6214 
6215 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
6216                                     Register tmp1, Register thread_tmp) {
6217   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
6218   decorators = AccessInternal::decorator_fixup(decorators);
6219   bool as_raw = (decorators & AS_RAW) != 0;
6220   if (as_raw) {
6221     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
6222   } else {
6223     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
6224   }
6225 }
6226 
6227 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
6228                                      Register tmp1, Register tmp2) {
6229   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
6230   decorators = AccessInternal::decorator_fixup(decorators);
6231   bool as_raw = (decorators & AS_RAW) != 0;
6232   if (as_raw) {
6233     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2);
6234   } else {
6235     bs->store_at(this, decorators, type, dst, src, tmp1, tmp2);
6236   }
6237 }
6238 
6239 void MacroAssembler::resolve(DecoratorSet decorators, Register obj) {
6240   // Use stronger ACCESS_WRITE|ACCESS_READ by default.
6241   if ((decorators & (ACCESS_READ | ACCESS_WRITE)) == 0) {
6242     decorators |= ACCESS_READ | ACCESS_WRITE;
6243   }
6244   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
6245   return bs->resolve(this, decorators, obj);
6246 }
6247 
6248 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
6249                                    Register thread_tmp, DecoratorSet decorators) {
6250   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
6251 }
6252 
6253 // Doesn't do verfication, generates fixed size code
6254 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
6255                                             Register thread_tmp, DecoratorSet decorators) {
6256   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
6257 }
6258 
6259 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
6260                                     Register tmp2, DecoratorSet decorators) {
6261   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
6262 }
6263 
6264 // Used for storing NULLs.
6265 void MacroAssembler::store_heap_oop_null(Address dst) {
6266   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg);
6267 }
6268 
6269 #ifdef _LP64
6270 void MacroAssembler::store_klass_gap(Register dst, Register src) {
6271   if (UseCompressedClassPointers) {
6272     // Store to klass gap in destination
6273     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
6274   }
6275 }
6276 
6277 #ifdef ASSERT
6278 void MacroAssembler::verify_heapbase(const char* msg) {
6279   assert (UseCompressedOops, "should be compressed");
6280   assert (Universe::heap() != NULL, "java heap should be initialized");
6281   if (CheckCompressedOops) {
6282     Label ok;
6283     push(rscratch1); // cmpptr trashes rscratch1
6284     cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6285     jcc(Assembler::equal, ok);
6286     STOP(msg);
6287     bind(ok);
6288     pop(rscratch1);
6289   }
6290 }
6291 #endif
6292 
6293 // Algorithm must match oop.inline.hpp encode_heap_oop.
6294 void MacroAssembler::encode_heap_oop(Register r) {
6295 #ifdef ASSERT
6296   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
6297 #endif
6298   verify_oop(r, "broken oop in encode_heap_oop");
6299   if (Universe::narrow_oop_base() == NULL) {
6300     if (Universe::narrow_oop_shift() != 0) {
6301       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6302       shrq(r, LogMinObjAlignmentInBytes);
6303     }
6304     return;
6305   }
6306   testq(r, r);
6307   cmovq(Assembler::equal, r, r12_heapbase);
6308   subq(r, r12_heapbase);
6309   shrq(r, LogMinObjAlignmentInBytes);
6310 }
6311 
6312 void MacroAssembler::encode_heap_oop_not_null(Register r) {
6313 #ifdef ASSERT
6314   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
6315   if (CheckCompressedOops) {
6316     Label ok;
6317     testq(r, r);
6318     jcc(Assembler::notEqual, ok);
6319     STOP("null oop passed to encode_heap_oop_not_null");
6320     bind(ok);
6321   }
6322 #endif
6323   verify_oop(r, "broken oop in encode_heap_oop_not_null");
6324   if (Universe::narrow_oop_base() != NULL) {
6325     subq(r, r12_heapbase);
6326   }
6327   if (Universe::narrow_oop_shift() != 0) {
6328     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6329     shrq(r, LogMinObjAlignmentInBytes);
6330   }
6331 }
6332 
6333 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
6334 #ifdef ASSERT
6335   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
6336   if (CheckCompressedOops) {
6337     Label ok;
6338     testq(src, src);
6339     jcc(Assembler::notEqual, ok);
6340     STOP("null oop passed to encode_heap_oop_not_null2");
6341     bind(ok);
6342   }
6343 #endif
6344   verify_oop(src, "broken oop in encode_heap_oop_not_null2");
6345   if (dst != src) {
6346     movq(dst, src);
6347   }
6348   if (Universe::narrow_oop_base() != NULL) {
6349     subq(dst, r12_heapbase);
6350   }
6351   if (Universe::narrow_oop_shift() != 0) {
6352     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6353     shrq(dst, LogMinObjAlignmentInBytes);
6354   }
6355 }
6356 
6357 void  MacroAssembler::decode_heap_oop(Register r) {
6358 #ifdef ASSERT
6359   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
6360 #endif
6361   if (Universe::narrow_oop_base() == NULL) {
6362     if (Universe::narrow_oop_shift() != 0) {
6363       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6364       shlq(r, LogMinObjAlignmentInBytes);
6365     }
6366   } else {
6367     Label done;
6368     shlq(r, LogMinObjAlignmentInBytes);
6369     jccb(Assembler::equal, done);
6370     addq(r, r12_heapbase);
6371     bind(done);
6372   }
6373   verify_oop(r, "broken oop in decode_heap_oop");
6374 }
6375 
6376 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
6377   // Note: it will change flags
6378   assert (UseCompressedOops, "should only be used for compressed headers");
6379   assert (Universe::heap() != NULL, "java heap should be initialized");
6380   // Cannot assert, unverified entry point counts instructions (see .ad file)
6381   // vtableStubs also counts instructions in pd_code_size_limit.
6382   // Also do not verify_oop as this is called by verify_oop.
6383   if (Universe::narrow_oop_shift() != 0) {
6384     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6385     shlq(r, LogMinObjAlignmentInBytes);
6386     if (Universe::narrow_oop_base() != NULL) {
6387       addq(r, r12_heapbase);
6388     }
6389   } else {
6390     assert (Universe::narrow_oop_base() == NULL, "sanity");
6391   }
6392 }
6393 
6394 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
6395   // Note: it will change flags
6396   assert (UseCompressedOops, "should only be used for compressed headers");
6397   assert (Universe::heap() != NULL, "java heap should be initialized");
6398   // Cannot assert, unverified entry point counts instructions (see .ad file)
6399   // vtableStubs also counts instructions in pd_code_size_limit.
6400   // Also do not verify_oop as this is called by verify_oop.
6401   if (Universe::narrow_oop_shift() != 0) {
6402     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6403     if (LogMinObjAlignmentInBytes == Address::times_8) {
6404       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
6405     } else {
6406       if (dst != src) {
6407         movq(dst, src);
6408       }
6409       shlq(dst, LogMinObjAlignmentInBytes);
6410       if (Universe::narrow_oop_base() != NULL) {
6411         addq(dst, r12_heapbase);
6412       }
6413     }
6414   } else {
6415     assert (Universe::narrow_oop_base() == NULL, "sanity");
6416     if (dst != src) {
6417       movq(dst, src);
6418     }
6419   }
6420 }
6421 
6422 void MacroAssembler::encode_klass_not_null(Register r) {
6423   if (Universe::narrow_klass_base() != NULL) {
6424     // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6425     assert(r != r12_heapbase, "Encoding a klass in r12");
6426     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6427     subq(r, r12_heapbase);
6428   }
6429   if (Universe::narrow_klass_shift() != 0) {
6430     assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6431     shrq(r, LogKlassAlignmentInBytes);
6432   }
6433   if (Universe::narrow_klass_base() != NULL) {
6434     reinit_heapbase();
6435   }
6436 }
6437 
6438 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
6439   if (dst == src) {
6440     encode_klass_not_null(src);
6441   } else {
6442     if (Universe::narrow_klass_base() != NULL) {
6443       mov64(dst, (int64_t)Universe::narrow_klass_base());
6444       negq(dst);
6445       addq(dst, src);
6446     } else {
6447       movptr(dst, src);
6448     }
6449     if (Universe::narrow_klass_shift() != 0) {
6450       assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6451       shrq(dst, LogKlassAlignmentInBytes);
6452     }
6453   }
6454 }
6455 
6456 // Function instr_size_for_decode_klass_not_null() counts the instructions
6457 // generated by decode_klass_not_null(register r) and reinit_heapbase(),
6458 // when (Universe::heap() != NULL).  Hence, if the instructions they
6459 // generate change, then this method needs to be updated.
6460 int MacroAssembler::instr_size_for_decode_klass_not_null() {
6461   assert (UseCompressedClassPointers, "only for compressed klass ptrs");
6462   if (Universe::narrow_klass_base() != NULL) {
6463     // mov64 + addq + shlq? + mov64  (for reinit_heapbase()).
6464     return (Universe::narrow_klass_shift() == 0 ? 20 : 24);
6465   } else {
6466     // longest load decode klass function, mov64, leaq
6467     return 16;
6468   }
6469 }
6470 
6471 // !!! If the instructions that get generated here change then function
6472 // instr_size_for_decode_klass_not_null() needs to get updated.
6473 void  MacroAssembler::decode_klass_not_null(Register r) {
6474   // Note: it will change flags
6475   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6476   assert(r != r12_heapbase, "Decoding a klass in r12");
6477   // Cannot assert, unverified entry point counts instructions (see .ad file)
6478   // vtableStubs also counts instructions in pd_code_size_limit.
6479   // Also do not verify_oop as this is called by verify_oop.
6480   if (Universe::narrow_klass_shift() != 0) {
6481     assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6482     shlq(r, LogKlassAlignmentInBytes);
6483   }
6484   // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6485   if (Universe::narrow_klass_base() != NULL) {
6486     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6487     addq(r, r12_heapbase);
6488     reinit_heapbase();
6489   }
6490 }
6491 
6492 void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
6493   // Note: it will change flags
6494   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6495   if (dst == src) {
6496     decode_klass_not_null(dst);
6497   } else {
6498     // Cannot assert, unverified entry point counts instructions (see .ad file)
6499     // vtableStubs also counts instructions in pd_code_size_limit.
6500     // Also do not verify_oop as this is called by verify_oop.
6501     mov64(dst, (int64_t)Universe::narrow_klass_base());
6502     if (Universe::narrow_klass_shift() != 0) {
6503       assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6504       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
6505       leaq(dst, Address(dst, src, Address::times_8, 0));
6506     } else {
6507       addq(dst, src);
6508     }
6509   }
6510 }
6511 
6512 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
6513   assert (UseCompressedOops, "should only be used for compressed headers");
6514   assert (Universe::heap() != NULL, "java heap should be initialized");
6515   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6516   int oop_index = oop_recorder()->find_index(obj);
6517   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6518   mov_narrow_oop(dst, oop_index, rspec);
6519 }
6520 
6521 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
6522   assert (UseCompressedOops, "should only be used for compressed headers");
6523   assert (Universe::heap() != NULL, "java heap should be initialized");
6524   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6525   int oop_index = oop_recorder()->find_index(obj);
6526   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6527   mov_narrow_oop(dst, oop_index, rspec);
6528 }
6529 
6530 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
6531   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6532   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6533   int klass_index = oop_recorder()->find_index(k);
6534   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6535   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6536 }
6537 
6538 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
6539   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6540   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6541   int klass_index = oop_recorder()->find_index(k);
6542   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6543   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6544 }
6545 
6546 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
6547   assert (UseCompressedOops, "should only be used for compressed headers");
6548   assert (Universe::heap() != NULL, "java heap should be initialized");
6549   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6550   int oop_index = oop_recorder()->find_index(obj);
6551   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6552   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6553 }
6554 
6555 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
6556   assert (UseCompressedOops, "should only be used for compressed headers");
6557   assert (Universe::heap() != NULL, "java heap should be initialized");
6558   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6559   int oop_index = oop_recorder()->find_index(obj);
6560   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6561   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6562 }
6563 
6564 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
6565   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6566   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6567   int klass_index = oop_recorder()->find_index(k);
6568   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6569   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6570 }
6571 
6572 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
6573   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6574   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6575   int klass_index = oop_recorder()->find_index(k);
6576   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6577   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6578 }
6579 
6580 void MacroAssembler::reinit_heapbase() {
6581   if (UseCompressedOops || UseCompressedClassPointers) {
6582     if (Universe::heap() != NULL) {
6583       if (Universe::narrow_oop_base() == NULL) {
6584         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
6585       } else {
6586         mov64(r12_heapbase, (int64_t)Universe::narrow_ptrs_base());
6587       }
6588     } else {
6589       movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6590     }
6591   }
6592 }
6593 
6594 #endif // _LP64
6595 
6596 // C2 compiled method's prolog code.
6597 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b) {
6598 
6599   // WARNING: Initial instruction MUST be 5 bytes or longer so that
6600   // NativeJump::patch_verified_entry will be able to patch out the entry
6601   // code safely. The push to verify stack depth is ok at 5 bytes,
6602   // the frame allocation can be either 3 or 6 bytes. So if we don't do
6603   // stack bang then we must use the 6 byte frame allocation even if
6604   // we have no frame. :-(
6605   assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
6606 
6607   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
6608   // Remove word for return addr
6609   framesize -= wordSize;
6610   stack_bang_size -= wordSize;
6611 
6612   // Calls to C2R adapters often do not accept exceptional returns.
6613   // We require that their callers must bang for them.  But be careful, because
6614   // some VM calls (such as call site linkage) can use several kilobytes of
6615   // stack.  But the stack safety zone should account for that.
6616   // See bugs 4446381, 4468289, 4497237.
6617   if (stack_bang_size > 0) {
6618     generate_stack_overflow_check(stack_bang_size);
6619 
6620     // We always push rbp, so that on return to interpreter rbp, will be
6621     // restored correctly and we can correct the stack.
6622     push(rbp);
6623     // Save caller's stack pointer into RBP if the frame pointer is preserved.
6624     if (PreserveFramePointer) {
6625       mov(rbp, rsp);
6626     }
6627     // Remove word for ebp
6628     framesize -= wordSize;
6629 
6630     // Create frame
6631     if (framesize) {
6632       subptr(rsp, framesize);
6633     }
6634   } else {
6635     // Create frame (force generation of a 4 byte immediate value)
6636     subptr_imm32(rsp, framesize);
6637 
6638     // Save RBP register now.
6639     framesize -= wordSize;
6640     movptr(Address(rsp, framesize), rbp);
6641     // Save caller's stack pointer into RBP if the frame pointer is preserved.
6642     if (PreserveFramePointer) {
6643       movptr(rbp, rsp);
6644       if (framesize > 0) {
6645         addptr(rbp, framesize);
6646       }
6647     }
6648   }
6649 
6650   if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
6651     framesize -= wordSize;
6652     movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
6653   }
6654 
6655 #ifndef _LP64
6656   // If method sets FPU control word do it now
6657   if (fp_mode_24b) {
6658     fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
6659   }
6660   if (UseSSE >= 2 && VerifyFPU) {
6661     verify_FPU(0, "FPU stack must be clean on entry");
6662   }
6663 #endif
6664 
6665 #ifdef ASSERT
6666   if (VerifyStackAtCalls) {
6667     Label L;
6668     push(rax);
6669     mov(rax, rsp);
6670     andptr(rax, StackAlignmentInBytes-1);
6671     cmpptr(rax, StackAlignmentInBytes-wordSize);
6672     pop(rax);
6673     jcc(Assembler::equal, L);
6674     STOP("Stack is not properly aligned!");
6675     bind(L);
6676   }
6677 #endif
6678 
6679 }
6680 
6681 // Add null checks for all value type arguments
6682 void MacroAssembler::null_check_value_args(Method* method) {
6683   // Get registers/stack slots for arguments
6684   assert(method->has_value_args(), "must have value type args");
6685   Symbol* sig_ext = method->adapter()->get_sig_extended();
6686   assert(sig_ext != NULL, "must have extended signature");
6687   BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, 256);
6688   VMRegPair* regs = NEW_RESOURCE_ARRAY(VMRegPair, 256);
6689   int num = 0;
6690   for (SignatureStream ss(sig_ext); !ss.at_return_type(); ss.next()) {
6691     BasicType bt = ss.type();
6692     sig_bt[num++] = bt;
6693     if (type2size[bt] == 2) {
6694       sig_bt[num++] = T_VOID;
6695     }
6696   }
6697   SharedRuntime::java_calling_convention(sig_bt, regs, num, false);
6698 
6699   // Jump to c2i adapter if a value type argument is null
6700   bool has_receiver = !method->is_static();
6701   RuntimeAddress interpreter_entry = RuntimeAddress(method->get_c2i_entry());
6702   num = 0;
6703   for (SignatureStream ss(sig_ext); !ss.at_return_type(); num += type2size[ss.type()], ss.next()) {
6704     if ((has_receiver && num == 0) || ss.type() != T_VALUETYPEPTR) {
6705       continue; // Skip receiver and non value type args
6706     }
6707     VMReg r = regs[num].first();
6708     if (r->is_reg()) {
6709       testptr(r->as_Register(), r->as_Register());
6710     } else {
6711       if (!r->is_stack()) {
6712         r->print();
6713       }
6714       int st_off = r->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
6715       cmpptr(Address(rsp, st_off), NULL_WORD);
6716     }
6717     jump_cc(Assembler::zero, interpreter_entry);
6718   }
6719 }
6720 
6721 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM registers
6722 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp) {
6723   // cnt - number of qwords (8-byte words).
6724   // base - start address, qword aligned.
6725   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
6726   movdq(xtmp, val);
6727   if (UseAVX >= 2) {
6728     punpcklqdq(xtmp, xtmp);
6729     vinserti128_high(xtmp, xtmp);
6730   } else {
6731     punpcklqdq(xtmp, xtmp);
6732   }
6733   jmp(L_zero_64_bytes);
6734 
6735   BIND(L_loop);
6736   if (UseAVX >= 2) {
6737     vmovdqu(Address(base,  0), xtmp);
6738     vmovdqu(Address(base, 32), xtmp);
6739   } else {
6740     movdqu(Address(base,  0), xtmp);
6741     movdqu(Address(base, 16), xtmp);
6742     movdqu(Address(base, 32), xtmp);
6743     movdqu(Address(base, 48), xtmp);
6744   }
6745   addptr(base, 64);
6746 
6747   BIND(L_zero_64_bytes);
6748   subptr(cnt, 8);
6749   jccb(Assembler::greaterEqual, L_loop);
6750   addptr(cnt, 4);
6751   jccb(Assembler::less, L_tail);
6752   // Copy trailing 32 bytes
6753   if (UseAVX >= 2) {
6754     vmovdqu(Address(base, 0), xtmp);
6755   } else {
6756     movdqu(Address(base,  0), xtmp);
6757     movdqu(Address(base, 16), xtmp);
6758   }
6759   addptr(base, 32);
6760   subptr(cnt, 4);
6761 
6762   BIND(L_tail);
6763   addptr(cnt, 4);
6764   jccb(Assembler::lessEqual, L_end);
6765   decrement(cnt);
6766 
6767   BIND(L_sloop);
6768   movq(Address(base, 0), xtmp);
6769   addptr(base, 8);
6770   decrement(cnt);
6771   jccb(Assembler::greaterEqual, L_sloop);
6772   BIND(L_end);
6773 }
6774 
6775 void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, bool is_large, bool word_copy_only) {
6776   // cnt - number of qwords (8-byte words).
6777   // base - start address, qword aligned.
6778   // is_large - if optimizers know cnt is larger than InitArrayShortSize
6779   assert(base==rdi, "base register must be edi for rep stos");
6780   assert(val==rax,   "tmp register must be eax for rep stos");
6781   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
6782   assert(InitArrayShortSize % BytesPerLong == 0,
6783     "InitArrayShortSize should be the multiple of BytesPerLong");
6784 
6785   Label DONE;
6786 
6787   if (!is_large) {
6788     Label LOOP, LONG;
6789     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
6790     jccb(Assembler::greater, LONG);
6791 
6792     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
6793 
6794     decrement(cnt);
6795     jccb(Assembler::negative, DONE); // Zero length
6796 
6797     // Use individual pointer-sized stores for small counts:
6798     BIND(LOOP);
6799     movptr(Address(base, cnt, Address::times_ptr), val);
6800     decrement(cnt);
6801     jccb(Assembler::greaterEqual, LOOP);
6802     jmpb(DONE);
6803 
6804     BIND(LONG);
6805   }
6806 
6807   // Use longer rep-prefixed ops for non-small counts:
6808   if (UseFastStosb && !word_copy_only) {
6809     shlptr(cnt, 3); // convert to number of bytes
6810     rep_stosb();
6811   } else if (UseXMMForObjInit) {
6812     xmm_clear_mem(base, cnt, val, xtmp);
6813   } else {
6814     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
6815     rep_stos();
6816   }
6817 
6818   BIND(DONE);
6819 }
6820 
6821 #ifdef COMPILER2
6822 
6823 // IndexOf for constant substrings with size >= 8 chars
6824 // which don't need to be loaded through stack.
6825 void MacroAssembler::string_indexofC8(Register str1, Register str2,
6826                                       Register cnt1, Register cnt2,
6827                                       int int_cnt2,  Register result,
6828                                       XMMRegister vec, Register tmp,
6829                                       int ae) {
6830   ShortBranchVerifier sbv(this);
6831   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
6832   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
6833 
6834   // This method uses the pcmpestri instruction with bound registers
6835   //   inputs:
6836   //     xmm - substring
6837   //     rax - substring length (elements count)
6838   //     mem - scanned string
6839   //     rdx - string length (elements count)
6840   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
6841   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
6842   //   outputs:
6843   //     rcx - matched index in string
6844   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
6845   int mode   = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
6846   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
6847   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
6848   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
6849 
6850   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR,
6851         RET_FOUND, RET_NOT_FOUND, EXIT, FOUND_SUBSTR,
6852         MATCH_SUBSTR_HEAD, RELOAD_STR, FOUND_CANDIDATE;
6853 
6854   // Note, inline_string_indexOf() generates checks:
6855   // if (substr.count > string.count) return -1;
6856   // if (substr.count == 0) return 0;
6857   assert(int_cnt2 >= stride, "this code is used only for cnt2 >= 8 chars");
6858 
6859   // Load substring.
6860   if (ae == StrIntrinsicNode::UL) {
6861     pmovzxbw(vec, Address(str2, 0));
6862   } else {
6863     movdqu(vec, Address(str2, 0));
6864   }
6865   movl(cnt2, int_cnt2);
6866   movptr(result, str1); // string addr
6867 
6868   if (int_cnt2 > stride) {
6869     jmpb(SCAN_TO_SUBSTR);
6870 
6871     // Reload substr for rescan, this code
6872     // is executed only for large substrings (> 8 chars)
6873     bind(RELOAD_SUBSTR);
6874     if (ae == StrIntrinsicNode::UL) {
6875       pmovzxbw(vec, Address(str2, 0));
6876     } else {
6877       movdqu(vec, Address(str2, 0));
6878     }
6879     negptr(cnt2); // Jumped here with negative cnt2, convert to positive
6880 
6881     bind(RELOAD_STR);
6882     // We came here after the beginning of the substring was
6883     // matched but the rest of it was not so we need to search
6884     // again. Start from the next element after the previous match.
6885 
6886     // cnt2 is number of substring reminding elements and
6887     // cnt1 is number of string reminding elements when cmp failed.
6888     // Restored cnt1 = cnt1 - cnt2 + int_cnt2
6889     subl(cnt1, cnt2);
6890     addl(cnt1, int_cnt2);
6891     movl(cnt2, int_cnt2); // Now restore cnt2
6892 
6893     decrementl(cnt1);     // Shift to next element
6894     cmpl(cnt1, cnt2);
6895     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6896 
6897     addptr(result, (1<<scale1));
6898 
6899   } // (int_cnt2 > 8)
6900 
6901   // Scan string for start of substr in 16-byte vectors
6902   bind(SCAN_TO_SUBSTR);
6903   pcmpestri(vec, Address(result, 0), mode);
6904   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
6905   subl(cnt1, stride);
6906   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
6907   cmpl(cnt1, cnt2);
6908   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6909   addptr(result, 16);
6910   jmpb(SCAN_TO_SUBSTR);
6911 
6912   // Found a potential substr
6913   bind(FOUND_CANDIDATE);
6914   // Matched whole vector if first element matched (tmp(rcx) == 0).
6915   if (int_cnt2 == stride) {
6916     jccb(Assembler::overflow, RET_FOUND);    // OF == 1
6917   } else { // int_cnt2 > 8
6918     jccb(Assembler::overflow, FOUND_SUBSTR);
6919   }
6920   // After pcmpestri tmp(rcx) contains matched element index
6921   // Compute start addr of substr
6922   lea(result, Address(result, tmp, scale1));
6923 
6924   // Make sure string is still long enough
6925   subl(cnt1, tmp);
6926   cmpl(cnt1, cnt2);
6927   if (int_cnt2 == stride) {
6928     jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
6929   } else { // int_cnt2 > 8
6930     jccb(Assembler::greaterEqual, MATCH_SUBSTR_HEAD);
6931   }
6932   // Left less then substring.
6933 
6934   bind(RET_NOT_FOUND);
6935   movl(result, -1);
6936   jmp(EXIT);
6937 
6938   if (int_cnt2 > stride) {
6939     // This code is optimized for the case when whole substring
6940     // is matched if its head is matched.
6941     bind(MATCH_SUBSTR_HEAD);
6942     pcmpestri(vec, Address(result, 0), mode);
6943     // Reload only string if does not match
6944     jcc(Assembler::noOverflow, RELOAD_STR); // OF == 0
6945 
6946     Label CONT_SCAN_SUBSTR;
6947     // Compare the rest of substring (> 8 chars).
6948     bind(FOUND_SUBSTR);
6949     // First 8 chars are already matched.
6950     negptr(cnt2);
6951     addptr(cnt2, stride);
6952 
6953     bind(SCAN_SUBSTR);
6954     subl(cnt1, stride);
6955     cmpl(cnt2, -stride); // Do not read beyond substring
6956     jccb(Assembler::lessEqual, CONT_SCAN_SUBSTR);
6957     // Back-up strings to avoid reading beyond substring:
6958     // cnt1 = cnt1 - cnt2 + 8
6959     addl(cnt1, cnt2); // cnt2 is negative
6960     addl(cnt1, stride);
6961     movl(cnt2, stride); negptr(cnt2);
6962     bind(CONT_SCAN_SUBSTR);
6963     if (int_cnt2 < (int)G) {
6964       int tail_off1 = int_cnt2<<scale1;
6965       int tail_off2 = int_cnt2<<scale2;
6966       if (ae == StrIntrinsicNode::UL) {
6967         pmovzxbw(vec, Address(str2, cnt2, scale2, tail_off2));
6968       } else {
6969         movdqu(vec, Address(str2, cnt2, scale2, tail_off2));
6970       }
6971       pcmpestri(vec, Address(result, cnt2, scale1, tail_off1), mode);
6972     } else {
6973       // calculate index in register to avoid integer overflow (int_cnt2*2)
6974       movl(tmp, int_cnt2);
6975       addptr(tmp, cnt2);
6976       if (ae == StrIntrinsicNode::UL) {
6977         pmovzxbw(vec, Address(str2, tmp, scale2, 0));
6978       } else {
6979         movdqu(vec, Address(str2, tmp, scale2, 0));
6980       }
6981       pcmpestri(vec, Address(result, tmp, scale1, 0), mode);
6982     }
6983     // Need to reload strings pointers if not matched whole vector
6984     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
6985     addptr(cnt2, stride);
6986     jcc(Assembler::negative, SCAN_SUBSTR);
6987     // Fall through if found full substring
6988 
6989   } // (int_cnt2 > 8)
6990 
6991   bind(RET_FOUND);
6992   // Found result if we matched full small substring.
6993   // Compute substr offset
6994   subptr(result, str1);
6995   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
6996     shrl(result, 1); // index
6997   }
6998   bind(EXIT);
6999 
7000 } // string_indexofC8
7001 
7002 // Small strings are loaded through stack if they cross page boundary.
7003 void MacroAssembler::string_indexof(Register str1, Register str2,
7004                                     Register cnt1, Register cnt2,
7005                                     int int_cnt2,  Register result,
7006                                     XMMRegister vec, Register tmp,
7007                                     int ae) {
7008   ShortBranchVerifier sbv(this);
7009   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7010   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
7011 
7012   //
7013   // int_cnt2 is length of small (< 8 chars) constant substring
7014   // or (-1) for non constant substring in which case its length
7015   // is in cnt2 register.
7016   //
7017   // Note, inline_string_indexOf() generates checks:
7018   // if (substr.count > string.count) return -1;
7019   // if (substr.count == 0) return 0;
7020   //
7021   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
7022   assert(int_cnt2 == -1 || (0 < int_cnt2 && int_cnt2 < stride), "should be != 0");
7023   // This method uses the pcmpestri instruction with bound registers
7024   //   inputs:
7025   //     xmm - substring
7026   //     rax - substring length (elements count)
7027   //     mem - scanned string
7028   //     rdx - string length (elements count)
7029   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
7030   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
7031   //   outputs:
7032   //     rcx - matched index in string
7033   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
7034   int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
7035   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
7036   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
7037 
7038   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, ADJUST_STR,
7039         RET_FOUND, RET_NOT_FOUND, CLEANUP, FOUND_SUBSTR,
7040         FOUND_CANDIDATE;
7041 
7042   { //========================================================
7043     // We don't know where these strings are located
7044     // and we can't read beyond them. Load them through stack.
7045     Label BIG_STRINGS, CHECK_STR, COPY_SUBSTR, COPY_STR;
7046 
7047     movptr(tmp, rsp); // save old SP
7048 
7049     if (int_cnt2 > 0) {     // small (< 8 chars) constant substring
7050       if (int_cnt2 == (1>>scale2)) { // One byte
7051         assert((ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL), "Only possible for latin1 encoding");
7052         load_unsigned_byte(result, Address(str2, 0));
7053         movdl(vec, result); // move 32 bits
7054       } else if (ae == StrIntrinsicNode::LL && int_cnt2 == 3) {  // Three bytes
7055         // Not enough header space in 32-bit VM: 12+3 = 15.
7056         movl(result, Address(str2, -1));
7057         shrl(result, 8);
7058         movdl(vec, result); // move 32 bits
7059       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (2>>scale2)) {  // One char
7060         load_unsigned_short(result, Address(str2, 0));
7061         movdl(vec, result); // move 32 bits
7062       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (4>>scale2)) { // Two chars
7063         movdl(vec, Address(str2, 0)); // move 32 bits
7064       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (8>>scale2)) { // Four chars
7065         movq(vec, Address(str2, 0));  // move 64 bits
7066       } else { // cnt2 = { 3, 5, 6, 7 } || (ae == StrIntrinsicNode::UL && cnt2 ={2, ..., 7})
7067         // Array header size is 12 bytes in 32-bit VM
7068         // + 6 bytes for 3 chars == 18 bytes,
7069         // enough space to load vec and shift.
7070         assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity");
7071         if (ae == StrIntrinsicNode::UL) {
7072           int tail_off = int_cnt2-8;
7073           pmovzxbw(vec, Address(str2, tail_off));
7074           psrldq(vec, -2*tail_off);
7075         }
7076         else {
7077           int tail_off = int_cnt2*(1<<scale2);
7078           movdqu(vec, Address(str2, tail_off-16));
7079           psrldq(vec, 16-tail_off);
7080         }
7081       }
7082     } else { // not constant substring
7083       cmpl(cnt2, stride);
7084       jccb(Assembler::aboveEqual, BIG_STRINGS); // Both strings are big enough
7085 
7086       // We can read beyond string if srt+16 does not cross page boundary
7087       // since heaps are aligned and mapped by pages.
7088       assert(os::vm_page_size() < (int)G, "default page should be small");
7089       movl(result, str2); // We need only low 32 bits
7090       andl(result, (os::vm_page_size()-1));
7091       cmpl(result, (os::vm_page_size()-16));
7092       jccb(Assembler::belowEqual, CHECK_STR);
7093 
7094       // Move small strings to stack to allow load 16 bytes into vec.
7095       subptr(rsp, 16);
7096       int stk_offset = wordSize-(1<<scale2);
7097       push(cnt2);
7098 
7099       bind(COPY_SUBSTR);
7100       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL) {
7101         load_unsigned_byte(result, Address(str2, cnt2, scale2, -1));
7102         movb(Address(rsp, cnt2, scale2, stk_offset), result);
7103       } else if (ae == StrIntrinsicNode::UU) {
7104         load_unsigned_short(result, Address(str2, cnt2, scale2, -2));
7105         movw(Address(rsp, cnt2, scale2, stk_offset), result);
7106       }
7107       decrement(cnt2);
7108       jccb(Assembler::notZero, COPY_SUBSTR);
7109 
7110       pop(cnt2);
7111       movptr(str2, rsp);  // New substring address
7112     } // non constant
7113 
7114     bind(CHECK_STR);
7115     cmpl(cnt1, stride);
7116     jccb(Assembler::aboveEqual, BIG_STRINGS);
7117 
7118     // Check cross page boundary.
7119     movl(result, str1); // We need only low 32 bits
7120     andl(result, (os::vm_page_size()-1));
7121     cmpl(result, (os::vm_page_size()-16));
7122     jccb(Assembler::belowEqual, BIG_STRINGS);
7123 
7124     subptr(rsp, 16);
7125     int stk_offset = -(1<<scale1);
7126     if (int_cnt2 < 0) { // not constant
7127       push(cnt2);
7128       stk_offset += wordSize;
7129     }
7130     movl(cnt2, cnt1);
7131 
7132     bind(COPY_STR);
7133     if (ae == StrIntrinsicNode::LL) {
7134       load_unsigned_byte(result, Address(str1, cnt2, scale1, -1));
7135       movb(Address(rsp, cnt2, scale1, stk_offset), result);
7136     } else {
7137       load_unsigned_short(result, Address(str1, cnt2, scale1, -2));
7138       movw(Address(rsp, cnt2, scale1, stk_offset), result);
7139     }
7140     decrement(cnt2);
7141     jccb(Assembler::notZero, COPY_STR);
7142 
7143     if (int_cnt2 < 0) { // not constant
7144       pop(cnt2);
7145     }
7146     movptr(str1, rsp);  // New string address
7147 
7148     bind(BIG_STRINGS);
7149     // Load substring.
7150     if (int_cnt2 < 0) { // -1
7151       if (ae == StrIntrinsicNode::UL) {
7152         pmovzxbw(vec, Address(str2, 0));
7153       } else {
7154         movdqu(vec, Address(str2, 0));
7155       }
7156       push(cnt2);       // substr count
7157       push(str2);       // substr addr
7158       push(str1);       // string addr
7159     } else {
7160       // Small (< 8 chars) constant substrings are loaded already.
7161       movl(cnt2, int_cnt2);
7162     }
7163     push(tmp);  // original SP
7164 
7165   } // Finished loading
7166 
7167   //========================================================
7168   // Start search
7169   //
7170 
7171   movptr(result, str1); // string addr
7172 
7173   if (int_cnt2  < 0) {  // Only for non constant substring
7174     jmpb(SCAN_TO_SUBSTR);
7175 
7176     // SP saved at sp+0
7177     // String saved at sp+1*wordSize
7178     // Substr saved at sp+2*wordSize
7179     // Substr count saved at sp+3*wordSize
7180 
7181     // Reload substr for rescan, this code
7182     // is executed only for large substrings (> 8 chars)
7183     bind(RELOAD_SUBSTR);
7184     movptr(str2, Address(rsp, 2*wordSize));
7185     movl(cnt2, Address(rsp, 3*wordSize));
7186     if (ae == StrIntrinsicNode::UL) {
7187       pmovzxbw(vec, Address(str2, 0));
7188     } else {
7189       movdqu(vec, Address(str2, 0));
7190     }
7191     // We came here after the beginning of the substring was
7192     // matched but the rest of it was not so we need to search
7193     // again. Start from the next element after the previous match.
7194     subptr(str1, result); // Restore counter
7195     if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7196       shrl(str1, 1);
7197     }
7198     addl(cnt1, str1);
7199     decrementl(cnt1);   // Shift to next element
7200     cmpl(cnt1, cnt2);
7201     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7202 
7203     addptr(result, (1<<scale1));
7204   } // non constant
7205 
7206   // Scan string for start of substr in 16-byte vectors
7207   bind(SCAN_TO_SUBSTR);
7208   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
7209   pcmpestri(vec, Address(result, 0), mode);
7210   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
7211   subl(cnt1, stride);
7212   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
7213   cmpl(cnt1, cnt2);
7214   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7215   addptr(result, 16);
7216 
7217   bind(ADJUST_STR);
7218   cmpl(cnt1, stride); // Do not read beyond string
7219   jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
7220   // Back-up string to avoid reading beyond string.
7221   lea(result, Address(result, cnt1, scale1, -16));
7222   movl(cnt1, stride);
7223   jmpb(SCAN_TO_SUBSTR);
7224 
7225   // Found a potential substr
7226   bind(FOUND_CANDIDATE);
7227   // After pcmpestri tmp(rcx) contains matched element index
7228 
7229   // Make sure string is still long enough
7230   subl(cnt1, tmp);
7231   cmpl(cnt1, cnt2);
7232   jccb(Assembler::greaterEqual, FOUND_SUBSTR);
7233   // Left less then substring.
7234 
7235   bind(RET_NOT_FOUND);
7236   movl(result, -1);
7237   jmp(CLEANUP);
7238 
7239   bind(FOUND_SUBSTR);
7240   // Compute start addr of substr
7241   lea(result, Address(result, tmp, scale1));
7242   if (int_cnt2 > 0) { // Constant substring
7243     // Repeat search for small substring (< 8 chars)
7244     // from new point without reloading substring.
7245     // Have to check that we don't read beyond string.
7246     cmpl(tmp, stride-int_cnt2);
7247     jccb(Assembler::greater, ADJUST_STR);
7248     // Fall through if matched whole substring.
7249   } else { // non constant
7250     assert(int_cnt2 == -1, "should be != 0");
7251 
7252     addl(tmp, cnt2);
7253     // Found result if we matched whole substring.
7254     cmpl(tmp, stride);
7255     jcc(Assembler::lessEqual, RET_FOUND);
7256 
7257     // Repeat search for small substring (<= 8 chars)
7258     // from new point 'str1' without reloading substring.
7259     cmpl(cnt2, stride);
7260     // Have to check that we don't read beyond string.
7261     jccb(Assembler::lessEqual, ADJUST_STR);
7262 
7263     Label CHECK_NEXT, CONT_SCAN_SUBSTR, RET_FOUND_LONG;
7264     // Compare the rest of substring (> 8 chars).
7265     movptr(str1, result);
7266 
7267     cmpl(tmp, cnt2);
7268     // First 8 chars are already matched.
7269     jccb(Assembler::equal, CHECK_NEXT);
7270 
7271     bind(SCAN_SUBSTR);
7272     pcmpestri(vec, Address(str1, 0), mode);
7273     // Need to reload strings pointers if not matched whole vector
7274     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
7275 
7276     bind(CHECK_NEXT);
7277     subl(cnt2, stride);
7278     jccb(Assembler::lessEqual, RET_FOUND_LONG); // Found full substring
7279     addptr(str1, 16);
7280     if (ae == StrIntrinsicNode::UL) {
7281       addptr(str2, 8);
7282     } else {
7283       addptr(str2, 16);
7284     }
7285     subl(cnt1, stride);
7286     cmpl(cnt2, stride); // Do not read beyond substring
7287     jccb(Assembler::greaterEqual, CONT_SCAN_SUBSTR);
7288     // Back-up strings to avoid reading beyond substring.
7289 
7290     if (ae == StrIntrinsicNode::UL) {
7291       lea(str2, Address(str2, cnt2, scale2, -8));
7292       lea(str1, Address(str1, cnt2, scale1, -16));
7293     } else {
7294       lea(str2, Address(str2, cnt2, scale2, -16));
7295       lea(str1, Address(str1, cnt2, scale1, -16));
7296     }
7297     subl(cnt1, cnt2);
7298     movl(cnt2, stride);
7299     addl(cnt1, stride);
7300     bind(CONT_SCAN_SUBSTR);
7301     if (ae == StrIntrinsicNode::UL) {
7302       pmovzxbw(vec, Address(str2, 0));
7303     } else {
7304       movdqu(vec, Address(str2, 0));
7305     }
7306     jmp(SCAN_SUBSTR);
7307 
7308     bind(RET_FOUND_LONG);
7309     movptr(str1, Address(rsp, wordSize));
7310   } // non constant
7311 
7312   bind(RET_FOUND);
7313   // Compute substr offset
7314   subptr(result, str1);
7315   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7316     shrl(result, 1); // index
7317   }
7318   bind(CLEANUP);
7319   pop(rsp); // restore SP
7320 
7321 } // string_indexof
7322 
7323 void MacroAssembler::string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
7324                                          XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp) {
7325   ShortBranchVerifier sbv(this);
7326   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7327 
7328   int stride = 8;
7329 
7330   Label FOUND_CHAR, SCAN_TO_CHAR, SCAN_TO_CHAR_LOOP,
7331         SCAN_TO_8_CHAR, SCAN_TO_8_CHAR_LOOP, SCAN_TO_16_CHAR_LOOP,
7332         RET_NOT_FOUND, SCAN_TO_8_CHAR_INIT,
7333         FOUND_SEQ_CHAR, DONE_LABEL;
7334 
7335   movptr(result, str1);
7336   if (UseAVX >= 2) {
7337     cmpl(cnt1, stride);
7338     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7339     cmpl(cnt1, 2*stride);
7340     jcc(Assembler::less, SCAN_TO_8_CHAR_INIT);
7341     movdl(vec1, ch);
7342     vpbroadcastw(vec1, vec1);
7343     vpxor(vec2, vec2);
7344     movl(tmp, cnt1);
7345     andl(tmp, 0xFFFFFFF0);  //vector count (in chars)
7346     andl(cnt1,0x0000000F);  //tail count (in chars)
7347 
7348     bind(SCAN_TO_16_CHAR_LOOP);
7349     vmovdqu(vec3, Address(result, 0));
7350     vpcmpeqw(vec3, vec3, vec1, 1);
7351     vptest(vec2, vec3);
7352     jcc(Assembler::carryClear, FOUND_CHAR);
7353     addptr(result, 32);
7354     subl(tmp, 2*stride);
7355     jcc(Assembler::notZero, SCAN_TO_16_CHAR_LOOP);
7356     jmp(SCAN_TO_8_CHAR);
7357     bind(SCAN_TO_8_CHAR_INIT);
7358     movdl(vec1, ch);
7359     pshuflw(vec1, vec1, 0x00);
7360     pshufd(vec1, vec1, 0);
7361     pxor(vec2, vec2);
7362   }
7363   bind(SCAN_TO_8_CHAR);
7364   cmpl(cnt1, stride);
7365   if (UseAVX >= 2) {
7366     jcc(Assembler::less, SCAN_TO_CHAR);
7367   } else {
7368     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7369     movdl(vec1, ch);
7370     pshuflw(vec1, vec1, 0x00);
7371     pshufd(vec1, vec1, 0);
7372     pxor(vec2, vec2);
7373   }
7374   movl(tmp, cnt1);
7375   andl(tmp, 0xFFFFFFF8);  //vector count (in chars)
7376   andl(cnt1,0x00000007);  //tail count (in chars)
7377 
7378   bind(SCAN_TO_8_CHAR_LOOP);
7379   movdqu(vec3, Address(result, 0));
7380   pcmpeqw(vec3, vec1);
7381   ptest(vec2, vec3);
7382   jcc(Assembler::carryClear, FOUND_CHAR);
7383   addptr(result, 16);
7384   subl(tmp, stride);
7385   jcc(Assembler::notZero, SCAN_TO_8_CHAR_LOOP);
7386   bind(SCAN_TO_CHAR);
7387   testl(cnt1, cnt1);
7388   jcc(Assembler::zero, RET_NOT_FOUND);
7389   bind(SCAN_TO_CHAR_LOOP);
7390   load_unsigned_short(tmp, Address(result, 0));
7391   cmpl(ch, tmp);
7392   jccb(Assembler::equal, FOUND_SEQ_CHAR);
7393   addptr(result, 2);
7394   subl(cnt1, 1);
7395   jccb(Assembler::zero, RET_NOT_FOUND);
7396   jmp(SCAN_TO_CHAR_LOOP);
7397 
7398   bind(RET_NOT_FOUND);
7399   movl(result, -1);
7400   jmpb(DONE_LABEL);
7401 
7402   bind(FOUND_CHAR);
7403   if (UseAVX >= 2) {
7404     vpmovmskb(tmp, vec3);
7405   } else {
7406     pmovmskb(tmp, vec3);
7407   }
7408   bsfl(ch, tmp);
7409   addl(result, ch);
7410 
7411   bind(FOUND_SEQ_CHAR);
7412   subptr(result, str1);
7413   shrl(result, 1);
7414 
7415   bind(DONE_LABEL);
7416 } // string_indexof_char
7417 
7418 // helper function for string_compare
7419 void MacroAssembler::load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
7420                                         Address::ScaleFactor scale, Address::ScaleFactor scale1,
7421                                         Address::ScaleFactor scale2, Register index, int ae) {
7422   if (ae == StrIntrinsicNode::LL) {
7423     load_unsigned_byte(elem1, Address(str1, index, scale, 0));
7424     load_unsigned_byte(elem2, Address(str2, index, scale, 0));
7425   } else if (ae == StrIntrinsicNode::UU) {
7426     load_unsigned_short(elem1, Address(str1, index, scale, 0));
7427     load_unsigned_short(elem2, Address(str2, index, scale, 0));
7428   } else {
7429     load_unsigned_byte(elem1, Address(str1, index, scale1, 0));
7430     load_unsigned_short(elem2, Address(str2, index, scale2, 0));
7431   }
7432 }
7433 
7434 // Compare strings, used for char[] and byte[].
7435 void MacroAssembler::string_compare(Register str1, Register str2,
7436                                     Register cnt1, Register cnt2, Register result,
7437                                     XMMRegister vec1, int ae) {
7438   ShortBranchVerifier sbv(this);
7439   Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
7440   Label COMPARE_WIDE_VECTORS_LOOP_FAILED;  // used only _LP64 && AVX3
7441   int stride, stride2, adr_stride, adr_stride1, adr_stride2;
7442   int stride2x2 = 0x40;
7443   Address::ScaleFactor scale = Address::no_scale;
7444   Address::ScaleFactor scale1 = Address::no_scale;
7445   Address::ScaleFactor scale2 = Address::no_scale;
7446 
7447   if (ae != StrIntrinsicNode::LL) {
7448     stride2x2 = 0x20;
7449   }
7450 
7451   if (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL) {
7452     shrl(cnt2, 1);
7453   }
7454   // Compute the minimum of the string lengths and the
7455   // difference of the string lengths (stack).
7456   // Do the conditional move stuff
7457   movl(result, cnt1);
7458   subl(cnt1, cnt2);
7459   push(cnt1);
7460   cmov32(Assembler::lessEqual, cnt2, result);    // cnt2 = min(cnt1, cnt2)
7461 
7462   // Is the minimum length zero?
7463   testl(cnt2, cnt2);
7464   jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7465   if (ae == StrIntrinsicNode::LL) {
7466     // Load first bytes
7467     load_unsigned_byte(result, Address(str1, 0));  // result = str1[0]
7468     load_unsigned_byte(cnt1, Address(str2, 0));    // cnt1   = str2[0]
7469   } else if (ae == StrIntrinsicNode::UU) {
7470     // Load first characters
7471     load_unsigned_short(result, Address(str1, 0));
7472     load_unsigned_short(cnt1, Address(str2, 0));
7473   } else {
7474     load_unsigned_byte(result, Address(str1, 0));
7475     load_unsigned_short(cnt1, Address(str2, 0));
7476   }
7477   subl(result, cnt1);
7478   jcc(Assembler::notZero,  POP_LABEL);
7479 
7480   if (ae == StrIntrinsicNode::UU) {
7481     // Divide length by 2 to get number of chars
7482     shrl(cnt2, 1);
7483   }
7484   cmpl(cnt2, 1);
7485   jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7486 
7487   // Check if the strings start at the same location and setup scale and stride
7488   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7489     cmpptr(str1, str2);
7490     jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7491     if (ae == StrIntrinsicNode::LL) {
7492       scale = Address::times_1;
7493       stride = 16;
7494     } else {
7495       scale = Address::times_2;
7496       stride = 8;
7497     }
7498   } else {
7499     scale1 = Address::times_1;
7500     scale2 = Address::times_2;
7501     // scale not used
7502     stride = 8;
7503   }
7504 
7505   if (UseAVX >= 2 && UseSSE42Intrinsics) {
7506     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_WIDE_TAIL, COMPARE_SMALL_STR;
7507     Label COMPARE_WIDE_VECTORS_LOOP, COMPARE_16_CHARS, COMPARE_INDEX_CHAR;
7508     Label COMPARE_WIDE_VECTORS_LOOP_AVX2;
7509     Label COMPARE_TAIL_LONG;
7510     Label COMPARE_WIDE_VECTORS_LOOP_AVX3;  // used only _LP64 && AVX3
7511 
7512     int pcmpmask = 0x19;
7513     if (ae == StrIntrinsicNode::LL) {
7514       pcmpmask &= ~0x01;
7515     }
7516 
7517     // Setup to compare 16-chars (32-bytes) vectors,
7518     // start from first character again because it has aligned address.
7519     if (ae == StrIntrinsicNode::LL) {
7520       stride2 = 32;
7521     } else {
7522       stride2 = 16;
7523     }
7524     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7525       adr_stride = stride << scale;
7526     } else {
7527       adr_stride1 = 8;  //stride << scale1;
7528       adr_stride2 = 16; //stride << scale2;
7529     }
7530 
7531     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
7532     // rax and rdx are used by pcmpestri as elements counters
7533     movl(result, cnt2);
7534     andl(cnt2, ~(stride2-1));   // cnt2 holds the vector count
7535     jcc(Assembler::zero, COMPARE_TAIL_LONG);
7536 
7537     // fast path : compare first 2 8-char vectors.
7538     bind(COMPARE_16_CHARS);
7539     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7540       movdqu(vec1, Address(str1, 0));
7541     } else {
7542       pmovzxbw(vec1, Address(str1, 0));
7543     }
7544     pcmpestri(vec1, Address(str2, 0), pcmpmask);
7545     jccb(Assembler::below, COMPARE_INDEX_CHAR);
7546 
7547     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7548       movdqu(vec1, Address(str1, adr_stride));
7549       pcmpestri(vec1, Address(str2, adr_stride), pcmpmask);
7550     } else {
7551       pmovzxbw(vec1, Address(str1, adr_stride1));
7552       pcmpestri(vec1, Address(str2, adr_stride2), pcmpmask);
7553     }
7554     jccb(Assembler::aboveEqual, COMPARE_WIDE_VECTORS);
7555     addl(cnt1, stride);
7556 
7557     // Compare the characters at index in cnt1
7558     bind(COMPARE_INDEX_CHAR); // cnt1 has the offset of the mismatching character
7559     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
7560     subl(result, cnt2);
7561     jmp(POP_LABEL);
7562 
7563     // Setup the registers to start vector comparison loop
7564     bind(COMPARE_WIDE_VECTORS);
7565     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7566       lea(str1, Address(str1, result, scale));
7567       lea(str2, Address(str2, result, scale));
7568     } else {
7569       lea(str1, Address(str1, result, scale1));
7570       lea(str2, Address(str2, result, scale2));
7571     }
7572     subl(result, stride2);
7573     subl(cnt2, stride2);
7574     jcc(Assembler::zero, COMPARE_WIDE_TAIL);
7575     negptr(result);
7576 
7577     //  In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest)
7578     bind(COMPARE_WIDE_VECTORS_LOOP);
7579 
7580 #ifdef _LP64
7581     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
7582       cmpl(cnt2, stride2x2);
7583       jccb(Assembler::below, COMPARE_WIDE_VECTORS_LOOP_AVX2);
7584       testl(cnt2, stride2x2-1);   // cnt2 holds the vector count
7585       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX2);   // means we cannot subtract by 0x40
7586 
7587       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
7588       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7589         evmovdquq(vec1, Address(str1, result, scale), Assembler::AVX_512bit);
7590         evpcmpeqb(k7, vec1, Address(str2, result, scale), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7591       } else {
7592         vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_512bit);
7593         evpcmpeqb(k7, vec1, Address(str2, result, scale2), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7594       }
7595       kortestql(k7, k7);
7596       jcc(Assembler::aboveEqual, COMPARE_WIDE_VECTORS_LOOP_FAILED);     // miscompare
7597       addptr(result, stride2x2);  // update since we already compared at this addr
7598       subl(cnt2, stride2x2);      // and sub the size too
7599       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX3);
7600 
7601       vpxor(vec1, vec1);
7602       jmpb(COMPARE_WIDE_TAIL);
7603     }//if (VM_Version::supports_avx512vlbw())
7604 #endif // _LP64
7605 
7606 
7607     bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7608     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7609       vmovdqu(vec1, Address(str1, result, scale));
7610       vpxor(vec1, Address(str2, result, scale));
7611     } else {
7612       vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_256bit);
7613       vpxor(vec1, Address(str2, result, scale2));
7614     }
7615     vptest(vec1, vec1);
7616     jcc(Assembler::notZero, VECTOR_NOT_EQUAL);
7617     addptr(result, stride2);
7618     subl(cnt2, stride2);
7619     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP);
7620     // clean upper bits of YMM registers
7621     vpxor(vec1, vec1);
7622 
7623     // compare wide vectors tail
7624     bind(COMPARE_WIDE_TAIL);
7625     testptr(result, result);
7626     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7627 
7628     movl(result, stride2);
7629     movl(cnt2, result);
7630     negptr(result);
7631     jmp(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7632 
7633     // Identifies the mismatching (higher or lower)16-bytes in the 32-byte vectors.
7634     bind(VECTOR_NOT_EQUAL);
7635     // clean upper bits of YMM registers
7636     vpxor(vec1, vec1);
7637     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7638       lea(str1, Address(str1, result, scale));
7639       lea(str2, Address(str2, result, scale));
7640     } else {
7641       lea(str1, Address(str1, result, scale1));
7642       lea(str2, Address(str2, result, scale2));
7643     }
7644     jmp(COMPARE_16_CHARS);
7645 
7646     // Compare tail chars, length between 1 to 15 chars
7647     bind(COMPARE_TAIL_LONG);
7648     movl(cnt2, result);
7649     cmpl(cnt2, stride);
7650     jcc(Assembler::less, COMPARE_SMALL_STR);
7651 
7652     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7653       movdqu(vec1, Address(str1, 0));
7654     } else {
7655       pmovzxbw(vec1, Address(str1, 0));
7656     }
7657     pcmpestri(vec1, Address(str2, 0), pcmpmask);
7658     jcc(Assembler::below, COMPARE_INDEX_CHAR);
7659     subptr(cnt2, stride);
7660     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7661     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7662       lea(str1, Address(str1, result, scale));
7663       lea(str2, Address(str2, result, scale));
7664     } else {
7665       lea(str1, Address(str1, result, scale1));
7666       lea(str2, Address(str2, result, scale2));
7667     }
7668     negptr(cnt2);
7669     jmpb(WHILE_HEAD_LABEL);
7670 
7671     bind(COMPARE_SMALL_STR);
7672   } else if (UseSSE42Intrinsics) {
7673     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
7674     int pcmpmask = 0x19;
7675     // Setup to compare 8-char (16-byte) vectors,
7676     // start from first character again because it has aligned address.
7677     movl(result, cnt2);
7678     andl(cnt2, ~(stride - 1));   // cnt2 holds the vector count
7679     if (ae == StrIntrinsicNode::LL) {
7680       pcmpmask &= ~0x01;
7681     }
7682     jcc(Assembler::zero, COMPARE_TAIL);
7683     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7684       lea(str1, Address(str1, result, scale));
7685       lea(str2, Address(str2, result, scale));
7686     } else {
7687       lea(str1, Address(str1, result, scale1));
7688       lea(str2, Address(str2, result, scale2));
7689     }
7690     negptr(result);
7691 
7692     // pcmpestri
7693     //   inputs:
7694     //     vec1- substring
7695     //     rax - negative string length (elements count)
7696     //     mem - scanned string
7697     //     rdx - string length (elements count)
7698     //     pcmpmask - cmp mode: 11000 (string compare with negated result)
7699     //               + 00 (unsigned bytes) or  + 01 (unsigned shorts)
7700     //   outputs:
7701     //     rcx - first mismatched element index
7702     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
7703 
7704     bind(COMPARE_WIDE_VECTORS);
7705     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7706       movdqu(vec1, Address(str1, result, scale));
7707       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
7708     } else {
7709       pmovzxbw(vec1, Address(str1, result, scale1));
7710       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
7711     }
7712     // After pcmpestri cnt1(rcx) contains mismatched element index
7713 
7714     jccb(Assembler::below, VECTOR_NOT_EQUAL);  // CF==1
7715     addptr(result, stride);
7716     subptr(cnt2, stride);
7717     jccb(Assembler::notZero, COMPARE_WIDE_VECTORS);
7718 
7719     // compare wide vectors tail
7720     testptr(result, result);
7721     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7722 
7723     movl(cnt2, stride);
7724     movl(result, stride);
7725     negptr(result);
7726     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7727       movdqu(vec1, Address(str1, result, scale));
7728       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
7729     } else {
7730       pmovzxbw(vec1, Address(str1, result, scale1));
7731       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
7732     }
7733     jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL);
7734 
7735     // Mismatched characters in the vectors
7736     bind(VECTOR_NOT_EQUAL);
7737     addptr(cnt1, result);
7738     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
7739     subl(result, cnt2);
7740     jmpb(POP_LABEL);
7741 
7742     bind(COMPARE_TAIL); // limit is zero
7743     movl(cnt2, result);
7744     // Fallthru to tail compare
7745   }
7746   // Shift str2 and str1 to the end of the arrays, negate min
7747   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7748     lea(str1, Address(str1, cnt2, scale));
7749     lea(str2, Address(str2, cnt2, scale));
7750   } else {
7751     lea(str1, Address(str1, cnt2, scale1));
7752     lea(str2, Address(str2, cnt2, scale2));
7753   }
7754   decrementl(cnt2);  // first character was compared already
7755   negptr(cnt2);
7756 
7757   // Compare the rest of the elements
7758   bind(WHILE_HEAD_LABEL);
7759   load_next_elements(result, cnt1, str1, str2, scale, scale1, scale2, cnt2, ae);
7760   subl(result, cnt1);
7761   jccb(Assembler::notZero, POP_LABEL);
7762   increment(cnt2);
7763   jccb(Assembler::notZero, WHILE_HEAD_LABEL);
7764 
7765   // Strings are equal up to min length.  Return the length difference.
7766   bind(LENGTH_DIFF_LABEL);
7767   pop(result);
7768   if (ae == StrIntrinsicNode::UU) {
7769     // Divide diff by 2 to get number of chars
7770     sarl(result, 1);
7771   }
7772   jmpb(DONE_LABEL);
7773 
7774 #ifdef _LP64
7775   if (VM_Version::supports_avx512vlbw()) {
7776 
7777     bind(COMPARE_WIDE_VECTORS_LOOP_FAILED);
7778 
7779     kmovql(cnt1, k7);
7780     notq(cnt1);
7781     bsfq(cnt2, cnt1);
7782     if (ae != StrIntrinsicNode::LL) {
7783       // Divide diff by 2 to get number of chars
7784       sarl(cnt2, 1);
7785     }
7786     addq(result, cnt2);
7787     if (ae == StrIntrinsicNode::LL) {
7788       load_unsigned_byte(cnt1, Address(str2, result));
7789       load_unsigned_byte(result, Address(str1, result));
7790     } else if (ae == StrIntrinsicNode::UU) {
7791       load_unsigned_short(cnt1, Address(str2, result, scale));
7792       load_unsigned_short(result, Address(str1, result, scale));
7793     } else {
7794       load_unsigned_short(cnt1, Address(str2, result, scale2));
7795       load_unsigned_byte(result, Address(str1, result, scale1));
7796     }
7797     subl(result, cnt1);
7798     jmpb(POP_LABEL);
7799   }//if (VM_Version::supports_avx512vlbw())
7800 #endif // _LP64
7801 
7802   // Discard the stored length difference
7803   bind(POP_LABEL);
7804   pop(cnt1);
7805 
7806   // That's it
7807   bind(DONE_LABEL);
7808   if(ae == StrIntrinsicNode::UL) {
7809     negl(result);
7810   }
7811 
7812 }
7813 
7814 // Search for Non-ASCII character (Negative byte value) in a byte array,
7815 // return true if it has any and false otherwise.
7816 //   ..\jdk\src\java.base\share\classes\java\lang\StringCoding.java
7817 //   @HotSpotIntrinsicCandidate
7818 //   private static boolean hasNegatives(byte[] ba, int off, int len) {
7819 //     for (int i = off; i < off + len; i++) {
7820 //       if (ba[i] < 0) {
7821 //         return true;
7822 //       }
7823 //     }
7824 //     return false;
7825 //   }
7826 void MacroAssembler::has_negatives(Register ary1, Register len,
7827   Register result, Register tmp1,
7828   XMMRegister vec1, XMMRegister vec2) {
7829   // rsi: byte array
7830   // rcx: len
7831   // rax: result
7832   ShortBranchVerifier sbv(this);
7833   assert_different_registers(ary1, len, result, tmp1);
7834   assert_different_registers(vec1, vec2);
7835   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_CHAR, COMPARE_VECTORS, COMPARE_BYTE;
7836 
7837   // len == 0
7838   testl(len, len);
7839   jcc(Assembler::zero, FALSE_LABEL);
7840 
7841   if ((UseAVX > 2) && // AVX512
7842     VM_Version::supports_avx512vlbw() &&
7843     VM_Version::supports_bmi2()) {
7844 
7845     set_vector_masking();  // opening of the stub context for programming mask registers
7846 
7847     Label test_64_loop, test_tail;
7848     Register tmp3_aliased = len;
7849 
7850     movl(tmp1, len);
7851     vpxor(vec2, vec2, vec2, Assembler::AVX_512bit);
7852 
7853     andl(tmp1, 64 - 1);   // tail count (in chars) 0x3F
7854     andl(len, ~(64 - 1));    // vector count (in chars)
7855     jccb(Assembler::zero, test_tail);
7856 
7857     lea(ary1, Address(ary1, len, Address::times_1));
7858     negptr(len);
7859 
7860     bind(test_64_loop);
7861     // Check whether our 64 elements of size byte contain negatives
7862     evpcmpgtb(k2, vec2, Address(ary1, len, Address::times_1), Assembler::AVX_512bit);
7863     kortestql(k2, k2);
7864     jcc(Assembler::notZero, TRUE_LABEL);
7865 
7866     addptr(len, 64);
7867     jccb(Assembler::notZero, test_64_loop);
7868 
7869 
7870     bind(test_tail);
7871     // bail out when there is nothing to be done
7872     testl(tmp1, -1);
7873     jcc(Assembler::zero, FALSE_LABEL);
7874 
7875     // Save k1
7876     kmovql(k3, k1);
7877 
7878     // ~(~0 << len) applied up to two times (for 32-bit scenario)
7879 #ifdef _LP64
7880     mov64(tmp3_aliased, 0xFFFFFFFFFFFFFFFF);
7881     shlxq(tmp3_aliased, tmp3_aliased, tmp1);
7882     notq(tmp3_aliased);
7883     kmovql(k1, tmp3_aliased);
7884 #else
7885     Label k_init;
7886     jmp(k_init);
7887 
7888     // We could not read 64-bits from a general purpose register thus we move
7889     // data required to compose 64 1's to the instruction stream
7890     // We emit 64 byte wide series of elements from 0..63 which later on would
7891     // be used as a compare targets with tail count contained in tmp1 register.
7892     // Result would be a k1 register having tmp1 consecutive number or 1
7893     // counting from least significant bit.
7894     address tmp = pc();
7895     emit_int64(0x0706050403020100);
7896     emit_int64(0x0F0E0D0C0B0A0908);
7897     emit_int64(0x1716151413121110);
7898     emit_int64(0x1F1E1D1C1B1A1918);
7899     emit_int64(0x2726252423222120);
7900     emit_int64(0x2F2E2D2C2B2A2928);
7901     emit_int64(0x3736353433323130);
7902     emit_int64(0x3F3E3D3C3B3A3938);
7903 
7904     bind(k_init);
7905     lea(len, InternalAddress(tmp));
7906     // create mask to test for negative byte inside a vector
7907     evpbroadcastb(vec1, tmp1, Assembler::AVX_512bit);
7908     evpcmpgtb(k1, vec1, Address(len, 0), Assembler::AVX_512bit);
7909 
7910 #endif
7911     evpcmpgtb(k2, k1, vec2, Address(ary1, 0), Assembler::AVX_512bit);
7912     ktestq(k2, k1);
7913     // Restore k1
7914     kmovql(k1, k3);
7915     jcc(Assembler::notZero, TRUE_LABEL);
7916 
7917     jmp(FALSE_LABEL);
7918 
7919     clear_vector_masking();   // closing of the stub context for programming mask registers
7920   } else {
7921     movl(result, len); // copy
7922 
7923     if (UseAVX == 2 && UseSSE >= 2) {
7924       // With AVX2, use 32-byte vector compare
7925       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7926 
7927       // Compare 32-byte vectors
7928       andl(result, 0x0000001f);  //   tail count (in bytes)
7929       andl(len, 0xffffffe0);   // vector count (in bytes)
7930       jccb(Assembler::zero, COMPARE_TAIL);
7931 
7932       lea(ary1, Address(ary1, len, Address::times_1));
7933       negptr(len);
7934 
7935       movl(tmp1, 0x80808080);   // create mask to test for Unicode chars in vector
7936       movdl(vec2, tmp1);
7937       vpbroadcastd(vec2, vec2);
7938 
7939       bind(COMPARE_WIDE_VECTORS);
7940       vmovdqu(vec1, Address(ary1, len, Address::times_1));
7941       vptest(vec1, vec2);
7942       jccb(Assembler::notZero, TRUE_LABEL);
7943       addptr(len, 32);
7944       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7945 
7946       testl(result, result);
7947       jccb(Assembler::zero, FALSE_LABEL);
7948 
7949       vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
7950       vptest(vec1, vec2);
7951       jccb(Assembler::notZero, TRUE_LABEL);
7952       jmpb(FALSE_LABEL);
7953 
7954       bind(COMPARE_TAIL); // len is zero
7955       movl(len, result);
7956       // Fallthru to tail compare
7957     } else if (UseSSE42Intrinsics) {
7958       // With SSE4.2, use double quad vector compare
7959       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7960 
7961       // Compare 16-byte vectors
7962       andl(result, 0x0000000f);  //   tail count (in bytes)
7963       andl(len, 0xfffffff0);   // vector count (in bytes)
7964       jcc(Assembler::zero, COMPARE_TAIL);
7965 
7966       lea(ary1, Address(ary1, len, Address::times_1));
7967       negptr(len);
7968 
7969       movl(tmp1, 0x80808080);
7970       movdl(vec2, tmp1);
7971       pshufd(vec2, vec2, 0);
7972 
7973       bind(COMPARE_WIDE_VECTORS);
7974       movdqu(vec1, Address(ary1, len, Address::times_1));
7975       ptest(vec1, vec2);
7976       jcc(Assembler::notZero, TRUE_LABEL);
7977       addptr(len, 16);
7978       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7979 
7980       testl(result, result);
7981       jcc(Assembler::zero, FALSE_LABEL);
7982 
7983       movdqu(vec1, Address(ary1, result, Address::times_1, -16));
7984       ptest(vec1, vec2);
7985       jccb(Assembler::notZero, TRUE_LABEL);
7986       jmpb(FALSE_LABEL);
7987 
7988       bind(COMPARE_TAIL); // len is zero
7989       movl(len, result);
7990       // Fallthru to tail compare
7991     }
7992   }
7993   // Compare 4-byte vectors
7994   andl(len, 0xfffffffc); // vector count (in bytes)
7995   jccb(Assembler::zero, COMPARE_CHAR);
7996 
7997   lea(ary1, Address(ary1, len, Address::times_1));
7998   negptr(len);
7999 
8000   bind(COMPARE_VECTORS);
8001   movl(tmp1, Address(ary1, len, Address::times_1));
8002   andl(tmp1, 0x80808080);
8003   jccb(Assembler::notZero, TRUE_LABEL);
8004   addptr(len, 4);
8005   jcc(Assembler::notZero, COMPARE_VECTORS);
8006 
8007   // Compare trailing char (final 2 bytes), if any
8008   bind(COMPARE_CHAR);
8009   testl(result, 0x2);   // tail  char
8010   jccb(Assembler::zero, COMPARE_BYTE);
8011   load_unsigned_short(tmp1, Address(ary1, 0));
8012   andl(tmp1, 0x00008080);
8013   jccb(Assembler::notZero, TRUE_LABEL);
8014   subptr(result, 2);
8015   lea(ary1, Address(ary1, 2));
8016 
8017   bind(COMPARE_BYTE);
8018   testl(result, 0x1);   // tail  byte
8019   jccb(Assembler::zero, FALSE_LABEL);
8020   load_unsigned_byte(tmp1, Address(ary1, 0));
8021   andl(tmp1, 0x00000080);
8022   jccb(Assembler::notEqual, TRUE_LABEL);
8023   jmpb(FALSE_LABEL);
8024 
8025   bind(TRUE_LABEL);
8026   movl(result, 1);   // return true
8027   jmpb(DONE);
8028 
8029   bind(FALSE_LABEL);
8030   xorl(result, result); // return false
8031 
8032   // That's it
8033   bind(DONE);
8034   if (UseAVX >= 2 && UseSSE >= 2) {
8035     // clean upper bits of YMM registers
8036     vpxor(vec1, vec1);
8037     vpxor(vec2, vec2);
8038   }
8039 }
8040 // Compare char[] or byte[] arrays aligned to 4 bytes or substrings.
8041 void MacroAssembler::arrays_equals(bool is_array_equ, Register ary1, Register ary2,
8042                                    Register limit, Register result, Register chr,
8043                                    XMMRegister vec1, XMMRegister vec2, bool is_char) {
8044   ShortBranchVerifier sbv(this);
8045   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR, COMPARE_BYTE;
8046 
8047   int length_offset  = arrayOopDesc::length_offset_in_bytes();
8048   int base_offset    = arrayOopDesc::base_offset_in_bytes(is_char ? T_CHAR : T_BYTE);
8049 
8050   if (is_array_equ) {
8051     // Check the input args
8052     cmpoop(ary1, ary2);
8053     jcc(Assembler::equal, TRUE_LABEL);
8054 
8055     // Need additional checks for arrays_equals.
8056     testptr(ary1, ary1);
8057     jcc(Assembler::zero, FALSE_LABEL);
8058     testptr(ary2, ary2);
8059     jcc(Assembler::zero, FALSE_LABEL);
8060 
8061     // Check the lengths
8062     movl(limit, Address(ary1, length_offset));
8063     cmpl(limit, Address(ary2, length_offset));
8064     jcc(Assembler::notEqual, FALSE_LABEL);
8065   }
8066 
8067   // count == 0
8068   testl(limit, limit);
8069   jcc(Assembler::zero, TRUE_LABEL);
8070 
8071   if (is_array_equ) {
8072     // Load array address
8073     lea(ary1, Address(ary1, base_offset));
8074     lea(ary2, Address(ary2, base_offset));
8075   }
8076 
8077   if (is_array_equ && is_char) {
8078     // arrays_equals when used for char[].
8079     shll(limit, 1);      // byte count != 0
8080   }
8081   movl(result, limit); // copy
8082 
8083   if (UseAVX >= 2) {
8084     // With AVX2, use 32-byte vector compare
8085     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8086 
8087     // Compare 32-byte vectors
8088     andl(result, 0x0000001f);  //   tail count (in bytes)
8089     andl(limit, 0xffffffe0);   // vector count (in bytes)
8090     jcc(Assembler::zero, COMPARE_TAIL);
8091 
8092     lea(ary1, Address(ary1, limit, Address::times_1));
8093     lea(ary2, Address(ary2, limit, Address::times_1));
8094     negptr(limit);
8095 
8096     bind(COMPARE_WIDE_VECTORS);
8097 
8098 #ifdef _LP64
8099     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
8100       Label COMPARE_WIDE_VECTORS_LOOP_AVX2, COMPARE_WIDE_VECTORS_LOOP_AVX3;
8101 
8102       cmpl(limit, -64);
8103       jccb(Assembler::greater, COMPARE_WIDE_VECTORS_LOOP_AVX2);
8104 
8105       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
8106 
8107       evmovdquq(vec1, Address(ary1, limit, Address::times_1), Assembler::AVX_512bit);
8108       evpcmpeqb(k7, vec1, Address(ary2, limit, Address::times_1), Assembler::AVX_512bit);
8109       kortestql(k7, k7);
8110       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
8111       addptr(limit, 64);  // update since we already compared at this addr
8112       cmpl(limit, -64);
8113       jccb(Assembler::lessEqual, COMPARE_WIDE_VECTORS_LOOP_AVX3);
8114 
8115       // At this point we may still need to compare -limit+result bytes.
8116       // We could execute the next two instruction and just continue via non-wide path:
8117       //  cmpl(limit, 0);
8118       //  jcc(Assembler::equal, COMPARE_TAIL);  // true
8119       // But since we stopped at the points ary{1,2}+limit which are
8120       // not farther than 64 bytes from the ends of arrays ary{1,2}+result
8121       // (|limit| <= 32 and result < 32),
8122       // we may just compare the last 64 bytes.
8123       //
8124       addptr(result, -64);   // it is safe, bc we just came from this area
8125       evmovdquq(vec1, Address(ary1, result, Address::times_1), Assembler::AVX_512bit);
8126       evpcmpeqb(k7, vec1, Address(ary2, result, Address::times_1), Assembler::AVX_512bit);
8127       kortestql(k7, k7);
8128       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
8129 
8130       jmp(TRUE_LABEL);
8131 
8132       bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
8133 
8134     }//if (VM_Version::supports_avx512vlbw())
8135 #endif //_LP64
8136 
8137     vmovdqu(vec1, Address(ary1, limit, Address::times_1));
8138     vmovdqu(vec2, Address(ary2, limit, Address::times_1));
8139     vpxor(vec1, vec2);
8140 
8141     vptest(vec1, vec1);
8142     jcc(Assembler::notZero, FALSE_LABEL);
8143     addptr(limit, 32);
8144     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8145 
8146     testl(result, result);
8147     jcc(Assembler::zero, TRUE_LABEL);
8148 
8149     vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
8150     vmovdqu(vec2, Address(ary2, result, Address::times_1, -32));
8151     vpxor(vec1, vec2);
8152 
8153     vptest(vec1, vec1);
8154     jccb(Assembler::notZero, FALSE_LABEL);
8155     jmpb(TRUE_LABEL);
8156 
8157     bind(COMPARE_TAIL); // limit is zero
8158     movl(limit, result);
8159     // Fallthru to tail compare
8160   } else if (UseSSE42Intrinsics) {
8161     // With SSE4.2, use double quad vector compare
8162     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8163 
8164     // Compare 16-byte vectors
8165     andl(result, 0x0000000f);  //   tail count (in bytes)
8166     andl(limit, 0xfffffff0);   // vector count (in bytes)
8167     jcc(Assembler::zero, COMPARE_TAIL);
8168 
8169     lea(ary1, Address(ary1, limit, Address::times_1));
8170     lea(ary2, Address(ary2, limit, Address::times_1));
8171     negptr(limit);
8172 
8173     bind(COMPARE_WIDE_VECTORS);
8174     movdqu(vec1, Address(ary1, limit, Address::times_1));
8175     movdqu(vec2, Address(ary2, limit, Address::times_1));
8176     pxor(vec1, vec2);
8177 
8178     ptest(vec1, vec1);
8179     jcc(Assembler::notZero, FALSE_LABEL);
8180     addptr(limit, 16);
8181     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8182 
8183     testl(result, result);
8184     jcc(Assembler::zero, TRUE_LABEL);
8185 
8186     movdqu(vec1, Address(ary1, result, Address::times_1, -16));
8187     movdqu(vec2, Address(ary2, result, Address::times_1, -16));
8188     pxor(vec1, vec2);
8189 
8190     ptest(vec1, vec1);
8191     jccb(Assembler::notZero, FALSE_LABEL);
8192     jmpb(TRUE_LABEL);
8193 
8194     bind(COMPARE_TAIL); // limit is zero
8195     movl(limit, result);
8196     // Fallthru to tail compare
8197   }
8198 
8199   // Compare 4-byte vectors
8200   andl(limit, 0xfffffffc); // vector count (in bytes)
8201   jccb(Assembler::zero, COMPARE_CHAR);
8202 
8203   lea(ary1, Address(ary1, limit, Address::times_1));
8204   lea(ary2, Address(ary2, limit, Address::times_1));
8205   negptr(limit);
8206 
8207   bind(COMPARE_VECTORS);
8208   movl(chr, Address(ary1, limit, Address::times_1));
8209   cmpl(chr, Address(ary2, limit, Address::times_1));
8210   jccb(Assembler::notEqual, FALSE_LABEL);
8211   addptr(limit, 4);
8212   jcc(Assembler::notZero, COMPARE_VECTORS);
8213 
8214   // Compare trailing char (final 2 bytes), if any
8215   bind(COMPARE_CHAR);
8216   testl(result, 0x2);   // tail  char
8217   jccb(Assembler::zero, COMPARE_BYTE);
8218   load_unsigned_short(chr, Address(ary1, 0));
8219   load_unsigned_short(limit, Address(ary2, 0));
8220   cmpl(chr, limit);
8221   jccb(Assembler::notEqual, FALSE_LABEL);
8222 
8223   if (is_array_equ && is_char) {
8224     bind(COMPARE_BYTE);
8225   } else {
8226     lea(ary1, Address(ary1, 2));
8227     lea(ary2, Address(ary2, 2));
8228 
8229     bind(COMPARE_BYTE);
8230     testl(result, 0x1);   // tail  byte
8231     jccb(Assembler::zero, TRUE_LABEL);
8232     load_unsigned_byte(chr, Address(ary1, 0));
8233     load_unsigned_byte(limit, Address(ary2, 0));
8234     cmpl(chr, limit);
8235     jccb(Assembler::notEqual, FALSE_LABEL);
8236   }
8237   bind(TRUE_LABEL);
8238   movl(result, 1);   // return true
8239   jmpb(DONE);
8240 
8241   bind(FALSE_LABEL);
8242   xorl(result, result); // return false
8243 
8244   // That's it
8245   bind(DONE);
8246   if (UseAVX >= 2) {
8247     // clean upper bits of YMM registers
8248     vpxor(vec1, vec1);
8249     vpxor(vec2, vec2);
8250   }
8251 }
8252 
8253 #endif
8254 
8255 void MacroAssembler::generate_fill(BasicType t, bool aligned,
8256                                    Register to, Register value, Register count,
8257                                    Register rtmp, XMMRegister xtmp) {
8258   ShortBranchVerifier sbv(this);
8259   assert_different_registers(to, value, count, rtmp);
8260   Label L_exit, L_skip_align1, L_skip_align2, L_fill_byte;
8261   Label L_fill_2_bytes, L_fill_4_bytes;
8262 
8263   int shift = -1;
8264   switch (t) {
8265     case T_BYTE:
8266       shift = 2;
8267       break;
8268     case T_SHORT:
8269       shift = 1;
8270       break;
8271     case T_INT:
8272       shift = 0;
8273       break;
8274     default: ShouldNotReachHere();
8275   }
8276 
8277   if (t == T_BYTE) {
8278     andl(value, 0xff);
8279     movl(rtmp, value);
8280     shll(rtmp, 8);
8281     orl(value, rtmp);
8282   }
8283   if (t == T_SHORT) {
8284     andl(value, 0xffff);
8285   }
8286   if (t == T_BYTE || t == T_SHORT) {
8287     movl(rtmp, value);
8288     shll(rtmp, 16);
8289     orl(value, rtmp);
8290   }
8291 
8292   cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
8293   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
8294   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
8295     // align source address at 4 bytes address boundary
8296     if (t == T_BYTE) {
8297       // One byte misalignment happens only for byte arrays
8298       testptr(to, 1);
8299       jccb(Assembler::zero, L_skip_align1);
8300       movb(Address(to, 0), value);
8301       increment(to);
8302       decrement(count);
8303       BIND(L_skip_align1);
8304     }
8305     // Two bytes misalignment happens only for byte and short (char) arrays
8306     testptr(to, 2);
8307     jccb(Assembler::zero, L_skip_align2);
8308     movw(Address(to, 0), value);
8309     addptr(to, 2);
8310     subl(count, 1<<(shift-1));
8311     BIND(L_skip_align2);
8312   }
8313   if (UseSSE < 2) {
8314     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8315     // Fill 32-byte chunks
8316     subl(count, 8 << shift);
8317     jcc(Assembler::less, L_check_fill_8_bytes);
8318     align(16);
8319 
8320     BIND(L_fill_32_bytes_loop);
8321 
8322     for (int i = 0; i < 32; i += 4) {
8323       movl(Address(to, i), value);
8324     }
8325 
8326     addptr(to, 32);
8327     subl(count, 8 << shift);
8328     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8329     BIND(L_check_fill_8_bytes);
8330     addl(count, 8 << shift);
8331     jccb(Assembler::zero, L_exit);
8332     jmpb(L_fill_8_bytes);
8333 
8334     //
8335     // length is too short, just fill qwords
8336     //
8337     BIND(L_fill_8_bytes_loop);
8338     movl(Address(to, 0), value);
8339     movl(Address(to, 4), value);
8340     addptr(to, 8);
8341     BIND(L_fill_8_bytes);
8342     subl(count, 1 << (shift + 1));
8343     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8344     // fall through to fill 4 bytes
8345   } else {
8346     Label L_fill_32_bytes;
8347     if (!UseUnalignedLoadStores) {
8348       // align to 8 bytes, we know we are 4 byte aligned to start
8349       testptr(to, 4);
8350       jccb(Assembler::zero, L_fill_32_bytes);
8351       movl(Address(to, 0), value);
8352       addptr(to, 4);
8353       subl(count, 1<<shift);
8354     }
8355     BIND(L_fill_32_bytes);
8356     {
8357       assert( UseSSE >= 2, "supported cpu only" );
8358       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8359       if (UseAVX > 2) {
8360         movl(rtmp, 0xffff);
8361         kmovwl(k1, rtmp);
8362       }
8363       movdl(xtmp, value);
8364       if (UseAVX > 2 && UseUnalignedLoadStores) {
8365         // Fill 64-byte chunks
8366         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8367         evpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
8368 
8369         subl(count, 16 << shift);
8370         jcc(Assembler::less, L_check_fill_32_bytes);
8371         align(16);
8372 
8373         BIND(L_fill_64_bytes_loop);
8374         evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
8375         addptr(to, 64);
8376         subl(count, 16 << shift);
8377         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8378 
8379         BIND(L_check_fill_32_bytes);
8380         addl(count, 8 << shift);
8381         jccb(Assembler::less, L_check_fill_8_bytes);
8382         vmovdqu(Address(to, 0), xtmp);
8383         addptr(to, 32);
8384         subl(count, 8 << shift);
8385 
8386         BIND(L_check_fill_8_bytes);
8387       } else if (UseAVX == 2 && UseUnalignedLoadStores) {
8388         // Fill 64-byte chunks
8389         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8390         vpbroadcastd(xtmp, xtmp);
8391 
8392         subl(count, 16 << shift);
8393         jcc(Assembler::less, L_check_fill_32_bytes);
8394         align(16);
8395 
8396         BIND(L_fill_64_bytes_loop);
8397         vmovdqu(Address(to, 0), xtmp);
8398         vmovdqu(Address(to, 32), xtmp);
8399         addptr(to, 64);
8400         subl(count, 16 << shift);
8401         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8402 
8403         BIND(L_check_fill_32_bytes);
8404         addl(count, 8 << shift);
8405         jccb(Assembler::less, L_check_fill_8_bytes);
8406         vmovdqu(Address(to, 0), xtmp);
8407         addptr(to, 32);
8408         subl(count, 8 << shift);
8409 
8410         BIND(L_check_fill_8_bytes);
8411         // clean upper bits of YMM registers
8412         movdl(xtmp, value);
8413         pshufd(xtmp, xtmp, 0);
8414       } else {
8415         // Fill 32-byte chunks
8416         pshufd(xtmp, xtmp, 0);
8417 
8418         subl(count, 8 << shift);
8419         jcc(Assembler::less, L_check_fill_8_bytes);
8420         align(16);
8421 
8422         BIND(L_fill_32_bytes_loop);
8423 
8424         if (UseUnalignedLoadStores) {
8425           movdqu(Address(to, 0), xtmp);
8426           movdqu(Address(to, 16), xtmp);
8427         } else {
8428           movq(Address(to, 0), xtmp);
8429           movq(Address(to, 8), xtmp);
8430           movq(Address(to, 16), xtmp);
8431           movq(Address(to, 24), xtmp);
8432         }
8433 
8434         addptr(to, 32);
8435         subl(count, 8 << shift);
8436         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8437 
8438         BIND(L_check_fill_8_bytes);
8439       }
8440       addl(count, 8 << shift);
8441       jccb(Assembler::zero, L_exit);
8442       jmpb(L_fill_8_bytes);
8443 
8444       //
8445       // length is too short, just fill qwords
8446       //
8447       BIND(L_fill_8_bytes_loop);
8448       movq(Address(to, 0), xtmp);
8449       addptr(to, 8);
8450       BIND(L_fill_8_bytes);
8451       subl(count, 1 << (shift + 1));
8452       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8453     }
8454   }
8455   // fill trailing 4 bytes
8456   BIND(L_fill_4_bytes);
8457   testl(count, 1<<shift);
8458   jccb(Assembler::zero, L_fill_2_bytes);
8459   movl(Address(to, 0), value);
8460   if (t == T_BYTE || t == T_SHORT) {
8461     addptr(to, 4);
8462     BIND(L_fill_2_bytes);
8463     // fill trailing 2 bytes
8464     testl(count, 1<<(shift-1));
8465     jccb(Assembler::zero, L_fill_byte);
8466     movw(Address(to, 0), value);
8467     if (t == T_BYTE) {
8468       addptr(to, 2);
8469       BIND(L_fill_byte);
8470       // fill trailing byte
8471       testl(count, 1);
8472       jccb(Assembler::zero, L_exit);
8473       movb(Address(to, 0), value);
8474     } else {
8475       BIND(L_fill_byte);
8476     }
8477   } else {
8478     BIND(L_fill_2_bytes);
8479   }
8480   BIND(L_exit);
8481 }
8482 
8483 // encode char[] to byte[] in ISO_8859_1
8484    //@HotSpotIntrinsicCandidate
8485    //private static int implEncodeISOArray(byte[] sa, int sp,
8486    //byte[] da, int dp, int len) {
8487    //  int i = 0;
8488    //  for (; i < len; i++) {
8489    //    char c = StringUTF16.getChar(sa, sp++);
8490    //    if (c > '\u00FF')
8491    //      break;
8492    //    da[dp++] = (byte)c;
8493    //  }
8494    //  return i;
8495    //}
8496 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
8497   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
8498   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
8499   Register tmp5, Register result) {
8500 
8501   // rsi: src
8502   // rdi: dst
8503   // rdx: len
8504   // rcx: tmp5
8505   // rax: result
8506   ShortBranchVerifier sbv(this);
8507   assert_different_registers(src, dst, len, tmp5, result);
8508   Label L_done, L_copy_1_char, L_copy_1_char_exit;
8509 
8510   // set result
8511   xorl(result, result);
8512   // check for zero length
8513   testl(len, len);
8514   jcc(Assembler::zero, L_done);
8515 
8516   movl(result, len);
8517 
8518   // Setup pointers
8519   lea(src, Address(src, len, Address::times_2)); // char[]
8520   lea(dst, Address(dst, len, Address::times_1)); // byte[]
8521   negptr(len);
8522 
8523   if (UseSSE42Intrinsics || UseAVX >= 2) {
8524     Label L_chars_8_check, L_copy_8_chars, L_copy_8_chars_exit;
8525     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
8526 
8527     if (UseAVX >= 2) {
8528       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
8529       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8530       movdl(tmp1Reg, tmp5);
8531       vpbroadcastd(tmp1Reg, tmp1Reg);
8532       jmp(L_chars_32_check);
8533 
8534       bind(L_copy_32_chars);
8535       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
8536       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
8537       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8538       vptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8539       jccb(Assembler::notZero, L_copy_32_chars_exit);
8540       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8541       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
8542       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
8543 
8544       bind(L_chars_32_check);
8545       addptr(len, 32);
8546       jcc(Assembler::lessEqual, L_copy_32_chars);
8547 
8548       bind(L_copy_32_chars_exit);
8549       subptr(len, 16);
8550       jccb(Assembler::greater, L_copy_16_chars_exit);
8551 
8552     } else if (UseSSE42Intrinsics) {
8553       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8554       movdl(tmp1Reg, tmp5);
8555       pshufd(tmp1Reg, tmp1Reg, 0);
8556       jmpb(L_chars_16_check);
8557     }
8558 
8559     bind(L_copy_16_chars);
8560     if (UseAVX >= 2) {
8561       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
8562       vptest(tmp2Reg, tmp1Reg);
8563       jcc(Assembler::notZero, L_copy_16_chars_exit);
8564       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
8565       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
8566     } else {
8567       if (UseAVX > 0) {
8568         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8569         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8570         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
8571       } else {
8572         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8573         por(tmp2Reg, tmp3Reg);
8574         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8575         por(tmp2Reg, tmp4Reg);
8576       }
8577       ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8578       jccb(Assembler::notZero, L_copy_16_chars_exit);
8579       packuswb(tmp3Reg, tmp4Reg);
8580     }
8581     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
8582 
8583     bind(L_chars_16_check);
8584     addptr(len, 16);
8585     jcc(Assembler::lessEqual, L_copy_16_chars);
8586 
8587     bind(L_copy_16_chars_exit);
8588     if (UseAVX >= 2) {
8589       // clean upper bits of YMM registers
8590       vpxor(tmp2Reg, tmp2Reg);
8591       vpxor(tmp3Reg, tmp3Reg);
8592       vpxor(tmp4Reg, tmp4Reg);
8593       movdl(tmp1Reg, tmp5);
8594       pshufd(tmp1Reg, tmp1Reg, 0);
8595     }
8596     subptr(len, 8);
8597     jccb(Assembler::greater, L_copy_8_chars_exit);
8598 
8599     bind(L_copy_8_chars);
8600     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
8601     ptest(tmp3Reg, tmp1Reg);
8602     jccb(Assembler::notZero, L_copy_8_chars_exit);
8603     packuswb(tmp3Reg, tmp1Reg);
8604     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
8605     addptr(len, 8);
8606     jccb(Assembler::lessEqual, L_copy_8_chars);
8607 
8608     bind(L_copy_8_chars_exit);
8609     subptr(len, 8);
8610     jccb(Assembler::zero, L_done);
8611   }
8612 
8613   bind(L_copy_1_char);
8614   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
8615   testl(tmp5, 0xff00);      // check if Unicode char
8616   jccb(Assembler::notZero, L_copy_1_char_exit);
8617   movb(Address(dst, len, Address::times_1, 0), tmp5);
8618   addptr(len, 1);
8619   jccb(Assembler::less, L_copy_1_char);
8620 
8621   bind(L_copy_1_char_exit);
8622   addptr(result, len); // len is negative count of not processed elements
8623 
8624   bind(L_done);
8625 }
8626 
8627 #ifdef _LP64
8628 /**
8629  * Helper for multiply_to_len().
8630  */
8631 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
8632   addq(dest_lo, src1);
8633   adcq(dest_hi, 0);
8634   addq(dest_lo, src2);
8635   adcq(dest_hi, 0);
8636 }
8637 
8638 /**
8639  * Multiply 64 bit by 64 bit first loop.
8640  */
8641 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
8642                                            Register y, Register y_idx, Register z,
8643                                            Register carry, Register product,
8644                                            Register idx, Register kdx) {
8645   //
8646   //  jlong carry, x[], y[], z[];
8647   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
8648   //    huge_128 product = y[idx] * x[xstart] + carry;
8649   //    z[kdx] = (jlong)product;
8650   //    carry  = (jlong)(product >>> 64);
8651   //  }
8652   //  z[xstart] = carry;
8653   //
8654 
8655   Label L_first_loop, L_first_loop_exit;
8656   Label L_one_x, L_one_y, L_multiply;
8657 
8658   decrementl(xstart);
8659   jcc(Assembler::negative, L_one_x);
8660 
8661   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
8662   rorq(x_xstart, 32); // convert big-endian to little-endian
8663 
8664   bind(L_first_loop);
8665   decrementl(idx);
8666   jcc(Assembler::negative, L_first_loop_exit);
8667   decrementl(idx);
8668   jcc(Assembler::negative, L_one_y);
8669   movq(y_idx, Address(y, idx, Address::times_4,  0));
8670   rorq(y_idx, 32); // convert big-endian to little-endian
8671   bind(L_multiply);
8672   movq(product, x_xstart);
8673   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
8674   addq(product, carry);
8675   adcq(rdx, 0);
8676   subl(kdx, 2);
8677   movl(Address(z, kdx, Address::times_4,  4), product);
8678   shrq(product, 32);
8679   movl(Address(z, kdx, Address::times_4,  0), product);
8680   movq(carry, rdx);
8681   jmp(L_first_loop);
8682 
8683   bind(L_one_y);
8684   movl(y_idx, Address(y,  0));
8685   jmp(L_multiply);
8686 
8687   bind(L_one_x);
8688   movl(x_xstart, Address(x,  0));
8689   jmp(L_first_loop);
8690 
8691   bind(L_first_loop_exit);
8692 }
8693 
8694 /**
8695  * Multiply 64 bit by 64 bit and add 128 bit.
8696  */
8697 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
8698                                             Register yz_idx, Register idx,
8699                                             Register carry, Register product, int offset) {
8700   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
8701   //     z[kdx] = (jlong)product;
8702 
8703   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
8704   rorq(yz_idx, 32); // convert big-endian to little-endian
8705   movq(product, x_xstart);
8706   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
8707   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
8708   rorq(yz_idx, 32); // convert big-endian to little-endian
8709 
8710   add2_with_carry(rdx, product, carry, yz_idx);
8711 
8712   movl(Address(z, idx, Address::times_4,  offset+4), product);
8713   shrq(product, 32);
8714   movl(Address(z, idx, Address::times_4,  offset), product);
8715 
8716 }
8717 
8718 /**
8719  * Multiply 128 bit by 128 bit. Unrolled inner loop.
8720  */
8721 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
8722                                              Register yz_idx, Register idx, Register jdx,
8723                                              Register carry, Register product,
8724                                              Register carry2) {
8725   //   jlong carry, x[], y[], z[];
8726   //   int kdx = ystart+1;
8727   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
8728   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
8729   //     z[kdx+idx+1] = (jlong)product;
8730   //     jlong carry2  = (jlong)(product >>> 64);
8731   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
8732   //     z[kdx+idx] = (jlong)product;
8733   //     carry  = (jlong)(product >>> 64);
8734   //   }
8735   //   idx += 2;
8736   //   if (idx > 0) {
8737   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
8738   //     z[kdx+idx] = (jlong)product;
8739   //     carry  = (jlong)(product >>> 64);
8740   //   }
8741   //
8742 
8743   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
8744 
8745   movl(jdx, idx);
8746   andl(jdx, 0xFFFFFFFC);
8747   shrl(jdx, 2);
8748 
8749   bind(L_third_loop);
8750   subl(jdx, 1);
8751   jcc(Assembler::negative, L_third_loop_exit);
8752   subl(idx, 4);
8753 
8754   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
8755   movq(carry2, rdx);
8756 
8757   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
8758   movq(carry, rdx);
8759   jmp(L_third_loop);
8760 
8761   bind (L_third_loop_exit);
8762 
8763   andl (idx, 0x3);
8764   jcc(Assembler::zero, L_post_third_loop_done);
8765 
8766   Label L_check_1;
8767   subl(idx, 2);
8768   jcc(Assembler::negative, L_check_1);
8769 
8770   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
8771   movq(carry, rdx);
8772 
8773   bind (L_check_1);
8774   addl (idx, 0x2);
8775   andl (idx, 0x1);
8776   subl(idx, 1);
8777   jcc(Assembler::negative, L_post_third_loop_done);
8778 
8779   movl(yz_idx, Address(y, idx, Address::times_4,  0));
8780   movq(product, x_xstart);
8781   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
8782   movl(yz_idx, Address(z, idx, Address::times_4,  0));
8783 
8784   add2_with_carry(rdx, product, yz_idx, carry);
8785 
8786   movl(Address(z, idx, Address::times_4,  0), product);
8787   shrq(product, 32);
8788 
8789   shlq(rdx, 32);
8790   orq(product, rdx);
8791   movq(carry, product);
8792 
8793   bind(L_post_third_loop_done);
8794 }
8795 
8796 /**
8797  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
8798  *
8799  */
8800 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
8801                                                   Register carry, Register carry2,
8802                                                   Register idx, Register jdx,
8803                                                   Register yz_idx1, Register yz_idx2,
8804                                                   Register tmp, Register tmp3, Register tmp4) {
8805   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
8806 
8807   //   jlong carry, x[], y[], z[];
8808   //   int kdx = ystart+1;
8809   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
8810   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
8811   //     jlong carry2  = (jlong)(tmp3 >>> 64);
8812   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
8813   //     carry  = (jlong)(tmp4 >>> 64);
8814   //     z[kdx+idx+1] = (jlong)tmp3;
8815   //     z[kdx+idx] = (jlong)tmp4;
8816   //   }
8817   //   idx += 2;
8818   //   if (idx > 0) {
8819   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
8820   //     z[kdx+idx] = (jlong)yz_idx1;
8821   //     carry  = (jlong)(yz_idx1 >>> 64);
8822   //   }
8823   //
8824 
8825   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
8826 
8827   movl(jdx, idx);
8828   andl(jdx, 0xFFFFFFFC);
8829   shrl(jdx, 2);
8830 
8831   bind(L_third_loop);
8832   subl(jdx, 1);
8833   jcc(Assembler::negative, L_third_loop_exit);
8834   subl(idx, 4);
8835 
8836   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
8837   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
8838   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
8839   rorxq(yz_idx2, yz_idx2, 32);
8840 
8841   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
8842   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
8843 
8844   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
8845   rorxq(yz_idx1, yz_idx1, 32);
8846   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
8847   rorxq(yz_idx2, yz_idx2, 32);
8848 
8849   if (VM_Version::supports_adx()) {
8850     adcxq(tmp3, carry);
8851     adoxq(tmp3, yz_idx1);
8852 
8853     adcxq(tmp4, tmp);
8854     adoxq(tmp4, yz_idx2);
8855 
8856     movl(carry, 0); // does not affect flags
8857     adcxq(carry2, carry);
8858     adoxq(carry2, carry);
8859   } else {
8860     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
8861     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
8862   }
8863   movq(carry, carry2);
8864 
8865   movl(Address(z, idx, Address::times_4, 12), tmp3);
8866   shrq(tmp3, 32);
8867   movl(Address(z, idx, Address::times_4,  8), tmp3);
8868 
8869   movl(Address(z, idx, Address::times_4,  4), tmp4);
8870   shrq(tmp4, 32);
8871   movl(Address(z, idx, Address::times_4,  0), tmp4);
8872 
8873   jmp(L_third_loop);
8874 
8875   bind (L_third_loop_exit);
8876 
8877   andl (idx, 0x3);
8878   jcc(Assembler::zero, L_post_third_loop_done);
8879 
8880   Label L_check_1;
8881   subl(idx, 2);
8882   jcc(Assembler::negative, L_check_1);
8883 
8884   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
8885   rorxq(yz_idx1, yz_idx1, 32);
8886   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
8887   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
8888   rorxq(yz_idx2, yz_idx2, 32);
8889 
8890   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
8891 
8892   movl(Address(z, idx, Address::times_4,  4), tmp3);
8893   shrq(tmp3, 32);
8894   movl(Address(z, idx, Address::times_4,  0), tmp3);
8895   movq(carry, tmp4);
8896 
8897   bind (L_check_1);
8898   addl (idx, 0x2);
8899   andl (idx, 0x1);
8900   subl(idx, 1);
8901   jcc(Assembler::negative, L_post_third_loop_done);
8902   movl(tmp4, Address(y, idx, Address::times_4,  0));
8903   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
8904   movl(tmp4, Address(z, idx, Address::times_4,  0));
8905 
8906   add2_with_carry(carry2, tmp3, tmp4, carry);
8907 
8908   movl(Address(z, idx, Address::times_4,  0), tmp3);
8909   shrq(tmp3, 32);
8910 
8911   shlq(carry2, 32);
8912   orq(tmp3, carry2);
8913   movq(carry, tmp3);
8914 
8915   bind(L_post_third_loop_done);
8916 }
8917 
8918 /**
8919  * Code for BigInteger::multiplyToLen() instrinsic.
8920  *
8921  * rdi: x
8922  * rax: xlen
8923  * rsi: y
8924  * rcx: ylen
8925  * r8:  z
8926  * r11: zlen
8927  * r12: tmp1
8928  * r13: tmp2
8929  * r14: tmp3
8930  * r15: tmp4
8931  * rbx: tmp5
8932  *
8933  */
8934 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
8935                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
8936   ShortBranchVerifier sbv(this);
8937   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
8938 
8939   push(tmp1);
8940   push(tmp2);
8941   push(tmp3);
8942   push(tmp4);
8943   push(tmp5);
8944 
8945   push(xlen);
8946   push(zlen);
8947 
8948   const Register idx = tmp1;
8949   const Register kdx = tmp2;
8950   const Register xstart = tmp3;
8951 
8952   const Register y_idx = tmp4;
8953   const Register carry = tmp5;
8954   const Register product  = xlen;
8955   const Register x_xstart = zlen;  // reuse register
8956 
8957   // First Loop.
8958   //
8959   //  final static long LONG_MASK = 0xffffffffL;
8960   //  int xstart = xlen - 1;
8961   //  int ystart = ylen - 1;
8962   //  long carry = 0;
8963   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
8964   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
8965   //    z[kdx] = (int)product;
8966   //    carry = product >>> 32;
8967   //  }
8968   //  z[xstart] = (int)carry;
8969   //
8970 
8971   movl(idx, ylen);      // idx = ylen;
8972   movl(kdx, zlen);      // kdx = xlen+ylen;
8973   xorq(carry, carry);   // carry = 0;
8974 
8975   Label L_done;
8976 
8977   movl(xstart, xlen);
8978   decrementl(xstart);
8979   jcc(Assembler::negative, L_done);
8980 
8981   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
8982 
8983   Label L_second_loop;
8984   testl(kdx, kdx);
8985   jcc(Assembler::zero, L_second_loop);
8986 
8987   Label L_carry;
8988   subl(kdx, 1);
8989   jcc(Assembler::zero, L_carry);
8990 
8991   movl(Address(z, kdx, Address::times_4,  0), carry);
8992   shrq(carry, 32);
8993   subl(kdx, 1);
8994 
8995   bind(L_carry);
8996   movl(Address(z, kdx, Address::times_4,  0), carry);
8997 
8998   // Second and third (nested) loops.
8999   //
9000   // for (int i = xstart-1; i >= 0; i--) { // Second loop
9001   //   carry = 0;
9002   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
9003   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
9004   //                    (z[k] & LONG_MASK) + carry;
9005   //     z[k] = (int)product;
9006   //     carry = product >>> 32;
9007   //   }
9008   //   z[i] = (int)carry;
9009   // }
9010   //
9011   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
9012 
9013   const Register jdx = tmp1;
9014 
9015   bind(L_second_loop);
9016   xorl(carry, carry);    // carry = 0;
9017   movl(jdx, ylen);       // j = ystart+1
9018 
9019   subl(xstart, 1);       // i = xstart-1;
9020   jcc(Assembler::negative, L_done);
9021 
9022   push (z);
9023 
9024   Label L_last_x;
9025   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
9026   subl(xstart, 1);       // i = xstart-1;
9027   jcc(Assembler::negative, L_last_x);
9028 
9029   if (UseBMI2Instructions) {
9030     movq(rdx,  Address(x, xstart, Address::times_4,  0));
9031     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
9032   } else {
9033     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
9034     rorq(x_xstart, 32);  // convert big-endian to little-endian
9035   }
9036 
9037   Label L_third_loop_prologue;
9038   bind(L_third_loop_prologue);
9039 
9040   push (x);
9041   push (xstart);
9042   push (ylen);
9043 
9044 
9045   if (UseBMI2Instructions) {
9046     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
9047   } else { // !UseBMI2Instructions
9048     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
9049   }
9050 
9051   pop(ylen);
9052   pop(xlen);
9053   pop(x);
9054   pop(z);
9055 
9056   movl(tmp3, xlen);
9057   addl(tmp3, 1);
9058   movl(Address(z, tmp3, Address::times_4,  0), carry);
9059   subl(tmp3, 1);
9060   jccb(Assembler::negative, L_done);
9061 
9062   shrq(carry, 32);
9063   movl(Address(z, tmp3, Address::times_4,  0), carry);
9064   jmp(L_second_loop);
9065 
9066   // Next infrequent code is moved outside loops.
9067   bind(L_last_x);
9068   if (UseBMI2Instructions) {
9069     movl(rdx, Address(x,  0));
9070   } else {
9071     movl(x_xstart, Address(x,  0));
9072   }
9073   jmp(L_third_loop_prologue);
9074 
9075   bind(L_done);
9076 
9077   pop(zlen);
9078   pop(xlen);
9079 
9080   pop(tmp5);
9081   pop(tmp4);
9082   pop(tmp3);
9083   pop(tmp2);
9084   pop(tmp1);
9085 }
9086 
9087 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
9088   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
9089   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
9090   Label VECTOR64_LOOP, VECTOR64_TAIL, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
9091   Label VECTOR32_LOOP, VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
9092   Label VECTOR16_TAIL, VECTOR8_TAIL, VECTOR4_TAIL;
9093   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
9094   Label SAME_TILL_END, DONE;
9095   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
9096 
9097   //scale is in rcx in both Win64 and Unix
9098   ShortBranchVerifier sbv(this);
9099 
9100   shlq(length);
9101   xorq(result, result);
9102 
9103   if ((UseAVX > 2) &&
9104       VM_Version::supports_avx512vlbw()) {
9105     set_vector_masking();  // opening of the stub context for programming mask registers
9106     cmpq(length, 64);
9107     jcc(Assembler::less, VECTOR32_TAIL);
9108     movq(tmp1, length);
9109     andq(tmp1, 0x3F);      // tail count
9110     andq(length, ~(0x3F)); //vector count
9111 
9112     bind(VECTOR64_LOOP);
9113     // AVX512 code to compare 64 byte vectors.
9114     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
9115     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
9116     kortestql(k7, k7);
9117     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
9118     addq(result, 64);
9119     subq(length, 64);
9120     jccb(Assembler::notZero, VECTOR64_LOOP);
9121 
9122     //bind(VECTOR64_TAIL);
9123     testq(tmp1, tmp1);
9124     jcc(Assembler::zero, SAME_TILL_END);
9125 
9126     bind(VECTOR64_TAIL);
9127     // AVX512 code to compare upto 63 byte vectors.
9128     // Save k1
9129     kmovql(k3, k1);
9130     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
9131     shlxq(tmp2, tmp2, tmp1);
9132     notq(tmp2);
9133     kmovql(k1, tmp2);
9134 
9135     evmovdqub(rymm0, k1, Address(obja, result), Assembler::AVX_512bit);
9136     evpcmpeqb(k7, k1, rymm0, Address(objb, result), Assembler::AVX_512bit);
9137 
9138     ktestql(k7, k1);
9139     // Restore k1
9140     kmovql(k1, k3);
9141     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
9142 
9143     bind(VECTOR64_NOT_EQUAL);
9144     kmovql(tmp1, k7);
9145     notq(tmp1);
9146     tzcntq(tmp1, tmp1);
9147     addq(result, tmp1);
9148     shrq(result);
9149     jmp(DONE);
9150     bind(VECTOR32_TAIL);
9151     clear_vector_masking();   // closing of the stub context for programming mask registers
9152   }
9153 
9154   cmpq(length, 8);
9155   jcc(Assembler::equal, VECTOR8_LOOP);
9156   jcc(Assembler::less, VECTOR4_TAIL);
9157 
9158   if (UseAVX >= 2) {
9159 
9160     cmpq(length, 16);
9161     jcc(Assembler::equal, VECTOR16_LOOP);
9162     jcc(Assembler::less, VECTOR8_LOOP);
9163 
9164     cmpq(length, 32);
9165     jccb(Assembler::less, VECTOR16_TAIL);
9166 
9167     subq(length, 32);
9168     bind(VECTOR32_LOOP);
9169     vmovdqu(rymm0, Address(obja, result));
9170     vmovdqu(rymm1, Address(objb, result));
9171     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
9172     vptest(rymm2, rymm2);
9173     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
9174     addq(result, 32);
9175     subq(length, 32);
9176     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
9177     addq(length, 32);
9178     jcc(Assembler::equal, SAME_TILL_END);
9179     //falling through if less than 32 bytes left //close the branch here.
9180 
9181     bind(VECTOR16_TAIL);
9182     cmpq(length, 16);
9183     jccb(Assembler::less, VECTOR8_TAIL);
9184     bind(VECTOR16_LOOP);
9185     movdqu(rymm0, Address(obja, result));
9186     movdqu(rymm1, Address(objb, result));
9187     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
9188     ptest(rymm2, rymm2);
9189     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
9190     addq(result, 16);
9191     subq(length, 16);
9192     jcc(Assembler::equal, SAME_TILL_END);
9193     //falling through if less than 16 bytes left
9194   } else {//regular intrinsics
9195 
9196     cmpq(length, 16);
9197     jccb(Assembler::less, VECTOR8_TAIL);
9198 
9199     subq(length, 16);
9200     bind(VECTOR16_LOOP);
9201     movdqu(rymm0, Address(obja, result));
9202     movdqu(rymm1, Address(objb, result));
9203     pxor(rymm0, rymm1);
9204     ptest(rymm0, rymm0);
9205     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
9206     addq(result, 16);
9207     subq(length, 16);
9208     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
9209     addq(length, 16);
9210     jcc(Assembler::equal, SAME_TILL_END);
9211     //falling through if less than 16 bytes left
9212   }
9213 
9214   bind(VECTOR8_TAIL);
9215   cmpq(length, 8);
9216   jccb(Assembler::less, VECTOR4_TAIL);
9217   bind(VECTOR8_LOOP);
9218   movq(tmp1, Address(obja, result));
9219   movq(tmp2, Address(objb, result));
9220   xorq(tmp1, tmp2);
9221   testq(tmp1, tmp1);
9222   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
9223   addq(result, 8);
9224   subq(length, 8);
9225   jcc(Assembler::equal, SAME_TILL_END);
9226   //falling through if less than 8 bytes left
9227 
9228   bind(VECTOR4_TAIL);
9229   cmpq(length, 4);
9230   jccb(Assembler::less, BYTES_TAIL);
9231   bind(VECTOR4_LOOP);
9232   movl(tmp1, Address(obja, result));
9233   xorl(tmp1, Address(objb, result));
9234   testl(tmp1, tmp1);
9235   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
9236   addq(result, 4);
9237   subq(length, 4);
9238   jcc(Assembler::equal, SAME_TILL_END);
9239   //falling through if less than 4 bytes left
9240 
9241   bind(BYTES_TAIL);
9242   bind(BYTES_LOOP);
9243   load_unsigned_byte(tmp1, Address(obja, result));
9244   load_unsigned_byte(tmp2, Address(objb, result));
9245   xorl(tmp1, tmp2);
9246   testl(tmp1, tmp1);
9247   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9248   decq(length);
9249   jcc(Assembler::zero, SAME_TILL_END);
9250   incq(result);
9251   load_unsigned_byte(tmp1, Address(obja, result));
9252   load_unsigned_byte(tmp2, Address(objb, result));
9253   xorl(tmp1, tmp2);
9254   testl(tmp1, tmp1);
9255   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9256   decq(length);
9257   jcc(Assembler::zero, SAME_TILL_END);
9258   incq(result);
9259   load_unsigned_byte(tmp1, Address(obja, result));
9260   load_unsigned_byte(tmp2, Address(objb, result));
9261   xorl(tmp1, tmp2);
9262   testl(tmp1, tmp1);
9263   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9264   jmp(SAME_TILL_END);
9265 
9266   if (UseAVX >= 2) {
9267     bind(VECTOR32_NOT_EQUAL);
9268     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
9269     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
9270     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
9271     vpmovmskb(tmp1, rymm0);
9272     bsfq(tmp1, tmp1);
9273     addq(result, tmp1);
9274     shrq(result);
9275     jmp(DONE);
9276   }
9277 
9278   bind(VECTOR16_NOT_EQUAL);
9279   if (UseAVX >= 2) {
9280     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
9281     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
9282     pxor(rymm0, rymm2);
9283   } else {
9284     pcmpeqb(rymm2, rymm2);
9285     pxor(rymm0, rymm1);
9286     pcmpeqb(rymm0, rymm1);
9287     pxor(rymm0, rymm2);
9288   }
9289   pmovmskb(tmp1, rymm0);
9290   bsfq(tmp1, tmp1);
9291   addq(result, tmp1);
9292   shrq(result);
9293   jmpb(DONE);
9294 
9295   bind(VECTOR8_NOT_EQUAL);
9296   bind(VECTOR4_NOT_EQUAL);
9297   bsfq(tmp1, tmp1);
9298   shrq(tmp1, 3);
9299   addq(result, tmp1);
9300   bind(BYTES_NOT_EQUAL);
9301   shrq(result);
9302   jmpb(DONE);
9303 
9304   bind(SAME_TILL_END);
9305   mov64(result, -1);
9306 
9307   bind(DONE);
9308 }
9309 
9310 //Helper functions for square_to_len()
9311 
9312 /**
9313  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
9314  * Preserves x and z and modifies rest of the registers.
9315  */
9316 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9317   // Perform square and right shift by 1
9318   // Handle odd xlen case first, then for even xlen do the following
9319   // jlong carry = 0;
9320   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
9321   //     huge_128 product = x[j:j+1] * x[j:j+1];
9322   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
9323   //     z[i+2:i+3] = (jlong)(product >>> 1);
9324   //     carry = (jlong)product;
9325   // }
9326 
9327   xorq(tmp5, tmp5);     // carry
9328   xorq(rdxReg, rdxReg);
9329   xorl(tmp1, tmp1);     // index for x
9330   xorl(tmp4, tmp4);     // index for z
9331 
9332   Label L_first_loop, L_first_loop_exit;
9333 
9334   testl(xlen, 1);
9335   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
9336 
9337   // Square and right shift by 1 the odd element using 32 bit multiply
9338   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
9339   imulq(raxReg, raxReg);
9340   shrq(raxReg, 1);
9341   adcq(tmp5, 0);
9342   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
9343   incrementl(tmp1);
9344   addl(tmp4, 2);
9345 
9346   // Square and  right shift by 1 the rest using 64 bit multiply
9347   bind(L_first_loop);
9348   cmpptr(tmp1, xlen);
9349   jccb(Assembler::equal, L_first_loop_exit);
9350 
9351   // Square
9352   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
9353   rorq(raxReg, 32);    // convert big-endian to little-endian
9354   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
9355 
9356   // Right shift by 1 and save carry
9357   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
9358   rcrq(rdxReg, 1);
9359   rcrq(raxReg, 1);
9360   adcq(tmp5, 0);
9361 
9362   // Store result in z
9363   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
9364   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
9365 
9366   // Update indices for x and z
9367   addl(tmp1, 2);
9368   addl(tmp4, 4);
9369   jmp(L_first_loop);
9370 
9371   bind(L_first_loop_exit);
9372 }
9373 
9374 
9375 /**
9376  * Perform the following multiply add operation using BMI2 instructions
9377  * carry:sum = sum + op1*op2 + carry
9378  * op2 should be in rdx
9379  * op2 is preserved, all other registers are modified
9380  */
9381 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
9382   // assert op2 is rdx
9383   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
9384   addq(sum, carry);
9385   adcq(tmp2, 0);
9386   addq(sum, op1);
9387   adcq(tmp2, 0);
9388   movq(carry, tmp2);
9389 }
9390 
9391 /**
9392  * Perform the following multiply add operation:
9393  * carry:sum = sum + op1*op2 + carry
9394  * Preserves op1, op2 and modifies rest of registers
9395  */
9396 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
9397   // rdx:rax = op1 * op2
9398   movq(raxReg, op2);
9399   mulq(op1);
9400 
9401   //  rdx:rax = sum + carry + rdx:rax
9402   addq(sum, carry);
9403   adcq(rdxReg, 0);
9404   addq(sum, raxReg);
9405   adcq(rdxReg, 0);
9406 
9407   // carry:sum = rdx:sum
9408   movq(carry, rdxReg);
9409 }
9410 
9411 /**
9412  * Add 64 bit long carry into z[] with carry propogation.
9413  * Preserves z and carry register values and modifies rest of registers.
9414  *
9415  */
9416 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
9417   Label L_fourth_loop, L_fourth_loop_exit;
9418 
9419   movl(tmp1, 1);
9420   subl(zlen, 2);
9421   addq(Address(z, zlen, Address::times_4, 0), carry);
9422 
9423   bind(L_fourth_loop);
9424   jccb(Assembler::carryClear, L_fourth_loop_exit);
9425   subl(zlen, 2);
9426   jccb(Assembler::negative, L_fourth_loop_exit);
9427   addq(Address(z, zlen, Address::times_4, 0), tmp1);
9428   jmp(L_fourth_loop);
9429   bind(L_fourth_loop_exit);
9430 }
9431 
9432 /**
9433  * Shift z[] left by 1 bit.
9434  * Preserves x, len, z and zlen registers and modifies rest of the registers.
9435  *
9436  */
9437 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
9438 
9439   Label L_fifth_loop, L_fifth_loop_exit;
9440 
9441   // Fifth loop
9442   // Perform primitiveLeftShift(z, zlen, 1)
9443 
9444   const Register prev_carry = tmp1;
9445   const Register new_carry = tmp4;
9446   const Register value = tmp2;
9447   const Register zidx = tmp3;
9448 
9449   // int zidx, carry;
9450   // long value;
9451   // carry = 0;
9452   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
9453   //    (carry:value)  = (z[i] << 1) | carry ;
9454   //    z[i] = value;
9455   // }
9456 
9457   movl(zidx, zlen);
9458   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
9459 
9460   bind(L_fifth_loop);
9461   decl(zidx);  // Use decl to preserve carry flag
9462   decl(zidx);
9463   jccb(Assembler::negative, L_fifth_loop_exit);
9464 
9465   if (UseBMI2Instructions) {
9466      movq(value, Address(z, zidx, Address::times_4, 0));
9467      rclq(value, 1);
9468      rorxq(value, value, 32);
9469      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9470   }
9471   else {
9472     // clear new_carry
9473     xorl(new_carry, new_carry);
9474 
9475     // Shift z[i] by 1, or in previous carry and save new carry
9476     movq(value, Address(z, zidx, Address::times_4, 0));
9477     shlq(value, 1);
9478     adcl(new_carry, 0);
9479 
9480     orq(value, prev_carry);
9481     rorq(value, 0x20);
9482     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9483 
9484     // Set previous carry = new carry
9485     movl(prev_carry, new_carry);
9486   }
9487   jmp(L_fifth_loop);
9488 
9489   bind(L_fifth_loop_exit);
9490 }
9491 
9492 
9493 /**
9494  * Code for BigInteger::squareToLen() intrinsic
9495  *
9496  * rdi: x
9497  * rsi: len
9498  * r8:  z
9499  * rcx: zlen
9500  * r12: tmp1
9501  * r13: tmp2
9502  * r14: tmp3
9503  * r15: tmp4
9504  * rbx: tmp5
9505  *
9506  */
9507 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9508 
9509   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, fifth_loop, fifth_loop_exit, L_last_x, L_multiply;
9510   push(tmp1);
9511   push(tmp2);
9512   push(tmp3);
9513   push(tmp4);
9514   push(tmp5);
9515 
9516   // First loop
9517   // Store the squares, right shifted one bit (i.e., divided by 2).
9518   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
9519 
9520   // Add in off-diagonal sums.
9521   //
9522   // Second, third (nested) and fourth loops.
9523   // zlen +=2;
9524   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
9525   //    carry = 0;
9526   //    long op2 = x[xidx:xidx+1];
9527   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
9528   //       k -= 2;
9529   //       long op1 = x[j:j+1];
9530   //       long sum = z[k:k+1];
9531   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
9532   //       z[k:k+1] = sum;
9533   //    }
9534   //    add_one_64(z, k, carry, tmp_regs);
9535   // }
9536 
9537   const Register carry = tmp5;
9538   const Register sum = tmp3;
9539   const Register op1 = tmp4;
9540   Register op2 = tmp2;
9541 
9542   push(zlen);
9543   push(len);
9544   addl(zlen,2);
9545   bind(L_second_loop);
9546   xorq(carry, carry);
9547   subl(zlen, 4);
9548   subl(len, 2);
9549   push(zlen);
9550   push(len);
9551   cmpl(len, 0);
9552   jccb(Assembler::lessEqual, L_second_loop_exit);
9553 
9554   // Multiply an array by one 64 bit long.
9555   if (UseBMI2Instructions) {
9556     op2 = rdxReg;
9557     movq(op2, Address(x, len, Address::times_4,  0));
9558     rorxq(op2, op2, 32);
9559   }
9560   else {
9561     movq(op2, Address(x, len, Address::times_4,  0));
9562     rorq(op2, 32);
9563   }
9564 
9565   bind(L_third_loop);
9566   decrementl(len);
9567   jccb(Assembler::negative, L_third_loop_exit);
9568   decrementl(len);
9569   jccb(Assembler::negative, L_last_x);
9570 
9571   movq(op1, Address(x, len, Address::times_4,  0));
9572   rorq(op1, 32);
9573 
9574   bind(L_multiply);
9575   subl(zlen, 2);
9576   movq(sum, Address(z, zlen, Address::times_4,  0));
9577 
9578   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
9579   if (UseBMI2Instructions) {
9580     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
9581   }
9582   else {
9583     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9584   }
9585 
9586   movq(Address(z, zlen, Address::times_4, 0), sum);
9587 
9588   jmp(L_third_loop);
9589   bind(L_third_loop_exit);
9590 
9591   // Fourth loop
9592   // Add 64 bit long carry into z with carry propogation.
9593   // Uses offsetted zlen.
9594   add_one_64(z, zlen, carry, tmp1);
9595 
9596   pop(len);
9597   pop(zlen);
9598   jmp(L_second_loop);
9599 
9600   // Next infrequent code is moved outside loops.
9601   bind(L_last_x);
9602   movl(op1, Address(x, 0));
9603   jmp(L_multiply);
9604 
9605   bind(L_second_loop_exit);
9606   pop(len);
9607   pop(zlen);
9608   pop(len);
9609   pop(zlen);
9610 
9611   // Fifth loop
9612   // Shift z left 1 bit.
9613   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
9614 
9615   // z[zlen-1] |= x[len-1] & 1;
9616   movl(tmp3, Address(x, len, Address::times_4, -4));
9617   andl(tmp3, 1);
9618   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
9619 
9620   pop(tmp5);
9621   pop(tmp4);
9622   pop(tmp3);
9623   pop(tmp2);
9624   pop(tmp1);
9625 }
9626 
9627 /**
9628  * Helper function for mul_add()
9629  * Multiply the in[] by int k and add to out[] starting at offset offs using
9630  * 128 bit by 32 bit multiply and return the carry in tmp5.
9631  * Only quad int aligned length of in[] is operated on in this function.
9632  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
9633  * This function preserves out, in and k registers.
9634  * len and offset point to the appropriate index in "in" & "out" correspondingly
9635  * tmp5 has the carry.
9636  * other registers are temporary and are modified.
9637  *
9638  */
9639 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
9640   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
9641   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9642 
9643   Label L_first_loop, L_first_loop_exit;
9644 
9645   movl(tmp1, len);
9646   shrl(tmp1, 2);
9647 
9648   bind(L_first_loop);
9649   subl(tmp1, 1);
9650   jccb(Assembler::negative, L_first_loop_exit);
9651 
9652   subl(len, 4);
9653   subl(offset, 4);
9654 
9655   Register op2 = tmp2;
9656   const Register sum = tmp3;
9657   const Register op1 = tmp4;
9658   const Register carry = tmp5;
9659 
9660   if (UseBMI2Instructions) {
9661     op2 = rdxReg;
9662   }
9663 
9664   movq(op1, Address(in, len, Address::times_4,  8));
9665   rorq(op1, 32);
9666   movq(sum, Address(out, offset, Address::times_4,  8));
9667   rorq(sum, 32);
9668   if (UseBMI2Instructions) {
9669     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9670   }
9671   else {
9672     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9673   }
9674   // Store back in big endian from little endian
9675   rorq(sum, 0x20);
9676   movq(Address(out, offset, Address::times_4,  8), sum);
9677 
9678   movq(op1, Address(in, len, Address::times_4,  0));
9679   rorq(op1, 32);
9680   movq(sum, Address(out, offset, Address::times_4,  0));
9681   rorq(sum, 32);
9682   if (UseBMI2Instructions) {
9683     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9684   }
9685   else {
9686     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9687   }
9688   // Store back in big endian from little endian
9689   rorq(sum, 0x20);
9690   movq(Address(out, offset, Address::times_4,  0), sum);
9691 
9692   jmp(L_first_loop);
9693   bind(L_first_loop_exit);
9694 }
9695 
9696 /**
9697  * Code for BigInteger::mulAdd() intrinsic
9698  *
9699  * rdi: out
9700  * rsi: in
9701  * r11: offs (out.length - offset)
9702  * rcx: len
9703  * r8:  k
9704  * r12: tmp1
9705  * r13: tmp2
9706  * r14: tmp3
9707  * r15: tmp4
9708  * rbx: tmp5
9709  * Multiply the in[] by word k and add to out[], return the carry in rax
9710  */
9711 void MacroAssembler::mul_add(Register out, Register in, Register offs,
9712    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
9713    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9714 
9715   Label L_carry, L_last_in, L_done;
9716 
9717 // carry = 0;
9718 // for (int j=len-1; j >= 0; j--) {
9719 //    long product = (in[j] & LONG_MASK) * kLong +
9720 //                   (out[offs] & LONG_MASK) + carry;
9721 //    out[offs--] = (int)product;
9722 //    carry = product >>> 32;
9723 // }
9724 //
9725   push(tmp1);
9726   push(tmp2);
9727   push(tmp3);
9728   push(tmp4);
9729   push(tmp5);
9730 
9731   Register op2 = tmp2;
9732   const Register sum = tmp3;
9733   const Register op1 = tmp4;
9734   const Register carry =  tmp5;
9735 
9736   if (UseBMI2Instructions) {
9737     op2 = rdxReg;
9738     movl(op2, k);
9739   }
9740   else {
9741     movl(op2, k);
9742   }
9743 
9744   xorq(carry, carry);
9745 
9746   //First loop
9747 
9748   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
9749   //The carry is in tmp5
9750   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
9751 
9752   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
9753   decrementl(len);
9754   jccb(Assembler::negative, L_carry);
9755   decrementl(len);
9756   jccb(Assembler::negative, L_last_in);
9757 
9758   movq(op1, Address(in, len, Address::times_4,  0));
9759   rorq(op1, 32);
9760 
9761   subl(offs, 2);
9762   movq(sum, Address(out, offs, Address::times_4,  0));
9763   rorq(sum, 32);
9764 
9765   if (UseBMI2Instructions) {
9766     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9767   }
9768   else {
9769     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9770   }
9771 
9772   // Store back in big endian from little endian
9773   rorq(sum, 0x20);
9774   movq(Address(out, offs, Address::times_4,  0), sum);
9775 
9776   testl(len, len);
9777   jccb(Assembler::zero, L_carry);
9778 
9779   //Multiply the last in[] entry, if any
9780   bind(L_last_in);
9781   movl(op1, Address(in, 0));
9782   movl(sum, Address(out, offs, Address::times_4,  -4));
9783 
9784   movl(raxReg, k);
9785   mull(op1); //tmp4 * eax -> edx:eax
9786   addl(sum, carry);
9787   adcl(rdxReg, 0);
9788   addl(sum, raxReg);
9789   adcl(rdxReg, 0);
9790   movl(carry, rdxReg);
9791 
9792   movl(Address(out, offs, Address::times_4,  -4), sum);
9793 
9794   bind(L_carry);
9795   //return tmp5/carry as carry in rax
9796   movl(rax, carry);
9797 
9798   bind(L_done);
9799   pop(tmp5);
9800   pop(tmp4);
9801   pop(tmp3);
9802   pop(tmp2);
9803   pop(tmp1);
9804 }
9805 #endif
9806 
9807 /**
9808  * Emits code to update CRC-32 with a byte value according to constants in table
9809  *
9810  * @param [in,out]crc   Register containing the crc.
9811  * @param [in]val       Register containing the byte to fold into the CRC.
9812  * @param [in]table     Register containing the table of crc constants.
9813  *
9814  * uint32_t crc;
9815  * val = crc_table[(val ^ crc) & 0xFF];
9816  * crc = val ^ (crc >> 8);
9817  *
9818  */
9819 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
9820   xorl(val, crc);
9821   andl(val, 0xFF);
9822   shrl(crc, 8); // unsigned shift
9823   xorl(crc, Address(table, val, Address::times_4, 0));
9824 }
9825 
9826 /**
9827 * Fold four 128-bit data chunks
9828 */
9829 void MacroAssembler::fold_128bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
9830   evpclmulhdq(xtmp, xK, xcrc, Assembler::AVX_512bit); // [123:64]
9831   evpclmulldq(xcrc, xK, xcrc, Assembler::AVX_512bit); // [63:0]
9832   evpxorq(xcrc, xcrc, Address(buf, offset), Assembler::AVX_512bit /* vector_len */);
9833   evpxorq(xcrc, xcrc, xtmp, Assembler::AVX_512bit /* vector_len */);
9834 }
9835 
9836 /**
9837  * Fold 128-bit data chunk
9838  */
9839 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
9840   if (UseAVX > 0) {
9841     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
9842     vpclmulldq(xcrc, xK, xcrc); // [63:0]
9843     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
9844     pxor(xcrc, xtmp);
9845   } else {
9846     movdqa(xtmp, xcrc);
9847     pclmulhdq(xtmp, xK);   // [123:64]
9848     pclmulldq(xcrc, xK);   // [63:0]
9849     pxor(xcrc, xtmp);
9850     movdqu(xtmp, Address(buf, offset));
9851     pxor(xcrc, xtmp);
9852   }
9853 }
9854 
9855 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
9856   if (UseAVX > 0) {
9857     vpclmulhdq(xtmp, xK, xcrc);
9858     vpclmulldq(xcrc, xK, xcrc);
9859     pxor(xcrc, xbuf);
9860     pxor(xcrc, xtmp);
9861   } else {
9862     movdqa(xtmp, xcrc);
9863     pclmulhdq(xtmp, xK);
9864     pclmulldq(xcrc, xK);
9865     pxor(xcrc, xbuf);
9866     pxor(xcrc, xtmp);
9867   }
9868 }
9869 
9870 /**
9871  * 8-bit folds to compute 32-bit CRC
9872  *
9873  * uint64_t xcrc;
9874  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
9875  */
9876 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
9877   movdl(tmp, xcrc);
9878   andl(tmp, 0xFF);
9879   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
9880   psrldq(xcrc, 1); // unsigned shift one byte
9881   pxor(xcrc, xtmp);
9882 }
9883 
9884 /**
9885  * uint32_t crc;
9886  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
9887  */
9888 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
9889   movl(tmp, crc);
9890   andl(tmp, 0xFF);
9891   shrl(crc, 8);
9892   xorl(crc, Address(table, tmp, Address::times_4, 0));
9893 }
9894 
9895 /**
9896  * @param crc   register containing existing CRC (32-bit)
9897  * @param buf   register pointing to input byte buffer (byte*)
9898  * @param len   register containing number of bytes
9899  * @param table register that will contain address of CRC table
9900  * @param tmp   scratch register
9901  */
9902 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
9903   assert_different_registers(crc, buf, len, table, tmp, rax);
9904 
9905   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
9906   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
9907 
9908   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
9909   // context for the registers used, where all instructions below are using 128-bit mode
9910   // On EVEX without VL and BW, these instructions will all be AVX.
9911   if (VM_Version::supports_avx512vlbw()) {
9912     movl(tmp, 0xffff);
9913     kmovwl(k1, tmp);
9914   }
9915 
9916   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
9917   notl(crc); // ~crc
9918   cmpl(len, 16);
9919   jcc(Assembler::less, L_tail);
9920 
9921   // Align buffer to 16 bytes
9922   movl(tmp, buf);
9923   andl(tmp, 0xF);
9924   jccb(Assembler::zero, L_aligned);
9925   subl(tmp,  16);
9926   addl(len, tmp);
9927 
9928   align(4);
9929   BIND(L_align_loop);
9930   movsbl(rax, Address(buf, 0)); // load byte with sign extension
9931   update_byte_crc32(crc, rax, table);
9932   increment(buf);
9933   incrementl(tmp);
9934   jccb(Assembler::less, L_align_loop);
9935 
9936   BIND(L_aligned);
9937   movl(tmp, len); // save
9938   shrl(len, 4);
9939   jcc(Assembler::zero, L_tail_restore);
9940 
9941   // Fold total 512 bits of polynomial on each iteration
9942   if (VM_Version::supports_vpclmulqdq()) {
9943     Label Parallel_loop, L_No_Parallel;
9944 
9945     cmpl(len, 8);
9946     jccb(Assembler::less, L_No_Parallel);
9947 
9948     movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
9949     evmovdquq(xmm1, Address(buf, 0), Assembler::AVX_512bit);
9950     movdl(xmm5, crc);
9951     evpxorq(xmm1, xmm1, xmm5, Assembler::AVX_512bit);
9952     addptr(buf, 64);
9953     subl(len, 7);
9954     evshufi64x2(xmm0, xmm0, xmm0, 0x00, Assembler::AVX_512bit); //propagate the mask from 128 bits to 512 bits
9955 
9956     BIND(Parallel_loop);
9957     fold_128bit_crc32_avx512(xmm1, xmm0, xmm5, buf, 0);
9958     addptr(buf, 64);
9959     subl(len, 4);
9960     jcc(Assembler::greater, Parallel_loop);
9961 
9962     vextracti64x2(xmm2, xmm1, 0x01);
9963     vextracti64x2(xmm3, xmm1, 0x02);
9964     vextracti64x2(xmm4, xmm1, 0x03);
9965     jmp(L_fold_512b);
9966 
9967     BIND(L_No_Parallel);
9968   }
9969   // Fold crc into first bytes of vector
9970   movdqa(xmm1, Address(buf, 0));
9971   movdl(rax, xmm1);
9972   xorl(crc, rax);
9973   if (VM_Version::supports_sse4_1()) {
9974     pinsrd(xmm1, crc, 0);
9975   } else {
9976     pinsrw(xmm1, crc, 0);
9977     shrl(crc, 16);
9978     pinsrw(xmm1, crc, 1);
9979   }
9980   addptr(buf, 16);
9981   subl(len, 4); // len > 0
9982   jcc(Assembler::less, L_fold_tail);
9983 
9984   movdqa(xmm2, Address(buf,  0));
9985   movdqa(xmm3, Address(buf, 16));
9986   movdqa(xmm4, Address(buf, 32));
9987   addptr(buf, 48);
9988   subl(len, 3);
9989   jcc(Assembler::lessEqual, L_fold_512b);
9990 
9991   // Fold total 512 bits of polynomial on each iteration,
9992   // 128 bits per each of 4 parallel streams.
9993   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
9994 
9995   align(32);
9996   BIND(L_fold_512b_loop);
9997   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
9998   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
9999   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
10000   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
10001   addptr(buf, 64);
10002   subl(len, 4);
10003   jcc(Assembler::greater, L_fold_512b_loop);
10004 
10005   // Fold 512 bits to 128 bits.
10006   BIND(L_fold_512b);
10007   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
10008   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
10009   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
10010   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
10011 
10012   // Fold the rest of 128 bits data chunks
10013   BIND(L_fold_tail);
10014   addl(len, 3);
10015   jccb(Assembler::lessEqual, L_fold_128b);
10016   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
10017 
10018   BIND(L_fold_tail_loop);
10019   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
10020   addptr(buf, 16);
10021   decrementl(len);
10022   jccb(Assembler::greater, L_fold_tail_loop);
10023 
10024   // Fold 128 bits in xmm1 down into 32 bits in crc register.
10025   BIND(L_fold_128b);
10026   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()));
10027   if (UseAVX > 0) {
10028     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
10029     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
10030     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
10031   } else {
10032     movdqa(xmm2, xmm0);
10033     pclmulqdq(xmm2, xmm1, 0x1);
10034     movdqa(xmm3, xmm0);
10035     pand(xmm3, xmm2);
10036     pclmulqdq(xmm0, xmm3, 0x1);
10037   }
10038   psrldq(xmm1, 8);
10039   psrldq(xmm2, 4);
10040   pxor(xmm0, xmm1);
10041   pxor(xmm0, xmm2);
10042 
10043   // 8 8-bit folds to compute 32-bit CRC.
10044   for (int j = 0; j < 4; j++) {
10045     fold_8bit_crc32(xmm0, table, xmm1, rax);
10046   }
10047   movdl(crc, xmm0); // mov 32 bits to general register
10048   for (int j = 0; j < 4; j++) {
10049     fold_8bit_crc32(crc, table, rax);
10050   }
10051 
10052   BIND(L_tail_restore);
10053   movl(len, tmp); // restore
10054   BIND(L_tail);
10055   andl(len, 0xf);
10056   jccb(Assembler::zero, L_exit);
10057 
10058   // Fold the rest of bytes
10059   align(4);
10060   BIND(L_tail_loop);
10061   movsbl(rax, Address(buf, 0)); // load byte with sign extension
10062   update_byte_crc32(crc, rax, table);
10063   increment(buf);
10064   decrementl(len);
10065   jccb(Assembler::greater, L_tail_loop);
10066 
10067   BIND(L_exit);
10068   notl(crc); // ~c
10069 }
10070 
10071 #ifdef _LP64
10072 // S. Gueron / Information Processing Letters 112 (2012) 184
10073 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
10074 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
10075 // Output: the 64-bit carry-less product of B * CONST
10076 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
10077                                      Register tmp1, Register tmp2, Register tmp3) {
10078   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
10079   if (n > 0) {
10080     addq(tmp3, n * 256 * 8);
10081   }
10082   //    Q1 = TABLEExt[n][B & 0xFF];
10083   movl(tmp1, in);
10084   andl(tmp1, 0x000000FF);
10085   shll(tmp1, 3);
10086   addq(tmp1, tmp3);
10087   movq(tmp1, Address(tmp1, 0));
10088 
10089   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
10090   movl(tmp2, in);
10091   shrl(tmp2, 8);
10092   andl(tmp2, 0x000000FF);
10093   shll(tmp2, 3);
10094   addq(tmp2, tmp3);
10095   movq(tmp2, Address(tmp2, 0));
10096 
10097   shlq(tmp2, 8);
10098   xorq(tmp1, tmp2);
10099 
10100   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
10101   movl(tmp2, in);
10102   shrl(tmp2, 16);
10103   andl(tmp2, 0x000000FF);
10104   shll(tmp2, 3);
10105   addq(tmp2, tmp3);
10106   movq(tmp2, Address(tmp2, 0));
10107 
10108   shlq(tmp2, 16);
10109   xorq(tmp1, tmp2);
10110 
10111   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
10112   shrl(in, 24);
10113   andl(in, 0x000000FF);
10114   shll(in, 3);
10115   addq(in, tmp3);
10116   movq(in, Address(in, 0));
10117 
10118   shlq(in, 24);
10119   xorq(in, tmp1);
10120   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
10121 }
10122 
10123 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
10124                                       Register in_out,
10125                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
10126                                       XMMRegister w_xtmp2,
10127                                       Register tmp1,
10128                                       Register n_tmp2, Register n_tmp3) {
10129   if (is_pclmulqdq_supported) {
10130     movdl(w_xtmp1, in_out); // modified blindly
10131 
10132     movl(tmp1, const_or_pre_comp_const_index);
10133     movdl(w_xtmp2, tmp1);
10134     pclmulqdq(w_xtmp1, w_xtmp2, 0);
10135 
10136     movdq(in_out, w_xtmp1);
10137   } else {
10138     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
10139   }
10140 }
10141 
10142 // Recombination Alternative 2: No bit-reflections
10143 // T1 = (CRC_A * U1) << 1
10144 // T2 = (CRC_B * U2) << 1
10145 // C1 = T1 >> 32
10146 // C2 = T2 >> 32
10147 // T1 = T1 & 0xFFFFFFFF
10148 // T2 = T2 & 0xFFFFFFFF
10149 // T1 = CRC32(0, T1)
10150 // T2 = CRC32(0, T2)
10151 // C1 = C1 ^ T1
10152 // C2 = C2 ^ T2
10153 // CRC = C1 ^ C2 ^ CRC_C
10154 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
10155                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10156                                      Register tmp1, Register tmp2,
10157                                      Register n_tmp3) {
10158   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10159   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10160   shlq(in_out, 1);
10161   movl(tmp1, in_out);
10162   shrq(in_out, 32);
10163   xorl(tmp2, tmp2);
10164   crc32(tmp2, tmp1, 4);
10165   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
10166   shlq(in1, 1);
10167   movl(tmp1, in1);
10168   shrq(in1, 32);
10169   xorl(tmp2, tmp2);
10170   crc32(tmp2, tmp1, 4);
10171   xorl(in1, tmp2);
10172   xorl(in_out, in1);
10173   xorl(in_out, in2);
10174 }
10175 
10176 // Set N to predefined value
10177 // Subtract from a lenght of a buffer
10178 // execute in a loop:
10179 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
10180 // for i = 1 to N do
10181 //  CRC_A = CRC32(CRC_A, A[i])
10182 //  CRC_B = CRC32(CRC_B, B[i])
10183 //  CRC_C = CRC32(CRC_C, C[i])
10184 // end for
10185 // Recombine
10186 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
10187                                        Register in_out1, Register in_out2, Register in_out3,
10188                                        Register tmp1, Register tmp2, Register tmp3,
10189                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10190                                        Register tmp4, Register tmp5,
10191                                        Register n_tmp6) {
10192   Label L_processPartitions;
10193   Label L_processPartition;
10194   Label L_exit;
10195 
10196   bind(L_processPartitions);
10197   cmpl(in_out1, 3 * size);
10198   jcc(Assembler::less, L_exit);
10199     xorl(tmp1, tmp1);
10200     xorl(tmp2, tmp2);
10201     movq(tmp3, in_out2);
10202     addq(tmp3, size);
10203 
10204     bind(L_processPartition);
10205       crc32(in_out3, Address(in_out2, 0), 8);
10206       crc32(tmp1, Address(in_out2, size), 8);
10207       crc32(tmp2, Address(in_out2, size * 2), 8);
10208       addq(in_out2, 8);
10209       cmpq(in_out2, tmp3);
10210       jcc(Assembler::less, L_processPartition);
10211     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
10212             w_xtmp1, w_xtmp2, w_xtmp3,
10213             tmp4, tmp5,
10214             n_tmp6);
10215     addq(in_out2, 2 * size);
10216     subl(in_out1, 3 * size);
10217     jmp(L_processPartitions);
10218 
10219   bind(L_exit);
10220 }
10221 #else
10222 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
10223                                      Register tmp1, Register tmp2, Register tmp3,
10224                                      XMMRegister xtmp1, XMMRegister xtmp2) {
10225   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
10226   if (n > 0) {
10227     addl(tmp3, n * 256 * 8);
10228   }
10229   //    Q1 = TABLEExt[n][B & 0xFF];
10230   movl(tmp1, in_out);
10231   andl(tmp1, 0x000000FF);
10232   shll(tmp1, 3);
10233   addl(tmp1, tmp3);
10234   movq(xtmp1, Address(tmp1, 0));
10235 
10236   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
10237   movl(tmp2, in_out);
10238   shrl(tmp2, 8);
10239   andl(tmp2, 0x000000FF);
10240   shll(tmp2, 3);
10241   addl(tmp2, tmp3);
10242   movq(xtmp2, Address(tmp2, 0));
10243 
10244   psllq(xtmp2, 8);
10245   pxor(xtmp1, xtmp2);
10246 
10247   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
10248   movl(tmp2, in_out);
10249   shrl(tmp2, 16);
10250   andl(tmp2, 0x000000FF);
10251   shll(tmp2, 3);
10252   addl(tmp2, tmp3);
10253   movq(xtmp2, Address(tmp2, 0));
10254 
10255   psllq(xtmp2, 16);
10256   pxor(xtmp1, xtmp2);
10257 
10258   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
10259   shrl(in_out, 24);
10260   andl(in_out, 0x000000FF);
10261   shll(in_out, 3);
10262   addl(in_out, tmp3);
10263   movq(xtmp2, Address(in_out, 0));
10264 
10265   psllq(xtmp2, 24);
10266   pxor(xtmp1, xtmp2); // Result in CXMM
10267   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
10268 }
10269 
10270 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
10271                                       Register in_out,
10272                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
10273                                       XMMRegister w_xtmp2,
10274                                       Register tmp1,
10275                                       Register n_tmp2, Register n_tmp3) {
10276   if (is_pclmulqdq_supported) {
10277     movdl(w_xtmp1, in_out);
10278 
10279     movl(tmp1, const_or_pre_comp_const_index);
10280     movdl(w_xtmp2, tmp1);
10281     pclmulqdq(w_xtmp1, w_xtmp2, 0);
10282     // Keep result in XMM since GPR is 32 bit in length
10283   } else {
10284     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
10285   }
10286 }
10287 
10288 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
10289                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10290                                      Register tmp1, Register tmp2,
10291                                      Register n_tmp3) {
10292   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10293   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10294 
10295   psllq(w_xtmp1, 1);
10296   movdl(tmp1, w_xtmp1);
10297   psrlq(w_xtmp1, 32);
10298   movdl(in_out, w_xtmp1);
10299 
10300   xorl(tmp2, tmp2);
10301   crc32(tmp2, tmp1, 4);
10302   xorl(in_out, tmp2);
10303 
10304   psllq(w_xtmp2, 1);
10305   movdl(tmp1, w_xtmp2);
10306   psrlq(w_xtmp2, 32);
10307   movdl(in1, w_xtmp2);
10308 
10309   xorl(tmp2, tmp2);
10310   crc32(tmp2, tmp1, 4);
10311   xorl(in1, tmp2);
10312   xorl(in_out, in1);
10313   xorl(in_out, in2);
10314 }
10315 
10316 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
10317                                        Register in_out1, Register in_out2, Register in_out3,
10318                                        Register tmp1, Register tmp2, Register tmp3,
10319                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10320                                        Register tmp4, Register tmp5,
10321                                        Register n_tmp6) {
10322   Label L_processPartitions;
10323   Label L_processPartition;
10324   Label L_exit;
10325 
10326   bind(L_processPartitions);
10327   cmpl(in_out1, 3 * size);
10328   jcc(Assembler::less, L_exit);
10329     xorl(tmp1, tmp1);
10330     xorl(tmp2, tmp2);
10331     movl(tmp3, in_out2);
10332     addl(tmp3, size);
10333 
10334     bind(L_processPartition);
10335       crc32(in_out3, Address(in_out2, 0), 4);
10336       crc32(tmp1, Address(in_out2, size), 4);
10337       crc32(tmp2, Address(in_out2, size*2), 4);
10338       crc32(in_out3, Address(in_out2, 0+4), 4);
10339       crc32(tmp1, Address(in_out2, size+4), 4);
10340       crc32(tmp2, Address(in_out2, size*2+4), 4);
10341       addl(in_out2, 8);
10342       cmpl(in_out2, tmp3);
10343       jcc(Assembler::less, L_processPartition);
10344 
10345         push(tmp3);
10346         push(in_out1);
10347         push(in_out2);
10348         tmp4 = tmp3;
10349         tmp5 = in_out1;
10350         n_tmp6 = in_out2;
10351 
10352       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
10353             w_xtmp1, w_xtmp2, w_xtmp3,
10354             tmp4, tmp5,
10355             n_tmp6);
10356 
10357         pop(in_out2);
10358         pop(in_out1);
10359         pop(tmp3);
10360 
10361     addl(in_out2, 2 * size);
10362     subl(in_out1, 3 * size);
10363     jmp(L_processPartitions);
10364 
10365   bind(L_exit);
10366 }
10367 #endif //LP64
10368 
10369 #ifdef _LP64
10370 // Algorithm 2: Pipelined usage of the CRC32 instruction.
10371 // Input: A buffer I of L bytes.
10372 // Output: the CRC32C value of the buffer.
10373 // Notations:
10374 // Write L = 24N + r, with N = floor (L/24).
10375 // r = L mod 24 (0 <= r < 24).
10376 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
10377 // N quadwords, and R consists of r bytes.
10378 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
10379 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
10380 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
10381 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
10382 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10383                                           Register tmp1, Register tmp2, Register tmp3,
10384                                           Register tmp4, Register tmp5, Register tmp6,
10385                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10386                                           bool is_pclmulqdq_supported) {
10387   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10388   Label L_wordByWord;
10389   Label L_byteByByteProlog;
10390   Label L_byteByByte;
10391   Label L_exit;
10392 
10393   if (is_pclmulqdq_supported ) {
10394     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10395     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
10396 
10397     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10398     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10399 
10400     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10401     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10402     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
10403   } else {
10404     const_or_pre_comp_const_index[0] = 1;
10405     const_or_pre_comp_const_index[1] = 0;
10406 
10407     const_or_pre_comp_const_index[2] = 3;
10408     const_or_pre_comp_const_index[3] = 2;
10409 
10410     const_or_pre_comp_const_index[4] = 5;
10411     const_or_pre_comp_const_index[5] = 4;
10412    }
10413   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10414                     in2, in1, in_out,
10415                     tmp1, tmp2, tmp3,
10416                     w_xtmp1, w_xtmp2, w_xtmp3,
10417                     tmp4, tmp5,
10418                     tmp6);
10419   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10420                     in2, in1, in_out,
10421                     tmp1, tmp2, tmp3,
10422                     w_xtmp1, w_xtmp2, w_xtmp3,
10423                     tmp4, tmp5,
10424                     tmp6);
10425   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10426                     in2, in1, in_out,
10427                     tmp1, tmp2, tmp3,
10428                     w_xtmp1, w_xtmp2, w_xtmp3,
10429                     tmp4, tmp5,
10430                     tmp6);
10431   movl(tmp1, in2);
10432   andl(tmp1, 0x00000007);
10433   negl(tmp1);
10434   addl(tmp1, in2);
10435   addq(tmp1, in1);
10436 
10437   BIND(L_wordByWord);
10438   cmpq(in1, tmp1);
10439   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10440     crc32(in_out, Address(in1, 0), 4);
10441     addq(in1, 4);
10442     jmp(L_wordByWord);
10443 
10444   BIND(L_byteByByteProlog);
10445   andl(in2, 0x00000007);
10446   movl(tmp2, 1);
10447 
10448   BIND(L_byteByByte);
10449   cmpl(tmp2, in2);
10450   jccb(Assembler::greater, L_exit);
10451     crc32(in_out, Address(in1, 0), 1);
10452     incq(in1);
10453     incl(tmp2);
10454     jmp(L_byteByByte);
10455 
10456   BIND(L_exit);
10457 }
10458 #else
10459 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10460                                           Register tmp1, Register  tmp2, Register tmp3,
10461                                           Register tmp4, Register  tmp5, Register tmp6,
10462                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10463                                           bool is_pclmulqdq_supported) {
10464   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10465   Label L_wordByWord;
10466   Label L_byteByByteProlog;
10467   Label L_byteByByte;
10468   Label L_exit;
10469 
10470   if (is_pclmulqdq_supported) {
10471     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10472     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
10473 
10474     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10475     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10476 
10477     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10478     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10479   } else {
10480     const_or_pre_comp_const_index[0] = 1;
10481     const_or_pre_comp_const_index[1] = 0;
10482 
10483     const_or_pre_comp_const_index[2] = 3;
10484     const_or_pre_comp_const_index[3] = 2;
10485 
10486     const_or_pre_comp_const_index[4] = 5;
10487     const_or_pre_comp_const_index[5] = 4;
10488   }
10489   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10490                     in2, in1, in_out,
10491                     tmp1, tmp2, tmp3,
10492                     w_xtmp1, w_xtmp2, w_xtmp3,
10493                     tmp4, tmp5,
10494                     tmp6);
10495   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10496                     in2, in1, in_out,
10497                     tmp1, tmp2, tmp3,
10498                     w_xtmp1, w_xtmp2, w_xtmp3,
10499                     tmp4, tmp5,
10500                     tmp6);
10501   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10502                     in2, in1, in_out,
10503                     tmp1, tmp2, tmp3,
10504                     w_xtmp1, w_xtmp2, w_xtmp3,
10505                     tmp4, tmp5,
10506                     tmp6);
10507   movl(tmp1, in2);
10508   andl(tmp1, 0x00000007);
10509   negl(tmp1);
10510   addl(tmp1, in2);
10511   addl(tmp1, in1);
10512 
10513   BIND(L_wordByWord);
10514   cmpl(in1, tmp1);
10515   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10516     crc32(in_out, Address(in1,0), 4);
10517     addl(in1, 4);
10518     jmp(L_wordByWord);
10519 
10520   BIND(L_byteByByteProlog);
10521   andl(in2, 0x00000007);
10522   movl(tmp2, 1);
10523 
10524   BIND(L_byteByByte);
10525   cmpl(tmp2, in2);
10526   jccb(Assembler::greater, L_exit);
10527     movb(tmp1, Address(in1, 0));
10528     crc32(in_out, tmp1, 1);
10529     incl(in1);
10530     incl(tmp2);
10531     jmp(L_byteByByte);
10532 
10533   BIND(L_exit);
10534 }
10535 #endif // LP64
10536 #undef BIND
10537 #undef BLOCK_COMMENT
10538 
10539 // Compress char[] array to byte[].
10540 //   ..\jdk\src\java.base\share\classes\java\lang\StringUTF16.java
10541 //   @HotSpotIntrinsicCandidate
10542 //   private static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
10543 //     for (int i = 0; i < len; i++) {
10544 //       int c = src[srcOff++];
10545 //       if (c >>> 8 != 0) {
10546 //         return 0;
10547 //       }
10548 //       dst[dstOff++] = (byte)c;
10549 //     }
10550 //     return len;
10551 //   }
10552 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
10553   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
10554   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
10555   Register tmp5, Register result) {
10556   Label copy_chars_loop, return_length, return_zero, done;
10557 
10558   // rsi: src
10559   // rdi: dst
10560   // rdx: len
10561   // rcx: tmp5
10562   // rax: result
10563 
10564   // rsi holds start addr of source char[] to be compressed
10565   // rdi holds start addr of destination byte[]
10566   // rdx holds length
10567 
10568   assert(len != result, "");
10569 
10570   // save length for return
10571   push(len);
10572 
10573   if ((UseAVX > 2) && // AVX512
10574     VM_Version::supports_avx512vlbw() &&
10575     VM_Version::supports_bmi2()) {
10576 
10577     set_vector_masking();  // opening of the stub context for programming mask registers
10578 
10579     Label copy_32_loop, copy_loop_tail, restore_k1_return_zero, below_threshold;
10580 
10581     // alignment
10582     Label post_alignment;
10583 
10584     // if length of the string is less than 16, handle it in an old fashioned way
10585     testl(len, -32);
10586     jcc(Assembler::zero, below_threshold);
10587 
10588     // First check whether a character is compressable ( <= 0xFF).
10589     // Create mask to test for Unicode chars inside zmm vector
10590     movl(result, 0x00FF);
10591     evpbroadcastw(tmp2Reg, result, Assembler::AVX_512bit);
10592 
10593     // Save k1
10594     kmovql(k3, k1);
10595 
10596     testl(len, -64);
10597     jcc(Assembler::zero, post_alignment);
10598 
10599     movl(tmp5, dst);
10600     andl(tmp5, (32 - 1));
10601     negl(tmp5);
10602     andl(tmp5, (32 - 1));
10603 
10604     // bail out when there is nothing to be done
10605     testl(tmp5, 0xFFFFFFFF);
10606     jcc(Assembler::zero, post_alignment);
10607 
10608     // ~(~0 << len), where len is the # of remaining elements to process
10609     movl(result, 0xFFFFFFFF);
10610     shlxl(result, result, tmp5);
10611     notl(result);
10612     kmovdl(k1, result);
10613 
10614     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
10615     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10616     ktestd(k2, k1);
10617     jcc(Assembler::carryClear, restore_k1_return_zero);
10618 
10619     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
10620 
10621     addptr(src, tmp5);
10622     addptr(src, tmp5);
10623     addptr(dst, tmp5);
10624     subl(len, tmp5);
10625 
10626     bind(post_alignment);
10627     // end of alignment
10628 
10629     movl(tmp5, len);
10630     andl(tmp5, (32 - 1));    // tail count (in chars)
10631     andl(len, ~(32 - 1));    // vector count (in chars)
10632     jcc(Assembler::zero, copy_loop_tail);
10633 
10634     lea(src, Address(src, len, Address::times_2));
10635     lea(dst, Address(dst, len, Address::times_1));
10636     negptr(len);
10637 
10638     bind(copy_32_loop);
10639     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
10640     evpcmpuw(k2, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10641     kortestdl(k2, k2);
10642     jcc(Assembler::carryClear, restore_k1_return_zero);
10643 
10644     // All elements in current processed chunk are valid candidates for
10645     // compression. Write a truncated byte elements to the memory.
10646     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
10647     addptr(len, 32);
10648     jcc(Assembler::notZero, copy_32_loop);
10649 
10650     bind(copy_loop_tail);
10651     // bail out when there is nothing to be done
10652     testl(tmp5, 0xFFFFFFFF);
10653     // Restore k1
10654     kmovql(k1, k3);
10655     jcc(Assembler::zero, return_length);
10656 
10657     movl(len, tmp5);
10658 
10659     // ~(~0 << len), where len is the # of remaining elements to process
10660     movl(result, 0xFFFFFFFF);
10661     shlxl(result, result, len);
10662     notl(result);
10663 
10664     kmovdl(k1, result);
10665 
10666     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
10667     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10668     ktestd(k2, k1);
10669     jcc(Assembler::carryClear, restore_k1_return_zero);
10670 
10671     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
10672     // Restore k1
10673     kmovql(k1, k3);
10674     jmp(return_length);
10675 
10676     bind(restore_k1_return_zero);
10677     // Restore k1
10678     kmovql(k1, k3);
10679     jmp(return_zero);
10680 
10681     clear_vector_masking();   // closing of the stub context for programming mask registers
10682 
10683     bind(below_threshold);
10684   }
10685 
10686   if (UseSSE42Intrinsics) {
10687     Label copy_32_loop, copy_16, copy_tail;
10688 
10689     movl(result, len);
10690 
10691     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
10692 
10693     // vectored compression
10694     andl(len, 0xfffffff0);    // vector count (in chars)
10695     andl(result, 0x0000000f);    // tail count (in chars)
10696     testl(len, len);
10697     jcc(Assembler::zero, copy_16);
10698 
10699     // compress 16 chars per iter
10700     movdl(tmp1Reg, tmp5);
10701     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
10702     pxor(tmp4Reg, tmp4Reg);
10703 
10704     lea(src, Address(src, len, Address::times_2));
10705     lea(dst, Address(dst, len, Address::times_1));
10706     negptr(len);
10707 
10708     bind(copy_32_loop);
10709     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
10710     por(tmp4Reg, tmp2Reg);
10711     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
10712     por(tmp4Reg, tmp3Reg);
10713     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
10714     jcc(Assembler::notZero, return_zero);
10715     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
10716     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
10717     addptr(len, 16);
10718     jcc(Assembler::notZero, copy_32_loop);
10719 
10720     // compress next vector of 8 chars (if any)
10721     bind(copy_16);
10722     movl(len, result);
10723     andl(len, 0xfffffff8);    // vector count (in chars)
10724     andl(result, 0x00000007);    // tail count (in chars)
10725     testl(len, len);
10726     jccb(Assembler::zero, copy_tail);
10727 
10728     movdl(tmp1Reg, tmp5);
10729     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
10730     pxor(tmp3Reg, tmp3Reg);
10731 
10732     movdqu(tmp2Reg, Address(src, 0));
10733     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
10734     jccb(Assembler::notZero, return_zero);
10735     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
10736     movq(Address(dst, 0), tmp2Reg);
10737     addptr(src, 16);
10738     addptr(dst, 8);
10739 
10740     bind(copy_tail);
10741     movl(len, result);
10742   }
10743   // compress 1 char per iter
10744   testl(len, len);
10745   jccb(Assembler::zero, return_length);
10746   lea(src, Address(src, len, Address::times_2));
10747   lea(dst, Address(dst, len, Address::times_1));
10748   negptr(len);
10749 
10750   bind(copy_chars_loop);
10751   load_unsigned_short(result, Address(src, len, Address::times_2));
10752   testl(result, 0xff00);      // check if Unicode char
10753   jccb(Assembler::notZero, return_zero);
10754   movb(Address(dst, len, Address::times_1), result);  // ASCII char; compress to 1 byte
10755   increment(len);
10756   jcc(Assembler::notZero, copy_chars_loop);
10757 
10758   // if compression succeeded, return length
10759   bind(return_length);
10760   pop(result);
10761   jmpb(done);
10762 
10763   // if compression failed, return 0
10764   bind(return_zero);
10765   xorl(result, result);
10766   addptr(rsp, wordSize);
10767 
10768   bind(done);
10769 }
10770 
10771 // Inflate byte[] array to char[].
10772 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
10773 //   @HotSpotIntrinsicCandidate
10774 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
10775 //     for (int i = 0; i < len; i++) {
10776 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
10777 //     }
10778 //   }
10779 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
10780   XMMRegister tmp1, Register tmp2) {
10781   Label copy_chars_loop, done, below_threshold;
10782   // rsi: src
10783   // rdi: dst
10784   // rdx: len
10785   // rcx: tmp2
10786 
10787   // rsi holds start addr of source byte[] to be inflated
10788   // rdi holds start addr of destination char[]
10789   // rdx holds length
10790   assert_different_registers(src, dst, len, tmp2);
10791 
10792   if ((UseAVX > 2) && // AVX512
10793     VM_Version::supports_avx512vlbw() &&
10794     VM_Version::supports_bmi2()) {
10795 
10796     set_vector_masking();  // opening of the stub context for programming mask registers
10797 
10798     Label copy_32_loop, copy_tail;
10799     Register tmp3_aliased = len;
10800 
10801     // if length of the string is less than 16, handle it in an old fashioned way
10802     testl(len, -16);
10803     jcc(Assembler::zero, below_threshold);
10804 
10805     // In order to use only one arithmetic operation for the main loop we use
10806     // this pre-calculation
10807     movl(tmp2, len);
10808     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
10809     andl(len, -32);     // vector count
10810     jccb(Assembler::zero, copy_tail);
10811 
10812     lea(src, Address(src, len, Address::times_1));
10813     lea(dst, Address(dst, len, Address::times_2));
10814     negptr(len);
10815 
10816 
10817     // inflate 32 chars per iter
10818     bind(copy_32_loop);
10819     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
10820     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
10821     addptr(len, 32);
10822     jcc(Assembler::notZero, copy_32_loop);
10823 
10824     bind(copy_tail);
10825     // bail out when there is nothing to be done
10826     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
10827     jcc(Assembler::zero, done);
10828 
10829     // Save k1
10830     kmovql(k2, k1);
10831 
10832     // ~(~0 << length), where length is the # of remaining elements to process
10833     movl(tmp3_aliased, -1);
10834     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
10835     notl(tmp3_aliased);
10836     kmovdl(k1, tmp3_aliased);
10837     evpmovzxbw(tmp1, k1, Address(src, 0), Assembler::AVX_512bit);
10838     evmovdquw(Address(dst, 0), k1, tmp1, Assembler::AVX_512bit);
10839 
10840     // Restore k1
10841     kmovql(k1, k2);
10842     jmp(done);
10843 
10844     clear_vector_masking();   // closing of the stub context for programming mask registers
10845   }
10846   if (UseSSE42Intrinsics) {
10847     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
10848 
10849     movl(tmp2, len);
10850 
10851     if (UseAVX > 1) {
10852       andl(tmp2, (16 - 1));
10853       andl(len, -16);
10854       jccb(Assembler::zero, copy_new_tail);
10855     } else {
10856       andl(tmp2, 0x00000007);   // tail count (in chars)
10857       andl(len, 0xfffffff8);    // vector count (in chars)
10858       jccb(Assembler::zero, copy_tail);
10859     }
10860 
10861     // vectored inflation
10862     lea(src, Address(src, len, Address::times_1));
10863     lea(dst, Address(dst, len, Address::times_2));
10864     negptr(len);
10865 
10866     if (UseAVX > 1) {
10867       bind(copy_16_loop);
10868       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
10869       vmovdqu(Address(dst, len, Address::times_2), tmp1);
10870       addptr(len, 16);
10871       jcc(Assembler::notZero, copy_16_loop);
10872 
10873       bind(below_threshold);
10874       bind(copy_new_tail);
10875       if ((UseAVX > 2) &&
10876         VM_Version::supports_avx512vlbw() &&
10877         VM_Version::supports_bmi2()) {
10878         movl(tmp2, len);
10879       } else {
10880         movl(len, tmp2);
10881       }
10882       andl(tmp2, 0x00000007);
10883       andl(len, 0xFFFFFFF8);
10884       jccb(Assembler::zero, copy_tail);
10885 
10886       pmovzxbw(tmp1, Address(src, 0));
10887       movdqu(Address(dst, 0), tmp1);
10888       addptr(src, 8);
10889       addptr(dst, 2 * 8);
10890 
10891       jmp(copy_tail, true);
10892     }
10893 
10894     // inflate 8 chars per iter
10895     bind(copy_8_loop);
10896     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
10897     movdqu(Address(dst, len, Address::times_2), tmp1);
10898     addptr(len, 8);
10899     jcc(Assembler::notZero, copy_8_loop);
10900 
10901     bind(copy_tail);
10902     movl(len, tmp2);
10903 
10904     cmpl(len, 4);
10905     jccb(Assembler::less, copy_bytes);
10906 
10907     movdl(tmp1, Address(src, 0));  // load 4 byte chars
10908     pmovzxbw(tmp1, tmp1);
10909     movq(Address(dst, 0), tmp1);
10910     subptr(len, 4);
10911     addptr(src, 4);
10912     addptr(dst, 8);
10913 
10914     bind(copy_bytes);
10915   } else {
10916     bind(below_threshold);
10917   }
10918 
10919   testl(len, len);
10920   jccb(Assembler::zero, done);
10921   lea(src, Address(src, len, Address::times_1));
10922   lea(dst, Address(dst, len, Address::times_2));
10923   negptr(len);
10924 
10925   // inflate 1 char per iter
10926   bind(copy_chars_loop);
10927   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
10928   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
10929   increment(len);
10930   jcc(Assembler::notZero, copy_chars_loop);
10931 
10932   bind(done);
10933 }
10934 
10935 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
10936   switch (cond) {
10937     // Note some conditions are synonyms for others
10938     case Assembler::zero:         return Assembler::notZero;
10939     case Assembler::notZero:      return Assembler::zero;
10940     case Assembler::less:         return Assembler::greaterEqual;
10941     case Assembler::lessEqual:    return Assembler::greater;
10942     case Assembler::greater:      return Assembler::lessEqual;
10943     case Assembler::greaterEqual: return Assembler::less;
10944     case Assembler::below:        return Assembler::aboveEqual;
10945     case Assembler::belowEqual:   return Assembler::above;
10946     case Assembler::above:        return Assembler::belowEqual;
10947     case Assembler::aboveEqual:   return Assembler::below;
10948     case Assembler::overflow:     return Assembler::noOverflow;
10949     case Assembler::noOverflow:   return Assembler::overflow;
10950     case Assembler::negative:     return Assembler::positive;
10951     case Assembler::positive:     return Assembler::negative;
10952     case Assembler::parity:       return Assembler::noParity;
10953     case Assembler::noParity:     return Assembler::parity;
10954   }
10955   ShouldNotReachHere(); return Assembler::overflow;
10956 }
10957 
10958 SkipIfEqual::SkipIfEqual(
10959     MacroAssembler* masm, const bool* flag_addr, bool value) {
10960   _masm = masm;
10961   _masm->cmp8(ExternalAddress((address)flag_addr), value);
10962   _masm->jcc(Assembler::equal, _label);
10963 }
10964 
10965 SkipIfEqual::~SkipIfEqual() {
10966   _masm->bind(_label);
10967 }
10968 
10969 // 32-bit Windows has its own fast-path implementation
10970 // of get_thread
10971 #if !defined(WIN32) || defined(_LP64)
10972 
10973 // This is simply a call to Thread::current()
10974 void MacroAssembler::get_thread(Register thread) {
10975   if (thread != rax) {
10976     push(rax);
10977   }
10978   LP64_ONLY(push(rdi);)
10979   LP64_ONLY(push(rsi);)
10980   push(rdx);
10981   push(rcx);
10982 #ifdef _LP64
10983   push(r8);
10984   push(r9);
10985   push(r10);
10986   push(r11);
10987 #endif
10988 
10989   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
10990 
10991 #ifdef _LP64
10992   pop(r11);
10993   pop(r10);
10994   pop(r9);
10995   pop(r8);
10996 #endif
10997   pop(rcx);
10998   pop(rdx);
10999   LP64_ONLY(pop(rsi);)
11000   LP64_ONLY(pop(rdi);)
11001   if (thread != rax) {
11002     mov(thread, rax);
11003     pop(rax);
11004   }
11005 }
11006 
11007 #endif