1 /*
   2  * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "asm/assembler.hpp"
  28 #include "asm/assembler.inline.hpp"
  29 #include "compiler/disassembler.hpp"
  30 #include "gc/shared/barrierSet.hpp"
  31 #include "gc/shared/barrierSetAssembler.hpp"
  32 #include "gc/shared/collectedHeap.inline.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "memory/universe.hpp"
  36 #include "oops/access.hpp"
  37 #include "oops/klass.inline.hpp"
  38 #include "prims/methodHandles.hpp"
  39 #include "runtime/biasedLocking.hpp"
  40 #include "runtime/flags/flagSetting.hpp"
  41 #include "runtime/interfaceSupport.inline.hpp"
  42 #include "runtime/objectMonitor.hpp"
  43 #include "runtime/os.hpp"
  44 #include "runtime/safepoint.hpp"
  45 #include "runtime/safepointMechanism.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/stubRoutines.hpp"
  48 #include "runtime/thread.hpp"
  49 #include "utilities/macros.hpp"
  50 #include "crc32c.h"
  51 #ifdef COMPILER2
  52 #include "opto/intrinsicnode.hpp"
  53 #endif
  54 
  55 #ifdef PRODUCT
  56 #define BLOCK_COMMENT(str) /* nothing */
  57 #define STOP(error) stop(error)
  58 #else
  59 #define BLOCK_COMMENT(str) block_comment(str)
  60 #define STOP(error) block_comment(error); stop(error)
  61 #endif
  62 
  63 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  64 
  65 #ifdef ASSERT
  66 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  67 #endif
  68 
  69 static Assembler::Condition reverse[] = {
  70     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  71     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  72     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  73     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  74     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
  75     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
  76     Assembler::above          /* belowEqual    = 0x6 */ ,
  77     Assembler::belowEqual     /* above         = 0x7 */ ,
  78     Assembler::positive       /* negative      = 0x8 */ ,
  79     Assembler::negative       /* positive      = 0x9 */ ,
  80     Assembler::noParity       /* parity        = 0xa */ ,
  81     Assembler::parity         /* noParity      = 0xb */ ,
  82     Assembler::greaterEqual   /* less          = 0xc */ ,
  83     Assembler::less           /* greaterEqual  = 0xd */ ,
  84     Assembler::greater        /* lessEqual     = 0xe */ ,
  85     Assembler::lessEqual      /* greater       = 0xf, */
  86 
  87 };
  88 
  89 
  90 // Implementation of MacroAssembler
  91 
  92 // First all the versions that have distinct versions depending on 32/64 bit
  93 // Unless the difference is trivial (1 line or so).
  94 
  95 #ifndef _LP64
  96 
  97 // 32bit versions
  98 
  99 Address MacroAssembler::as_Address(AddressLiteral adr) {
 100   return Address(adr.target(), adr.rspec());
 101 }
 102 
 103 Address MacroAssembler::as_Address(ArrayAddress adr) {
 104   return Address::make_array(adr);
 105 }
 106 
 107 void MacroAssembler::call_VM_leaf_base(address entry_point,
 108                                        int number_of_arguments) {
 109   call(RuntimeAddress(entry_point));
 110   increment(rsp, number_of_arguments * wordSize);
 111 }
 112 
 113 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
 114   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 115 }
 116 
 117 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
 118   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 119 }
 120 
 121 void MacroAssembler::cmpoop_raw(Address src1, jobject obj) {
 122   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 123 }
 124 
 125 void MacroAssembler::cmpoop_raw(Register src1, jobject obj) {
 126   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 127 }
 128 
 129 void MacroAssembler::cmpoop(Address src1, jobject obj) {
 130   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 131   bs->obj_equals(this, IN_HEAP, src1, obj);
 132 }
 133 
 134 void MacroAssembler::cmpoop(Register src1, jobject obj) {
 135   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 136   bs->obj_equals(this, IN_HEAP, src1, obj);
 137 }
 138 
 139 void MacroAssembler::extend_sign(Register hi, Register lo) {
 140   // According to Intel Doc. AP-526, "Integer Divide", p.18.
 141   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
 142     cdql();
 143   } else {
 144     movl(hi, lo);
 145     sarl(hi, 31);
 146   }
 147 }
 148 
 149 void MacroAssembler::jC2(Register tmp, Label& L) {
 150   // set parity bit if FPU flag C2 is set (via rax)
 151   save_rax(tmp);
 152   fwait(); fnstsw_ax();
 153   sahf();
 154   restore_rax(tmp);
 155   // branch
 156   jcc(Assembler::parity, L);
 157 }
 158 
 159 void MacroAssembler::jnC2(Register tmp, Label& L) {
 160   // set parity bit if FPU flag C2 is set (via rax)
 161   save_rax(tmp);
 162   fwait(); fnstsw_ax();
 163   sahf();
 164   restore_rax(tmp);
 165   // branch
 166   jcc(Assembler::noParity, L);
 167 }
 168 
 169 // 32bit can do a case table jump in one instruction but we no longer allow the base
 170 // to be installed in the Address class
 171 void MacroAssembler::jump(ArrayAddress entry) {
 172   jmp(as_Address(entry));
 173 }
 174 
 175 // Note: y_lo will be destroyed
 176 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 177   // Long compare for Java (semantics as described in JVM spec.)
 178   Label high, low, done;
 179 
 180   cmpl(x_hi, y_hi);
 181   jcc(Assembler::less, low);
 182   jcc(Assembler::greater, high);
 183   // x_hi is the return register
 184   xorl(x_hi, x_hi);
 185   cmpl(x_lo, y_lo);
 186   jcc(Assembler::below, low);
 187   jcc(Assembler::equal, done);
 188 
 189   bind(high);
 190   xorl(x_hi, x_hi);
 191   increment(x_hi);
 192   jmp(done);
 193 
 194   bind(low);
 195   xorl(x_hi, x_hi);
 196   decrementl(x_hi);
 197 
 198   bind(done);
 199 }
 200 
 201 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 202     mov_literal32(dst, (int32_t)src.target(), src.rspec());
 203 }
 204 
 205 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 206   // leal(dst, as_Address(adr));
 207   // see note in movl as to why we must use a move
 208   mov_literal32(dst, (int32_t) adr.target(), adr.rspec());
 209 }
 210 
 211 void MacroAssembler::leave() {
 212   mov(rsp, rbp);
 213   pop(rbp);
 214 }
 215 
 216 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
 217   // Multiplication of two Java long values stored on the stack
 218   // as illustrated below. Result is in rdx:rax.
 219   //
 220   // rsp ---> [  ??  ] \               \
 221   //            ....    | y_rsp_offset  |
 222   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
 223   //          [ y_hi ]                  | (in bytes)
 224   //            ....                    |
 225   //          [ x_lo ]                 /
 226   //          [ x_hi ]
 227   //            ....
 228   //
 229   // Basic idea: lo(result) = lo(x_lo * y_lo)
 230   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
 231   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
 232   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
 233   Label quick;
 234   // load x_hi, y_hi and check if quick
 235   // multiplication is possible
 236   movl(rbx, x_hi);
 237   movl(rcx, y_hi);
 238   movl(rax, rbx);
 239   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
 240   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
 241   // do full multiplication
 242   // 1st step
 243   mull(y_lo);                                    // x_hi * y_lo
 244   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
 245   // 2nd step
 246   movl(rax, x_lo);
 247   mull(rcx);                                     // x_lo * y_hi
 248   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
 249   // 3rd step
 250   bind(quick);                                   // note: rbx, = 0 if quick multiply!
 251   movl(rax, x_lo);
 252   mull(y_lo);                                    // x_lo * y_lo
 253   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
 254 }
 255 
 256 void MacroAssembler::lneg(Register hi, Register lo) {
 257   negl(lo);
 258   adcl(hi, 0);
 259   negl(hi);
 260 }
 261 
 262 void MacroAssembler::lshl(Register hi, Register lo) {
 263   // Java shift left long support (semantics as described in JVM spec., p.305)
 264   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
 265   // shift value is in rcx !
 266   assert(hi != rcx, "must not use rcx");
 267   assert(lo != rcx, "must not use rcx");
 268   const Register s = rcx;                        // shift count
 269   const int      n = BitsPerWord;
 270   Label L;
 271   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 272   cmpl(s, n);                                    // if (s < n)
 273   jcc(Assembler::less, L);                       // else (s >= n)
 274   movl(hi, lo);                                  // x := x << n
 275   xorl(lo, lo);
 276   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 277   bind(L);                                       // s (mod n) < n
 278   shldl(hi, lo);                                 // x := x << s
 279   shll(lo);
 280 }
 281 
 282 
 283 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
 284   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
 285   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
 286   assert(hi != rcx, "must not use rcx");
 287   assert(lo != rcx, "must not use rcx");
 288   const Register s = rcx;                        // shift count
 289   const int      n = BitsPerWord;
 290   Label L;
 291   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 292   cmpl(s, n);                                    // if (s < n)
 293   jcc(Assembler::less, L);                       // else (s >= n)
 294   movl(lo, hi);                                  // x := x >> n
 295   if (sign_extension) sarl(hi, 31);
 296   else                xorl(hi, hi);
 297   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 298   bind(L);                                       // s (mod n) < n
 299   shrdl(lo, hi);                                 // x := x >> s
 300   if (sign_extension) sarl(hi);
 301   else                shrl(hi);
 302 }
 303 
 304 void MacroAssembler::movoop(Register dst, jobject obj) {
 305   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 306 }
 307 
 308 void MacroAssembler::movoop(Address dst, jobject obj) {
 309   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 310 }
 311 
 312 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 313   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 314 }
 315 
 316 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 317   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 318 }
 319 
 320 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 321   // scratch register is not used,
 322   // it is defined to match parameters of 64-bit version of this method.
 323   if (src.is_lval()) {
 324     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
 325   } else {
 326     movl(dst, as_Address(src));
 327   }
 328 }
 329 
 330 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 331   movl(as_Address(dst), src);
 332 }
 333 
 334 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 335   movl(dst, as_Address(src));
 336 }
 337 
 338 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 339 void MacroAssembler::movptr(Address dst, intptr_t src) {
 340   movl(dst, src);
 341 }
 342 
 343 
 344 void MacroAssembler::pop_callee_saved_registers() {
 345   pop(rcx);
 346   pop(rdx);
 347   pop(rdi);
 348   pop(rsi);
 349 }
 350 
 351 void MacroAssembler::pop_fTOS() {
 352   fld_d(Address(rsp, 0));
 353   addl(rsp, 2 * wordSize);
 354 }
 355 
 356 void MacroAssembler::push_callee_saved_registers() {
 357   push(rsi);
 358   push(rdi);
 359   push(rdx);
 360   push(rcx);
 361 }
 362 
 363 void MacroAssembler::push_fTOS() {
 364   subl(rsp, 2 * wordSize);
 365   fstp_d(Address(rsp, 0));
 366 }
 367 
 368 
 369 void MacroAssembler::pushoop(jobject obj) {
 370   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
 371 }
 372 
 373 void MacroAssembler::pushklass(Metadata* obj) {
 374   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
 375 }
 376 
 377 void MacroAssembler::pushptr(AddressLiteral src) {
 378   if (src.is_lval()) {
 379     push_literal32((int32_t)src.target(), src.rspec());
 380   } else {
 381     pushl(as_Address(src));
 382   }
 383 }
 384 
 385 void MacroAssembler::set_word_if_not_zero(Register dst) {
 386   xorl(dst, dst);
 387   set_byte_if_not_zero(dst);
 388 }
 389 
 390 static void pass_arg0(MacroAssembler* masm, Register arg) {
 391   masm->push(arg);
 392 }
 393 
 394 static void pass_arg1(MacroAssembler* masm, Register arg) {
 395   masm->push(arg);
 396 }
 397 
 398 static void pass_arg2(MacroAssembler* masm, Register arg) {
 399   masm->push(arg);
 400 }
 401 
 402 static void pass_arg3(MacroAssembler* masm, Register arg) {
 403   masm->push(arg);
 404 }
 405 
 406 #ifndef PRODUCT
 407 extern "C" void findpc(intptr_t x);
 408 #endif
 409 
 410 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
 411   // In order to get locks to work, we need to fake a in_VM state
 412   JavaThread* thread = JavaThread::current();
 413   JavaThreadState saved_state = thread->thread_state();
 414   thread->set_thread_state(_thread_in_vm);
 415   if (ShowMessageBoxOnError) {
 416     JavaThread* thread = JavaThread::current();
 417     JavaThreadState saved_state = thread->thread_state();
 418     thread->set_thread_state(_thread_in_vm);
 419     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 420       ttyLocker ttyl;
 421       BytecodeCounter::print();
 422     }
 423     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 424     // This is the value of eip which points to where verify_oop will return.
 425     if (os::message_box(msg, "Execution stopped, print registers?")) {
 426       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
 427       BREAKPOINT;
 428     }
 429   } else {
 430     ttyLocker ttyl;
 431     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
 432   }
 433   // Don't assert holding the ttyLock
 434     assert(false, "DEBUG MESSAGE: %s", msg);
 435   ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 436 }
 437 
 438 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
 439   ttyLocker ttyl;
 440   FlagSetting fs(Debugging, true);
 441   tty->print_cr("eip = 0x%08x", eip);
 442 #ifndef PRODUCT
 443   if ((WizardMode || Verbose) && PrintMiscellaneous) {
 444     tty->cr();
 445     findpc(eip);
 446     tty->cr();
 447   }
 448 #endif
 449 #define PRINT_REG(rax) \
 450   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
 451   PRINT_REG(rax);
 452   PRINT_REG(rbx);
 453   PRINT_REG(rcx);
 454   PRINT_REG(rdx);
 455   PRINT_REG(rdi);
 456   PRINT_REG(rsi);
 457   PRINT_REG(rbp);
 458   PRINT_REG(rsp);
 459 #undef PRINT_REG
 460   // Print some words near top of staack.
 461   int* dump_sp = (int*) rsp;
 462   for (int col1 = 0; col1 < 8; col1++) {
 463     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 464     os::print_location(tty, *dump_sp++);
 465   }
 466   for (int row = 0; row < 16; row++) {
 467     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 468     for (int col = 0; col < 8; col++) {
 469       tty->print(" 0x%08x", *dump_sp++);
 470     }
 471     tty->cr();
 472   }
 473   // Print some instructions around pc:
 474   Disassembler::decode((address)eip-64, (address)eip);
 475   tty->print_cr("--------");
 476   Disassembler::decode((address)eip, (address)eip+32);
 477 }
 478 
 479 void MacroAssembler::stop(const char* msg) {
 480   ExternalAddress message((address)msg);
 481   // push address of message
 482   pushptr(message.addr());
 483   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 484   pusha();                                            // push registers
 485   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 486   hlt();
 487 }
 488 
 489 void MacroAssembler::warn(const char* msg) {
 490   push_CPU_state();
 491 
 492   ExternalAddress message((address) msg);
 493   // push address of message
 494   pushptr(message.addr());
 495 
 496   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
 497   addl(rsp, wordSize);       // discard argument
 498   pop_CPU_state();
 499 }
 500 
 501 void MacroAssembler::print_state() {
 502   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 503   pusha();                                            // push registers
 504 
 505   push_CPU_state();
 506   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
 507   pop_CPU_state();
 508 
 509   popa();
 510   addl(rsp, wordSize);
 511 }
 512 
 513 #else // _LP64
 514 
 515 // 64 bit versions
 516 
 517 Address MacroAssembler::as_Address(AddressLiteral adr) {
 518   // amd64 always does this as a pc-rel
 519   // we can be absolute or disp based on the instruction type
 520   // jmp/call are displacements others are absolute
 521   assert(!adr.is_lval(), "must be rval");
 522   assert(reachable(adr), "must be");
 523   return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
 524 
 525 }
 526 
 527 Address MacroAssembler::as_Address(ArrayAddress adr) {
 528   AddressLiteral base = adr.base();
 529   lea(rscratch1, base);
 530   Address index = adr.index();
 531   assert(index._disp == 0, "must not have disp"); // maybe it can?
 532   Address array(rscratch1, index._index, index._scale, index._disp);
 533   return array;
 534 }
 535 
 536 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
 537   Label L, E;
 538 
 539 #ifdef _WIN64
 540   // Windows always allocates space for it's register args
 541   assert(num_args <= 4, "only register arguments supported");
 542   subq(rsp,  frame::arg_reg_save_area_bytes);
 543 #endif
 544 
 545   // Align stack if necessary
 546   testl(rsp, 15);
 547   jcc(Assembler::zero, L);
 548 
 549   subq(rsp, 8);
 550   {
 551     call(RuntimeAddress(entry_point));
 552   }
 553   addq(rsp, 8);
 554   jmp(E);
 555 
 556   bind(L);
 557   {
 558     call(RuntimeAddress(entry_point));
 559   }
 560 
 561   bind(E);
 562 
 563 #ifdef _WIN64
 564   // restore stack pointer
 565   addq(rsp, frame::arg_reg_save_area_bytes);
 566 #endif
 567 
 568 }
 569 
 570 void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
 571   assert(!src2.is_lval(), "should use cmpptr");
 572 
 573   if (reachable(src2)) {
 574     cmpq(src1, as_Address(src2));
 575   } else {
 576     lea(rscratch1, src2);
 577     Assembler::cmpq(src1, Address(rscratch1, 0));
 578   }
 579 }
 580 
 581 int MacroAssembler::corrected_idivq(Register reg) {
 582   // Full implementation of Java ldiv and lrem; checks for special
 583   // case as described in JVM spec., p.243 & p.271.  The function
 584   // returns the (pc) offset of the idivl instruction - may be needed
 585   // for implicit exceptions.
 586   //
 587   //         normal case                           special case
 588   //
 589   // input : rax: dividend                         min_long
 590   //         reg: divisor   (may not be eax/edx)   -1
 591   //
 592   // output: rax: quotient  (= rax idiv reg)       min_long
 593   //         rdx: remainder (= rax irem reg)       0
 594   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
 595   static const int64_t min_long = 0x8000000000000000;
 596   Label normal_case, special_case;
 597 
 598   // check for special case
 599   cmp64(rax, ExternalAddress((address) &min_long));
 600   jcc(Assembler::notEqual, normal_case);
 601   xorl(rdx, rdx); // prepare rdx for possible special case (where
 602                   // remainder = 0)
 603   cmpq(reg, -1);
 604   jcc(Assembler::equal, special_case);
 605 
 606   // handle normal case
 607   bind(normal_case);
 608   cdqq();
 609   int idivq_offset = offset();
 610   idivq(reg);
 611 
 612   // normal and special case exit
 613   bind(special_case);
 614 
 615   return idivq_offset;
 616 }
 617 
 618 void MacroAssembler::decrementq(Register reg, int value) {
 619   if (value == min_jint) { subq(reg, value); return; }
 620   if (value <  0) { incrementq(reg, -value); return; }
 621   if (value == 0) {                        ; return; }
 622   if (value == 1 && UseIncDec) { decq(reg) ; return; }
 623   /* else */      { subq(reg, value)       ; return; }
 624 }
 625 
 626 void MacroAssembler::decrementq(Address dst, int value) {
 627   if (value == min_jint) { subq(dst, value); return; }
 628   if (value <  0) { incrementq(dst, -value); return; }
 629   if (value == 0) {                        ; return; }
 630   if (value == 1 && UseIncDec) { decq(dst) ; return; }
 631   /* else */      { subq(dst, value)       ; return; }
 632 }
 633 
 634 void MacroAssembler::incrementq(AddressLiteral dst) {
 635   if (reachable(dst)) {
 636     incrementq(as_Address(dst));
 637   } else {
 638     lea(rscratch1, dst);
 639     incrementq(Address(rscratch1, 0));
 640   }
 641 }
 642 
 643 void MacroAssembler::incrementq(Register reg, int value) {
 644   if (value == min_jint) { addq(reg, value); return; }
 645   if (value <  0) { decrementq(reg, -value); return; }
 646   if (value == 0) {                        ; return; }
 647   if (value == 1 && UseIncDec) { incq(reg) ; return; }
 648   /* else */      { addq(reg, value)       ; return; }
 649 }
 650 
 651 void MacroAssembler::incrementq(Address dst, int value) {
 652   if (value == min_jint) { addq(dst, value); return; }
 653   if (value <  0) { decrementq(dst, -value); return; }
 654   if (value == 0) {                        ; return; }
 655   if (value == 1 && UseIncDec) { incq(dst) ; return; }
 656   /* else */      { addq(dst, value)       ; return; }
 657 }
 658 
 659 // 32bit can do a case table jump in one instruction but we no longer allow the base
 660 // to be installed in the Address class
 661 void MacroAssembler::jump(ArrayAddress entry) {
 662   lea(rscratch1, entry.base());
 663   Address dispatch = entry.index();
 664   assert(dispatch._base == noreg, "must be");
 665   dispatch._base = rscratch1;
 666   jmp(dispatch);
 667 }
 668 
 669 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 670   ShouldNotReachHere(); // 64bit doesn't use two regs
 671   cmpq(x_lo, y_lo);
 672 }
 673 
 674 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 675     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 676 }
 677 
 678 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 679   mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());
 680   movptr(dst, rscratch1);
 681 }
 682 
 683 void MacroAssembler::leave() {
 684   // %%% is this really better? Why not on 32bit too?
 685   emit_int8((unsigned char)0xC9); // LEAVE
 686 }
 687 
 688 void MacroAssembler::lneg(Register hi, Register lo) {
 689   ShouldNotReachHere(); // 64bit doesn't use two regs
 690   negq(lo);
 691 }
 692 
 693 void MacroAssembler::movoop(Register dst, jobject obj) {
 694   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 695 }
 696 
 697 void MacroAssembler::movoop(Address dst, jobject obj) {
 698   mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 699   movq(dst, rscratch1);
 700 }
 701 
 702 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 703   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 704 }
 705 
 706 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 707   mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 708   movq(dst, rscratch1);
 709 }
 710 
 711 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 712   if (src.is_lval()) {
 713     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 714   } else {
 715     if (reachable(src)) {
 716       movq(dst, as_Address(src));
 717     } else {
 718       lea(scratch, src);
 719       movq(dst, Address(scratch, 0));
 720     }
 721   }
 722 }
 723 
 724 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 725   movq(as_Address(dst), src);
 726 }
 727 
 728 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 729   movq(dst, as_Address(src));
 730 }
 731 
 732 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 733 void MacroAssembler::movptr(Address dst, intptr_t src) {
 734   mov64(rscratch1, src);
 735   movq(dst, rscratch1);
 736 }
 737 
 738 // These are mostly for initializing NULL
 739 void MacroAssembler::movptr(Address dst, int32_t src) {
 740   movslq(dst, src);
 741 }
 742 
 743 void MacroAssembler::movptr(Register dst, int32_t src) {
 744   mov64(dst, (intptr_t)src);
 745 }
 746 
 747 void MacroAssembler::pushoop(jobject obj) {
 748   movoop(rscratch1, obj);
 749   push(rscratch1);
 750 }
 751 
 752 void MacroAssembler::pushklass(Metadata* obj) {
 753   mov_metadata(rscratch1, obj);
 754   push(rscratch1);
 755 }
 756 
 757 void MacroAssembler::pushptr(AddressLiteral src) {
 758   lea(rscratch1, src);
 759   if (src.is_lval()) {
 760     push(rscratch1);
 761   } else {
 762     pushq(Address(rscratch1, 0));
 763   }
 764 }
 765 
 766 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
 767   // we must set sp to zero to clear frame
 768   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 769   // must clear fp, so that compiled frames are not confused; it is
 770   // possible that we need it only for debugging
 771   if (clear_fp) {
 772     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 773   }
 774 
 775   // Always clear the pc because it could have been set by make_walkable()
 776   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 777   vzeroupper();
 778 }
 779 
 780 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 781                                          Register last_java_fp,
 782                                          address  last_java_pc) {
 783   vzeroupper();
 784   // determine last_java_sp register
 785   if (!last_java_sp->is_valid()) {
 786     last_java_sp = rsp;
 787   }
 788 
 789   // last_java_fp is optional
 790   if (last_java_fp->is_valid()) {
 791     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),
 792            last_java_fp);
 793   }
 794 
 795   // last_java_pc is optional
 796   if (last_java_pc != NULL) {
 797     Address java_pc(r15_thread,
 798                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 799     lea(rscratch1, InternalAddress(last_java_pc));
 800     movptr(java_pc, rscratch1);
 801   }
 802 
 803   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 804 }
 805 
 806 static void pass_arg0(MacroAssembler* masm, Register arg) {
 807   if (c_rarg0 != arg ) {
 808     masm->mov(c_rarg0, arg);
 809   }
 810 }
 811 
 812 static void pass_arg1(MacroAssembler* masm, Register arg) {
 813   if (c_rarg1 != arg ) {
 814     masm->mov(c_rarg1, arg);
 815   }
 816 }
 817 
 818 static void pass_arg2(MacroAssembler* masm, Register arg) {
 819   if (c_rarg2 != arg ) {
 820     masm->mov(c_rarg2, arg);
 821   }
 822 }
 823 
 824 static void pass_arg3(MacroAssembler* masm, Register arg) {
 825   if (c_rarg3 != arg ) {
 826     masm->mov(c_rarg3, arg);
 827   }
 828 }
 829 
 830 void MacroAssembler::stop(const char* msg) {
 831   address rip = pc();
 832   pusha(); // get regs on stack
 833   lea(c_rarg0, ExternalAddress((address) msg));
 834   lea(c_rarg1, InternalAddress(rip));
 835   movq(c_rarg2, rsp); // pass pointer to regs array
 836   andq(rsp, -16); // align stack as required by ABI
 837   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
 838   hlt();
 839 }
 840 
 841 void MacroAssembler::warn(const char* msg) {
 842   push(rbp);
 843   movq(rbp, rsp);
 844   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 845   push_CPU_state();   // keeps alignment at 16 bytes
 846   lea(c_rarg0, ExternalAddress((address) msg));
 847   lea(rax, ExternalAddress(CAST_FROM_FN_PTR(address, warning)));
 848   call(rax);
 849   pop_CPU_state();
 850   mov(rsp, rbp);
 851   pop(rbp);
 852 }
 853 
 854 void MacroAssembler::print_state() {
 855   address rip = pc();
 856   pusha();            // get regs on stack
 857   push(rbp);
 858   movq(rbp, rsp);
 859   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 860   push_CPU_state();   // keeps alignment at 16 bytes
 861 
 862   lea(c_rarg0, InternalAddress(rip));
 863   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
 864   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
 865 
 866   pop_CPU_state();
 867   mov(rsp, rbp);
 868   pop(rbp);
 869   popa();
 870 }
 871 
 872 #ifndef PRODUCT
 873 extern "C" void findpc(intptr_t x);
 874 #endif
 875 
 876 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
 877   // In order to get locks to work, we need to fake a in_VM state
 878   if (ShowMessageBoxOnError) {
 879     JavaThread* thread = JavaThread::current();
 880     JavaThreadState saved_state = thread->thread_state();
 881     thread->set_thread_state(_thread_in_vm);
 882 #ifndef PRODUCT
 883     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 884       ttyLocker ttyl;
 885       BytecodeCounter::print();
 886     }
 887 #endif
 888     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 889     // XXX correct this offset for amd64
 890     // This is the value of eip which points to where verify_oop will return.
 891     if (os::message_box(msg, "Execution stopped, print registers?")) {
 892       print_state64(pc, regs);
 893       BREAKPOINT;
 894       assert(false, "start up GDB");
 895     }
 896     ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 897   } else {
 898     ttyLocker ttyl;
 899     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
 900                     msg);
 901     assert(false, "DEBUG MESSAGE: %s", msg);
 902   }
 903 }
 904 
 905 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
 906   ttyLocker ttyl;
 907   FlagSetting fs(Debugging, true);
 908   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
 909 #ifndef PRODUCT
 910   tty->cr();
 911   findpc(pc);
 912   tty->cr();
 913 #endif
 914 #define PRINT_REG(rax, value) \
 915   { tty->print("%s = ", #rax); os::print_location(tty, value); }
 916   PRINT_REG(rax, regs[15]);
 917   PRINT_REG(rbx, regs[12]);
 918   PRINT_REG(rcx, regs[14]);
 919   PRINT_REG(rdx, regs[13]);
 920   PRINT_REG(rdi, regs[8]);
 921   PRINT_REG(rsi, regs[9]);
 922   PRINT_REG(rbp, regs[10]);
 923   PRINT_REG(rsp, regs[11]);
 924   PRINT_REG(r8 , regs[7]);
 925   PRINT_REG(r9 , regs[6]);
 926   PRINT_REG(r10, regs[5]);
 927   PRINT_REG(r11, regs[4]);
 928   PRINT_REG(r12, regs[3]);
 929   PRINT_REG(r13, regs[2]);
 930   PRINT_REG(r14, regs[1]);
 931   PRINT_REG(r15, regs[0]);
 932 #undef PRINT_REG
 933   // Print some words near top of staack.
 934   int64_t* rsp = (int64_t*) regs[11];
 935   int64_t* dump_sp = rsp;
 936   for (int col1 = 0; col1 < 8; col1++) {
 937     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 938     os::print_location(tty, *dump_sp++);
 939   }
 940   for (int row = 0; row < 25; row++) {
 941     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 942     for (int col = 0; col < 4; col++) {
 943       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
 944     }
 945     tty->cr();
 946   }
 947   // Print some instructions around pc:
 948   Disassembler::decode((address)pc-64, (address)pc);
 949   tty->print_cr("--------");
 950   Disassembler::decode((address)pc, (address)pc+32);
 951 }
 952 
 953 #endif // _LP64
 954 
 955 // Now versions that are common to 32/64 bit
 956 
 957 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 958   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 959 }
 960 
 961 void MacroAssembler::addptr(Register dst, Register src) {
 962   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 963 }
 964 
 965 void MacroAssembler::addptr(Address dst, Register src) {
 966   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 967 }
 968 
 969 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {
 970   if (reachable(src)) {
 971     Assembler::addsd(dst, as_Address(src));
 972   } else {
 973     lea(rscratch1, src);
 974     Assembler::addsd(dst, Address(rscratch1, 0));
 975   }
 976 }
 977 
 978 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {
 979   if (reachable(src)) {
 980     addss(dst, as_Address(src));
 981   } else {
 982     lea(rscratch1, src);
 983     addss(dst, Address(rscratch1, 0));
 984   }
 985 }
 986 
 987 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) {
 988   if (reachable(src)) {
 989     Assembler::addpd(dst, as_Address(src));
 990   } else {
 991     lea(rscratch1, src);
 992     Assembler::addpd(dst, Address(rscratch1, 0));
 993   }
 994 }
 995 
 996 void MacroAssembler::align(int modulus) {
 997   align(modulus, offset());
 998 }
 999 
1000 void MacroAssembler::align(int modulus, int target) {
1001   if (target % modulus != 0) {
1002     nop(modulus - (target % modulus));
1003   }
1004 }
1005 
1006 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {
1007   // Used in sign-masking with aligned address.
1008   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1009   if (reachable(src)) {
1010     Assembler::andpd(dst, as_Address(src));
1011   } else {
1012     lea(rscratch1, src);
1013     Assembler::andpd(dst, Address(rscratch1, 0));
1014   }
1015 }
1016 
1017 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {
1018   // Used in sign-masking with aligned address.
1019   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1020   if (reachable(src)) {
1021     Assembler::andps(dst, as_Address(src));
1022   } else {
1023     lea(rscratch1, src);
1024     Assembler::andps(dst, Address(rscratch1, 0));
1025   }
1026 }
1027 
1028 void MacroAssembler::andptr(Register dst, int32_t imm32) {
1029   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
1030 }
1031 
1032 void MacroAssembler::atomic_incl(Address counter_addr) {
1033   if (os::is_MP())
1034     lock();
1035   incrementl(counter_addr);
1036 }
1037 
1038 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
1039   if (reachable(counter_addr)) {
1040     atomic_incl(as_Address(counter_addr));
1041   } else {
1042     lea(scr, counter_addr);
1043     atomic_incl(Address(scr, 0));
1044   }
1045 }
1046 
1047 #ifdef _LP64
1048 void MacroAssembler::atomic_incq(Address counter_addr) {
1049   if (os::is_MP())
1050     lock();
1051   incrementq(counter_addr);
1052 }
1053 
1054 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) {
1055   if (reachable(counter_addr)) {
1056     atomic_incq(as_Address(counter_addr));
1057   } else {
1058     lea(scr, counter_addr);
1059     atomic_incq(Address(scr, 0));
1060   }
1061 }
1062 #endif
1063 
1064 // Writes to stack successive pages until offset reached to check for
1065 // stack overflow + shadow pages.  This clobbers tmp.
1066 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
1067   movptr(tmp, rsp);
1068   // Bang stack for total size given plus shadow page size.
1069   // Bang one page at a time because large size can bang beyond yellow and
1070   // red zones.
1071   Label loop;
1072   bind(loop);
1073   movl(Address(tmp, (-os::vm_page_size())), size );
1074   subptr(tmp, os::vm_page_size());
1075   subl(size, os::vm_page_size());
1076   jcc(Assembler::greater, loop);
1077 
1078   // Bang down shadow pages too.
1079   // At this point, (tmp-0) is the last address touched, so don't
1080   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
1081   // was post-decremented.)  Skip this address by starting at i=1, and
1082   // touch a few more pages below.  N.B.  It is important to touch all
1083   // the way down including all pages in the shadow zone.
1084   for (int i = 1; i < ((int)JavaThread::stack_shadow_zone_size() / os::vm_page_size()); i++) {
1085     // this could be any sized move but this is can be a debugging crumb
1086     // so the bigger the better.
1087     movptr(Address(tmp, (-i*os::vm_page_size())), size );
1088   }
1089 }
1090 
1091 void MacroAssembler::reserved_stack_check() {
1092     // testing if reserved zone needs to be enabled
1093     Label no_reserved_zone_enabling;
1094     Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
1095     NOT_LP64(get_thread(rsi);)
1096 
1097     cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
1098     jcc(Assembler::below, no_reserved_zone_enabling);
1099 
1100     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
1101     jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
1102     should_not_reach_here();
1103 
1104     bind(no_reserved_zone_enabling);
1105 }
1106 
1107 int MacroAssembler::biased_locking_enter(Register lock_reg,
1108                                          Register obj_reg,
1109                                          Register swap_reg,
1110                                          Register tmp_reg,
1111                                          bool swap_reg_contains_mark,
1112                                          Label& done,
1113                                          Label* slow_case,
1114                                          BiasedLockingCounters* counters) {
1115   assert(UseBiasedLocking, "why call this otherwise?");
1116   assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
1117   assert(tmp_reg != noreg, "tmp_reg must be supplied");
1118   assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
1119   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
1120   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
1121   NOT_LP64( Address saved_mark_addr(lock_reg, 0); )
1122 
1123   if (PrintBiasedLockingStatistics && counters == NULL) {
1124     counters = BiasedLocking::counters();
1125   }
1126   // Biased locking
1127   // See whether the lock is currently biased toward our thread and
1128   // whether the epoch is still valid
1129   // Note that the runtime guarantees sufficient alignment of JavaThread
1130   // pointers to allow age to be placed into low bits
1131   // First check to see whether biasing is even enabled for this object
1132   Label cas_label;
1133   int null_check_offset = -1;
1134   if (!swap_reg_contains_mark) {
1135     null_check_offset = offset();
1136     movptr(swap_reg, mark_addr);
1137   }
1138   movptr(tmp_reg, swap_reg);
1139   andptr(tmp_reg, markOopDesc::biased_lock_mask_in_place);
1140   cmpptr(tmp_reg, markOopDesc::biased_lock_pattern);
1141   jcc(Assembler::notEqual, cas_label);
1142   // The bias pattern is present in the object's header. Need to check
1143   // whether the bias owner and the epoch are both still current.
1144 #ifndef _LP64
1145   // Note that because there is no current thread register on x86_32 we
1146   // need to store off the mark word we read out of the object to
1147   // avoid reloading it and needing to recheck invariants below. This
1148   // store is unfortunate but it makes the overall code shorter and
1149   // simpler.
1150   movptr(saved_mark_addr, swap_reg);
1151 #endif
1152   if (swap_reg_contains_mark) {
1153     null_check_offset = offset();
1154   }
1155   load_prototype_header(tmp_reg, obj_reg);
1156 #ifdef _LP64
1157   orptr(tmp_reg, r15_thread);
1158   xorptr(tmp_reg, swap_reg);
1159   Register header_reg = tmp_reg;
1160 #else
1161   xorptr(tmp_reg, swap_reg);
1162   get_thread(swap_reg);
1163   xorptr(swap_reg, tmp_reg);
1164   Register header_reg = swap_reg;
1165 #endif
1166   andptr(header_reg, ~((int) markOopDesc::age_mask_in_place));
1167   if (counters != NULL) {
1168     cond_inc32(Assembler::zero,
1169                ExternalAddress((address) counters->biased_lock_entry_count_addr()));
1170   }
1171   jcc(Assembler::equal, done);
1172 
1173   Label try_revoke_bias;
1174   Label try_rebias;
1175 
1176   // At this point we know that the header has the bias pattern and
1177   // that we are not the bias owner in the current epoch. We need to
1178   // figure out more details about the state of the header in order to
1179   // know what operations can be legally performed on the object's
1180   // header.
1181 
1182   // If the low three bits in the xor result aren't clear, that means
1183   // the prototype header is no longer biased and we have to revoke
1184   // the bias on this object.
1185   testptr(header_reg, markOopDesc::biased_lock_mask_in_place);
1186   jccb(Assembler::notZero, try_revoke_bias);
1187 
1188   // Biasing is still enabled for this data type. See whether the
1189   // epoch of the current bias is still valid, meaning that the epoch
1190   // bits of the mark word are equal to the epoch bits of the
1191   // prototype header. (Note that the prototype header's epoch bits
1192   // only change at a safepoint.) If not, attempt to rebias the object
1193   // toward the current thread. Note that we must be absolutely sure
1194   // that the current epoch is invalid in order to do this because
1195   // otherwise the manipulations it performs on the mark word are
1196   // illegal.
1197   testptr(header_reg, markOopDesc::epoch_mask_in_place);
1198   jccb(Assembler::notZero, try_rebias);
1199 
1200   // The epoch of the current bias is still valid but we know nothing
1201   // about the owner; it might be set or it might be clear. Try to
1202   // acquire the bias of the object using an atomic operation. If this
1203   // fails we will go in to the runtime to revoke the object's bias.
1204   // Note that we first construct the presumed unbiased header so we
1205   // don't accidentally blow away another thread's valid bias.
1206   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1207   andptr(swap_reg,
1208          markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
1209 #ifdef _LP64
1210   movptr(tmp_reg, swap_reg);
1211   orptr(tmp_reg, r15_thread);
1212 #else
1213   get_thread(tmp_reg);
1214   orptr(tmp_reg, swap_reg);
1215 #endif
1216   if (os::is_MP()) {
1217     lock();
1218   }
1219   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1220   // If the biasing toward our thread failed, this means that
1221   // another thread succeeded in biasing it toward itself and we
1222   // need to revoke that bias. The revocation will occur in the
1223   // interpreter runtime in the slow case.
1224   if (counters != NULL) {
1225     cond_inc32(Assembler::zero,
1226                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
1227   }
1228   if (slow_case != NULL) {
1229     jcc(Assembler::notZero, *slow_case);
1230   }
1231   jmp(done);
1232 
1233   bind(try_rebias);
1234   // At this point we know the epoch has expired, meaning that the
1235   // current "bias owner", if any, is actually invalid. Under these
1236   // circumstances _only_, we are allowed to use the current header's
1237   // value as the comparison value when doing the cas to acquire the
1238   // bias in the current epoch. In other words, we allow transfer of
1239   // the bias from one thread to another directly in this situation.
1240   //
1241   // FIXME: due to a lack of registers we currently blow away the age
1242   // bits in this situation. Should attempt to preserve them.
1243   load_prototype_header(tmp_reg, obj_reg);
1244 #ifdef _LP64
1245   orptr(tmp_reg, r15_thread);
1246 #else
1247   get_thread(swap_reg);
1248   orptr(tmp_reg, swap_reg);
1249   movptr(swap_reg, saved_mark_addr);
1250 #endif
1251   if (os::is_MP()) {
1252     lock();
1253   }
1254   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1255   // If the biasing toward our thread failed, then another thread
1256   // succeeded in biasing it toward itself and we need to revoke that
1257   // bias. The revocation will occur in the runtime in the slow case.
1258   if (counters != NULL) {
1259     cond_inc32(Assembler::zero,
1260                ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
1261   }
1262   if (slow_case != NULL) {
1263     jcc(Assembler::notZero, *slow_case);
1264   }
1265   jmp(done);
1266 
1267   bind(try_revoke_bias);
1268   // The prototype mark in the klass doesn't have the bias bit set any
1269   // more, indicating that objects of this data type are not supposed
1270   // to be biased any more. We are going to try to reset the mark of
1271   // this object to the prototype value and fall through to the
1272   // CAS-based locking scheme. Note that if our CAS fails, it means
1273   // that another thread raced us for the privilege of revoking the
1274   // bias of this particular object, so it's okay to continue in the
1275   // normal locking code.
1276   //
1277   // FIXME: due to a lack of registers we currently blow away the age
1278   // bits in this situation. Should attempt to preserve them.
1279   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1280   load_prototype_header(tmp_reg, obj_reg);
1281   if (os::is_MP()) {
1282     lock();
1283   }
1284   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1285   // Fall through to the normal CAS-based lock, because no matter what
1286   // the result of the above CAS, some thread must have succeeded in
1287   // removing the bias bit from the object's header.
1288   if (counters != NULL) {
1289     cond_inc32(Assembler::zero,
1290                ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
1291   }
1292 
1293   bind(cas_label);
1294 
1295   return null_check_offset;
1296 }
1297 
1298 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
1299   assert(UseBiasedLocking, "why call this otherwise?");
1300 
1301   // Check for biased locking unlock case, which is a no-op
1302   // Note: we do not have to check the thread ID for two reasons.
1303   // First, the interpreter checks for IllegalMonitorStateException at
1304   // a higher level. Second, if the bias was revoked while we held the
1305   // lock, the object could not be rebiased toward another thread, so
1306   // the bias bit would be clear.
1307   movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1308   andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);
1309   cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
1310   jcc(Assembler::equal, done);
1311 }
1312 
1313 #ifdef COMPILER2
1314 
1315 #if INCLUDE_RTM_OPT
1316 
1317 // Update rtm_counters based on abort status
1318 // input: abort_status
1319 //        rtm_counters (RTMLockingCounters*)
1320 // flags are killed
1321 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) {
1322 
1323   atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset()));
1324   if (PrintPreciseRTMLockingStatistics) {
1325     for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
1326       Label check_abort;
1327       testl(abort_status, (1<<i));
1328       jccb(Assembler::equal, check_abort);
1329       atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx))));
1330       bind(check_abort);
1331     }
1332   }
1333 }
1334 
1335 // Branch if (random & (count-1) != 0), count is 2^n
1336 // tmp, scr and flags are killed
1337 void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) {
1338   assert(tmp == rax, "");
1339   assert(scr == rdx, "");
1340   rdtsc(); // modifies EDX:EAX
1341   andptr(tmp, count-1);
1342   jccb(Assembler::notZero, brLabel);
1343 }
1344 
1345 // Perform abort ratio calculation, set no_rtm bit if high ratio
1346 // input:  rtm_counters_Reg (RTMLockingCounters* address)
1347 // tmpReg, rtm_counters_Reg and flags are killed
1348 void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg,
1349                                                  Register rtm_counters_Reg,
1350                                                  RTMLockingCounters* rtm_counters,
1351                                                  Metadata* method_data) {
1352   Label L_done, L_check_always_rtm1, L_check_always_rtm2;
1353 
1354   if (RTMLockingCalculationDelay > 0) {
1355     // Delay calculation
1356     movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg);
1357     testptr(tmpReg, tmpReg);
1358     jccb(Assembler::equal, L_done);
1359   }
1360   // Abort ratio calculation only if abort_count > RTMAbortThreshold
1361   //   Aborted transactions = abort_count * 100
1362   //   All transactions = total_count *  RTMTotalCountIncrRate
1363   //   Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio)
1364 
1365   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset()));
1366   cmpptr(tmpReg, RTMAbortThreshold);
1367   jccb(Assembler::below, L_check_always_rtm2);
1368   imulptr(tmpReg, tmpReg, 100);
1369 
1370   Register scrReg = rtm_counters_Reg;
1371   movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1372   imulptr(scrReg, scrReg, RTMTotalCountIncrRate);
1373   imulptr(scrReg, scrReg, RTMAbortRatio);
1374   cmpptr(tmpReg, scrReg);
1375   jccb(Assembler::below, L_check_always_rtm1);
1376   if (method_data != NULL) {
1377     // set rtm_state to "no rtm" in MDO
1378     mov_metadata(tmpReg, method_data);
1379     if (os::is_MP()) {
1380       lock();
1381     }
1382     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM);
1383   }
1384   jmpb(L_done);
1385   bind(L_check_always_rtm1);
1386   // Reload RTMLockingCounters* address
1387   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1388   bind(L_check_always_rtm2);
1389   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1390   cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate);
1391   jccb(Assembler::below, L_done);
1392   if (method_data != NULL) {
1393     // set rtm_state to "always rtm" in MDO
1394     mov_metadata(tmpReg, method_data);
1395     if (os::is_MP()) {
1396       lock();
1397     }
1398     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM);
1399   }
1400   bind(L_done);
1401 }
1402 
1403 // Update counters and perform abort ratio calculation
1404 // input:  abort_status_Reg
1405 // rtm_counters_Reg, flags are killed
1406 void MacroAssembler::rtm_profiling(Register abort_status_Reg,
1407                                    Register rtm_counters_Reg,
1408                                    RTMLockingCounters* rtm_counters,
1409                                    Metadata* method_data,
1410                                    bool profile_rtm) {
1411 
1412   assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1413   // update rtm counters based on rax value at abort
1414   // reads abort_status_Reg, updates flags
1415   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1416   rtm_counters_update(abort_status_Reg, rtm_counters_Reg);
1417   if (profile_rtm) {
1418     // Save abort status because abort_status_Reg is used by following code.
1419     if (RTMRetryCount > 0) {
1420       push(abort_status_Reg);
1421     }
1422     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1423     rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data);
1424     // restore abort status
1425     if (RTMRetryCount > 0) {
1426       pop(abort_status_Reg);
1427     }
1428   }
1429 }
1430 
1431 // Retry on abort if abort's status is 0x6: can retry (0x2) | memory conflict (0x4)
1432 // inputs: retry_count_Reg
1433 //       : abort_status_Reg
1434 // output: retry_count_Reg decremented by 1
1435 // flags are killed
1436 void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) {
1437   Label doneRetry;
1438   assert(abort_status_Reg == rax, "");
1439   // The abort reason bits are in eax (see all states in rtmLocking.hpp)
1440   // 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4)
1441   // if reason is in 0x6 and retry count != 0 then retry
1442   andptr(abort_status_Reg, 0x6);
1443   jccb(Assembler::zero, doneRetry);
1444   testl(retry_count_Reg, retry_count_Reg);
1445   jccb(Assembler::zero, doneRetry);
1446   pause();
1447   decrementl(retry_count_Reg);
1448   jmp(retryLabel);
1449   bind(doneRetry);
1450 }
1451 
1452 // Spin and retry if lock is busy,
1453 // inputs: box_Reg (monitor address)
1454 //       : retry_count_Reg
1455 // output: retry_count_Reg decremented by 1
1456 //       : clear z flag if retry count exceeded
1457 // tmp_Reg, scr_Reg, flags are killed
1458 void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg,
1459                                             Register tmp_Reg, Register scr_Reg, Label& retryLabel) {
1460   Label SpinLoop, SpinExit, doneRetry;
1461   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1462 
1463   testl(retry_count_Reg, retry_count_Reg);
1464   jccb(Assembler::zero, doneRetry);
1465   decrementl(retry_count_Reg);
1466   movptr(scr_Reg, RTMSpinLoopCount);
1467 
1468   bind(SpinLoop);
1469   pause();
1470   decrementl(scr_Reg);
1471   jccb(Assembler::lessEqual, SpinExit);
1472   movptr(tmp_Reg, Address(box_Reg, owner_offset));
1473   testptr(tmp_Reg, tmp_Reg);
1474   jccb(Assembler::notZero, SpinLoop);
1475 
1476   bind(SpinExit);
1477   jmp(retryLabel);
1478   bind(doneRetry);
1479   incrementl(retry_count_Reg); // clear z flag
1480 }
1481 
1482 // Use RTM for normal stack locks
1483 // Input: objReg (object to lock)
1484 void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg,
1485                                        Register retry_on_abort_count_Reg,
1486                                        RTMLockingCounters* stack_rtm_counters,
1487                                        Metadata* method_data, bool profile_rtm,
1488                                        Label& DONE_LABEL, Label& IsInflated) {
1489   assert(UseRTMForStackLocks, "why call this otherwise?");
1490   assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1491   assert(tmpReg == rax, "");
1492   assert(scrReg == rdx, "");
1493   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1494 
1495   if (RTMRetryCount > 0) {
1496     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1497     bind(L_rtm_retry);
1498   }
1499   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1500   testptr(tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased
1501   jcc(Assembler::notZero, IsInflated);
1502 
1503   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1504     Label L_noincrement;
1505     if (RTMTotalCountIncrRate > 1) {
1506       // tmpReg, scrReg and flags are killed
1507       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1508     }
1509     assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM");
1510     atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg);
1511     bind(L_noincrement);
1512   }
1513   xbegin(L_on_abort);
1514   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));       // fetch markword
1515   andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
1516   cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
1517   jcc(Assembler::equal, DONE_LABEL);        // all done if unlocked
1518 
1519   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1520   if (UseRTMXendForLockBusy) {
1521     xend();
1522     movptr(abort_status_Reg, 0x2);   // Set the abort status to 2 (so we can retry)
1523     jmp(L_decrement_retry);
1524   }
1525   else {
1526     xabort(0);
1527   }
1528   bind(L_on_abort);
1529   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1530     rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm);
1531   }
1532   bind(L_decrement_retry);
1533   if (RTMRetryCount > 0) {
1534     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1535     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1536   }
1537 }
1538 
1539 // Use RTM for inflating locks
1540 // inputs: objReg (object to lock)
1541 //         boxReg (on-stack box address (displaced header location) - KILLED)
1542 //         tmpReg (ObjectMonitor address + markOopDesc::monitor_value)
1543 void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg,
1544                                           Register scrReg, Register retry_on_busy_count_Reg,
1545                                           Register retry_on_abort_count_Reg,
1546                                           RTMLockingCounters* rtm_counters,
1547                                           Metadata* method_data, bool profile_rtm,
1548                                           Label& DONE_LABEL) {
1549   assert(UseRTMLocking, "why call this otherwise?");
1550   assert(tmpReg == rax, "");
1551   assert(scrReg == rdx, "");
1552   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1553   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1554 
1555   // Without cast to int32_t a movptr will destroy r10 which is typically obj
1556   movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1557   movptr(boxReg, tmpReg); // Save ObjectMonitor address
1558 
1559   if (RTMRetryCount > 0) {
1560     movl(retry_on_busy_count_Reg, RTMRetryCount);  // Retry on lock busy
1561     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1562     bind(L_rtm_retry);
1563   }
1564   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1565     Label L_noincrement;
1566     if (RTMTotalCountIncrRate > 1) {
1567       // tmpReg, scrReg and flags are killed
1568       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1569     }
1570     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1571     atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);
1572     bind(L_noincrement);
1573   }
1574   xbegin(L_on_abort);
1575   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1576   movptr(tmpReg, Address(tmpReg, owner_offset));
1577   testptr(tmpReg, tmpReg);
1578   jcc(Assembler::zero, DONE_LABEL);
1579   if (UseRTMXendForLockBusy) {
1580     xend();
1581     jmp(L_decrement_retry);
1582   }
1583   else {
1584     xabort(0);
1585   }
1586   bind(L_on_abort);
1587   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1588   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1589     rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm);
1590   }
1591   if (RTMRetryCount > 0) {
1592     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1593     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1594   }
1595 
1596   movptr(tmpReg, Address(boxReg, owner_offset)) ;
1597   testptr(tmpReg, tmpReg) ;
1598   jccb(Assembler::notZero, L_decrement_retry) ;
1599 
1600   // Appears unlocked - try to swing _owner from null to non-null.
1601   // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1602 #ifdef _LP64
1603   Register threadReg = r15_thread;
1604 #else
1605   get_thread(scrReg);
1606   Register threadReg = scrReg;
1607 #endif
1608   if (os::is_MP()) {
1609     lock();
1610   }
1611   cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg
1612 
1613   if (RTMRetryCount > 0) {
1614     // success done else retry
1615     jccb(Assembler::equal, DONE_LABEL) ;
1616     bind(L_decrement_retry);
1617     // Spin and retry if lock is busy.
1618     rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
1619   }
1620   else {
1621     bind(L_decrement_retry);
1622   }
1623 }
1624 
1625 #endif //  INCLUDE_RTM_OPT
1626 
1627 // Fast_Lock and Fast_Unlock used by C2
1628 
1629 // Because the transitions from emitted code to the runtime
1630 // monitorenter/exit helper stubs are so slow it's critical that
1631 // we inline both the stack-locking fast-path and the inflated fast path.
1632 //
1633 // See also: cmpFastLock and cmpFastUnlock.
1634 //
1635 // What follows is a specialized inline transliteration of the code
1636 // in slow_enter() and slow_exit().  If we're concerned about I$ bloat
1637 // another option would be to emit TrySlowEnter and TrySlowExit methods
1638 // at startup-time.  These methods would accept arguments as
1639 // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure
1640 // indications in the icc.ZFlag.  Fast_Lock and Fast_Unlock would simply
1641 // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.
1642 // In practice, however, the # of lock sites is bounded and is usually small.
1643 // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer
1644 // if the processor uses simple bimodal branch predictors keyed by EIP
1645 // Since the helper routines would be called from multiple synchronization
1646 // sites.
1647 //
1648 // An even better approach would be write "MonitorEnter()" and "MonitorExit()"
1649 // in java - using j.u.c and unsafe - and just bind the lock and unlock sites
1650 // to those specialized methods.  That'd give us a mostly platform-independent
1651 // implementation that the JITs could optimize and inline at their pleasure.
1652 // Done correctly, the only time we'd need to cross to native could would be
1653 // to park() or unpark() threads.  We'd also need a few more unsafe operators
1654 // to (a) prevent compiler-JIT reordering of non-volatile accesses, and
1655 // (b) explicit barriers or fence operations.
1656 //
1657 // TODO:
1658 //
1659 // *  Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr).
1660 //    This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals.
1661 //    Given TLAB allocation, Self is usually manifested in a register, so passing it into
1662 //    the lock operators would typically be faster than reifying Self.
1663 //
1664 // *  Ideally I'd define the primitives as:
1665 //       fast_lock   (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.
1666 //       fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED
1667 //    Unfortunately ADLC bugs prevent us from expressing the ideal form.
1668 //    Instead, we're stuck with a rather awkward and brittle register assignments below.
1669 //    Furthermore the register assignments are overconstrained, possibly resulting in
1670 //    sub-optimal code near the synchronization site.
1671 //
1672 // *  Eliminate the sp-proximity tests and just use "== Self" tests instead.
1673 //    Alternately, use a better sp-proximity test.
1674 //
1675 // *  Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.
1676 //    Either one is sufficient to uniquely identify a thread.
1677 //    TODO: eliminate use of sp in _owner and use get_thread(tr) instead.
1678 //
1679 // *  Intrinsify notify() and notifyAll() for the common cases where the
1680 //    object is locked by the calling thread but the waitlist is empty.
1681 //    avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().
1682 //
1683 // *  use jccb and jmpb instead of jcc and jmp to improve code density.
1684 //    But beware of excessive branch density on AMD Opterons.
1685 //
1686 // *  Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success
1687 //    or failure of the fast-path.  If the fast-path fails then we pass
1688 //    control to the slow-path, typically in C.  In Fast_Lock and
1689 //    Fast_Unlock we often branch to DONE_LABEL, just to find that C2
1690 //    will emit a conditional branch immediately after the node.
1691 //    So we have branches to branches and lots of ICC.ZF games.
1692 //    Instead, it might be better to have C2 pass a "FailureLabel"
1693 //    into Fast_Lock and Fast_Unlock.  In the case of success, control
1694 //    will drop through the node.  ICC.ZF is undefined at exit.
1695 //    In the case of failure, the node will branch directly to the
1696 //    FailureLabel
1697 
1698 
1699 // obj: object to lock
1700 // box: on-stack box address (displaced header location) - KILLED
1701 // rax,: tmp -- KILLED
1702 // scr: tmp -- KILLED
1703 void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
1704                                Register scrReg, Register cx1Reg, Register cx2Reg,
1705                                BiasedLockingCounters* counters,
1706                                RTMLockingCounters* rtm_counters,
1707                                RTMLockingCounters* stack_rtm_counters,
1708                                Metadata* method_data,
1709                                bool use_rtm, bool profile_rtm) {
1710   // Ensure the register assignments are disjoint
1711   assert(tmpReg == rax, "");
1712 
1713   if (use_rtm) {
1714     assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg);
1715   } else {
1716     assert(cx1Reg == noreg, "");
1717     assert(cx2Reg == noreg, "");
1718     assert_different_registers(objReg, boxReg, tmpReg, scrReg);
1719   }
1720 
1721   if (counters != NULL) {
1722     atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);
1723   }
1724   if (EmitSync & 1) {
1725       // set box->dhw = markOopDesc::unused_mark()
1726       // Force all sync thru slow-path: slow_enter() and slow_exit()
1727       movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1728       cmpptr (rsp, (int32_t)NULL_WORD);
1729   } else {
1730     // Possible cases that we'll encounter in fast_lock
1731     // ------------------------------------------------
1732     // * Inflated
1733     //    -- unlocked
1734     //    -- Locked
1735     //       = by self
1736     //       = by other
1737     // * biased
1738     //    -- by Self
1739     //    -- by other
1740     // * neutral
1741     // * stack-locked
1742     //    -- by self
1743     //       = sp-proximity test hits
1744     //       = sp-proximity test generates false-negative
1745     //    -- by other
1746     //
1747 
1748     Label IsInflated, DONE_LABEL;
1749 
1750     // it's stack-locked, biased or neutral
1751     // TODO: optimize away redundant LDs of obj->mark and improve the markword triage
1752     // order to reduce the number of conditional branches in the most common cases.
1753     // Beware -- there's a subtle invariant that fetch of the markword
1754     // at [FETCH], below, will never observe a biased encoding (*101b).
1755     // If this invariant is not held we risk exclusion (safety) failure.
1756     if (UseBiasedLocking && !UseOptoBiasInlining) {
1757       biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters);
1758     }
1759 
1760 #if INCLUDE_RTM_OPT
1761     if (UseRTMForStackLocks && use_rtm) {
1762       rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
1763                         stack_rtm_counters, method_data, profile_rtm,
1764                         DONE_LABEL, IsInflated);
1765     }
1766 #endif // INCLUDE_RTM_OPT
1767 
1768     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));          // [FETCH]
1769     testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
1770     jccb(Assembler::notZero, IsInflated);
1771 
1772     // Attempt stack-locking ...
1773     orptr (tmpReg, markOopDesc::unlocked_value);
1774     movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
1775     if (os::is_MP()) {
1776       lock();
1777     }
1778     cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes()));      // Updates tmpReg
1779     if (counters != NULL) {
1780       cond_inc32(Assembler::equal,
1781                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
1782     }
1783     jcc(Assembler::equal, DONE_LABEL);           // Success
1784 
1785     // Recursive locking.
1786     // The object is stack-locked: markword contains stack pointer to BasicLock.
1787     // Locked by current thread if difference with current SP is less than one page.
1788     subptr(tmpReg, rsp);
1789     // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
1790     andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
1791     movptr(Address(boxReg, 0), tmpReg);
1792     if (counters != NULL) {
1793       cond_inc32(Assembler::equal,
1794                  ExternalAddress((address)counters->fast_path_entry_count_addr()));
1795     }
1796     jmp(DONE_LABEL);
1797 
1798     bind(IsInflated);
1799     // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markOopDesc::monitor_value
1800 
1801 #if INCLUDE_RTM_OPT
1802     // Use the same RTM locking code in 32- and 64-bit VM.
1803     if (use_rtm) {
1804       rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg,
1805                            rtm_counters, method_data, profile_rtm, DONE_LABEL);
1806     } else {
1807 #endif // INCLUDE_RTM_OPT
1808 
1809 #ifndef _LP64
1810     // The object is inflated.
1811 
1812     // boxReg refers to the on-stack BasicLock in the current frame.
1813     // We'd like to write:
1814     //   set box->_displaced_header = markOopDesc::unused_mark().  Any non-0 value suffices.
1815     // This is convenient but results a ST-before-CAS penalty.  The following CAS suffers
1816     // additional latency as we have another ST in the store buffer that must drain.
1817 
1818     if (EmitSync & 8192) {
1819        movptr(Address(boxReg, 0), 3);            // results in ST-before-CAS penalty
1820        get_thread (scrReg);
1821        movptr(boxReg, tmpReg);                    // consider: LEA box, [tmp-2]
1822        movptr(tmpReg, NULL_WORD);                 // consider: xor vs mov
1823        if (os::is_MP()) {
1824          lock();
1825        }
1826        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1827     } else
1828     if ((EmitSync & 128) == 0) {                      // avoid ST-before-CAS
1829        // register juggle because we need tmpReg for cmpxchgptr below
1830        movptr(scrReg, boxReg);
1831        movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2]
1832 
1833        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
1834        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
1835           // prefetchw [eax + Offset(_owner)-2]
1836           prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1837        }
1838 
1839        if ((EmitSync & 64) == 0) {
1840          // Optimistic form: consider XORL tmpReg,tmpReg
1841          movptr(tmpReg, NULL_WORD);
1842        } else {
1843          // Can suffer RTS->RTO upgrades on shared or cold $ lines
1844          // Test-And-CAS instead of CAS
1845          movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));   // rax, = m->_owner
1846          testptr(tmpReg, tmpReg);                   // Locked ?
1847          jccb  (Assembler::notZero, DONE_LABEL);
1848        }
1849 
1850        // Appears unlocked - try to swing _owner from null to non-null.
1851        // Ideally, I'd manifest "Self" with get_thread and then attempt
1852        // to CAS the register containing Self into m->Owner.
1853        // But we don't have enough registers, so instead we can either try to CAS
1854        // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
1855        // we later store "Self" into m->Owner.  Transiently storing a stack address
1856        // (rsp or the address of the box) into  m->owner is harmless.
1857        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1858        if (os::is_MP()) {
1859          lock();
1860        }
1861        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1862        movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3
1863        // If we weren't able to swing _owner from NULL to the BasicLock
1864        // then take the slow path.
1865        jccb  (Assembler::notZero, DONE_LABEL);
1866        // update _owner from BasicLock to thread
1867        get_thread (scrReg);                    // beware: clobbers ICCs
1868        movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
1869        xorptr(boxReg, boxReg);                 // set icc.ZFlag = 1 to indicate success
1870 
1871        // If the CAS fails we can either retry or pass control to the slow-path.
1872        // We use the latter tactic.
1873        // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1874        // If the CAS was successful ...
1875        //   Self has acquired the lock
1876        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1877        // Intentional fall-through into DONE_LABEL ...
1878     } else {
1879        movptr(Address(boxReg, 0), intptr_t(markOopDesc::unused_mark()));  // results in ST-before-CAS penalty
1880        movptr(boxReg, tmpReg);
1881 
1882        // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes
1883        if ((EmitSync & 2048) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
1884           // prefetchw [eax + Offset(_owner)-2]
1885           prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1886        }
1887 
1888        if ((EmitSync & 64) == 0) {
1889          // Optimistic form
1890          xorptr  (tmpReg, tmpReg);
1891        } else {
1892          // Can suffer RTS->RTO upgrades on shared or cold $ lines
1893          movptr(tmpReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));   // rax, = m->_owner
1894          testptr(tmpReg, tmpReg);                   // Locked ?
1895          jccb  (Assembler::notZero, DONE_LABEL);
1896        }
1897 
1898        // Appears unlocked - try to swing _owner from null to non-null.
1899        // Use either "Self" (in scr) or rsp as thread identity in _owner.
1900        // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1901        get_thread (scrReg);
1902        if (os::is_MP()) {
1903          lock();
1904        }
1905        cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1906 
1907        // If the CAS fails we can either retry or pass control to the slow-path.
1908        // We use the latter tactic.
1909        // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1910        // If the CAS was successful ...
1911        //   Self has acquired the lock
1912        //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1913        // Intentional fall-through into DONE_LABEL ...
1914     }
1915 #else // _LP64
1916     // It's inflated
1917     movq(scrReg, tmpReg);
1918     xorq(tmpReg, tmpReg);
1919 
1920     if (os::is_MP()) {
1921       lock();
1922     }
1923     cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1924     // Unconditionally set box->_displaced_header = markOopDesc::unused_mark().
1925     // Without cast to int32_t movptr will destroy r10 which is typically obj.
1926     movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1927     // Intentional fall-through into DONE_LABEL ...
1928     // Propagate ICC.ZF from CAS above into DONE_LABEL.
1929 #endif // _LP64
1930 #if INCLUDE_RTM_OPT
1931     } // use_rtm()
1932 #endif
1933     // DONE_LABEL is a hot target - we'd really like to place it at the
1934     // start of cache line by padding with NOPs.
1935     // See the AMD and Intel software optimization manuals for the
1936     // most efficient "long" NOP encodings.
1937     // Unfortunately none of our alignment mechanisms suffice.
1938     bind(DONE_LABEL);
1939 
1940     // At DONE_LABEL the icc ZFlag is set as follows ...
1941     // Fast_Unlock uses the same protocol.
1942     // ZFlag == 1 -> Success
1943     // ZFlag == 0 -> Failure - force control through the slow-path
1944   }
1945 }
1946 
1947 // obj: object to unlock
1948 // box: box address (displaced header location), killed.  Must be EAX.
1949 // tmp: killed, cannot be obj nor box.
1950 //
1951 // Some commentary on balanced locking:
1952 //
1953 // Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites.
1954 // Methods that don't have provably balanced locking are forced to run in the
1955 // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
1956 // The interpreter provides two properties:
1957 // I1:  At return-time the interpreter automatically and quietly unlocks any
1958 //      objects acquired the current activation (frame).  Recall that the
1959 //      interpreter maintains an on-stack list of locks currently held by
1960 //      a frame.
1961 // I2:  If a method attempts to unlock an object that is not held by the
1962 //      the frame the interpreter throws IMSX.
1963 //
1964 // Lets say A(), which has provably balanced locking, acquires O and then calls B().
1965 // B() doesn't have provably balanced locking so it runs in the interpreter.
1966 // Control returns to A() and A() unlocks O.  By I1 and I2, above, we know that O
1967 // is still locked by A().
1968 //
1969 // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:
1970 // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
1971 // should not be unlocked by "normal" java-level locking and vice-versa.  The specification
1972 // doesn't specify what will occur if a program engages in such mixed-mode locking, however.
1973 // Arguably given that the spec legislates the JNI case as undefined our implementation
1974 // could reasonably *avoid* checking owner in Fast_Unlock().
1975 // In the interest of performance we elide m->Owner==Self check in unlock.
1976 // A perfectly viable alternative is to elide the owner check except when
1977 // Xcheck:jni is enabled.
1978 
1979 void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
1980   assert(boxReg == rax, "");
1981   assert_different_registers(objReg, boxReg, tmpReg);
1982 
1983   if (EmitSync & 4) {
1984     // Disable - inhibit all inlining.  Force control through the slow-path
1985     cmpptr (rsp, 0);
1986   } else {
1987     Label DONE_LABEL, Stacked, CheckSucc;
1988 
1989     // Critically, the biased locking test must have precedence over
1990     // and appear before the (box->dhw == 0) recursive stack-lock test.
1991     if (UseBiasedLocking && !UseOptoBiasInlining) {
1992        biased_locking_exit(objReg, tmpReg, DONE_LABEL);
1993     }
1994 
1995 #if INCLUDE_RTM_OPT
1996     if (UseRTMForStackLocks && use_rtm) {
1997       assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1998       Label L_regular_unlock;
1999       movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));           // fetch markword
2000       andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
2001       cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
2002       jccb(Assembler::notEqual, L_regular_unlock);  // if !HLE RegularLock
2003       xend();                                       // otherwise end...
2004       jmp(DONE_LABEL);                              // ... and we're done
2005       bind(L_regular_unlock);
2006     }
2007 #endif
2008 
2009     cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
2010     jcc   (Assembler::zero, DONE_LABEL);            // 0 indicates recursive stack-lock
2011     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));             // Examine the object's markword
2012     testptr(tmpReg, markOopDesc::monitor_value);    // Inflated?
2013     jccb  (Assembler::zero, Stacked);
2014 
2015     // It's inflated.
2016 #if INCLUDE_RTM_OPT
2017     if (use_rtm) {
2018       Label L_regular_inflated_unlock;
2019       int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
2020       movptr(boxReg, Address(tmpReg, owner_offset));
2021       testptr(boxReg, boxReg);
2022       jccb(Assembler::notZero, L_regular_inflated_unlock);
2023       xend();
2024       jmpb(DONE_LABEL);
2025       bind(L_regular_inflated_unlock);
2026     }
2027 #endif
2028 
2029     // Despite our balanced locking property we still check that m->_owner == Self
2030     // as java routines or native JNI code called by this thread might
2031     // have released the lock.
2032     // Refer to the comments in synchronizer.cpp for how we might encode extra
2033     // state in _succ so we can avoid fetching EntryList|cxq.
2034     //
2035     // I'd like to add more cases in fast_lock() and fast_unlock() --
2036     // such as recursive enter and exit -- but we have to be wary of
2037     // I$ bloat, T$ effects and BP$ effects.
2038     //
2039     // If there's no contention try a 1-0 exit.  That is, exit without
2040     // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how
2041     // we detect and recover from the race that the 1-0 exit admits.
2042     //
2043     // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier
2044     // before it STs null into _owner, releasing the lock.  Updates
2045     // to data protected by the critical section must be visible before
2046     // we drop the lock (and thus before any other thread could acquire
2047     // the lock and observe the fields protected by the lock).
2048     // IA32's memory-model is SPO, so STs are ordered with respect to
2049     // each other and there's no need for an explicit barrier (fence).
2050     // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
2051 #ifndef _LP64
2052     get_thread (boxReg);
2053     if ((EmitSync & 4096) && VM_Version::supports_3dnow_prefetch() && os::is_MP()) {
2054       // prefetchw [ebx + Offset(_owner)-2]
2055       prefetchw(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2056     }
2057 
2058     // Note that we could employ various encoding schemes to reduce
2059     // the number of loads below (currently 4) to just 2 or 3.
2060     // Refer to the comments in synchronizer.cpp.
2061     // In practice the chain of fetches doesn't seem to impact performance, however.
2062     xorptr(boxReg, boxReg);
2063     if ((EmitSync & 65536) == 0 && (EmitSync & 256)) {
2064        // Attempt to reduce branch density - AMD's branch predictor.
2065        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2066        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2067        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2068        jccb  (Assembler::notZero, DONE_LABEL);
2069        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2070        jmpb  (DONE_LABEL);
2071     } else {
2072        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2073        jccb  (Assembler::notZero, DONE_LABEL);
2074        movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2075        orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2076        jccb  (Assembler::notZero, CheckSucc);
2077        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2078        jmpb  (DONE_LABEL);
2079     }
2080 
2081     // The Following code fragment (EmitSync & 65536) improves the performance of
2082     // contended applications and contended synchronization microbenchmarks.
2083     // Unfortunately the emission of the code - even though not executed - causes regressions
2084     // in scimark and jetstream, evidently because of $ effects.  Replacing the code
2085     // with an equal number of never-executed NOPs results in the same regression.
2086     // We leave it off by default.
2087 
2088     if ((EmitSync & 65536) != 0) {
2089        Label LSuccess, LGoSlowPath ;
2090 
2091        bind  (CheckSucc);
2092 
2093        // Optional pre-test ... it's safe to elide this
2094        cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2095        jccb(Assembler::zero, LGoSlowPath);
2096 
2097        // We have a classic Dekker-style idiom:
2098        //    ST m->_owner = 0 ; MEMBAR; LD m->_succ
2099        // There are a number of ways to implement the barrier:
2100        // (1) lock:andl &m->_owner, 0
2101        //     is fast, but mask doesn't currently support the "ANDL M,IMM32" form.
2102        //     LOCK: ANDL [ebx+Offset(_Owner)-2], 0
2103        //     Encodes as 81 31 OFF32 IMM32 or 83 63 OFF8 IMM8
2104        // (2) If supported, an explicit MFENCE is appealing.
2105        //     In older IA32 processors MFENCE is slower than lock:add or xchg
2106        //     particularly if the write-buffer is full as might be the case if
2107        //     if stores closely precede the fence or fence-equivalent instruction.
2108        //     See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences
2109        //     as the situation has changed with Nehalem and Shanghai.
2110        // (3) In lieu of an explicit fence, use lock:addl to the top-of-stack
2111        //     The $lines underlying the top-of-stack should be in M-state.
2112        //     The locked add instruction is serializing, of course.
2113        // (4) Use xchg, which is serializing
2114        //     mov boxReg, 0; xchgl boxReg, [tmpReg + Offset(_owner)-2] also works
2115        // (5) ST m->_owner = 0 and then execute lock:orl &m->_succ, 0.
2116        //     The integer condition codes will tell us if succ was 0.
2117        //     Since _succ and _owner should reside in the same $line and
2118        //     we just stored into _owner, it's likely that the $line
2119        //     remains in M-state for the lock:orl.
2120        //
2121        // We currently use (3), although it's likely that switching to (2)
2122        // is correct for the future.
2123 
2124        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
2125        if (os::is_MP()) {
2126          lock(); addptr(Address(rsp, 0), 0);
2127        }
2128        // Ratify _succ remains non-null
2129        cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), 0);
2130        jccb  (Assembler::notZero, LSuccess);
2131 
2132        xorptr(boxReg, boxReg);                  // box is really EAX
2133        if (os::is_MP()) { lock(); }
2134        cmpxchgptr(rsp, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2135        // There's no successor so we tried to regrab the lock with the
2136        // placeholder value. If that didn't work, then another thread
2137        // grabbed the lock so we're done (and exit was a success).
2138        jccb  (Assembler::notEqual, LSuccess);
2139        // Since we're low on registers we installed rsp as a placeholding in _owner.
2140        // Now install Self over rsp.  This is safe as we're transitioning from
2141        // non-null to non=null
2142        get_thread (boxReg);
2143        movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), boxReg);
2144        // Intentional fall-through into LGoSlowPath ...
2145 
2146        bind  (LGoSlowPath);
2147        orptr(boxReg, 1);                      // set ICC.ZF=0 to indicate failure
2148        jmpb  (DONE_LABEL);
2149 
2150        bind  (LSuccess);
2151        xorptr(boxReg, boxReg);                 // set ICC.ZF=1 to indicate success
2152        jmpb  (DONE_LABEL);
2153     }
2154 
2155     bind (Stacked);
2156     // It's not inflated and it's not recursively stack-locked and it's not biased.
2157     // It must be stack-locked.
2158     // Try to reset the header to displaced header.
2159     // The "box" value on the stack is stable, so we can reload
2160     // and be assured we observe the same value as above.
2161     movptr(tmpReg, Address(boxReg, 0));
2162     if (os::is_MP()) {
2163       lock();
2164     }
2165     cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
2166     // Intention fall-thru into DONE_LABEL
2167 
2168     // DONE_LABEL is a hot target - we'd really like to place it at the
2169     // start of cache line by padding with NOPs.
2170     // See the AMD and Intel software optimization manuals for the
2171     // most efficient "long" NOP encodings.
2172     // Unfortunately none of our alignment mechanisms suffice.
2173     if ((EmitSync & 65536) == 0) {
2174        bind (CheckSucc);
2175     }
2176 #else // _LP64
2177     // It's inflated
2178     if (EmitSync & 1024) {
2179       // Emit code to check that _owner == Self
2180       // We could fold the _owner test into subsequent code more efficiently
2181       // than using a stand-alone check, but since _owner checking is off by
2182       // default we don't bother. We also might consider predicating the
2183       // _owner==Self check on Xcheck:jni or running on a debug build.
2184       movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2185       xorptr(boxReg, r15_thread);
2186     } else {
2187       xorptr(boxReg, boxReg);
2188     }
2189     orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2190     jccb  (Assembler::notZero, DONE_LABEL);
2191     movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2192     orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2193     jccb  (Assembler::notZero, CheckSucc);
2194     movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2195     jmpb  (DONE_LABEL);
2196 
2197     if ((EmitSync & 65536) == 0) {
2198       // Try to avoid passing control into the slow_path ...
2199       Label LSuccess, LGoSlowPath ;
2200       bind  (CheckSucc);
2201 
2202       // The following optional optimization can be elided if necessary
2203       // Effectively: if (succ == null) goto SlowPath
2204       // The code reduces the window for a race, however,
2205       // and thus benefits performance.
2206       cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2207       jccb  (Assembler::zero, LGoSlowPath);
2208 
2209       xorptr(boxReg, boxReg);
2210       if ((EmitSync & 16) && os::is_MP()) {
2211         xchgptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2212       } else {
2213         movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2214         if (os::is_MP()) {
2215           // Memory barrier/fence
2216           // Dekker pivot point -- fulcrum : ST Owner; MEMBAR; LD Succ
2217           // Instead of MFENCE we use a dummy locked add of 0 to the top-of-stack.
2218           // This is faster on Nehalem and AMD Shanghai/Barcelona.
2219           // See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences
2220           // We might also restructure (ST Owner=0;barrier;LD _Succ) to
2221           // (mov box,0; xchgq box, &m->Owner; LD _succ) .
2222           lock(); addl(Address(rsp, 0), 0);
2223         }
2224       }
2225       cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2226       jccb  (Assembler::notZero, LSuccess);
2227 
2228       // Rare inopportune interleaving - race.
2229       // The successor vanished in the small window above.
2230       // The lock is contended -- (cxq|EntryList) != null -- and there's no apparent successor.
2231       // We need to ensure progress and succession.
2232       // Try to reacquire the lock.
2233       // If that fails then the new owner is responsible for succession and this
2234       // thread needs to take no further action and can exit via the fast path (success).
2235       // If the re-acquire succeeds then pass control into the slow path.
2236       // As implemented, this latter mode is horrible because we generated more
2237       // coherence traffic on the lock *and* artifically extended the critical section
2238       // length while by virtue of passing control into the slow path.
2239 
2240       // box is really RAX -- the following CMPXCHG depends on that binding
2241       // cmpxchg R,[M] is equivalent to rax = CAS(M,rax,R)
2242       if (os::is_MP()) { lock(); }
2243       cmpxchgptr(r15_thread, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2244       // There's no successor so we tried to regrab the lock.
2245       // If that didn't work, then another thread grabbed the
2246       // lock so we're done (and exit was a success).
2247       jccb  (Assembler::notEqual, LSuccess);
2248       // Intentional fall-through into slow-path
2249 
2250       bind  (LGoSlowPath);
2251       orl   (boxReg, 1);                      // set ICC.ZF=0 to indicate failure
2252       jmpb  (DONE_LABEL);
2253 
2254       bind  (LSuccess);
2255       testl (boxReg, 0);                      // set ICC.ZF=1 to indicate success
2256       jmpb  (DONE_LABEL);
2257     }
2258 
2259     bind  (Stacked);
2260     movptr(tmpReg, Address (boxReg, 0));      // re-fetch
2261     if (os::is_MP()) { lock(); }
2262     cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
2263 
2264     if (EmitSync & 65536) {
2265        bind (CheckSucc);
2266     }
2267 #endif
2268     bind(DONE_LABEL);
2269   }
2270 }
2271 #endif // COMPILER2
2272 
2273 void MacroAssembler::c2bool(Register x) {
2274   // implements x == 0 ? 0 : 1
2275   // note: must only look at least-significant byte of x
2276   //       since C-style booleans are stored in one byte
2277   //       only! (was bug)
2278   andl(x, 0xFF);
2279   setb(Assembler::notZero, x);
2280 }
2281 
2282 // Wouldn't need if AddressLiteral version had new name
2283 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
2284   Assembler::call(L, rtype);
2285 }
2286 
2287 void MacroAssembler::call(Register entry) {
2288   Assembler::call(entry);
2289 }
2290 
2291 void MacroAssembler::call(AddressLiteral entry) {
2292   if (reachable(entry)) {
2293     Assembler::call_literal(entry.target(), entry.rspec());
2294   } else {
2295     lea(rscratch1, entry);
2296     Assembler::call(rscratch1);
2297   }
2298 }
2299 
2300 void MacroAssembler::ic_call(address entry, jint method_index) {
2301   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
2302   movptr(rax, (intptr_t)Universe::non_oop_word());
2303   call(AddressLiteral(entry, rh));
2304 }
2305 
2306 // Implementation of call_VM versions
2307 
2308 void MacroAssembler::call_VM(Register oop_result,
2309                              address entry_point,
2310                              bool check_exceptions) {
2311   Label C, E;
2312   call(C, relocInfo::none);
2313   jmp(E);
2314 
2315   bind(C);
2316   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
2317   ret(0);
2318 
2319   bind(E);
2320 }
2321 
2322 void MacroAssembler::call_VM(Register oop_result,
2323                              address entry_point,
2324                              Register arg_1,
2325                              bool check_exceptions) {
2326   Label C, E;
2327   call(C, relocInfo::none);
2328   jmp(E);
2329 
2330   bind(C);
2331   pass_arg1(this, arg_1);
2332   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
2333   ret(0);
2334 
2335   bind(E);
2336 }
2337 
2338 void MacroAssembler::call_VM(Register oop_result,
2339                              address entry_point,
2340                              Register arg_1,
2341                              Register arg_2,
2342                              bool check_exceptions) {
2343   Label C, E;
2344   call(C, relocInfo::none);
2345   jmp(E);
2346 
2347   bind(C);
2348 
2349   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2350 
2351   pass_arg2(this, arg_2);
2352   pass_arg1(this, arg_1);
2353   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
2354   ret(0);
2355 
2356   bind(E);
2357 }
2358 
2359 void MacroAssembler::call_VM(Register oop_result,
2360                              address entry_point,
2361                              Register arg_1,
2362                              Register arg_2,
2363                              Register arg_3,
2364                              bool check_exceptions) {
2365   Label C, E;
2366   call(C, relocInfo::none);
2367   jmp(E);
2368 
2369   bind(C);
2370 
2371   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2372   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2373   pass_arg3(this, arg_3);
2374 
2375   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2376   pass_arg2(this, arg_2);
2377 
2378   pass_arg1(this, arg_1);
2379   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
2380   ret(0);
2381 
2382   bind(E);
2383 }
2384 
2385 void MacroAssembler::call_VM(Register oop_result,
2386                              Register last_java_sp,
2387                              address entry_point,
2388                              int number_of_arguments,
2389                              bool check_exceptions) {
2390   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2391   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2392 }
2393 
2394 void MacroAssembler::call_VM(Register oop_result,
2395                              Register last_java_sp,
2396                              address entry_point,
2397                              Register arg_1,
2398                              bool check_exceptions) {
2399   pass_arg1(this, arg_1);
2400   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2401 }
2402 
2403 void MacroAssembler::call_VM(Register oop_result,
2404                              Register last_java_sp,
2405                              address entry_point,
2406                              Register arg_1,
2407                              Register arg_2,
2408                              bool check_exceptions) {
2409 
2410   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2411   pass_arg2(this, arg_2);
2412   pass_arg1(this, arg_1);
2413   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2414 }
2415 
2416 void MacroAssembler::call_VM(Register oop_result,
2417                              Register last_java_sp,
2418                              address entry_point,
2419                              Register arg_1,
2420                              Register arg_2,
2421                              Register arg_3,
2422                              bool check_exceptions) {
2423   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2424   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2425   pass_arg3(this, arg_3);
2426   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2427   pass_arg2(this, arg_2);
2428   pass_arg1(this, arg_1);
2429   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2430 }
2431 
2432 void MacroAssembler::super_call_VM(Register oop_result,
2433                                    Register last_java_sp,
2434                                    address entry_point,
2435                                    int number_of_arguments,
2436                                    bool check_exceptions) {
2437   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2438   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2439 }
2440 
2441 void MacroAssembler::super_call_VM(Register oop_result,
2442                                    Register last_java_sp,
2443                                    address entry_point,
2444                                    Register arg_1,
2445                                    bool check_exceptions) {
2446   pass_arg1(this, arg_1);
2447   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2448 }
2449 
2450 void MacroAssembler::super_call_VM(Register oop_result,
2451                                    Register last_java_sp,
2452                                    address entry_point,
2453                                    Register arg_1,
2454                                    Register arg_2,
2455                                    bool check_exceptions) {
2456 
2457   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2458   pass_arg2(this, arg_2);
2459   pass_arg1(this, arg_1);
2460   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2461 }
2462 
2463 void MacroAssembler::super_call_VM(Register oop_result,
2464                                    Register last_java_sp,
2465                                    address entry_point,
2466                                    Register arg_1,
2467                                    Register arg_2,
2468                                    Register arg_3,
2469                                    bool check_exceptions) {
2470   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2471   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2472   pass_arg3(this, arg_3);
2473   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2474   pass_arg2(this, arg_2);
2475   pass_arg1(this, arg_1);
2476   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2477 }
2478 
2479 void MacroAssembler::call_VM_base(Register oop_result,
2480                                   Register java_thread,
2481                                   Register last_java_sp,
2482                                   address  entry_point,
2483                                   int      number_of_arguments,
2484                                   bool     check_exceptions) {
2485   // determine java_thread register
2486   if (!java_thread->is_valid()) {
2487 #ifdef _LP64
2488     java_thread = r15_thread;
2489 #else
2490     java_thread = rdi;
2491     get_thread(java_thread);
2492 #endif // LP64
2493   }
2494   // determine last_java_sp register
2495   if (!last_java_sp->is_valid()) {
2496     last_java_sp = rsp;
2497   }
2498   // debugging support
2499   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
2500   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
2501 #ifdef ASSERT
2502   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
2503   // r12 is the heapbase.
2504   LP64_ONLY(if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
2505 #endif // ASSERT
2506 
2507   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
2508   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
2509 
2510   // push java thread (becomes first argument of C function)
2511 
2512   NOT_LP64(push(java_thread); number_of_arguments++);
2513   LP64_ONLY(mov(c_rarg0, r15_thread));
2514 
2515   // set last Java frame before call
2516   assert(last_java_sp != rbp, "can't use ebp/rbp");
2517 
2518   // Only interpreter should have to set fp
2519   set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);
2520 
2521   // do the call, remove parameters
2522   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
2523 
2524   // restore the thread (cannot use the pushed argument since arguments
2525   // may be overwritten by C code generated by an optimizing compiler);
2526   // however can use the register value directly if it is callee saved.
2527   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
2528     // rdi & rsi (also r15) are callee saved -> nothing to do
2529 #ifdef ASSERT
2530     guarantee(java_thread != rax, "change this code");
2531     push(rax);
2532     { Label L;
2533       get_thread(rax);
2534       cmpptr(java_thread, rax);
2535       jcc(Assembler::equal, L);
2536       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
2537       bind(L);
2538     }
2539     pop(rax);
2540 #endif
2541   } else {
2542     get_thread(java_thread);
2543   }
2544   // reset last Java frame
2545   // Only interpreter should have to clear fp
2546   reset_last_Java_frame(java_thread, true);
2547 
2548    // C++ interp handles this in the interpreter
2549   check_and_handle_popframe(java_thread);
2550   check_and_handle_earlyret(java_thread);
2551 
2552   if (check_exceptions) {
2553     // check for pending exceptions (java_thread is set upon return)
2554     cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
2555 #ifndef _LP64
2556     jump_cc(Assembler::notEqual,
2557             RuntimeAddress(StubRoutines::forward_exception_entry()));
2558 #else
2559     // This used to conditionally jump to forward_exception however it is
2560     // possible if we relocate that the branch will not reach. So we must jump
2561     // around so we can always reach
2562 
2563     Label ok;
2564     jcc(Assembler::equal, ok);
2565     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2566     bind(ok);
2567 #endif // LP64
2568   }
2569 
2570   // get oop result if there is one and reset the value in the thread
2571   if (oop_result->is_valid()) {
2572     get_vm_result(oop_result, java_thread);
2573   }
2574 }
2575 
2576 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
2577 
2578   // Calculate the value for last_Java_sp
2579   // somewhat subtle. call_VM does an intermediate call
2580   // which places a return address on the stack just under the
2581   // stack pointer as the user finsihed with it. This allows
2582   // use to retrieve last_Java_pc from last_Java_sp[-1].
2583   // On 32bit we then have to push additional args on the stack to accomplish
2584   // the actual requested call. On 64bit call_VM only can use register args
2585   // so the only extra space is the return address that call_VM created.
2586   // This hopefully explains the calculations here.
2587 
2588 #ifdef _LP64
2589   // We've pushed one address, correct last_Java_sp
2590   lea(rax, Address(rsp, wordSize));
2591 #else
2592   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
2593 #endif // LP64
2594 
2595   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
2596 
2597 }
2598 
2599 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
2600 void MacroAssembler::call_VM_leaf0(address entry_point) {
2601   MacroAssembler::call_VM_leaf_base(entry_point, 0);
2602 }
2603 
2604 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
2605   call_VM_leaf_base(entry_point, number_of_arguments);
2606 }
2607 
2608 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
2609   pass_arg0(this, arg_0);
2610   call_VM_leaf(entry_point, 1);
2611 }
2612 
2613 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2614 
2615   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2616   pass_arg1(this, arg_1);
2617   pass_arg0(this, arg_0);
2618   call_VM_leaf(entry_point, 2);
2619 }
2620 
2621 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2622   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2623   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2624   pass_arg2(this, arg_2);
2625   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2626   pass_arg1(this, arg_1);
2627   pass_arg0(this, arg_0);
2628   call_VM_leaf(entry_point, 3);
2629 }
2630 
2631 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
2632   pass_arg0(this, arg_0);
2633   MacroAssembler::call_VM_leaf_base(entry_point, 1);
2634 }
2635 
2636 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2637 
2638   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2639   pass_arg1(this, arg_1);
2640   pass_arg0(this, arg_0);
2641   MacroAssembler::call_VM_leaf_base(entry_point, 2);
2642 }
2643 
2644 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2645   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2646   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2647   pass_arg2(this, arg_2);
2648   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2649   pass_arg1(this, arg_1);
2650   pass_arg0(this, arg_0);
2651   MacroAssembler::call_VM_leaf_base(entry_point, 3);
2652 }
2653 
2654 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
2655   LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg"));
2656   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2657   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2658   pass_arg3(this, arg_3);
2659   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2660   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2661   pass_arg2(this, arg_2);
2662   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2663   pass_arg1(this, arg_1);
2664   pass_arg0(this, arg_0);
2665   MacroAssembler::call_VM_leaf_base(entry_point, 4);
2666 }
2667 
2668 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
2669   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
2670   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
2671   verify_oop(oop_result, "broken oop in call_VM_base");
2672 }
2673 
2674 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
2675   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
2676   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
2677 }
2678 
2679 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
2680 }
2681 
2682 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
2683 }
2684 
2685 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) {
2686   if (reachable(src1)) {
2687     cmpl(as_Address(src1), imm);
2688   } else {
2689     lea(rscratch1, src1);
2690     cmpl(Address(rscratch1, 0), imm);
2691   }
2692 }
2693 
2694 void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
2695   assert(!src2.is_lval(), "use cmpptr");
2696   if (reachable(src2)) {
2697     cmpl(src1, as_Address(src2));
2698   } else {
2699     lea(rscratch1, src2);
2700     cmpl(src1, Address(rscratch1, 0));
2701   }
2702 }
2703 
2704 void MacroAssembler::cmp32(Register src1, int32_t imm) {
2705   Assembler::cmpl(src1, imm);
2706 }
2707 
2708 void MacroAssembler::cmp32(Register src1, Address src2) {
2709   Assembler::cmpl(src1, src2);
2710 }
2711 
2712 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2713   ucomisd(opr1, opr2);
2714 
2715   Label L;
2716   if (unordered_is_less) {
2717     movl(dst, -1);
2718     jcc(Assembler::parity, L);
2719     jcc(Assembler::below , L);
2720     movl(dst, 0);
2721     jcc(Assembler::equal , L);
2722     increment(dst);
2723   } else { // unordered is greater
2724     movl(dst, 1);
2725     jcc(Assembler::parity, L);
2726     jcc(Assembler::above , L);
2727     movl(dst, 0);
2728     jcc(Assembler::equal , L);
2729     decrementl(dst);
2730   }
2731   bind(L);
2732 }
2733 
2734 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2735   ucomiss(opr1, opr2);
2736 
2737   Label L;
2738   if (unordered_is_less) {
2739     movl(dst, -1);
2740     jcc(Assembler::parity, L);
2741     jcc(Assembler::below , L);
2742     movl(dst, 0);
2743     jcc(Assembler::equal , L);
2744     increment(dst);
2745   } else { // unordered is greater
2746     movl(dst, 1);
2747     jcc(Assembler::parity, L);
2748     jcc(Assembler::above , L);
2749     movl(dst, 0);
2750     jcc(Assembler::equal , L);
2751     decrementl(dst);
2752   }
2753   bind(L);
2754 }
2755 
2756 
2757 void MacroAssembler::cmp8(AddressLiteral src1, int imm) {
2758   if (reachable(src1)) {
2759     cmpb(as_Address(src1), imm);
2760   } else {
2761     lea(rscratch1, src1);
2762     cmpb(Address(rscratch1, 0), imm);
2763   }
2764 }
2765 
2766 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
2767 #ifdef _LP64
2768   if (src2.is_lval()) {
2769     movptr(rscratch1, src2);
2770     Assembler::cmpq(src1, rscratch1);
2771   } else if (reachable(src2)) {
2772     cmpq(src1, as_Address(src2));
2773   } else {
2774     lea(rscratch1, src2);
2775     Assembler::cmpq(src1, Address(rscratch1, 0));
2776   }
2777 #else
2778   if (src2.is_lval()) {
2779     cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2780   } else {
2781     cmpl(src1, as_Address(src2));
2782   }
2783 #endif // _LP64
2784 }
2785 
2786 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
2787   assert(src2.is_lval(), "not a mem-mem compare");
2788 #ifdef _LP64
2789   // moves src2's literal address
2790   movptr(rscratch1, src2);
2791   Assembler::cmpq(src1, rscratch1);
2792 #else
2793   cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2794 #endif // _LP64
2795 }
2796 
2797 void MacroAssembler::cmpoop(Register src1, Register src2) {
2798   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2799   bs->obj_equals(this, IN_HEAP, src1, src2);
2800 }
2801 
2802 void MacroAssembler::cmpoop(Register src1, Address src2) {
2803   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2804   bs->obj_equals(this, IN_HEAP, src1, src2);
2805 }
2806 
2807 #ifdef _LP64
2808 void MacroAssembler::cmpoop(Register src1, jobject src2) {
2809   movoop(rscratch1, src2);
2810   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2811   bs->obj_equals(this, IN_HEAP, src1, rscratch1);
2812 }
2813 #endif
2814 
2815 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) {
2816   if (reachable(adr)) {
2817     if (os::is_MP())
2818       lock();
2819     cmpxchgptr(reg, as_Address(adr));
2820   } else {
2821     lea(rscratch1, adr);
2822     if (os::is_MP())
2823       lock();
2824     cmpxchgptr(reg, Address(rscratch1, 0));
2825   }
2826 }
2827 
2828 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
2829   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
2830 }
2831 
2832 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {
2833   if (reachable(src)) {
2834     Assembler::comisd(dst, as_Address(src));
2835   } else {
2836     lea(rscratch1, src);
2837     Assembler::comisd(dst, Address(rscratch1, 0));
2838   }
2839 }
2840 
2841 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {
2842   if (reachable(src)) {
2843     Assembler::comiss(dst, as_Address(src));
2844   } else {
2845     lea(rscratch1, src);
2846     Assembler::comiss(dst, Address(rscratch1, 0));
2847   }
2848 }
2849 
2850 
2851 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
2852   Condition negated_cond = negate_condition(cond);
2853   Label L;
2854   jcc(negated_cond, L);
2855   pushf(); // Preserve flags
2856   atomic_incl(counter_addr);
2857   popf();
2858   bind(L);
2859 }
2860 
2861 int MacroAssembler::corrected_idivl(Register reg) {
2862   // Full implementation of Java idiv and irem; checks for
2863   // special case as described in JVM spec., p.243 & p.271.
2864   // The function returns the (pc) offset of the idivl
2865   // instruction - may be needed for implicit exceptions.
2866   //
2867   //         normal case                           special case
2868   //
2869   // input : rax,: dividend                         min_int
2870   //         reg: divisor   (may not be rax,/rdx)   -1
2871   //
2872   // output: rax,: quotient  (= rax, idiv reg)       min_int
2873   //         rdx: remainder (= rax, irem reg)       0
2874   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
2875   const int min_int = 0x80000000;
2876   Label normal_case, special_case;
2877 
2878   // check for special case
2879   cmpl(rax, min_int);
2880   jcc(Assembler::notEqual, normal_case);
2881   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
2882   cmpl(reg, -1);
2883   jcc(Assembler::equal, special_case);
2884 
2885   // handle normal case
2886   bind(normal_case);
2887   cdql();
2888   int idivl_offset = offset();
2889   idivl(reg);
2890 
2891   // normal and special case exit
2892   bind(special_case);
2893 
2894   return idivl_offset;
2895 }
2896 
2897 
2898 
2899 void MacroAssembler::decrementl(Register reg, int value) {
2900   if (value == min_jint) {subl(reg, value) ; return; }
2901   if (value <  0) { incrementl(reg, -value); return; }
2902   if (value == 0) {                        ; return; }
2903   if (value == 1 && UseIncDec) { decl(reg) ; return; }
2904   /* else */      { subl(reg, value)       ; return; }
2905 }
2906 
2907 void MacroAssembler::decrementl(Address dst, int value) {
2908   if (value == min_jint) {subl(dst, value) ; return; }
2909   if (value <  0) { incrementl(dst, -value); return; }
2910   if (value == 0) {                        ; return; }
2911   if (value == 1 && UseIncDec) { decl(dst) ; return; }
2912   /* else */      { subl(dst, value)       ; return; }
2913 }
2914 
2915 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
2916   assert (shift_value > 0, "illegal shift value");
2917   Label _is_positive;
2918   testl (reg, reg);
2919   jcc (Assembler::positive, _is_positive);
2920   int offset = (1 << shift_value) - 1 ;
2921 
2922   if (offset == 1) {
2923     incrementl(reg);
2924   } else {
2925     addl(reg, offset);
2926   }
2927 
2928   bind (_is_positive);
2929   sarl(reg, shift_value);
2930 }
2931 
2932 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
2933   if (reachable(src)) {
2934     Assembler::divsd(dst, as_Address(src));
2935   } else {
2936     lea(rscratch1, src);
2937     Assembler::divsd(dst, Address(rscratch1, 0));
2938   }
2939 }
2940 
2941 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
2942   if (reachable(src)) {
2943     Assembler::divss(dst, as_Address(src));
2944   } else {
2945     lea(rscratch1, src);
2946     Assembler::divss(dst, Address(rscratch1, 0));
2947   }
2948 }
2949 
2950 // !defined(COMPILER2) is because of stupid core builds
2951 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) || INCLUDE_JVMCI
2952 void MacroAssembler::empty_FPU_stack() {
2953   if (VM_Version::supports_mmx()) {
2954     emms();
2955   } else {
2956     for (int i = 8; i-- > 0; ) ffree(i);
2957   }
2958 }
2959 #endif // !LP64 || C1 || !C2 || INCLUDE_JVMCI
2960 
2961 
2962 // Defines obj, preserves var_size_in_bytes
2963 void MacroAssembler::eden_allocate(Register obj,
2964                                    Register var_size_in_bytes,
2965                                    int con_size_in_bytes,
2966                                    Register t1,
2967                                    Label& slow_case) {
2968   assert(obj == rax, "obj must be in rax, for cmpxchg");
2969   assert_different_registers(obj, var_size_in_bytes, t1);
2970   if (!Universe::heap()->supports_inline_contig_alloc()) {
2971     jmp(slow_case);
2972   } else {
2973     Register end = t1;
2974     Label retry;
2975     bind(retry);
2976     ExternalAddress heap_top((address) Universe::heap()->top_addr());
2977     movptr(obj, heap_top);
2978     if (var_size_in_bytes == noreg) {
2979       lea(end, Address(obj, con_size_in_bytes));
2980     } else {
2981       lea(end, Address(obj, var_size_in_bytes, Address::times_1));
2982     }
2983     // if end < obj then we wrapped around => object too long => slow case
2984     cmpptr(end, obj);
2985     jcc(Assembler::below, slow_case);
2986     cmpptr(end, ExternalAddress((address) Universe::heap()->end_addr()));
2987     jcc(Assembler::above, slow_case);
2988     // Compare obj with the top addr, and if still equal, store the new top addr in
2989     // end at the address of the top addr pointer. Sets ZF if was equal, and clears
2990     // it otherwise. Use lock prefix for atomicity on MPs.
2991     locked_cmpxchgptr(end, heap_top);
2992     jcc(Assembler::notEqual, retry);
2993   }
2994 }
2995 
2996 void MacroAssembler::enter() {
2997   push(rbp);
2998   mov(rbp, rsp);
2999 }
3000 
3001 // A 5 byte nop that is safe for patching (see patch_verified_entry)
3002 void MacroAssembler::fat_nop() {
3003   if (UseAddressNop) {
3004     addr_nop_5();
3005   } else {
3006     emit_int8(0x26); // es:
3007     emit_int8(0x2e); // cs:
3008     emit_int8(0x64); // fs:
3009     emit_int8(0x65); // gs:
3010     emit_int8((unsigned char)0x90);
3011   }
3012 }
3013 
3014 void MacroAssembler::fcmp(Register tmp) {
3015   fcmp(tmp, 1, true, true);
3016 }
3017 
3018 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
3019   assert(!pop_right || pop_left, "usage error");
3020   if (VM_Version::supports_cmov()) {
3021     assert(tmp == noreg, "unneeded temp");
3022     if (pop_left) {
3023       fucomip(index);
3024     } else {
3025       fucomi(index);
3026     }
3027     if (pop_right) {
3028       fpop();
3029     }
3030   } else {
3031     assert(tmp != noreg, "need temp");
3032     if (pop_left) {
3033       if (pop_right) {
3034         fcompp();
3035       } else {
3036         fcomp(index);
3037       }
3038     } else {
3039       fcom(index);
3040     }
3041     // convert FPU condition into eflags condition via rax,
3042     save_rax(tmp);
3043     fwait(); fnstsw_ax();
3044     sahf();
3045     restore_rax(tmp);
3046   }
3047   // condition codes set as follows:
3048   //
3049   // CF (corresponds to C0) if x < y
3050   // PF (corresponds to C2) if unordered
3051   // ZF (corresponds to C3) if x = y
3052 }
3053 
3054 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
3055   fcmp2int(dst, unordered_is_less, 1, true, true);
3056 }
3057 
3058 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
3059   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
3060   Label L;
3061   if (unordered_is_less) {
3062     movl(dst, -1);
3063     jcc(Assembler::parity, L);
3064     jcc(Assembler::below , L);
3065     movl(dst, 0);
3066     jcc(Assembler::equal , L);
3067     increment(dst);
3068   } else { // unordered is greater
3069     movl(dst, 1);
3070     jcc(Assembler::parity, L);
3071     jcc(Assembler::above , L);
3072     movl(dst, 0);
3073     jcc(Assembler::equal , L);
3074     decrementl(dst);
3075   }
3076   bind(L);
3077 }
3078 
3079 void MacroAssembler::fld_d(AddressLiteral src) {
3080   fld_d(as_Address(src));
3081 }
3082 
3083 void MacroAssembler::fld_s(AddressLiteral src) {
3084   fld_s(as_Address(src));
3085 }
3086 
3087 void MacroAssembler::fld_x(AddressLiteral src) {
3088   Assembler::fld_x(as_Address(src));
3089 }
3090 
3091 void MacroAssembler::fldcw(AddressLiteral src) {
3092   Assembler::fldcw(as_Address(src));
3093 }
3094 
3095 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src) {
3096   if (reachable(src)) {
3097     Assembler::mulpd(dst, as_Address(src));
3098   } else {
3099     lea(rscratch1, src);
3100     Assembler::mulpd(dst, Address(rscratch1, 0));
3101   }
3102 }
3103 
3104 void MacroAssembler::increase_precision() {
3105   subptr(rsp, BytesPerWord);
3106   fnstcw(Address(rsp, 0));
3107   movl(rax, Address(rsp, 0));
3108   orl(rax, 0x300);
3109   push(rax);
3110   fldcw(Address(rsp, 0));
3111   pop(rax);
3112 }
3113 
3114 void MacroAssembler::restore_precision() {
3115   fldcw(Address(rsp, 0));
3116   addptr(rsp, BytesPerWord);
3117 }
3118 
3119 void MacroAssembler::fpop() {
3120   ffree();
3121   fincstp();
3122 }
3123 
3124 void MacroAssembler::load_float(Address src) {
3125   if (UseSSE >= 1) {
3126     movflt(xmm0, src);
3127   } else {
3128     LP64_ONLY(ShouldNotReachHere());
3129     NOT_LP64(fld_s(src));
3130   }
3131 }
3132 
3133 void MacroAssembler::store_float(Address dst) {
3134   if (UseSSE >= 1) {
3135     movflt(dst, xmm0);
3136   } else {
3137     LP64_ONLY(ShouldNotReachHere());
3138     NOT_LP64(fstp_s(dst));
3139   }
3140 }
3141 
3142 void MacroAssembler::load_double(Address src) {
3143   if (UseSSE >= 2) {
3144     movdbl(xmm0, src);
3145   } else {
3146     LP64_ONLY(ShouldNotReachHere());
3147     NOT_LP64(fld_d(src));
3148   }
3149 }
3150 
3151 void MacroAssembler::store_double(Address dst) {
3152   if (UseSSE >= 2) {
3153     movdbl(dst, xmm0);
3154   } else {
3155     LP64_ONLY(ShouldNotReachHere());
3156     NOT_LP64(fstp_d(dst));
3157   }
3158 }
3159 
3160 void MacroAssembler::fremr(Register tmp) {
3161   save_rax(tmp);
3162   { Label L;
3163     bind(L);
3164     fprem();
3165     fwait(); fnstsw_ax();
3166 #ifdef _LP64
3167     testl(rax, 0x400);
3168     jcc(Assembler::notEqual, L);
3169 #else
3170     sahf();
3171     jcc(Assembler::parity, L);
3172 #endif // _LP64
3173   }
3174   restore_rax(tmp);
3175   // Result is in ST0.
3176   // Note: fxch & fpop to get rid of ST1
3177   // (otherwise FPU stack could overflow eventually)
3178   fxch(1);
3179   fpop();
3180 }
3181 
3182 // dst = c = a * b + c
3183 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
3184   Assembler::vfmadd231sd(c, a, b);
3185   if (dst != c) {
3186     movdbl(dst, c);
3187   }
3188 }
3189 
3190 // dst = c = a * b + c
3191 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
3192   Assembler::vfmadd231ss(c, a, b);
3193   if (dst != c) {
3194     movflt(dst, c);
3195   }
3196 }
3197 
3198 // dst = c = a * b + c
3199 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
3200   Assembler::vfmadd231pd(c, a, b, vector_len);
3201   if (dst != c) {
3202     vmovdqu(dst, c);
3203   }
3204 }
3205 
3206 // dst = c = a * b + c
3207 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
3208   Assembler::vfmadd231ps(c, a, b, vector_len);
3209   if (dst != c) {
3210     vmovdqu(dst, c);
3211   }
3212 }
3213 
3214 // dst = c = a * b + c
3215 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
3216   Assembler::vfmadd231pd(c, a, b, vector_len);
3217   if (dst != c) {
3218     vmovdqu(dst, c);
3219   }
3220 }
3221 
3222 // dst = c = a * b + c
3223 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
3224   Assembler::vfmadd231ps(c, a, b, vector_len);
3225   if (dst != c) {
3226     vmovdqu(dst, c);
3227   }
3228 }
3229 
3230 void MacroAssembler::incrementl(AddressLiteral dst) {
3231   if (reachable(dst)) {
3232     incrementl(as_Address(dst));
3233   } else {
3234     lea(rscratch1, dst);
3235     incrementl(Address(rscratch1, 0));
3236   }
3237 }
3238 
3239 void MacroAssembler::incrementl(ArrayAddress dst) {
3240   incrementl(as_Address(dst));
3241 }
3242 
3243 void MacroAssembler::incrementl(Register reg, int value) {
3244   if (value == min_jint) {addl(reg, value) ; return; }
3245   if (value <  0) { decrementl(reg, -value); return; }
3246   if (value == 0) {                        ; return; }
3247   if (value == 1 && UseIncDec) { incl(reg) ; return; }
3248   /* else */      { addl(reg, value)       ; return; }
3249 }
3250 
3251 void MacroAssembler::incrementl(Address dst, int value) {
3252   if (value == min_jint) {addl(dst, value) ; return; }
3253   if (value <  0) { decrementl(dst, -value); return; }
3254   if (value == 0) {                        ; return; }
3255   if (value == 1 && UseIncDec) { incl(dst) ; return; }
3256   /* else */      { addl(dst, value)       ; return; }
3257 }
3258 
3259 void MacroAssembler::jump(AddressLiteral dst) {
3260   if (reachable(dst)) {
3261     jmp_literal(dst.target(), dst.rspec());
3262   } else {
3263     lea(rscratch1, dst);
3264     jmp(rscratch1);
3265   }
3266 }
3267 
3268 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
3269   if (reachable(dst)) {
3270     InstructionMark im(this);
3271     relocate(dst.reloc());
3272     const int short_size = 2;
3273     const int long_size = 6;
3274     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
3275     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
3276       // 0111 tttn #8-bit disp
3277       emit_int8(0x70 | cc);
3278       emit_int8((offs - short_size) & 0xFF);
3279     } else {
3280       // 0000 1111 1000 tttn #32-bit disp
3281       emit_int8(0x0F);
3282       emit_int8((unsigned char)(0x80 | cc));
3283       emit_int32(offs - long_size);
3284     }
3285   } else {
3286 #ifdef ASSERT
3287     warning("reversing conditional branch");
3288 #endif /* ASSERT */
3289     Label skip;
3290     jccb(reverse[cc], skip);
3291     lea(rscratch1, dst);
3292     Assembler::jmp(rscratch1);
3293     bind(skip);
3294   }
3295 }
3296 
3297 void MacroAssembler::ldmxcsr(AddressLiteral src) {
3298   if (reachable(src)) {
3299     Assembler::ldmxcsr(as_Address(src));
3300   } else {
3301     lea(rscratch1, src);
3302     Assembler::ldmxcsr(Address(rscratch1, 0));
3303   }
3304 }
3305 
3306 int MacroAssembler::load_signed_byte(Register dst, Address src) {
3307   int off;
3308   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3309     off = offset();
3310     movsbl(dst, src); // movsxb
3311   } else {
3312     off = load_unsigned_byte(dst, src);
3313     shll(dst, 24);
3314     sarl(dst, 24);
3315   }
3316   return off;
3317 }
3318 
3319 // Note: load_signed_short used to be called load_signed_word.
3320 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
3321 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
3322 // The term "word" in HotSpot means a 32- or 64-bit machine word.
3323 int MacroAssembler::load_signed_short(Register dst, Address src) {
3324   int off;
3325   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3326     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
3327     // version but this is what 64bit has always done. This seems to imply
3328     // that users are only using 32bits worth.
3329     off = offset();
3330     movswl(dst, src); // movsxw
3331   } else {
3332     off = load_unsigned_short(dst, src);
3333     shll(dst, 16);
3334     sarl(dst, 16);
3335   }
3336   return off;
3337 }
3338 
3339 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
3340   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3341   // and "3.9 Partial Register Penalties", p. 22).
3342   int off;
3343   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
3344     off = offset();
3345     movzbl(dst, src); // movzxb
3346   } else {
3347     xorl(dst, dst);
3348     off = offset();
3349     movb(dst, src);
3350   }
3351   return off;
3352 }
3353 
3354 // Note: load_unsigned_short used to be called load_unsigned_word.
3355 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
3356   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3357   // and "3.9 Partial Register Penalties", p. 22).
3358   int off;
3359   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
3360     off = offset();
3361     movzwl(dst, src); // movzxw
3362   } else {
3363     xorl(dst, dst);
3364     off = offset();
3365     movw(dst, src);
3366   }
3367   return off;
3368 }
3369 
3370 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
3371   switch (size_in_bytes) {
3372 #ifndef _LP64
3373   case  8:
3374     assert(dst2 != noreg, "second dest register required");
3375     movl(dst,  src);
3376     movl(dst2, src.plus_disp(BytesPerInt));
3377     break;
3378 #else
3379   case  8:  movq(dst, src); break;
3380 #endif
3381   case  4:  movl(dst, src); break;
3382   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
3383   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
3384   default:  ShouldNotReachHere();
3385   }
3386 }
3387 
3388 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
3389   switch (size_in_bytes) {
3390 #ifndef _LP64
3391   case  8:
3392     assert(src2 != noreg, "second source register required");
3393     movl(dst,                        src);
3394     movl(dst.plus_disp(BytesPerInt), src2);
3395     break;
3396 #else
3397   case  8:  movq(dst, src); break;
3398 #endif
3399   case  4:  movl(dst, src); break;
3400   case  2:  movw(dst, src); break;
3401   case  1:  movb(dst, src); break;
3402   default:  ShouldNotReachHere();
3403   }
3404 }
3405 
3406 void MacroAssembler::mov32(AddressLiteral dst, Register src) {
3407   if (reachable(dst)) {
3408     movl(as_Address(dst), src);
3409   } else {
3410     lea(rscratch1, dst);
3411     movl(Address(rscratch1, 0), src);
3412   }
3413 }
3414 
3415 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
3416   if (reachable(src)) {
3417     movl(dst, as_Address(src));
3418   } else {
3419     lea(rscratch1, src);
3420     movl(dst, Address(rscratch1, 0));
3421   }
3422 }
3423 
3424 // C++ bool manipulation
3425 
3426 void MacroAssembler::movbool(Register dst, Address src) {
3427   if(sizeof(bool) == 1)
3428     movb(dst, src);
3429   else if(sizeof(bool) == 2)
3430     movw(dst, src);
3431   else if(sizeof(bool) == 4)
3432     movl(dst, src);
3433   else
3434     // unsupported
3435     ShouldNotReachHere();
3436 }
3437 
3438 void MacroAssembler::movbool(Address dst, bool boolconst) {
3439   if(sizeof(bool) == 1)
3440     movb(dst, (int) boolconst);
3441   else if(sizeof(bool) == 2)
3442     movw(dst, (int) boolconst);
3443   else if(sizeof(bool) == 4)
3444     movl(dst, (int) boolconst);
3445   else
3446     // unsupported
3447     ShouldNotReachHere();
3448 }
3449 
3450 void MacroAssembler::movbool(Address dst, Register src) {
3451   if(sizeof(bool) == 1)
3452     movb(dst, src);
3453   else if(sizeof(bool) == 2)
3454     movw(dst, src);
3455   else if(sizeof(bool) == 4)
3456     movl(dst, src);
3457   else
3458     // unsupported
3459     ShouldNotReachHere();
3460 }
3461 
3462 void MacroAssembler::movbyte(ArrayAddress dst, int src) {
3463   movb(as_Address(dst), src);
3464 }
3465 
3466 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src) {
3467   if (reachable(src)) {
3468     movdl(dst, as_Address(src));
3469   } else {
3470     lea(rscratch1, src);
3471     movdl(dst, Address(rscratch1, 0));
3472   }
3473 }
3474 
3475 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src) {
3476   if (reachable(src)) {
3477     movq(dst, as_Address(src));
3478   } else {
3479     lea(rscratch1, src);
3480     movq(dst, Address(rscratch1, 0));
3481   }
3482 }
3483 
3484 void MacroAssembler::setvectmask(Register dst, Register src) {
3485   Assembler::movl(dst, 1);
3486   Assembler::shlxl(dst, dst, src);
3487   Assembler::decl(dst);
3488   Assembler::kmovdl(k1, dst);
3489   Assembler::movl(dst, src);
3490 }
3491 
3492 void MacroAssembler::restorevectmask() {
3493   Assembler::knotwl(k1, k0);
3494 }
3495 
3496 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
3497   if (reachable(src)) {
3498     if (UseXmmLoadAndClearUpper) {
3499       movsd (dst, as_Address(src));
3500     } else {
3501       movlpd(dst, as_Address(src));
3502     }
3503   } else {
3504     lea(rscratch1, src);
3505     if (UseXmmLoadAndClearUpper) {
3506       movsd (dst, Address(rscratch1, 0));
3507     } else {
3508       movlpd(dst, Address(rscratch1, 0));
3509     }
3510   }
3511 }
3512 
3513 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
3514   if (reachable(src)) {
3515     movss(dst, as_Address(src));
3516   } else {
3517     lea(rscratch1, src);
3518     movss(dst, Address(rscratch1, 0));
3519   }
3520 }
3521 
3522 void MacroAssembler::movptr(Register dst, Register src) {
3523   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3524 }
3525 
3526 void MacroAssembler::movptr(Register dst, Address src) {
3527   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3528 }
3529 
3530 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
3531 void MacroAssembler::movptr(Register dst, intptr_t src) {
3532   LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src));
3533 }
3534 
3535 void MacroAssembler::movptr(Address dst, Register src) {
3536   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3537 }
3538 
3539 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
3540   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3541     Assembler::vextractf32x4(dst, src, 0);
3542   } else {
3543     Assembler::movdqu(dst, src);
3544   }
3545 }
3546 
3547 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
3548   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3549     Assembler::vinsertf32x4(dst, dst, src, 0);
3550   } else {
3551     Assembler::movdqu(dst, src);
3552   }
3553 }
3554 
3555 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
3556   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3557     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3558   } else {
3559     Assembler::movdqu(dst, src);
3560   }
3561 }
3562 
3563 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register scratchReg) {
3564   if (reachable(src)) {
3565     movdqu(dst, as_Address(src));
3566   } else {
3567     lea(scratchReg, src);
3568     movdqu(dst, Address(scratchReg, 0));
3569   }
3570 }
3571 
3572 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
3573   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3574     vextractf64x4_low(dst, src);
3575   } else {
3576     Assembler::vmovdqu(dst, src);
3577   }
3578 }
3579 
3580 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
3581   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3582     vinsertf64x4_low(dst, src);
3583   } else {
3584     Assembler::vmovdqu(dst, src);
3585   }
3586 }
3587 
3588 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3589   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3590     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3591   }
3592   else {
3593     Assembler::vmovdqu(dst, src);
3594   }
3595 }
3596 
3597 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src) {
3598   if (reachable(src)) {
3599     vmovdqu(dst, as_Address(src));
3600   }
3601   else {
3602     lea(rscratch1, src);
3603     vmovdqu(dst, Address(rscratch1, 0));
3604   }
3605 }
3606 
3607 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) {
3608   if (reachable(src)) {
3609     Assembler::movdqa(dst, as_Address(src));
3610   } else {
3611     lea(rscratch1, src);
3612     Assembler::movdqa(dst, Address(rscratch1, 0));
3613   }
3614 }
3615 
3616 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
3617   if (reachable(src)) {
3618     Assembler::movsd(dst, as_Address(src));
3619   } else {
3620     lea(rscratch1, src);
3621     Assembler::movsd(dst, Address(rscratch1, 0));
3622   }
3623 }
3624 
3625 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
3626   if (reachable(src)) {
3627     Assembler::movss(dst, as_Address(src));
3628   } else {
3629     lea(rscratch1, src);
3630     Assembler::movss(dst, Address(rscratch1, 0));
3631   }
3632 }
3633 
3634 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) {
3635   if (reachable(src)) {
3636     Assembler::mulsd(dst, as_Address(src));
3637   } else {
3638     lea(rscratch1, src);
3639     Assembler::mulsd(dst, Address(rscratch1, 0));
3640   }
3641 }
3642 
3643 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) {
3644   if (reachable(src)) {
3645     Assembler::mulss(dst, as_Address(src));
3646   } else {
3647     lea(rscratch1, src);
3648     Assembler::mulss(dst, Address(rscratch1, 0));
3649   }
3650 }
3651 
3652 void MacroAssembler::null_check(Register reg, int offset) {
3653   if (needs_explicit_null_check(offset)) {
3654     // provoke OS NULL exception if reg = NULL by
3655     // accessing M[reg] w/o changing any (non-CC) registers
3656     // NOTE: cmpl is plenty here to provoke a segv
3657     cmpptr(rax, Address(reg, 0));
3658     // Note: should probably use testl(rax, Address(reg, 0));
3659     //       may be shorter code (however, this version of
3660     //       testl needs to be implemented first)
3661   } else {
3662     // nothing to do, (later) access of M[reg + offset]
3663     // will provoke OS NULL exception if reg = NULL
3664   }
3665 }
3666 
3667 void MacroAssembler::os_breakpoint() {
3668   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
3669   // (e.g., MSVC can't call ps() otherwise)
3670   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
3671 }
3672 
3673 void MacroAssembler::unimplemented(const char* what) {
3674   const char* buf = NULL;
3675   {
3676     ResourceMark rm;
3677     stringStream ss;
3678     ss.print("unimplemented: %s", what);
3679     buf = code_string(ss.as_string());
3680   }
3681   stop(buf);
3682 }
3683 
3684 #ifdef _LP64
3685 #define XSTATE_BV 0x200
3686 #endif
3687 
3688 void MacroAssembler::pop_CPU_state() {
3689   pop_FPU_state();
3690   pop_IU_state();
3691 }
3692 
3693 void MacroAssembler::pop_FPU_state() {
3694 #ifndef _LP64
3695   frstor(Address(rsp, 0));
3696 #else
3697   fxrstor(Address(rsp, 0));
3698 #endif
3699   addptr(rsp, FPUStateSizeInWords * wordSize);
3700 }
3701 
3702 void MacroAssembler::pop_IU_state() {
3703   popa();
3704   LP64_ONLY(addq(rsp, 8));
3705   popf();
3706 }
3707 
3708 // Save Integer and Float state
3709 // Warning: Stack must be 16 byte aligned (64bit)
3710 void MacroAssembler::push_CPU_state() {
3711   push_IU_state();
3712   push_FPU_state();
3713 }
3714 
3715 void MacroAssembler::push_FPU_state() {
3716   subptr(rsp, FPUStateSizeInWords * wordSize);
3717 #ifndef _LP64
3718   fnsave(Address(rsp, 0));
3719   fwait();
3720 #else
3721   fxsave(Address(rsp, 0));
3722 #endif // LP64
3723 }
3724 
3725 void MacroAssembler::push_IU_state() {
3726   // Push flags first because pusha kills them
3727   pushf();
3728   // Make sure rsp stays 16-byte aligned
3729   LP64_ONLY(subq(rsp, 8));
3730   pusha();
3731 }
3732 
3733 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
3734   if (!java_thread->is_valid()) {
3735     java_thread = rdi;
3736     get_thread(java_thread);
3737   }
3738   // we must set sp to zero to clear frame
3739   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
3740   if (clear_fp) {
3741     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
3742   }
3743 
3744   // Always clear the pc because it could have been set by make_walkable()
3745   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
3746 
3747   vzeroupper();
3748 }
3749 
3750 void MacroAssembler::restore_rax(Register tmp) {
3751   if (tmp == noreg) pop(rax);
3752   else if (tmp != rax) mov(rax, tmp);
3753 }
3754 
3755 void MacroAssembler::round_to(Register reg, int modulus) {
3756   addptr(reg, modulus - 1);
3757   andptr(reg, -modulus);
3758 }
3759 
3760 void MacroAssembler::save_rax(Register tmp) {
3761   if (tmp == noreg) push(rax);
3762   else if (tmp != rax) mov(tmp, rax);
3763 }
3764 
3765 // Write serialization page so VM thread can do a pseudo remote membar.
3766 // We use the current thread pointer to calculate a thread specific
3767 // offset to write to within the page. This minimizes bus traffic
3768 // due to cache line collision.
3769 void MacroAssembler::serialize_memory(Register thread, Register tmp) {
3770   movl(tmp, thread);
3771   shrl(tmp, os::get_serialize_page_shift_count());
3772   andl(tmp, (os::vm_page_size() - sizeof(int)));
3773 
3774   Address index(noreg, tmp, Address::times_1);
3775   ExternalAddress page(os::get_memory_serialize_page());
3776 
3777   // Size of store must match masking code above
3778   movl(as_Address(ArrayAddress(page, index)), tmp);
3779 }
3780 
3781 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, Register temp_reg) {
3782   if (SafepointMechanism::uses_thread_local_poll()) {
3783 #ifdef _LP64
3784     assert(thread_reg == r15_thread, "should be");
3785 #else
3786     if (thread_reg == noreg) {
3787       thread_reg = temp_reg;
3788       get_thread(thread_reg);
3789     }
3790 #endif
3791     testb(Address(thread_reg, Thread::polling_page_offset()), SafepointMechanism::poll_bit());
3792     jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
3793   } else {
3794     cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
3795         SafepointSynchronize::_not_synchronized);
3796     jcc(Assembler::notEqual, slow_path);
3797   }
3798 }
3799 
3800 // Calls to C land
3801 //
3802 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
3803 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
3804 // has to be reset to 0. This is required to allow proper stack traversal.
3805 void MacroAssembler::set_last_Java_frame(Register java_thread,
3806                                          Register last_java_sp,
3807                                          Register last_java_fp,
3808                                          address  last_java_pc) {
3809   vzeroupper();
3810   // determine java_thread register
3811   if (!java_thread->is_valid()) {
3812     java_thread = rdi;
3813     get_thread(java_thread);
3814   }
3815   // determine last_java_sp register
3816   if (!last_java_sp->is_valid()) {
3817     last_java_sp = rsp;
3818   }
3819 
3820   // last_java_fp is optional
3821 
3822   if (last_java_fp->is_valid()) {
3823     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
3824   }
3825 
3826   // last_java_pc is optional
3827 
3828   if (last_java_pc != NULL) {
3829     lea(Address(java_thread,
3830                  JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()),
3831         InternalAddress(last_java_pc));
3832 
3833   }
3834   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
3835 }
3836 
3837 void MacroAssembler::shlptr(Register dst, int imm8) {
3838   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
3839 }
3840 
3841 void MacroAssembler::shrptr(Register dst, int imm8) {
3842   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
3843 }
3844 
3845 void MacroAssembler::sign_extend_byte(Register reg) {
3846   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
3847     movsbl(reg, reg); // movsxb
3848   } else {
3849     shll(reg, 24);
3850     sarl(reg, 24);
3851   }
3852 }
3853 
3854 void MacroAssembler::sign_extend_short(Register reg) {
3855   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3856     movswl(reg, reg); // movsxw
3857   } else {
3858     shll(reg, 16);
3859     sarl(reg, 16);
3860   }
3861 }
3862 
3863 void MacroAssembler::testl(Register dst, AddressLiteral src) {
3864   assert(reachable(src), "Address should be reachable");
3865   testl(dst, as_Address(src));
3866 }
3867 
3868 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3869   int dst_enc = dst->encoding();
3870   int src_enc = src->encoding();
3871   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3872     Assembler::pcmpeqb(dst, src);
3873   } else if ((dst_enc < 16) && (src_enc < 16)) {
3874     Assembler::pcmpeqb(dst, src);
3875   } else if (src_enc < 16) {
3876     subptr(rsp, 64);
3877     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3878     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3879     Assembler::pcmpeqb(xmm0, src);
3880     movdqu(dst, xmm0);
3881     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3882     addptr(rsp, 64);
3883   } else if (dst_enc < 16) {
3884     subptr(rsp, 64);
3885     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3886     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3887     Assembler::pcmpeqb(dst, xmm0);
3888     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3889     addptr(rsp, 64);
3890   } else {
3891     subptr(rsp, 64);
3892     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3893     subptr(rsp, 64);
3894     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
3895     movdqu(xmm0, src);
3896     movdqu(xmm1, dst);
3897     Assembler::pcmpeqb(xmm1, xmm0);
3898     movdqu(dst, xmm1);
3899     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
3900     addptr(rsp, 64);
3901     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3902     addptr(rsp, 64);
3903   }
3904 }
3905 
3906 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3907   int dst_enc = dst->encoding();
3908   int src_enc = src->encoding();
3909   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3910     Assembler::pcmpeqw(dst, src);
3911   } else if ((dst_enc < 16) && (src_enc < 16)) {
3912     Assembler::pcmpeqw(dst, src);
3913   } else if (src_enc < 16) {
3914     subptr(rsp, 64);
3915     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3916     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3917     Assembler::pcmpeqw(xmm0, src);
3918     movdqu(dst, xmm0);
3919     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3920     addptr(rsp, 64);
3921   } else if (dst_enc < 16) {
3922     subptr(rsp, 64);
3923     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3924     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3925     Assembler::pcmpeqw(dst, xmm0);
3926     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3927     addptr(rsp, 64);
3928   } else {
3929     subptr(rsp, 64);
3930     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3931     subptr(rsp, 64);
3932     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
3933     movdqu(xmm0, src);
3934     movdqu(xmm1, dst);
3935     Assembler::pcmpeqw(xmm1, xmm0);
3936     movdqu(dst, xmm1);
3937     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
3938     addptr(rsp, 64);
3939     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3940     addptr(rsp, 64);
3941   }
3942 }
3943 
3944 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3945   int dst_enc = dst->encoding();
3946   if (dst_enc < 16) {
3947     Assembler::pcmpestri(dst, src, imm8);
3948   } else {
3949     subptr(rsp, 64);
3950     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3951     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3952     Assembler::pcmpestri(xmm0, src, imm8);
3953     movdqu(dst, xmm0);
3954     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3955     addptr(rsp, 64);
3956   }
3957 }
3958 
3959 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3960   int dst_enc = dst->encoding();
3961   int src_enc = src->encoding();
3962   if ((dst_enc < 16) && (src_enc < 16)) {
3963     Assembler::pcmpestri(dst, src, imm8);
3964   } else if (src_enc < 16) {
3965     subptr(rsp, 64);
3966     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3967     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3968     Assembler::pcmpestri(xmm0, src, imm8);
3969     movdqu(dst, xmm0);
3970     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3971     addptr(rsp, 64);
3972   } else if (dst_enc < 16) {
3973     subptr(rsp, 64);
3974     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3975     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3976     Assembler::pcmpestri(dst, xmm0, imm8);
3977     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3978     addptr(rsp, 64);
3979   } else {
3980     subptr(rsp, 64);
3981     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
3982     subptr(rsp, 64);
3983     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
3984     movdqu(xmm0, src);
3985     movdqu(xmm1, dst);
3986     Assembler::pcmpestri(xmm1, xmm0, imm8);
3987     movdqu(dst, xmm1);
3988     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
3989     addptr(rsp, 64);
3990     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
3991     addptr(rsp, 64);
3992   }
3993 }
3994 
3995 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3996   int dst_enc = dst->encoding();
3997   int src_enc = src->encoding();
3998   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3999     Assembler::pmovzxbw(dst, src);
4000   } else if ((dst_enc < 16) && (src_enc < 16)) {
4001     Assembler::pmovzxbw(dst, src);
4002   } else if (src_enc < 16) {
4003     subptr(rsp, 64);
4004     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4005     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4006     Assembler::pmovzxbw(xmm0, src);
4007     movdqu(dst, xmm0);
4008     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4009     addptr(rsp, 64);
4010   } else if (dst_enc < 16) {
4011     subptr(rsp, 64);
4012     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4013     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4014     Assembler::pmovzxbw(dst, xmm0);
4015     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4016     addptr(rsp, 64);
4017   } else {
4018     subptr(rsp, 64);
4019     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4020     subptr(rsp, 64);
4021     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4022     movdqu(xmm0, src);
4023     movdqu(xmm1, dst);
4024     Assembler::pmovzxbw(xmm1, xmm0);
4025     movdqu(dst, xmm1);
4026     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4027     addptr(rsp, 64);
4028     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4029     addptr(rsp, 64);
4030   }
4031 }
4032 
4033 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
4034   int dst_enc = dst->encoding();
4035   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4036     Assembler::pmovzxbw(dst, src);
4037   } else if (dst_enc < 16) {
4038     Assembler::pmovzxbw(dst, src);
4039   } else {
4040     subptr(rsp, 64);
4041     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4042     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4043     Assembler::pmovzxbw(xmm0, src);
4044     movdqu(dst, xmm0);
4045     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4046     addptr(rsp, 64);
4047   }
4048 }
4049 
4050 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
4051   int src_enc = src->encoding();
4052   if (src_enc < 16) {
4053     Assembler::pmovmskb(dst, src);
4054   } else {
4055     subptr(rsp, 64);
4056     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4057     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4058     Assembler::pmovmskb(dst, xmm0);
4059     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4060     addptr(rsp, 64);
4061   }
4062 }
4063 
4064 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
4065   int dst_enc = dst->encoding();
4066   int src_enc = src->encoding();
4067   if ((dst_enc < 16) && (src_enc < 16)) {
4068     Assembler::ptest(dst, src);
4069   } else if (src_enc < 16) {
4070     subptr(rsp, 64);
4071     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4072     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4073     Assembler::ptest(xmm0, src);
4074     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4075     addptr(rsp, 64);
4076   } else if (dst_enc < 16) {
4077     subptr(rsp, 64);
4078     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4079     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4080     Assembler::ptest(dst, xmm0);
4081     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4082     addptr(rsp, 64);
4083   } else {
4084     subptr(rsp, 64);
4085     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4086     subptr(rsp, 64);
4087     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4088     movdqu(xmm0, src);
4089     movdqu(xmm1, dst);
4090     Assembler::ptest(xmm1, xmm0);
4091     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4092     addptr(rsp, 64);
4093     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4094     addptr(rsp, 64);
4095   }
4096 }
4097 
4098 void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) {
4099   if (reachable(src)) {
4100     Assembler::sqrtsd(dst, as_Address(src));
4101   } else {
4102     lea(rscratch1, src);
4103     Assembler::sqrtsd(dst, Address(rscratch1, 0));
4104   }
4105 }
4106 
4107 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) {
4108   if (reachable(src)) {
4109     Assembler::sqrtss(dst, as_Address(src));
4110   } else {
4111     lea(rscratch1, src);
4112     Assembler::sqrtss(dst, Address(rscratch1, 0));
4113   }
4114 }
4115 
4116 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) {
4117   if (reachable(src)) {
4118     Assembler::subsd(dst, as_Address(src));
4119   } else {
4120     lea(rscratch1, src);
4121     Assembler::subsd(dst, Address(rscratch1, 0));
4122   }
4123 }
4124 
4125 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) {
4126   if (reachable(src)) {
4127     Assembler::subss(dst, as_Address(src));
4128   } else {
4129     lea(rscratch1, src);
4130     Assembler::subss(dst, Address(rscratch1, 0));
4131   }
4132 }
4133 
4134 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
4135   if (reachable(src)) {
4136     Assembler::ucomisd(dst, as_Address(src));
4137   } else {
4138     lea(rscratch1, src);
4139     Assembler::ucomisd(dst, Address(rscratch1, 0));
4140   }
4141 }
4142 
4143 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
4144   if (reachable(src)) {
4145     Assembler::ucomiss(dst, as_Address(src));
4146   } else {
4147     lea(rscratch1, src);
4148     Assembler::ucomiss(dst, Address(rscratch1, 0));
4149   }
4150 }
4151 
4152 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
4153   // Used in sign-bit flipping with aligned address.
4154   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
4155   if (reachable(src)) {
4156     Assembler::xorpd(dst, as_Address(src));
4157   } else {
4158     lea(rscratch1, src);
4159     Assembler::xorpd(dst, Address(rscratch1, 0));
4160   }
4161 }
4162 
4163 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
4164   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
4165     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
4166   }
4167   else {
4168     Assembler::xorpd(dst, src);
4169   }
4170 }
4171 
4172 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
4173   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
4174     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
4175   } else {
4176     Assembler::xorps(dst, src);
4177   }
4178 }
4179 
4180 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
4181   // Used in sign-bit flipping with aligned address.
4182   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
4183   if (reachable(src)) {
4184     Assembler::xorps(dst, as_Address(src));
4185   } else {
4186     lea(rscratch1, src);
4187     Assembler::xorps(dst, Address(rscratch1, 0));
4188   }
4189 }
4190 
4191 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) {
4192   // Used in sign-bit flipping with aligned address.
4193   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
4194   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
4195   if (reachable(src)) {
4196     Assembler::pshufb(dst, as_Address(src));
4197   } else {
4198     lea(rscratch1, src);
4199     Assembler::pshufb(dst, Address(rscratch1, 0));
4200   }
4201 }
4202 
4203 // AVX 3-operands instructions
4204 
4205 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4206   if (reachable(src)) {
4207     vaddsd(dst, nds, as_Address(src));
4208   } else {
4209     lea(rscratch1, src);
4210     vaddsd(dst, nds, Address(rscratch1, 0));
4211   }
4212 }
4213 
4214 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4215   if (reachable(src)) {
4216     vaddss(dst, nds, as_Address(src));
4217   } else {
4218     lea(rscratch1, src);
4219     vaddss(dst, nds, Address(rscratch1, 0));
4220   }
4221 }
4222 
4223 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
4224   int dst_enc = dst->encoding();
4225   int nds_enc = nds->encoding();
4226   int src_enc = src->encoding();
4227   if ((dst_enc < 16) && (nds_enc < 16)) {
4228     vandps(dst, nds, negate_field, vector_len);
4229   } else if ((src_enc < 16) && (dst_enc < 16)) {
4230     evmovdqul(src, nds, Assembler::AVX_512bit);
4231     vandps(dst, src, negate_field, vector_len);
4232   } else if (src_enc < 16) {
4233     evmovdqul(src, nds, Assembler::AVX_512bit);
4234     vandps(src, src, negate_field, vector_len);
4235     evmovdqul(dst, src, Assembler::AVX_512bit);
4236   } else if (dst_enc < 16) {
4237     evmovdqul(src, xmm0, Assembler::AVX_512bit);
4238     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4239     vandps(dst, xmm0, negate_field, vector_len);
4240     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4241   } else {
4242     if (src_enc != dst_enc) {
4243       evmovdqul(src, xmm0, Assembler::AVX_512bit);
4244       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4245       vandps(xmm0, xmm0, negate_field, vector_len);
4246       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4247       evmovdqul(xmm0, src, Assembler::AVX_512bit);
4248     } else {
4249       subptr(rsp, 64);
4250       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4251       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4252       vandps(xmm0, xmm0, negate_field, vector_len);
4253       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4254       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4255       addptr(rsp, 64);
4256     }
4257   }
4258 }
4259 
4260 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
4261   int dst_enc = dst->encoding();
4262   int nds_enc = nds->encoding();
4263   int src_enc = src->encoding();
4264   if ((dst_enc < 16) && (nds_enc < 16)) {
4265     vandpd(dst, nds, negate_field, vector_len);
4266   } else if ((src_enc < 16) && (dst_enc < 16)) {
4267     evmovdqul(src, nds, Assembler::AVX_512bit);
4268     vandpd(dst, src, negate_field, vector_len);
4269   } else if (src_enc < 16) {
4270     evmovdqul(src, nds, Assembler::AVX_512bit);
4271     vandpd(src, src, negate_field, vector_len);
4272     evmovdqul(dst, src, Assembler::AVX_512bit);
4273   } else if (dst_enc < 16) {
4274     evmovdqul(src, xmm0, Assembler::AVX_512bit);
4275     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4276     vandpd(dst, xmm0, negate_field, vector_len);
4277     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4278   } else {
4279     if (src_enc != dst_enc) {
4280       evmovdqul(src, xmm0, Assembler::AVX_512bit);
4281       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4282       vandpd(xmm0, xmm0, negate_field, vector_len);
4283       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4284       evmovdqul(xmm0, src, Assembler::AVX_512bit);
4285     } else {
4286       subptr(rsp, 64);
4287       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4288       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4289       vandpd(xmm0, xmm0, negate_field, vector_len);
4290       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4291       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4292       addptr(rsp, 64);
4293     }
4294   }
4295 }
4296 
4297 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4298   int dst_enc = dst->encoding();
4299   int nds_enc = nds->encoding();
4300   int src_enc = src->encoding();
4301   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4302     Assembler::vpaddb(dst, nds, src, vector_len);
4303   } else if ((dst_enc < 16) && (src_enc < 16)) {
4304     Assembler::vpaddb(dst, dst, src, vector_len);
4305   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4306     // use nds as scratch for src
4307     evmovdqul(nds, src, Assembler::AVX_512bit);
4308     Assembler::vpaddb(dst, dst, nds, vector_len);
4309   } else if ((src_enc < 16) && (nds_enc < 16)) {
4310     // use nds as scratch for dst
4311     evmovdqul(nds, dst, Assembler::AVX_512bit);
4312     Assembler::vpaddb(nds, nds, src, vector_len);
4313     evmovdqul(dst, nds, Assembler::AVX_512bit);
4314   } else if (dst_enc < 16) {
4315     // use nds as scatch for xmm0 to hold src
4316     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4317     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4318     Assembler::vpaddb(dst, dst, xmm0, vector_len);
4319     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4320   } else {
4321     // worse case scenario, all regs are in the upper bank
4322     subptr(rsp, 64);
4323     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4324     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4325     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4326     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4327     Assembler::vpaddb(xmm0, xmm0, xmm1, vector_len);
4328     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4329     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4330     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4331     addptr(rsp, 64);
4332   }
4333 }
4334 
4335 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4336   int dst_enc = dst->encoding();
4337   int nds_enc = nds->encoding();
4338   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4339     Assembler::vpaddb(dst, nds, src, vector_len);
4340   } else if (dst_enc < 16) {
4341     Assembler::vpaddb(dst, dst, src, vector_len);
4342   } else if (nds_enc < 16) {
4343     // implies dst_enc in upper bank with src as scratch
4344     evmovdqul(nds, dst, Assembler::AVX_512bit);
4345     Assembler::vpaddb(nds, nds, src, vector_len);
4346     evmovdqul(dst, nds, Assembler::AVX_512bit);
4347   } else {
4348     // worse case scenario, all regs in upper bank
4349     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4350     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4351     Assembler::vpaddb(xmm0, xmm0, src, vector_len);
4352     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4353   }
4354 }
4355 
4356 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4357   int dst_enc = dst->encoding();
4358   int nds_enc = nds->encoding();
4359   int src_enc = src->encoding();
4360   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4361     Assembler::vpaddw(dst, nds, src, vector_len);
4362   } else if ((dst_enc < 16) && (src_enc < 16)) {
4363     Assembler::vpaddw(dst, dst, src, vector_len);
4364   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4365     // use nds as scratch for src
4366     evmovdqul(nds, src, Assembler::AVX_512bit);
4367     Assembler::vpaddw(dst, dst, nds, vector_len);
4368   } else if ((src_enc < 16) && (nds_enc < 16)) {
4369     // use nds as scratch for dst
4370     evmovdqul(nds, dst, Assembler::AVX_512bit);
4371     Assembler::vpaddw(nds, nds, src, vector_len);
4372     evmovdqul(dst, nds, Assembler::AVX_512bit);
4373   } else if (dst_enc < 16) {
4374     // use nds as scatch for xmm0 to hold src
4375     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4376     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4377     Assembler::vpaddw(dst, dst, xmm0, vector_len);
4378     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4379   } else {
4380     // worse case scenario, all regs are in the upper bank
4381     subptr(rsp, 64);
4382     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4383     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4384     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4385     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4386     Assembler::vpaddw(xmm0, xmm0, xmm1, vector_len);
4387     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4388     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4389     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4390     addptr(rsp, 64);
4391   }
4392 }
4393 
4394 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4395   int dst_enc = dst->encoding();
4396   int nds_enc = nds->encoding();
4397   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4398     Assembler::vpaddw(dst, nds, src, vector_len);
4399   } else if (dst_enc < 16) {
4400     Assembler::vpaddw(dst, dst, src, vector_len);
4401   } else if (nds_enc < 16) {
4402     // implies dst_enc in upper bank with src as scratch
4403     evmovdqul(nds, dst, Assembler::AVX_512bit);
4404     Assembler::vpaddw(nds, nds, src, vector_len);
4405     evmovdqul(dst, nds, Assembler::AVX_512bit);
4406   } else {
4407     // worse case scenario, all regs in upper bank
4408     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4409     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4410     Assembler::vpaddw(xmm0, xmm0, src, vector_len);
4411     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4412   }
4413 }
4414 
4415 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4416   if (reachable(src)) {
4417     Assembler::vpand(dst, nds, as_Address(src), vector_len);
4418   } else {
4419     lea(rscratch1, src);
4420     Assembler::vpand(dst, nds, Address(rscratch1, 0), vector_len);
4421   }
4422 }
4423 
4424 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
4425   int dst_enc = dst->encoding();
4426   int src_enc = src->encoding();
4427   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4428     Assembler::vpbroadcastw(dst, src);
4429   } else if ((dst_enc < 16) && (src_enc < 16)) {
4430     Assembler::vpbroadcastw(dst, src);
4431   } else if (src_enc < 16) {
4432     subptr(rsp, 64);
4433     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4434     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4435     Assembler::vpbroadcastw(xmm0, src);
4436     movdqu(dst, xmm0);
4437     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4438     addptr(rsp, 64);
4439   } else if (dst_enc < 16) {
4440     subptr(rsp, 64);
4441     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4442     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4443     Assembler::vpbroadcastw(dst, xmm0);
4444     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4445     addptr(rsp, 64);
4446   } else {
4447     subptr(rsp, 64);
4448     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4449     subptr(rsp, 64);
4450     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4451     movdqu(xmm0, src);
4452     movdqu(xmm1, dst);
4453     Assembler::vpbroadcastw(xmm1, xmm0);
4454     movdqu(dst, xmm1);
4455     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4456     addptr(rsp, 64);
4457     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4458     addptr(rsp, 64);
4459   }
4460 }
4461 
4462 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4463   int dst_enc = dst->encoding();
4464   int nds_enc = nds->encoding();
4465   int src_enc = src->encoding();
4466   assert(dst_enc == nds_enc, "");
4467   if ((dst_enc < 16) && (src_enc < 16)) {
4468     Assembler::vpcmpeqb(dst, nds, src, vector_len);
4469   } else if (src_enc < 16) {
4470     subptr(rsp, 64);
4471     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4472     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4473     Assembler::vpcmpeqb(xmm0, xmm0, src, vector_len);
4474     movdqu(dst, xmm0);
4475     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4476     addptr(rsp, 64);
4477   } else if (dst_enc < 16) {
4478     subptr(rsp, 64);
4479     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4480     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4481     Assembler::vpcmpeqb(dst, dst, xmm0, vector_len);
4482     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4483     addptr(rsp, 64);
4484   } else {
4485     subptr(rsp, 64);
4486     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4487     subptr(rsp, 64);
4488     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4489     movdqu(xmm0, src);
4490     movdqu(xmm1, dst);
4491     Assembler::vpcmpeqb(xmm1, xmm1, xmm0, vector_len);
4492     movdqu(dst, xmm1);
4493     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4494     addptr(rsp, 64);
4495     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4496     addptr(rsp, 64);
4497   }
4498 }
4499 
4500 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4501   int dst_enc = dst->encoding();
4502   int nds_enc = nds->encoding();
4503   int src_enc = src->encoding();
4504   assert(dst_enc == nds_enc, "");
4505   if ((dst_enc < 16) && (src_enc < 16)) {
4506     Assembler::vpcmpeqw(dst, nds, src, vector_len);
4507   } else if (src_enc < 16) {
4508     subptr(rsp, 64);
4509     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4510     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4511     Assembler::vpcmpeqw(xmm0, xmm0, src, vector_len);
4512     movdqu(dst, xmm0);
4513     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4514     addptr(rsp, 64);
4515   } else if (dst_enc < 16) {
4516     subptr(rsp, 64);
4517     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4518     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4519     Assembler::vpcmpeqw(dst, dst, xmm0, vector_len);
4520     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4521     addptr(rsp, 64);
4522   } else {
4523     subptr(rsp, 64);
4524     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4525     subptr(rsp, 64);
4526     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4527     movdqu(xmm0, src);
4528     movdqu(xmm1, dst);
4529     Assembler::vpcmpeqw(xmm1, xmm1, xmm0, vector_len);
4530     movdqu(dst, xmm1);
4531     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4532     addptr(rsp, 64);
4533     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4534     addptr(rsp, 64);
4535   }
4536 }
4537 
4538 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
4539   int dst_enc = dst->encoding();
4540   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4541     Assembler::vpmovzxbw(dst, src, vector_len);
4542   } else if (dst_enc < 16) {
4543     Assembler::vpmovzxbw(dst, src, vector_len);
4544   } else {
4545     subptr(rsp, 64);
4546     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4547     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4548     Assembler::vpmovzxbw(xmm0, src, vector_len);
4549     movdqu(dst, xmm0);
4550     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4551     addptr(rsp, 64);
4552   }
4553 }
4554 
4555 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src) {
4556   int src_enc = src->encoding();
4557   if (src_enc < 16) {
4558     Assembler::vpmovmskb(dst, src);
4559   } else {
4560     subptr(rsp, 64);
4561     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4562     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4563     Assembler::vpmovmskb(dst, xmm0);
4564     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4565     addptr(rsp, 64);
4566   }
4567 }
4568 
4569 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4570   int dst_enc = dst->encoding();
4571   int nds_enc = nds->encoding();
4572   int src_enc = src->encoding();
4573   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4574     Assembler::vpmullw(dst, nds, src, vector_len);
4575   } else if ((dst_enc < 16) && (src_enc < 16)) {
4576     Assembler::vpmullw(dst, dst, src, vector_len);
4577   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4578     // use nds as scratch for src
4579     evmovdqul(nds, src, Assembler::AVX_512bit);
4580     Assembler::vpmullw(dst, dst, nds, vector_len);
4581   } else if ((src_enc < 16) && (nds_enc < 16)) {
4582     // use nds as scratch for dst
4583     evmovdqul(nds, dst, Assembler::AVX_512bit);
4584     Assembler::vpmullw(nds, nds, src, vector_len);
4585     evmovdqul(dst, nds, Assembler::AVX_512bit);
4586   } else if (dst_enc < 16) {
4587     // use nds as scatch for xmm0 to hold src
4588     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4589     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4590     Assembler::vpmullw(dst, dst, xmm0, vector_len);
4591     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4592   } else {
4593     // worse case scenario, all regs are in the upper bank
4594     subptr(rsp, 64);
4595     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4596     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4597     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4598     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4599     Assembler::vpmullw(xmm0, xmm0, xmm1, vector_len);
4600     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4601     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4602     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4603     addptr(rsp, 64);
4604   }
4605 }
4606 
4607 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4608   int dst_enc = dst->encoding();
4609   int nds_enc = nds->encoding();
4610   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4611     Assembler::vpmullw(dst, nds, src, vector_len);
4612   } else if (dst_enc < 16) {
4613     Assembler::vpmullw(dst, dst, src, vector_len);
4614   } else if (nds_enc < 16) {
4615     // implies dst_enc in upper bank with src as scratch
4616     evmovdqul(nds, dst, Assembler::AVX_512bit);
4617     Assembler::vpmullw(nds, nds, src, vector_len);
4618     evmovdqul(dst, nds, Assembler::AVX_512bit);
4619   } else {
4620     // worse case scenario, all regs in upper bank
4621     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4622     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4623     Assembler::vpmullw(xmm0, xmm0, src, vector_len);
4624     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4625   }
4626 }
4627 
4628 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4629   int dst_enc = dst->encoding();
4630   int nds_enc = nds->encoding();
4631   int src_enc = src->encoding();
4632   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4633     Assembler::vpsubb(dst, nds, src, vector_len);
4634   } else if ((dst_enc < 16) && (src_enc < 16)) {
4635     Assembler::vpsubb(dst, dst, src, vector_len);
4636   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4637     // use nds as scratch for src
4638     evmovdqul(nds, src, Assembler::AVX_512bit);
4639     Assembler::vpsubb(dst, dst, nds, vector_len);
4640   } else if ((src_enc < 16) && (nds_enc < 16)) {
4641     // use nds as scratch for dst
4642     evmovdqul(nds, dst, Assembler::AVX_512bit);
4643     Assembler::vpsubb(nds, nds, src, vector_len);
4644     evmovdqul(dst, nds, Assembler::AVX_512bit);
4645   } else if (dst_enc < 16) {
4646     // use nds as scatch for xmm0 to hold src
4647     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4648     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4649     Assembler::vpsubb(dst, dst, xmm0, vector_len);
4650     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4651   } else {
4652     // worse case scenario, all regs are in the upper bank
4653     subptr(rsp, 64);
4654     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4655     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4656     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4657     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4658     Assembler::vpsubb(xmm0, xmm0, xmm1, vector_len);
4659     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4660     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4661     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4662     addptr(rsp, 64);
4663   }
4664 }
4665 
4666 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4667   int dst_enc = dst->encoding();
4668   int nds_enc = nds->encoding();
4669   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4670     Assembler::vpsubb(dst, nds, src, vector_len);
4671   } else if (dst_enc < 16) {
4672     Assembler::vpsubb(dst, dst, src, vector_len);
4673   } else if (nds_enc < 16) {
4674     // implies dst_enc in upper bank with src as scratch
4675     evmovdqul(nds, dst, Assembler::AVX_512bit);
4676     Assembler::vpsubb(nds, nds, src, vector_len);
4677     evmovdqul(dst, nds, Assembler::AVX_512bit);
4678   } else {
4679     // worse case scenario, all regs in upper bank
4680     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4681     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4682     Assembler::vpsubw(xmm0, xmm0, src, vector_len);
4683     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4684   }
4685 }
4686 
4687 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4688   int dst_enc = dst->encoding();
4689   int nds_enc = nds->encoding();
4690   int src_enc = src->encoding();
4691   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4692     Assembler::vpsubw(dst, nds, src, vector_len);
4693   } else if ((dst_enc < 16) && (src_enc < 16)) {
4694     Assembler::vpsubw(dst, dst, src, vector_len);
4695   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4696     // use nds as scratch for src
4697     evmovdqul(nds, src, Assembler::AVX_512bit);
4698     Assembler::vpsubw(dst, dst, nds, vector_len);
4699   } else if ((src_enc < 16) && (nds_enc < 16)) {
4700     // use nds as scratch for dst
4701     evmovdqul(nds, dst, Assembler::AVX_512bit);
4702     Assembler::vpsubw(nds, nds, src, vector_len);
4703     evmovdqul(dst, nds, Assembler::AVX_512bit);
4704   } else if (dst_enc < 16) {
4705     // use nds as scatch for xmm0 to hold src
4706     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4707     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4708     Assembler::vpsubw(dst, dst, xmm0, vector_len);
4709     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4710   } else {
4711     // worse case scenario, all regs are in the upper bank
4712     subptr(rsp, 64);
4713     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4714     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4715     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4716     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4717     Assembler::vpsubw(xmm0, xmm0, xmm1, vector_len);
4718     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4719     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4720     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4721     addptr(rsp, 64);
4722   }
4723 }
4724 
4725 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4726   int dst_enc = dst->encoding();
4727   int nds_enc = nds->encoding();
4728   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4729     Assembler::vpsubw(dst, nds, src, vector_len);
4730   } else if (dst_enc < 16) {
4731     Assembler::vpsubw(dst, dst, src, vector_len);
4732   } else if (nds_enc < 16) {
4733     // implies dst_enc in upper bank with src as scratch
4734     evmovdqul(nds, dst, Assembler::AVX_512bit);
4735     Assembler::vpsubw(nds, nds, src, vector_len);
4736     evmovdqul(dst, nds, Assembler::AVX_512bit);
4737   } else {
4738     // worse case scenario, all regs in upper bank
4739     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4740     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4741     Assembler::vpsubw(xmm0, xmm0, src, vector_len);
4742     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4743   }
4744 }
4745 
4746 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4747   int dst_enc = dst->encoding();
4748   int nds_enc = nds->encoding();
4749   int shift_enc = shift->encoding();
4750   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4751     Assembler::vpsraw(dst, nds, shift, vector_len);
4752   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4753     Assembler::vpsraw(dst, dst, shift, vector_len);
4754   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4755     // use nds_enc as scratch with shift
4756     evmovdqul(nds, shift, Assembler::AVX_512bit);
4757     Assembler::vpsraw(dst, dst, nds, vector_len);
4758   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4759     // use nds as scratch with dst
4760     evmovdqul(nds, dst, Assembler::AVX_512bit);
4761     Assembler::vpsraw(nds, nds, shift, vector_len);
4762     evmovdqul(dst, nds, Assembler::AVX_512bit);
4763   } else if (dst_enc < 16) {
4764     // use nds to save a copy of xmm0 and hold shift
4765     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4766     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4767     Assembler::vpsraw(dst, dst, xmm0, vector_len);
4768     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4769   } else if (nds_enc < 16) {
4770     // use nds as dest as temps
4771     evmovdqul(nds, dst, Assembler::AVX_512bit);
4772     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4773     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4774     Assembler::vpsraw(nds, nds, xmm0, vector_len);
4775     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4776     evmovdqul(dst, nds, Assembler::AVX_512bit);
4777   } else {
4778     // worse case scenario, all regs are in the upper bank
4779     subptr(rsp, 64);
4780     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4781     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4782     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4783     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4784     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4785     evmovdqul(xmm1, dst, Assembler::AVX_512bit);
4786     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4787     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4788     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4789     addptr(rsp, 64);
4790   }
4791 }
4792 
4793 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4794   int dst_enc = dst->encoding();
4795   int nds_enc = nds->encoding();
4796   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4797     Assembler::vpsraw(dst, nds, shift, vector_len);
4798   } else if (dst_enc < 16) {
4799     Assembler::vpsraw(dst, dst, shift, vector_len);
4800   } else if (nds_enc < 16) {
4801     // use nds as scratch
4802     evmovdqul(nds, dst, Assembler::AVX_512bit);
4803     Assembler::vpsraw(nds, nds, shift, vector_len);
4804     evmovdqul(dst, nds, Assembler::AVX_512bit);
4805   } else {
4806     // use nds as scratch for xmm0
4807     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4808     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4809     Assembler::vpsraw(xmm0, xmm0, shift, vector_len);
4810     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4811   }
4812 }
4813 
4814 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4815   int dst_enc = dst->encoding();
4816   int nds_enc = nds->encoding();
4817   int shift_enc = shift->encoding();
4818   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4819     Assembler::vpsrlw(dst, nds, shift, vector_len);
4820   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4821     Assembler::vpsrlw(dst, dst, shift, vector_len);
4822   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4823     // use nds_enc as scratch with shift
4824     evmovdqul(nds, shift, Assembler::AVX_512bit);
4825     Assembler::vpsrlw(dst, dst, nds, vector_len);
4826   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4827     // use nds as scratch with dst
4828     evmovdqul(nds, dst, Assembler::AVX_512bit);
4829     Assembler::vpsrlw(nds, nds, shift, vector_len);
4830     evmovdqul(dst, nds, Assembler::AVX_512bit);
4831   } else if (dst_enc < 16) {
4832     // use nds to save a copy of xmm0 and hold shift
4833     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4834     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4835     Assembler::vpsrlw(dst, dst, xmm0, vector_len);
4836     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4837   } else if (nds_enc < 16) {
4838     // use nds as dest as temps
4839     evmovdqul(nds, dst, Assembler::AVX_512bit);
4840     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4841     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4842     Assembler::vpsrlw(nds, nds, xmm0, vector_len);
4843     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4844     evmovdqul(dst, nds, Assembler::AVX_512bit);
4845   } else {
4846     // worse case scenario, all regs are in the upper bank
4847     subptr(rsp, 64);
4848     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4849     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4850     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4851     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4852     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4853     evmovdqul(xmm1, dst, Assembler::AVX_512bit);
4854     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4855     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4856     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4857     addptr(rsp, 64);
4858   }
4859 }
4860 
4861 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4862   int dst_enc = dst->encoding();
4863   int nds_enc = nds->encoding();
4864   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4865     Assembler::vpsrlw(dst, nds, shift, vector_len);
4866   } else if (dst_enc < 16) {
4867     Assembler::vpsrlw(dst, dst, shift, vector_len);
4868   } else if (nds_enc < 16) {
4869     // use nds as scratch
4870     evmovdqul(nds, dst, Assembler::AVX_512bit);
4871     Assembler::vpsrlw(nds, nds, shift, vector_len);
4872     evmovdqul(dst, nds, Assembler::AVX_512bit);
4873   } else {
4874     // use nds as scratch for xmm0
4875     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4876     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4877     Assembler::vpsrlw(xmm0, xmm0, shift, vector_len);
4878     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4879   }
4880 }
4881 
4882 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4883   int dst_enc = dst->encoding();
4884   int nds_enc = nds->encoding();
4885   int shift_enc = shift->encoding();
4886   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4887     Assembler::vpsllw(dst, nds, shift, vector_len);
4888   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4889     Assembler::vpsllw(dst, dst, shift, vector_len);
4890   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4891     // use nds_enc as scratch with shift
4892     evmovdqul(nds, shift, Assembler::AVX_512bit);
4893     Assembler::vpsllw(dst, dst, nds, vector_len);
4894   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4895     // use nds as scratch with dst
4896     evmovdqul(nds, dst, Assembler::AVX_512bit);
4897     Assembler::vpsllw(nds, nds, shift, vector_len);
4898     evmovdqul(dst, nds, Assembler::AVX_512bit);
4899   } else if (dst_enc < 16) {
4900     // use nds to save a copy of xmm0 and hold shift
4901     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4902     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4903     Assembler::vpsllw(dst, dst, xmm0, vector_len);
4904     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4905   } else if (nds_enc < 16) {
4906     // use nds as dest as temps
4907     evmovdqul(nds, dst, Assembler::AVX_512bit);
4908     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4909     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4910     Assembler::vpsllw(nds, nds, xmm0, vector_len);
4911     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4912     evmovdqul(dst, nds, Assembler::AVX_512bit);
4913   } else {
4914     // worse case scenario, all regs are in the upper bank
4915     subptr(rsp, 64);
4916     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4917     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4918     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4919     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4920     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4921     evmovdqul(xmm1, dst, Assembler::AVX_512bit);
4922     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4923     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4924     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4925     addptr(rsp, 64);
4926   }
4927 }
4928 
4929 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4930   int dst_enc = dst->encoding();
4931   int nds_enc = nds->encoding();
4932   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4933     Assembler::vpsllw(dst, nds, shift, vector_len);
4934   } else if (dst_enc < 16) {
4935     Assembler::vpsllw(dst, dst, shift, vector_len);
4936   } else if (nds_enc < 16) {
4937     // use nds as scratch
4938     evmovdqul(nds, dst, Assembler::AVX_512bit);
4939     Assembler::vpsllw(nds, nds, shift, vector_len);
4940     evmovdqul(dst, nds, Assembler::AVX_512bit);
4941   } else {
4942     // use nds as scratch for xmm0
4943     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4944     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4945     Assembler::vpsllw(xmm0, xmm0, shift, vector_len);
4946     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4947   }
4948 }
4949 
4950 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
4951   int dst_enc = dst->encoding();
4952   int src_enc = src->encoding();
4953   if ((dst_enc < 16) && (src_enc < 16)) {
4954     Assembler::vptest(dst, src);
4955   } else if (src_enc < 16) {
4956     subptr(rsp, 64);
4957     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4958     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4959     Assembler::vptest(xmm0, src);
4960     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4961     addptr(rsp, 64);
4962   } else if (dst_enc < 16) {
4963     subptr(rsp, 64);
4964     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4965     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4966     Assembler::vptest(dst, xmm0);
4967     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4968     addptr(rsp, 64);
4969   } else {
4970     subptr(rsp, 64);
4971     evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4972     subptr(rsp, 64);
4973     evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
4974     movdqu(xmm0, src);
4975     movdqu(xmm1, dst);
4976     Assembler::vptest(xmm1, xmm0);
4977     evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
4978     addptr(rsp, 64);
4979     evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
4980     addptr(rsp, 64);
4981   }
4982 }
4983 
4984 // This instruction exists within macros, ergo we cannot control its input
4985 // when emitted through those patterns.
4986 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4987   if (VM_Version::supports_avx512nobw()) {
4988     int dst_enc = dst->encoding();
4989     int src_enc = src->encoding();
4990     if (dst_enc == src_enc) {
4991       if (dst_enc < 16) {
4992         Assembler::punpcklbw(dst, src);
4993       } else {
4994         subptr(rsp, 64);
4995         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
4996         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4997         Assembler::punpcklbw(xmm0, xmm0);
4998         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4999         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5000         addptr(rsp, 64);
5001       }
5002     } else {
5003       if ((src_enc < 16) && (dst_enc < 16)) {
5004         Assembler::punpcklbw(dst, src);
5005       } else if (src_enc < 16) {
5006         subptr(rsp, 64);
5007         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5008         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5009         Assembler::punpcklbw(xmm0, src);
5010         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5011         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5012         addptr(rsp, 64);
5013       } else if (dst_enc < 16) {
5014         subptr(rsp, 64);
5015         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5016         evmovdqul(xmm0, src, Assembler::AVX_512bit);
5017         Assembler::punpcklbw(dst, xmm0);
5018         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5019         addptr(rsp, 64);
5020       } else {
5021         subptr(rsp, 64);
5022         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5023         subptr(rsp, 64);
5024         evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
5025         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5026         evmovdqul(xmm1, src, Assembler::AVX_512bit);
5027         Assembler::punpcklbw(xmm0, xmm1);
5028         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5029         evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
5030         addptr(rsp, 64);
5031         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5032         addptr(rsp, 64);
5033       }
5034     }
5035   } else {
5036     Assembler::punpcklbw(dst, src);
5037   }
5038 }
5039 
5040 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
5041   if (VM_Version::supports_avx512vl()) {
5042     Assembler::pshufd(dst, src, mode);
5043   } else {
5044     int dst_enc = dst->encoding();
5045     if (dst_enc < 16) {
5046       Assembler::pshufd(dst, src, mode);
5047     } else {
5048       subptr(rsp, 64);
5049       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5050       Assembler::pshufd(xmm0, src, mode);
5051       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5052       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5053       addptr(rsp, 64);
5054     }
5055   }
5056 }
5057 
5058 // This instruction exists within macros, ergo we cannot control its input
5059 // when emitted through those patterns.
5060 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
5061   if (VM_Version::supports_avx512nobw()) {
5062     int dst_enc = dst->encoding();
5063     int src_enc = src->encoding();
5064     if (dst_enc == src_enc) {
5065       if (dst_enc < 16) {
5066         Assembler::pshuflw(dst, src, mode);
5067       } else {
5068         subptr(rsp, 64);
5069         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5070         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5071         Assembler::pshuflw(xmm0, xmm0, mode);
5072         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5073         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5074         addptr(rsp, 64);
5075       }
5076     } else {
5077       if ((src_enc < 16) && (dst_enc < 16)) {
5078         Assembler::pshuflw(dst, src, mode);
5079       } else if (src_enc < 16) {
5080         subptr(rsp, 64);
5081         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5082         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5083         Assembler::pshuflw(xmm0, src, mode);
5084         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5085         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5086         addptr(rsp, 64);
5087       } else if (dst_enc < 16) {
5088         subptr(rsp, 64);
5089         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5090         evmovdqul(xmm0, src, Assembler::AVX_512bit);
5091         Assembler::pshuflw(dst, xmm0, mode);
5092         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5093         addptr(rsp, 64);
5094       } else {
5095         subptr(rsp, 64);
5096         evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5097         subptr(rsp, 64);
5098         evmovdqul(Address(rsp, 0), xmm1, Assembler::AVX_512bit);
5099         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
5100         evmovdqul(xmm1, src, Assembler::AVX_512bit);
5101         Assembler::pshuflw(xmm0, xmm1, mode);
5102         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
5103         evmovdqul(xmm1, Address(rsp, 0), Assembler::AVX_512bit);
5104         addptr(rsp, 64);
5105         evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5106         addptr(rsp, 64);
5107       }
5108     }
5109   } else {
5110     Assembler::pshuflw(dst, src, mode);
5111   }
5112 }
5113 
5114 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5115   if (reachable(src)) {
5116     vandpd(dst, nds, as_Address(src), vector_len);
5117   } else {
5118     lea(rscratch1, src);
5119     vandpd(dst, nds, Address(rscratch1, 0), vector_len);
5120   }
5121 }
5122 
5123 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5124   if (reachable(src)) {
5125     vandps(dst, nds, as_Address(src), vector_len);
5126   } else {
5127     lea(rscratch1, src);
5128     vandps(dst, nds, Address(rscratch1, 0), vector_len);
5129   }
5130 }
5131 
5132 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5133   if (reachable(src)) {
5134     vdivsd(dst, nds, as_Address(src));
5135   } else {
5136     lea(rscratch1, src);
5137     vdivsd(dst, nds, Address(rscratch1, 0));
5138   }
5139 }
5140 
5141 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5142   if (reachable(src)) {
5143     vdivss(dst, nds, as_Address(src));
5144   } else {
5145     lea(rscratch1, src);
5146     vdivss(dst, nds, Address(rscratch1, 0));
5147   }
5148 }
5149 
5150 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5151   if (reachable(src)) {
5152     vmulsd(dst, nds, as_Address(src));
5153   } else {
5154     lea(rscratch1, src);
5155     vmulsd(dst, nds, Address(rscratch1, 0));
5156   }
5157 }
5158 
5159 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5160   if (reachable(src)) {
5161     vmulss(dst, nds, as_Address(src));
5162   } else {
5163     lea(rscratch1, src);
5164     vmulss(dst, nds, Address(rscratch1, 0));
5165   }
5166 }
5167 
5168 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5169   if (reachable(src)) {
5170     vsubsd(dst, nds, as_Address(src));
5171   } else {
5172     lea(rscratch1, src);
5173     vsubsd(dst, nds, Address(rscratch1, 0));
5174   }
5175 }
5176 
5177 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5178   if (reachable(src)) {
5179     vsubss(dst, nds, as_Address(src));
5180   } else {
5181     lea(rscratch1, src);
5182     vsubss(dst, nds, Address(rscratch1, 0));
5183   }
5184 }
5185 
5186 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5187   int nds_enc = nds->encoding();
5188   int dst_enc = dst->encoding();
5189   bool dst_upper_bank = (dst_enc > 15);
5190   bool nds_upper_bank = (nds_enc > 15);
5191   if (VM_Version::supports_avx512novl() &&
5192       (nds_upper_bank || dst_upper_bank)) {
5193     if (dst_upper_bank) {
5194       subptr(rsp, 64);
5195       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5196       movflt(xmm0, nds);
5197       vxorps(xmm0, xmm0, src, Assembler::AVX_128bit);
5198       movflt(dst, xmm0);
5199       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5200       addptr(rsp, 64);
5201     } else {
5202       movflt(dst, nds);
5203       vxorps(dst, dst, src, Assembler::AVX_128bit);
5204     }
5205   } else {
5206     vxorps(dst, nds, src, Assembler::AVX_128bit);
5207   }
5208 }
5209 
5210 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
5211   int nds_enc = nds->encoding();
5212   int dst_enc = dst->encoding();
5213   bool dst_upper_bank = (dst_enc > 15);
5214   bool nds_upper_bank = (nds_enc > 15);
5215   if (VM_Version::supports_avx512novl() &&
5216       (nds_upper_bank || dst_upper_bank)) {
5217     if (dst_upper_bank) {
5218       subptr(rsp, 64);
5219       evmovdqul(Address(rsp, 0), xmm0, Assembler::AVX_512bit);
5220       movdbl(xmm0, nds);
5221       vxorpd(xmm0, xmm0, src, Assembler::AVX_128bit);
5222       movdbl(dst, xmm0);
5223       evmovdqul(xmm0, Address(rsp, 0), Assembler::AVX_512bit);
5224       addptr(rsp, 64);
5225     } else {
5226       movdbl(dst, nds);
5227       vxorpd(dst, dst, src, Assembler::AVX_128bit);
5228     }
5229   } else {
5230     vxorpd(dst, nds, src, Assembler::AVX_128bit);
5231   }
5232 }
5233 
5234 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5235   if (reachable(src)) {
5236     vxorpd(dst, nds, as_Address(src), vector_len);
5237   } else {
5238     lea(rscratch1, src);
5239     vxorpd(dst, nds, Address(rscratch1, 0), vector_len);
5240   }
5241 }
5242 
5243 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
5244   if (reachable(src)) {
5245     vxorps(dst, nds, as_Address(src), vector_len);
5246   } else {
5247     lea(rscratch1, src);
5248     vxorps(dst, nds, Address(rscratch1, 0), vector_len);
5249   }
5250 }
5251 
5252 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) {
5253   const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask);
5254   STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code
5255   // The inverted mask is sign-extended
5256   andptr(possibly_jweak, inverted_jweak_mask);
5257 }
5258 
5259 void MacroAssembler::resolve_jobject(Register value,
5260                                      Register thread,
5261                                      Register tmp) {
5262   assert_different_registers(value, thread, tmp);
5263   Label done, not_weak;
5264   testptr(value, value);
5265   jcc(Assembler::zero, done);                // Use NULL as-is.
5266   testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag.
5267   jcc(Assembler::zero, not_weak);
5268   // Resolve jweak.
5269   access_load_at(T_OBJECT, IN_ROOT | ON_PHANTOM_OOP_REF,
5270                  value, Address(value, -JNIHandles::weak_tag_value), tmp, thread);
5271   verify_oop(value);
5272   jmp(done);
5273   bind(not_weak);
5274   // Resolve (untagged) jobject.
5275   access_load_at(T_OBJECT, IN_ROOT | IN_CONCURRENT_ROOT,
5276                  value, Address(value, 0), tmp, thread);
5277   verify_oop(value);
5278   bind(done);
5279 }
5280 
5281 void MacroAssembler::subptr(Register dst, int32_t imm32) {
5282   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
5283 }
5284 
5285 // Force generation of a 4 byte immediate value even if it fits into 8bit
5286 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
5287   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
5288 }
5289 
5290 void MacroAssembler::subptr(Register dst, Register src) {
5291   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
5292 }
5293 
5294 // C++ bool manipulation
5295 void MacroAssembler::testbool(Register dst) {
5296   if(sizeof(bool) == 1)
5297     testb(dst, 0xff);
5298   else if(sizeof(bool) == 2) {
5299     // testw implementation needed for two byte bools
5300     ShouldNotReachHere();
5301   } else if(sizeof(bool) == 4)
5302     testl(dst, dst);
5303   else
5304     // unsupported
5305     ShouldNotReachHere();
5306 }
5307 
5308 void MacroAssembler::testptr(Register dst, Register src) {
5309   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
5310 }
5311 
5312 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
5313 void MacroAssembler::tlab_allocate(Register obj,
5314                                    Register var_size_in_bytes,
5315                                    int con_size_in_bytes,
5316                                    Register t1,
5317                                    Register t2,
5318                                    Label& slow_case) {
5319   assert_different_registers(obj, t1, t2);
5320   assert_different_registers(obj, var_size_in_bytes, t1);
5321   Register end = t2;
5322   Register thread = NOT_LP64(t1) LP64_ONLY(r15_thread);
5323 
5324   verify_tlab();
5325 
5326   NOT_LP64(get_thread(thread));
5327 
5328   movptr(obj, Address(thread, JavaThread::tlab_top_offset()));
5329   if (var_size_in_bytes == noreg) {
5330     lea(end, Address(obj, con_size_in_bytes));
5331   } else {
5332     lea(end, Address(obj, var_size_in_bytes, Address::times_1));
5333   }
5334   cmpptr(end, Address(thread, JavaThread::tlab_end_offset()));
5335   jcc(Assembler::above, slow_case);
5336 
5337   // update the tlab top pointer
5338   movptr(Address(thread, JavaThread::tlab_top_offset()), end);
5339 
5340   // recover var_size_in_bytes if necessary
5341   if (var_size_in_bytes == end) {
5342     subptr(var_size_in_bytes, obj);
5343   }
5344   verify_tlab();
5345 }
5346 
5347 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
5348 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
5349   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
5350   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
5351   Label done;
5352 
5353   testptr(length_in_bytes, length_in_bytes);
5354   jcc(Assembler::zero, done);
5355 
5356   // initialize topmost word, divide index by 2, check if odd and test if zero
5357   // note: for the remaining code to work, index must be a multiple of BytesPerWord
5358 #ifdef ASSERT
5359   {
5360     Label L;
5361     testptr(length_in_bytes, BytesPerWord - 1);
5362     jcc(Assembler::zero, L);
5363     stop("length must be a multiple of BytesPerWord");
5364     bind(L);
5365   }
5366 #endif
5367   Register index = length_in_bytes;
5368   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
5369   if (UseIncDec) {
5370     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
5371   } else {
5372     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
5373     shrptr(index, 1);
5374   }
5375 #ifndef _LP64
5376   // index could have not been a multiple of 8 (i.e., bit 2 was set)
5377   {
5378     Label even;
5379     // note: if index was a multiple of 8, then it cannot
5380     //       be 0 now otherwise it must have been 0 before
5381     //       => if it is even, we don't need to check for 0 again
5382     jcc(Assembler::carryClear, even);
5383     // clear topmost word (no jump would be needed if conditional assignment worked here)
5384     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
5385     // index could be 0 now, must check again
5386     jcc(Assembler::zero, done);
5387     bind(even);
5388   }
5389 #endif // !_LP64
5390   // initialize remaining object fields: index is a multiple of 2 now
5391   {
5392     Label loop;
5393     bind(loop);
5394     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
5395     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
5396     decrement(index);
5397     jcc(Assembler::notZero, loop);
5398   }
5399 
5400   bind(done);
5401 }
5402 
5403 void MacroAssembler::incr_allocated_bytes(Register thread,
5404                                           Register var_size_in_bytes,
5405                                           int con_size_in_bytes,
5406                                           Register t1) {
5407   if (!thread->is_valid()) {
5408 #ifdef _LP64
5409     thread = r15_thread;
5410 #else
5411     assert(t1->is_valid(), "need temp reg");
5412     thread = t1;
5413     get_thread(thread);
5414 #endif
5415   }
5416 
5417 #ifdef _LP64
5418   if (var_size_in_bytes->is_valid()) {
5419     addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
5420   } else {
5421     addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
5422   }
5423 #else
5424   if (var_size_in_bytes->is_valid()) {
5425     addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
5426   } else {
5427     addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
5428   }
5429   adcl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())+4), 0);
5430 #endif
5431 }
5432 
5433 // Look up the method for a megamorphic invokeinterface call.
5434 // The target method is determined by <intf_klass, itable_index>.
5435 // The receiver klass is in recv_klass.
5436 // On success, the result will be in method_result, and execution falls through.
5437 // On failure, execution transfers to the given label.
5438 void MacroAssembler::lookup_interface_method(Register recv_klass,
5439                                              Register intf_klass,
5440                                              RegisterOrConstant itable_index,
5441                                              Register method_result,
5442                                              Register scan_temp,
5443                                              Label& L_no_such_interface,
5444                                              bool return_method) {
5445   assert_different_registers(recv_klass, intf_klass, scan_temp);
5446   assert_different_registers(method_result, intf_klass, scan_temp);
5447   assert(recv_klass != method_result || !return_method,
5448          "recv_klass can be destroyed when method isn't needed");
5449 
5450   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
5451          "caller must use same register for non-constant itable index as for method");
5452 
5453   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
5454   int vtable_base = in_bytes(Klass::vtable_start_offset());
5455   int itentry_off = itableMethodEntry::method_offset_in_bytes();
5456   int scan_step   = itableOffsetEntry::size() * wordSize;
5457   int vte_size    = vtableEntry::size_in_bytes();
5458   Address::ScaleFactor times_vte_scale = Address::times_ptr;
5459   assert(vte_size == wordSize, "else adjust times_vte_scale");
5460 
5461   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
5462 
5463   // %%% Could store the aligned, prescaled offset in the klassoop.
5464   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
5465 
5466   if (return_method) {
5467     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
5468     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
5469     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
5470   }
5471 
5472   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
5473   //   if (scan->interface() == intf) {
5474   //     result = (klass + scan->offset() + itable_index);
5475   //   }
5476   // }
5477   Label search, found_method;
5478 
5479   for (int peel = 1; peel >= 0; peel--) {
5480     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
5481     cmpptr(intf_klass, method_result);
5482 
5483     if (peel) {
5484       jccb(Assembler::equal, found_method);
5485     } else {
5486       jccb(Assembler::notEqual, search);
5487       // (invert the test to fall through to found_method...)
5488     }
5489 
5490     if (!peel)  break;
5491 
5492     bind(search);
5493 
5494     // Check that the previous entry is non-null.  A null entry means that
5495     // the receiver class doesn't implement the interface, and wasn't the
5496     // same as when the caller was compiled.
5497     testptr(method_result, method_result);
5498     jcc(Assembler::zero, L_no_such_interface);
5499     addptr(scan_temp, scan_step);
5500   }
5501 
5502   bind(found_method);
5503 
5504   if (return_method) {
5505     // Got a hit.
5506     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
5507     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
5508   }
5509 }
5510 
5511 
5512 // virtual method calling
5513 void MacroAssembler::lookup_virtual_method(Register recv_klass,
5514                                            RegisterOrConstant vtable_index,
5515                                            Register method_result) {
5516   const int base = in_bytes(Klass::vtable_start_offset());
5517   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
5518   Address vtable_entry_addr(recv_klass,
5519                             vtable_index, Address::times_ptr,
5520                             base + vtableEntry::method_offset_in_bytes());
5521   movptr(method_result, vtable_entry_addr);
5522 }
5523 
5524 
5525 void MacroAssembler::check_klass_subtype(Register sub_klass,
5526                            Register super_klass,
5527                            Register temp_reg,
5528                            Label& L_success) {
5529   Label L_failure;
5530   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, NULL);
5531   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
5532   bind(L_failure);
5533 }
5534 
5535 
5536 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
5537                                                    Register super_klass,
5538                                                    Register temp_reg,
5539                                                    Label* L_success,
5540                                                    Label* L_failure,
5541                                                    Label* L_slow_path,
5542                                         RegisterOrConstant super_check_offset) {
5543   assert_different_registers(sub_klass, super_klass, temp_reg);
5544   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
5545   if (super_check_offset.is_register()) {
5546     assert_different_registers(sub_klass, super_klass,
5547                                super_check_offset.as_register());
5548   } else if (must_load_sco) {
5549     assert(temp_reg != noreg, "supply either a temp or a register offset");
5550   }
5551 
5552   Label L_fallthrough;
5553   int label_nulls = 0;
5554   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
5555   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
5556   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
5557   assert(label_nulls <= 1, "at most one NULL in the batch");
5558 
5559   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
5560   int sco_offset = in_bytes(Klass::super_check_offset_offset());
5561   Address super_check_offset_addr(super_klass, sco_offset);
5562 
5563   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
5564   // range of a jccb.  If this routine grows larger, reconsider at
5565   // least some of these.
5566 #define local_jcc(assembler_cond, label)                                \
5567   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
5568   else                             jcc( assembler_cond, label) /*omit semi*/
5569 
5570   // Hacked jmp, which may only be used just before L_fallthrough.
5571 #define final_jmp(label)                                                \
5572   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
5573   else                            jmp(label)                /*omit semi*/
5574 
5575   // If the pointers are equal, we are done (e.g., String[] elements).
5576   // This self-check enables sharing of secondary supertype arrays among
5577   // non-primary types such as array-of-interface.  Otherwise, each such
5578   // type would need its own customized SSA.
5579   // We move this check to the front of the fast path because many
5580   // type checks are in fact trivially successful in this manner,
5581   // so we get a nicely predicted branch right at the start of the check.
5582   cmpptr(sub_klass, super_klass);
5583   local_jcc(Assembler::equal, *L_success);
5584 
5585   // Check the supertype display:
5586   if (must_load_sco) {
5587     // Positive movl does right thing on LP64.
5588     movl(temp_reg, super_check_offset_addr);
5589     super_check_offset = RegisterOrConstant(temp_reg);
5590   }
5591   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
5592   cmpptr(super_klass, super_check_addr); // load displayed supertype
5593 
5594   // This check has worked decisively for primary supers.
5595   // Secondary supers are sought in the super_cache ('super_cache_addr').
5596   // (Secondary supers are interfaces and very deeply nested subtypes.)
5597   // This works in the same check above because of a tricky aliasing
5598   // between the super_cache and the primary super display elements.
5599   // (The 'super_check_addr' can address either, as the case requires.)
5600   // Note that the cache is updated below if it does not help us find
5601   // what we need immediately.
5602   // So if it was a primary super, we can just fail immediately.
5603   // Otherwise, it's the slow path for us (no success at this point).
5604 
5605   if (super_check_offset.is_register()) {
5606     local_jcc(Assembler::equal, *L_success);
5607     cmpl(super_check_offset.as_register(), sc_offset);
5608     if (L_failure == &L_fallthrough) {
5609       local_jcc(Assembler::equal, *L_slow_path);
5610     } else {
5611       local_jcc(Assembler::notEqual, *L_failure);
5612       final_jmp(*L_slow_path);
5613     }
5614   } else if (super_check_offset.as_constant() == sc_offset) {
5615     // Need a slow path; fast failure is impossible.
5616     if (L_slow_path == &L_fallthrough) {
5617       local_jcc(Assembler::equal, *L_success);
5618     } else {
5619       local_jcc(Assembler::notEqual, *L_slow_path);
5620       final_jmp(*L_success);
5621     }
5622   } else {
5623     // No slow path; it's a fast decision.
5624     if (L_failure == &L_fallthrough) {
5625       local_jcc(Assembler::equal, *L_success);
5626     } else {
5627       local_jcc(Assembler::notEqual, *L_failure);
5628       final_jmp(*L_success);
5629     }
5630   }
5631 
5632   bind(L_fallthrough);
5633 
5634 #undef local_jcc
5635 #undef final_jmp
5636 }
5637 
5638 
5639 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
5640                                                    Register super_klass,
5641                                                    Register temp_reg,
5642                                                    Register temp2_reg,
5643                                                    Label* L_success,
5644                                                    Label* L_failure,
5645                                                    bool set_cond_codes) {
5646   assert_different_registers(sub_klass, super_klass, temp_reg);
5647   if (temp2_reg != noreg)
5648     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
5649 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
5650 
5651   Label L_fallthrough;
5652   int label_nulls = 0;
5653   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
5654   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
5655   assert(label_nulls <= 1, "at most one NULL in the batch");
5656 
5657   // a couple of useful fields in sub_klass:
5658   int ss_offset = in_bytes(Klass::secondary_supers_offset());
5659   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
5660   Address secondary_supers_addr(sub_klass, ss_offset);
5661   Address super_cache_addr(     sub_klass, sc_offset);
5662 
5663   // Do a linear scan of the secondary super-klass chain.
5664   // This code is rarely used, so simplicity is a virtue here.
5665   // The repne_scan instruction uses fixed registers, which we must spill.
5666   // Don't worry too much about pre-existing connections with the input regs.
5667 
5668   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
5669   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
5670 
5671   // Get super_klass value into rax (even if it was in rdi or rcx).
5672   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
5673   if (super_klass != rax || UseCompressedOops) {
5674     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
5675     mov(rax, super_klass);
5676   }
5677   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
5678   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
5679 
5680 #ifndef PRODUCT
5681   int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
5682   ExternalAddress pst_counter_addr((address) pst_counter);
5683   NOT_LP64(  incrementl(pst_counter_addr) );
5684   LP64_ONLY( lea(rcx, pst_counter_addr) );
5685   LP64_ONLY( incrementl(Address(rcx, 0)) );
5686 #endif //PRODUCT
5687 
5688   // We will consult the secondary-super array.
5689   movptr(rdi, secondary_supers_addr);
5690   // Load the array length.  (Positive movl does right thing on LP64.)
5691   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
5692   // Skip to start of data.
5693   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
5694 
5695   // Scan RCX words at [RDI] for an occurrence of RAX.
5696   // Set NZ/Z based on last compare.
5697   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
5698   // not change flags (only scas instruction which is repeated sets flags).
5699   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
5700 
5701     testptr(rax,rax); // Set Z = 0
5702     repne_scan();
5703 
5704   // Unspill the temp. registers:
5705   if (pushed_rdi)  pop(rdi);
5706   if (pushed_rcx)  pop(rcx);
5707   if (pushed_rax)  pop(rax);
5708 
5709   if (set_cond_codes) {
5710     // Special hack for the AD files:  rdi is guaranteed non-zero.
5711     assert(!pushed_rdi, "rdi must be left non-NULL");
5712     // Also, the condition codes are properly set Z/NZ on succeed/failure.
5713   }
5714 
5715   if (L_failure == &L_fallthrough)
5716         jccb(Assembler::notEqual, *L_failure);
5717   else  jcc(Assembler::notEqual, *L_failure);
5718 
5719   // Success.  Cache the super we found and proceed in triumph.
5720   movptr(super_cache_addr, super_klass);
5721 
5722   if (L_success != &L_fallthrough) {
5723     jmp(*L_success);
5724   }
5725 
5726 #undef IS_A_TEMP
5727 
5728   bind(L_fallthrough);
5729 }
5730 
5731 
5732 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
5733   if (VM_Version::supports_cmov()) {
5734     cmovl(cc, dst, src);
5735   } else {
5736     Label L;
5737     jccb(negate_condition(cc), L);
5738     movl(dst, src);
5739     bind(L);
5740   }
5741 }
5742 
5743 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
5744   if (VM_Version::supports_cmov()) {
5745     cmovl(cc, dst, src);
5746   } else {
5747     Label L;
5748     jccb(negate_condition(cc), L);
5749     movl(dst, src);
5750     bind(L);
5751   }
5752 }
5753 
5754 void MacroAssembler::verify_oop(Register reg, const char* s) {
5755   if (!VerifyOops) return;
5756 
5757   // Pass register number to verify_oop_subroutine
5758   const char* b = NULL;
5759   {
5760     ResourceMark rm;
5761     stringStream ss;
5762     ss.print("verify_oop: %s: %s", reg->name(), s);
5763     b = code_string(ss.as_string());
5764   }
5765   BLOCK_COMMENT("verify_oop {");
5766 #ifdef _LP64
5767   push(rscratch1);                    // save r10, trashed by movptr()
5768 #endif
5769   push(rax);                          // save rax,
5770   push(reg);                          // pass register argument
5771   ExternalAddress buffer((address) b);
5772   // avoid using pushptr, as it modifies scratch registers
5773   // and our contract is not to modify anything
5774   movptr(rax, buffer.addr());
5775   push(rax);
5776   // call indirectly to solve generation ordering problem
5777   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
5778   call(rax);
5779   // Caller pops the arguments (oop, message) and restores rax, r10
5780   BLOCK_COMMENT("} verify_oop");
5781 }
5782 
5783 
5784 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
5785                                                       Register tmp,
5786                                                       int offset) {
5787   intptr_t value = *delayed_value_addr;
5788   if (value != 0)
5789     return RegisterOrConstant(value + offset);
5790 
5791   // load indirectly to solve generation ordering problem
5792   movptr(tmp, ExternalAddress((address) delayed_value_addr));
5793 
5794 #ifdef ASSERT
5795   { Label L;
5796     testptr(tmp, tmp);
5797     if (WizardMode) {
5798       const char* buf = NULL;
5799       {
5800         ResourceMark rm;
5801         stringStream ss;
5802         ss.print("DelayedValue=" INTPTR_FORMAT, delayed_value_addr[1]);
5803         buf = code_string(ss.as_string());
5804       }
5805       jcc(Assembler::notZero, L);
5806       STOP(buf);
5807     } else {
5808       jccb(Assembler::notZero, L);
5809       hlt();
5810     }
5811     bind(L);
5812   }
5813 #endif
5814 
5815   if (offset != 0)
5816     addptr(tmp, offset);
5817 
5818   return RegisterOrConstant(tmp);
5819 }
5820 
5821 
5822 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
5823                                          int extra_slot_offset) {
5824   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
5825   int stackElementSize = Interpreter::stackElementSize;
5826   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
5827 #ifdef ASSERT
5828   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
5829   assert(offset1 - offset == stackElementSize, "correct arithmetic");
5830 #endif
5831   Register             scale_reg    = noreg;
5832   Address::ScaleFactor scale_factor = Address::no_scale;
5833   if (arg_slot.is_constant()) {
5834     offset += arg_slot.as_constant() * stackElementSize;
5835   } else {
5836     scale_reg    = arg_slot.as_register();
5837     scale_factor = Address::times(stackElementSize);
5838   }
5839   offset += wordSize;           // return PC is on stack
5840   return Address(rsp, scale_reg, scale_factor, offset);
5841 }
5842 
5843 
5844 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
5845   if (!VerifyOops) return;
5846 
5847   // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
5848   // Pass register number to verify_oop_subroutine
5849   const char* b = NULL;
5850   {
5851     ResourceMark rm;
5852     stringStream ss;
5853     ss.print("verify_oop_addr: %s", s);
5854     b = code_string(ss.as_string());
5855   }
5856 #ifdef _LP64
5857   push(rscratch1);                    // save r10, trashed by movptr()
5858 #endif
5859   push(rax);                          // save rax,
5860   // addr may contain rsp so we will have to adjust it based on the push
5861   // we just did (and on 64 bit we do two pushes)
5862   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
5863   // stores rax into addr which is backwards of what was intended.
5864   if (addr.uses(rsp)) {
5865     lea(rax, addr);
5866     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
5867   } else {
5868     pushptr(addr);
5869   }
5870 
5871   ExternalAddress buffer((address) b);
5872   // pass msg argument
5873   // avoid using pushptr, as it modifies scratch registers
5874   // and our contract is not to modify anything
5875   movptr(rax, buffer.addr());
5876   push(rax);
5877 
5878   // call indirectly to solve generation ordering problem
5879   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
5880   call(rax);
5881   // Caller pops the arguments (addr, message) and restores rax, r10.
5882 }
5883 
5884 void MacroAssembler::verify_tlab() {
5885 #ifdef ASSERT
5886   if (UseTLAB && VerifyOops) {
5887     Label next, ok;
5888     Register t1 = rsi;
5889     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
5890 
5891     push(t1);
5892     NOT_LP64(push(thread_reg));
5893     NOT_LP64(get_thread(thread_reg));
5894 
5895     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5896     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
5897     jcc(Assembler::aboveEqual, next);
5898     STOP("assert(top >= start)");
5899     should_not_reach_here();
5900 
5901     bind(next);
5902     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
5903     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5904     jcc(Assembler::aboveEqual, ok);
5905     STOP("assert(top <= end)");
5906     should_not_reach_here();
5907 
5908     bind(ok);
5909     NOT_LP64(pop(thread_reg));
5910     pop(t1);
5911   }
5912 #endif
5913 }
5914 
5915 class ControlWord {
5916  public:
5917   int32_t _value;
5918 
5919   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
5920   int  precision_control() const       { return  (_value >>  8) & 3      ; }
5921   bool precision() const               { return ((_value >>  5) & 1) != 0; }
5922   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
5923   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
5924   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
5925   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
5926   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
5927 
5928   void print() const {
5929     // rounding control
5930     const char* rc;
5931     switch (rounding_control()) {
5932       case 0: rc = "round near"; break;
5933       case 1: rc = "round down"; break;
5934       case 2: rc = "round up  "; break;
5935       case 3: rc = "chop      "; break;
5936     };
5937     // precision control
5938     const char* pc;
5939     switch (precision_control()) {
5940       case 0: pc = "24 bits "; break;
5941       case 1: pc = "reserved"; break;
5942       case 2: pc = "53 bits "; break;
5943       case 3: pc = "64 bits "; break;
5944     };
5945     // flags
5946     char f[9];
5947     f[0] = ' ';
5948     f[1] = ' ';
5949     f[2] = (precision   ()) ? 'P' : 'p';
5950     f[3] = (underflow   ()) ? 'U' : 'u';
5951     f[4] = (overflow    ()) ? 'O' : 'o';
5952     f[5] = (zero_divide ()) ? 'Z' : 'z';
5953     f[6] = (denormalized()) ? 'D' : 'd';
5954     f[7] = (invalid     ()) ? 'I' : 'i';
5955     f[8] = '\x0';
5956     // output
5957     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
5958   }
5959 
5960 };
5961 
5962 class StatusWord {
5963  public:
5964   int32_t _value;
5965 
5966   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
5967   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
5968   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
5969   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
5970   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
5971   int  top() const                     { return  (_value >> 11) & 7      ; }
5972   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
5973   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
5974   bool precision() const               { return ((_value >>  5) & 1) != 0; }
5975   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
5976   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
5977   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
5978   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
5979   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
5980 
5981   void print() const {
5982     // condition codes
5983     char c[5];
5984     c[0] = (C3()) ? '3' : '-';
5985     c[1] = (C2()) ? '2' : '-';
5986     c[2] = (C1()) ? '1' : '-';
5987     c[3] = (C0()) ? '0' : '-';
5988     c[4] = '\x0';
5989     // flags
5990     char f[9];
5991     f[0] = (error_status()) ? 'E' : '-';
5992     f[1] = (stack_fault ()) ? 'S' : '-';
5993     f[2] = (precision   ()) ? 'P' : '-';
5994     f[3] = (underflow   ()) ? 'U' : '-';
5995     f[4] = (overflow    ()) ? 'O' : '-';
5996     f[5] = (zero_divide ()) ? 'Z' : '-';
5997     f[6] = (denormalized()) ? 'D' : '-';
5998     f[7] = (invalid     ()) ? 'I' : '-';
5999     f[8] = '\x0';
6000     // output
6001     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
6002   }
6003 
6004 };
6005 
6006 class TagWord {
6007  public:
6008   int32_t _value;
6009 
6010   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
6011 
6012   void print() const {
6013     printf("%04x", _value & 0xFFFF);
6014   }
6015 
6016 };
6017 
6018 class FPU_Register {
6019  public:
6020   int32_t _m0;
6021   int32_t _m1;
6022   int16_t _ex;
6023 
6024   bool is_indefinite() const           {
6025     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
6026   }
6027 
6028   void print() const {
6029     char  sign = (_ex < 0) ? '-' : '+';
6030     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
6031     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
6032   };
6033 
6034 };
6035 
6036 class FPU_State {
6037  public:
6038   enum {
6039     register_size       = 10,
6040     number_of_registers =  8,
6041     register_mask       =  7
6042   };
6043 
6044   ControlWord  _control_word;
6045   StatusWord   _status_word;
6046   TagWord      _tag_word;
6047   int32_t      _error_offset;
6048   int32_t      _error_selector;
6049   int32_t      _data_offset;
6050   int32_t      _data_selector;
6051   int8_t       _register[register_size * number_of_registers];
6052 
6053   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
6054   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
6055 
6056   const char* tag_as_string(int tag) const {
6057     switch (tag) {
6058       case 0: return "valid";
6059       case 1: return "zero";
6060       case 2: return "special";
6061       case 3: return "empty";
6062     }
6063     ShouldNotReachHere();
6064     return NULL;
6065   }
6066 
6067   void print() const {
6068     // print computation registers
6069     { int t = _status_word.top();
6070       for (int i = 0; i < number_of_registers; i++) {
6071         int j = (i - t) & register_mask;
6072         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
6073         st(j)->print();
6074         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
6075       }
6076     }
6077     printf("\n");
6078     // print control registers
6079     printf("ctrl = "); _control_word.print(); printf("\n");
6080     printf("stat = "); _status_word .print(); printf("\n");
6081     printf("tags = "); _tag_word    .print(); printf("\n");
6082   }
6083 
6084 };
6085 
6086 class Flag_Register {
6087  public:
6088   int32_t _value;
6089 
6090   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
6091   bool direction() const               { return ((_value >> 10) & 1) != 0; }
6092   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
6093   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
6094   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
6095   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
6096   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
6097 
6098   void print() const {
6099     // flags
6100     char f[8];
6101     f[0] = (overflow       ()) ? 'O' : '-';
6102     f[1] = (direction      ()) ? 'D' : '-';
6103     f[2] = (sign           ()) ? 'S' : '-';
6104     f[3] = (zero           ()) ? 'Z' : '-';
6105     f[4] = (auxiliary_carry()) ? 'A' : '-';
6106     f[5] = (parity         ()) ? 'P' : '-';
6107     f[6] = (carry          ()) ? 'C' : '-';
6108     f[7] = '\x0';
6109     // output
6110     printf("%08x  flags = %s", _value, f);
6111   }
6112 
6113 };
6114 
6115 class IU_Register {
6116  public:
6117   int32_t _value;
6118 
6119   void print() const {
6120     printf("%08x  %11d", _value, _value);
6121   }
6122 
6123 };
6124 
6125 class IU_State {
6126  public:
6127   Flag_Register _eflags;
6128   IU_Register   _rdi;
6129   IU_Register   _rsi;
6130   IU_Register   _rbp;
6131   IU_Register   _rsp;
6132   IU_Register   _rbx;
6133   IU_Register   _rdx;
6134   IU_Register   _rcx;
6135   IU_Register   _rax;
6136 
6137   void print() const {
6138     // computation registers
6139     printf("rax,  = "); _rax.print(); printf("\n");
6140     printf("rbx,  = "); _rbx.print(); printf("\n");
6141     printf("rcx  = "); _rcx.print(); printf("\n");
6142     printf("rdx  = "); _rdx.print(); printf("\n");
6143     printf("rdi  = "); _rdi.print(); printf("\n");
6144     printf("rsi  = "); _rsi.print(); printf("\n");
6145     printf("rbp,  = "); _rbp.print(); printf("\n");
6146     printf("rsp  = "); _rsp.print(); printf("\n");
6147     printf("\n");
6148     // control registers
6149     printf("flgs = "); _eflags.print(); printf("\n");
6150   }
6151 };
6152 
6153 
6154 class CPU_State {
6155  public:
6156   FPU_State _fpu_state;
6157   IU_State  _iu_state;
6158 
6159   void print() const {
6160     printf("--------------------------------------------------\n");
6161     _iu_state .print();
6162     printf("\n");
6163     _fpu_state.print();
6164     printf("--------------------------------------------------\n");
6165   }
6166 
6167 };
6168 
6169 
6170 static void _print_CPU_state(CPU_State* state) {
6171   state->print();
6172 };
6173 
6174 
6175 void MacroAssembler::print_CPU_state() {
6176   push_CPU_state();
6177   push(rsp);                // pass CPU state
6178   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
6179   addptr(rsp, wordSize);       // discard argument
6180   pop_CPU_state();
6181 }
6182 
6183 
6184 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
6185   static int counter = 0;
6186   FPU_State* fs = &state->_fpu_state;
6187   counter++;
6188   // For leaf calls, only verify that the top few elements remain empty.
6189   // We only need 1 empty at the top for C2 code.
6190   if( stack_depth < 0 ) {
6191     if( fs->tag_for_st(7) != 3 ) {
6192       printf("FPR7 not empty\n");
6193       state->print();
6194       assert(false, "error");
6195       return false;
6196     }
6197     return true;                // All other stack states do not matter
6198   }
6199 
6200   assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std,
6201          "bad FPU control word");
6202 
6203   // compute stack depth
6204   int i = 0;
6205   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
6206   int d = i;
6207   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
6208   // verify findings
6209   if (i != FPU_State::number_of_registers) {
6210     // stack not contiguous
6211     printf("%s: stack not contiguous at ST%d\n", s, i);
6212     state->print();
6213     assert(false, "error");
6214     return false;
6215   }
6216   // check if computed stack depth corresponds to expected stack depth
6217   if (stack_depth < 0) {
6218     // expected stack depth is -stack_depth or less
6219     if (d > -stack_depth) {
6220       // too many elements on the stack
6221       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
6222       state->print();
6223       assert(false, "error");
6224       return false;
6225     }
6226   } else {
6227     // expected stack depth is stack_depth
6228     if (d != stack_depth) {
6229       // wrong stack depth
6230       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
6231       state->print();
6232       assert(false, "error");
6233       return false;
6234     }
6235   }
6236   // everything is cool
6237   return true;
6238 }
6239 
6240 
6241 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
6242   if (!VerifyFPU) return;
6243   push_CPU_state();
6244   push(rsp);                // pass CPU state
6245   ExternalAddress msg((address) s);
6246   // pass message string s
6247   pushptr(msg.addr());
6248   push(stack_depth);        // pass stack depth
6249   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
6250   addptr(rsp, 3 * wordSize);   // discard arguments
6251   // check for error
6252   { Label L;
6253     testl(rax, rax);
6254     jcc(Assembler::notZero, L);
6255     int3();                  // break if error condition
6256     bind(L);
6257   }
6258   pop_CPU_state();
6259 }
6260 
6261 void MacroAssembler::restore_cpu_control_state_after_jni() {
6262   // Either restore the MXCSR register after returning from the JNI Call
6263   // or verify that it wasn't changed (with -Xcheck:jni flag).
6264   if (VM_Version::supports_sse()) {
6265     if (RestoreMXCSROnJNICalls) {
6266       ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std()));
6267     } else if (CheckJNICalls) {
6268       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
6269     }
6270   }
6271   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
6272   vzeroupper();
6273   // Reset k1 to 0xffff.
6274   if (VM_Version::supports_evex()) {
6275     push(rcx);
6276     movl(rcx, 0xffff);
6277     kmovwl(k1, rcx);
6278     pop(rcx);
6279   }
6280 
6281 #ifndef _LP64
6282   // Either restore the x87 floating pointer control word after returning
6283   // from the JNI call or verify that it wasn't changed.
6284   if (CheckJNICalls) {
6285     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
6286   }
6287 #endif // _LP64
6288 }
6289 
6290 // ((OopHandle)result).resolve();
6291 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
6292   // Only 64 bit platforms support GCs that require a tmp register
6293   // Only IN_HEAP loads require a thread_tmp register
6294   // OopHandle::resolve is an indirection like jobject.
6295   access_load_at(T_OBJECT, IN_ROOT | IN_CONCURRENT_ROOT,
6296                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
6297 }
6298 
6299 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
6300   // get mirror
6301   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
6302   movptr(mirror, Address(method, Method::const_offset()));
6303   movptr(mirror, Address(mirror, ConstMethod::constants_offset()));
6304   movptr(mirror, Address(mirror, ConstantPool::pool_holder_offset_in_bytes()));
6305   movptr(mirror, Address(mirror, mirror_offset));
6306   resolve_oop_handle(mirror, tmp);
6307 }
6308 
6309 void MacroAssembler::load_klass(Register dst, Register src) {
6310 #ifdef _LP64
6311   if (UseCompressedClassPointers) {
6312     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
6313     decode_klass_not_null(dst);
6314   } else
6315 #endif
6316     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
6317 }
6318 
6319 void MacroAssembler::load_prototype_header(Register dst, Register src) {
6320   load_klass(dst, src);
6321   movptr(dst, Address(dst, Klass::prototype_header_offset()));
6322 }
6323 
6324 void MacroAssembler::store_klass(Register dst, Register src) {
6325 #ifdef _LP64
6326   if (UseCompressedClassPointers) {
6327     encode_klass_not_null(src);
6328     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
6329   } else
6330 #endif
6331     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
6332 }
6333 
6334 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
6335                                     Register tmp1, Register thread_tmp) {
6336   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
6337   bool as_raw = (decorators & AS_RAW) != 0;
6338   if (as_raw) {
6339     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
6340   } else {
6341     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
6342   }
6343 }
6344 
6345 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
6346                                      Register tmp1, Register tmp2) {
6347   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
6348   bool as_raw = (decorators & AS_RAW) != 0;
6349   if (as_raw) {
6350     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2);
6351   } else {
6352     bs->store_at(this, decorators, type, dst, src, tmp1, tmp2);
6353   }
6354 }
6355 
6356 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
6357                                    Register thread_tmp, DecoratorSet decorators) {
6358   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
6359 }
6360 
6361 // Doesn't do verfication, generates fixed size code
6362 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
6363                                             Register thread_tmp, DecoratorSet decorators) {
6364   access_load_at(T_OBJECT, IN_HEAP | OOP_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
6365 }
6366 
6367 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
6368                                     Register tmp2, DecoratorSet decorators) {
6369   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
6370 }
6371 
6372 // Used for storing NULLs.
6373 void MacroAssembler::store_heap_oop_null(Address dst) {
6374   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg);
6375 }
6376 
6377 #ifdef _LP64
6378 void MacroAssembler::store_klass_gap(Register dst, Register src) {
6379   if (UseCompressedClassPointers) {
6380     // Store to klass gap in destination
6381     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
6382   }
6383 }
6384 
6385 #ifdef ASSERT
6386 void MacroAssembler::verify_heapbase(const char* msg) {
6387   assert (UseCompressedOops, "should be compressed");
6388   assert (Universe::heap() != NULL, "java heap should be initialized");
6389   if (CheckCompressedOops) {
6390     Label ok;
6391     push(rscratch1); // cmpptr trashes rscratch1
6392     cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6393     jcc(Assembler::equal, ok);
6394     STOP(msg);
6395     bind(ok);
6396     pop(rscratch1);
6397   }
6398 }
6399 #endif
6400 
6401 // Algorithm must match oop.inline.hpp encode_heap_oop.
6402 void MacroAssembler::encode_heap_oop(Register r) {
6403 #ifdef ASSERT
6404   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
6405 #endif
6406   verify_oop(r, "broken oop in encode_heap_oop");
6407   if (Universe::narrow_oop_base() == NULL) {
6408     if (Universe::narrow_oop_shift() != 0) {
6409       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6410       shrq(r, LogMinObjAlignmentInBytes);
6411     }
6412     return;
6413   }
6414   testq(r, r);
6415   cmovq(Assembler::equal, r, r12_heapbase);
6416   subq(r, r12_heapbase);
6417   shrq(r, LogMinObjAlignmentInBytes);
6418 }
6419 
6420 void MacroAssembler::encode_heap_oop_not_null(Register r) {
6421 #ifdef ASSERT
6422   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
6423   if (CheckCompressedOops) {
6424     Label ok;
6425     testq(r, r);
6426     jcc(Assembler::notEqual, ok);
6427     STOP("null oop passed to encode_heap_oop_not_null");
6428     bind(ok);
6429   }
6430 #endif
6431   verify_oop(r, "broken oop in encode_heap_oop_not_null");
6432   if (Universe::narrow_oop_base() != NULL) {
6433     subq(r, r12_heapbase);
6434   }
6435   if (Universe::narrow_oop_shift() != 0) {
6436     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6437     shrq(r, LogMinObjAlignmentInBytes);
6438   }
6439 }
6440 
6441 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
6442 #ifdef ASSERT
6443   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
6444   if (CheckCompressedOops) {
6445     Label ok;
6446     testq(src, src);
6447     jcc(Assembler::notEqual, ok);
6448     STOP("null oop passed to encode_heap_oop_not_null2");
6449     bind(ok);
6450   }
6451 #endif
6452   verify_oop(src, "broken oop in encode_heap_oop_not_null2");
6453   if (dst != src) {
6454     movq(dst, src);
6455   }
6456   if (Universe::narrow_oop_base() != NULL) {
6457     subq(dst, r12_heapbase);
6458   }
6459   if (Universe::narrow_oop_shift() != 0) {
6460     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6461     shrq(dst, LogMinObjAlignmentInBytes);
6462   }
6463 }
6464 
6465 void  MacroAssembler::decode_heap_oop(Register r) {
6466 #ifdef ASSERT
6467   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
6468 #endif
6469   if (Universe::narrow_oop_base() == NULL) {
6470     if (Universe::narrow_oop_shift() != 0) {
6471       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6472       shlq(r, LogMinObjAlignmentInBytes);
6473     }
6474   } else {
6475     Label done;
6476     shlq(r, LogMinObjAlignmentInBytes);
6477     jccb(Assembler::equal, done);
6478     addq(r, r12_heapbase);
6479     bind(done);
6480   }
6481   verify_oop(r, "broken oop in decode_heap_oop");
6482 }
6483 
6484 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
6485   // Note: it will change flags
6486   assert (UseCompressedOops, "should only be used for compressed headers");
6487   assert (Universe::heap() != NULL, "java heap should be initialized");
6488   // Cannot assert, unverified entry point counts instructions (see .ad file)
6489   // vtableStubs also counts instructions in pd_code_size_limit.
6490   // Also do not verify_oop as this is called by verify_oop.
6491   if (Universe::narrow_oop_shift() != 0) {
6492     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6493     shlq(r, LogMinObjAlignmentInBytes);
6494     if (Universe::narrow_oop_base() != NULL) {
6495       addq(r, r12_heapbase);
6496     }
6497   } else {
6498     assert (Universe::narrow_oop_base() == NULL, "sanity");
6499   }
6500 }
6501 
6502 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
6503   // Note: it will change flags
6504   assert (UseCompressedOops, "should only be used for compressed headers");
6505   assert (Universe::heap() != NULL, "java heap should be initialized");
6506   // Cannot assert, unverified entry point counts instructions (see .ad file)
6507   // vtableStubs also counts instructions in pd_code_size_limit.
6508   // Also do not verify_oop as this is called by verify_oop.
6509   if (Universe::narrow_oop_shift() != 0) {
6510     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6511     if (LogMinObjAlignmentInBytes == Address::times_8) {
6512       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
6513     } else {
6514       if (dst != src) {
6515         movq(dst, src);
6516       }
6517       shlq(dst, LogMinObjAlignmentInBytes);
6518       if (Universe::narrow_oop_base() != NULL) {
6519         addq(dst, r12_heapbase);
6520       }
6521     }
6522   } else {
6523     assert (Universe::narrow_oop_base() == NULL, "sanity");
6524     if (dst != src) {
6525       movq(dst, src);
6526     }
6527   }
6528 }
6529 
6530 void MacroAssembler::encode_klass_not_null(Register r) {
6531   if (Universe::narrow_klass_base() != NULL) {
6532     // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6533     assert(r != r12_heapbase, "Encoding a klass in r12");
6534     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6535     subq(r, r12_heapbase);
6536   }
6537   if (Universe::narrow_klass_shift() != 0) {
6538     assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6539     shrq(r, LogKlassAlignmentInBytes);
6540   }
6541   if (Universe::narrow_klass_base() != NULL) {
6542     reinit_heapbase();
6543   }
6544 }
6545 
6546 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
6547   if (dst == src) {
6548     encode_klass_not_null(src);
6549   } else {
6550     if (Universe::narrow_klass_base() != NULL) {
6551       mov64(dst, (int64_t)Universe::narrow_klass_base());
6552       negq(dst);
6553       addq(dst, src);
6554     } else {
6555       movptr(dst, src);
6556     }
6557     if (Universe::narrow_klass_shift() != 0) {
6558       assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6559       shrq(dst, LogKlassAlignmentInBytes);
6560     }
6561   }
6562 }
6563 
6564 // Function instr_size_for_decode_klass_not_null() counts the instructions
6565 // generated by decode_klass_not_null(register r) and reinit_heapbase(),
6566 // when (Universe::heap() != NULL).  Hence, if the instructions they
6567 // generate change, then this method needs to be updated.
6568 int MacroAssembler::instr_size_for_decode_klass_not_null() {
6569   assert (UseCompressedClassPointers, "only for compressed klass ptrs");
6570   if (Universe::narrow_klass_base() != NULL) {
6571     // mov64 + addq + shlq? + mov64  (for reinit_heapbase()).
6572     return (Universe::narrow_klass_shift() == 0 ? 20 : 24);
6573   } else {
6574     // longest load decode klass function, mov64, leaq
6575     return 16;
6576   }
6577 }
6578 
6579 // !!! If the instructions that get generated here change then function
6580 // instr_size_for_decode_klass_not_null() needs to get updated.
6581 void  MacroAssembler::decode_klass_not_null(Register r) {
6582   // Note: it will change flags
6583   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6584   assert(r != r12_heapbase, "Decoding a klass in r12");
6585   // Cannot assert, unverified entry point counts instructions (see .ad file)
6586   // vtableStubs also counts instructions in pd_code_size_limit.
6587   // Also do not verify_oop as this is called by verify_oop.
6588   if (Universe::narrow_klass_shift() != 0) {
6589     assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6590     shlq(r, LogKlassAlignmentInBytes);
6591   }
6592   // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6593   if (Universe::narrow_klass_base() != NULL) {
6594     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6595     addq(r, r12_heapbase);
6596     reinit_heapbase();
6597   }
6598 }
6599 
6600 void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
6601   // Note: it will change flags
6602   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6603   if (dst == src) {
6604     decode_klass_not_null(dst);
6605   } else {
6606     // Cannot assert, unverified entry point counts instructions (see .ad file)
6607     // vtableStubs also counts instructions in pd_code_size_limit.
6608     // Also do not verify_oop as this is called by verify_oop.
6609     mov64(dst, (int64_t)Universe::narrow_klass_base());
6610     if (Universe::narrow_klass_shift() != 0) {
6611       assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6612       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
6613       leaq(dst, Address(dst, src, Address::times_8, 0));
6614     } else {
6615       addq(dst, src);
6616     }
6617   }
6618 }
6619 
6620 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
6621   assert (UseCompressedOops, "should only be used for compressed headers");
6622   assert (Universe::heap() != NULL, "java heap should be initialized");
6623   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6624   int oop_index = oop_recorder()->find_index(obj);
6625   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6626   mov_narrow_oop(dst, oop_index, rspec);
6627 }
6628 
6629 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
6630   assert (UseCompressedOops, "should only be used for compressed headers");
6631   assert (Universe::heap() != NULL, "java heap should be initialized");
6632   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6633   int oop_index = oop_recorder()->find_index(obj);
6634   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6635   mov_narrow_oop(dst, oop_index, rspec);
6636 }
6637 
6638 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
6639   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6640   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6641   int klass_index = oop_recorder()->find_index(k);
6642   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6643   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6644 }
6645 
6646 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
6647   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6648   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6649   int klass_index = oop_recorder()->find_index(k);
6650   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6651   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6652 }
6653 
6654 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
6655   assert (UseCompressedOops, "should only be used for compressed headers");
6656   assert (Universe::heap() != NULL, "java heap should be initialized");
6657   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6658   int oop_index = oop_recorder()->find_index(obj);
6659   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6660   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6661 }
6662 
6663 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
6664   assert (UseCompressedOops, "should only be used for compressed headers");
6665   assert (Universe::heap() != NULL, "java heap should be initialized");
6666   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6667   int oop_index = oop_recorder()->find_index(obj);
6668   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6669   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6670 }
6671 
6672 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
6673   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6674   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6675   int klass_index = oop_recorder()->find_index(k);
6676   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6677   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6678 }
6679 
6680 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
6681   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6682   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6683   int klass_index = oop_recorder()->find_index(k);
6684   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6685   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6686 }
6687 
6688 void MacroAssembler::reinit_heapbase() {
6689   if (UseCompressedOops || UseCompressedClassPointers) {
6690     if (Universe::heap() != NULL) {
6691       if (Universe::narrow_oop_base() == NULL) {
6692         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
6693       } else {
6694         mov64(r12_heapbase, (int64_t)Universe::narrow_ptrs_base());
6695       }
6696     } else {
6697       movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6698     }
6699   }
6700 }
6701 
6702 #endif // _LP64
6703 
6704 // C2 compiled method's prolog code.
6705 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b) {
6706 
6707   // WARNING: Initial instruction MUST be 5 bytes or longer so that
6708   // NativeJump::patch_verified_entry will be able to patch out the entry
6709   // code safely. The push to verify stack depth is ok at 5 bytes,
6710   // the frame allocation can be either 3 or 6 bytes. So if we don't do
6711   // stack bang then we must use the 6 byte frame allocation even if
6712   // we have no frame. :-(
6713   assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
6714 
6715   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
6716   // Remove word for return addr
6717   framesize -= wordSize;
6718   stack_bang_size -= wordSize;
6719 
6720   // Calls to C2R adapters often do not accept exceptional returns.
6721   // We require that their callers must bang for them.  But be careful, because
6722   // some VM calls (such as call site linkage) can use several kilobytes of
6723   // stack.  But the stack safety zone should account for that.
6724   // See bugs 4446381, 4468289, 4497237.
6725   if (stack_bang_size > 0) {
6726     generate_stack_overflow_check(stack_bang_size);
6727 
6728     // We always push rbp, so that on return to interpreter rbp, will be
6729     // restored correctly and we can correct the stack.
6730     push(rbp);
6731     // Save caller's stack pointer into RBP if the frame pointer is preserved.
6732     if (PreserveFramePointer) {
6733       mov(rbp, rsp);
6734     }
6735     // Remove word for ebp
6736     framesize -= wordSize;
6737 
6738     // Create frame
6739     if (framesize) {
6740       subptr(rsp, framesize);
6741     }
6742   } else {
6743     // Create frame (force generation of a 4 byte immediate value)
6744     subptr_imm32(rsp, framesize);
6745 
6746     // Save RBP register now.
6747     framesize -= wordSize;
6748     movptr(Address(rsp, framesize), rbp);
6749     // Save caller's stack pointer into RBP if the frame pointer is preserved.
6750     if (PreserveFramePointer) {
6751       movptr(rbp, rsp);
6752       if (framesize > 0) {
6753         addptr(rbp, framesize);
6754       }
6755     }
6756   }
6757 
6758   if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
6759     framesize -= wordSize;
6760     movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
6761   }
6762 
6763 #ifndef _LP64
6764   // If method sets FPU control word do it now
6765   if (fp_mode_24b) {
6766     fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
6767   }
6768   if (UseSSE >= 2 && VerifyFPU) {
6769     verify_FPU(0, "FPU stack must be clean on entry");
6770   }
6771 #endif
6772 
6773 #ifdef ASSERT
6774   if (VerifyStackAtCalls) {
6775     Label L;
6776     push(rax);
6777     mov(rax, rsp);
6778     andptr(rax, StackAlignmentInBytes-1);
6779     cmpptr(rax, StackAlignmentInBytes-wordSize);
6780     pop(rax);
6781     jcc(Assembler::equal, L);
6782     STOP("Stack is not properly aligned!");
6783     bind(L);
6784   }
6785 #endif
6786 
6787 }
6788 
6789 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, bool is_large) {
6790   // cnt - number of qwords (8-byte words).
6791   // base - start address, qword aligned.
6792   // is_large - if optimizers know cnt is larger than InitArrayShortSize
6793   assert(base==rdi, "base register must be edi for rep stos");
6794   assert(tmp==rax,   "tmp register must be eax for rep stos");
6795   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
6796   assert(InitArrayShortSize % BytesPerLong == 0,
6797     "InitArrayShortSize should be the multiple of BytesPerLong");
6798 
6799   Label DONE;
6800 
6801   xorptr(tmp, tmp);
6802 
6803   if (!is_large) {
6804     Label LOOP, LONG;
6805     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
6806     jccb(Assembler::greater, LONG);
6807 
6808     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
6809 
6810     decrement(cnt);
6811     jccb(Assembler::negative, DONE); // Zero length
6812 
6813     // Use individual pointer-sized stores for small counts:
6814     BIND(LOOP);
6815     movptr(Address(base, cnt, Address::times_ptr), tmp);
6816     decrement(cnt);
6817     jccb(Assembler::greaterEqual, LOOP);
6818     jmpb(DONE);
6819 
6820     BIND(LONG);
6821   }
6822 
6823   // Use longer rep-prefixed ops for non-small counts:
6824   if (UseFastStosb) {
6825     shlptr(cnt, 3); // convert to number of bytes
6826     rep_stosb();
6827   } else {
6828     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
6829     rep_stos();
6830   }
6831 
6832   BIND(DONE);
6833 }
6834 
6835 #ifdef COMPILER2
6836 
6837 // IndexOf for constant substrings with size >= 8 chars
6838 // which don't need to be loaded through stack.
6839 void MacroAssembler::string_indexofC8(Register str1, Register str2,
6840                                       Register cnt1, Register cnt2,
6841                                       int int_cnt2,  Register result,
6842                                       XMMRegister vec, Register tmp,
6843                                       int ae) {
6844   ShortBranchVerifier sbv(this);
6845   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
6846   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
6847 
6848   // This method uses the pcmpestri instruction with bound registers
6849   //   inputs:
6850   //     xmm - substring
6851   //     rax - substring length (elements count)
6852   //     mem - scanned string
6853   //     rdx - string length (elements count)
6854   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
6855   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
6856   //   outputs:
6857   //     rcx - matched index in string
6858   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
6859   int mode   = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
6860   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
6861   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
6862   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
6863 
6864   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR,
6865         RET_FOUND, RET_NOT_FOUND, EXIT, FOUND_SUBSTR,
6866         MATCH_SUBSTR_HEAD, RELOAD_STR, FOUND_CANDIDATE;
6867 
6868   // Note, inline_string_indexOf() generates checks:
6869   // if (substr.count > string.count) return -1;
6870   // if (substr.count == 0) return 0;
6871   assert(int_cnt2 >= stride, "this code is used only for cnt2 >= 8 chars");
6872 
6873   // Load substring.
6874   if (ae == StrIntrinsicNode::UL) {
6875     pmovzxbw(vec, Address(str2, 0));
6876   } else {
6877     movdqu(vec, Address(str2, 0));
6878   }
6879   movl(cnt2, int_cnt2);
6880   movptr(result, str1); // string addr
6881 
6882   if (int_cnt2 > stride) {
6883     jmpb(SCAN_TO_SUBSTR);
6884 
6885     // Reload substr for rescan, this code
6886     // is executed only for large substrings (> 8 chars)
6887     bind(RELOAD_SUBSTR);
6888     if (ae == StrIntrinsicNode::UL) {
6889       pmovzxbw(vec, Address(str2, 0));
6890     } else {
6891       movdqu(vec, Address(str2, 0));
6892     }
6893     negptr(cnt2); // Jumped here with negative cnt2, convert to positive
6894 
6895     bind(RELOAD_STR);
6896     // We came here after the beginning of the substring was
6897     // matched but the rest of it was not so we need to search
6898     // again. Start from the next element after the previous match.
6899 
6900     // cnt2 is number of substring reminding elements and
6901     // cnt1 is number of string reminding elements when cmp failed.
6902     // Restored cnt1 = cnt1 - cnt2 + int_cnt2
6903     subl(cnt1, cnt2);
6904     addl(cnt1, int_cnt2);
6905     movl(cnt2, int_cnt2); // Now restore cnt2
6906 
6907     decrementl(cnt1);     // Shift to next element
6908     cmpl(cnt1, cnt2);
6909     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6910 
6911     addptr(result, (1<<scale1));
6912 
6913   } // (int_cnt2 > 8)
6914 
6915   // Scan string for start of substr in 16-byte vectors
6916   bind(SCAN_TO_SUBSTR);
6917   pcmpestri(vec, Address(result, 0), mode);
6918   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
6919   subl(cnt1, stride);
6920   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
6921   cmpl(cnt1, cnt2);
6922   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6923   addptr(result, 16);
6924   jmpb(SCAN_TO_SUBSTR);
6925 
6926   // Found a potential substr
6927   bind(FOUND_CANDIDATE);
6928   // Matched whole vector if first element matched (tmp(rcx) == 0).
6929   if (int_cnt2 == stride) {
6930     jccb(Assembler::overflow, RET_FOUND);    // OF == 1
6931   } else { // int_cnt2 > 8
6932     jccb(Assembler::overflow, FOUND_SUBSTR);
6933   }
6934   // After pcmpestri tmp(rcx) contains matched element index
6935   // Compute start addr of substr
6936   lea(result, Address(result, tmp, scale1));
6937 
6938   // Make sure string is still long enough
6939   subl(cnt1, tmp);
6940   cmpl(cnt1, cnt2);
6941   if (int_cnt2 == stride) {
6942     jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
6943   } else { // int_cnt2 > 8
6944     jccb(Assembler::greaterEqual, MATCH_SUBSTR_HEAD);
6945   }
6946   // Left less then substring.
6947 
6948   bind(RET_NOT_FOUND);
6949   movl(result, -1);
6950   jmp(EXIT);
6951 
6952   if (int_cnt2 > stride) {
6953     // This code is optimized for the case when whole substring
6954     // is matched if its head is matched.
6955     bind(MATCH_SUBSTR_HEAD);
6956     pcmpestri(vec, Address(result, 0), mode);
6957     // Reload only string if does not match
6958     jcc(Assembler::noOverflow, RELOAD_STR); // OF == 0
6959 
6960     Label CONT_SCAN_SUBSTR;
6961     // Compare the rest of substring (> 8 chars).
6962     bind(FOUND_SUBSTR);
6963     // First 8 chars are already matched.
6964     negptr(cnt2);
6965     addptr(cnt2, stride);
6966 
6967     bind(SCAN_SUBSTR);
6968     subl(cnt1, stride);
6969     cmpl(cnt2, -stride); // Do not read beyond substring
6970     jccb(Assembler::lessEqual, CONT_SCAN_SUBSTR);
6971     // Back-up strings to avoid reading beyond substring:
6972     // cnt1 = cnt1 - cnt2 + 8
6973     addl(cnt1, cnt2); // cnt2 is negative
6974     addl(cnt1, stride);
6975     movl(cnt2, stride); negptr(cnt2);
6976     bind(CONT_SCAN_SUBSTR);
6977     if (int_cnt2 < (int)G) {
6978       int tail_off1 = int_cnt2<<scale1;
6979       int tail_off2 = int_cnt2<<scale2;
6980       if (ae == StrIntrinsicNode::UL) {
6981         pmovzxbw(vec, Address(str2, cnt2, scale2, tail_off2));
6982       } else {
6983         movdqu(vec, Address(str2, cnt2, scale2, tail_off2));
6984       }
6985       pcmpestri(vec, Address(result, cnt2, scale1, tail_off1), mode);
6986     } else {
6987       // calculate index in register to avoid integer overflow (int_cnt2*2)
6988       movl(tmp, int_cnt2);
6989       addptr(tmp, cnt2);
6990       if (ae == StrIntrinsicNode::UL) {
6991         pmovzxbw(vec, Address(str2, tmp, scale2, 0));
6992       } else {
6993         movdqu(vec, Address(str2, tmp, scale2, 0));
6994       }
6995       pcmpestri(vec, Address(result, tmp, scale1, 0), mode);
6996     }
6997     // Need to reload strings pointers if not matched whole vector
6998     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
6999     addptr(cnt2, stride);
7000     jcc(Assembler::negative, SCAN_SUBSTR);
7001     // Fall through if found full substring
7002 
7003   } // (int_cnt2 > 8)
7004 
7005   bind(RET_FOUND);
7006   // Found result if we matched full small substring.
7007   // Compute substr offset
7008   subptr(result, str1);
7009   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7010     shrl(result, 1); // index
7011   }
7012   bind(EXIT);
7013 
7014 } // string_indexofC8
7015 
7016 // Small strings are loaded through stack if they cross page boundary.
7017 void MacroAssembler::string_indexof(Register str1, Register str2,
7018                                     Register cnt1, Register cnt2,
7019                                     int int_cnt2,  Register result,
7020                                     XMMRegister vec, Register tmp,
7021                                     int ae) {
7022   ShortBranchVerifier sbv(this);
7023   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7024   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
7025 
7026   //
7027   // int_cnt2 is length of small (< 8 chars) constant substring
7028   // or (-1) for non constant substring in which case its length
7029   // is in cnt2 register.
7030   //
7031   // Note, inline_string_indexOf() generates checks:
7032   // if (substr.count > string.count) return -1;
7033   // if (substr.count == 0) return 0;
7034   //
7035   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
7036   assert(int_cnt2 == -1 || (0 < int_cnt2 && int_cnt2 < stride), "should be != 0");
7037   // This method uses the pcmpestri instruction with bound registers
7038   //   inputs:
7039   //     xmm - substring
7040   //     rax - substring length (elements count)
7041   //     mem - scanned string
7042   //     rdx - string length (elements count)
7043   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
7044   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
7045   //   outputs:
7046   //     rcx - matched index in string
7047   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
7048   int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
7049   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
7050   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
7051 
7052   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, ADJUST_STR,
7053         RET_FOUND, RET_NOT_FOUND, CLEANUP, FOUND_SUBSTR,
7054         FOUND_CANDIDATE;
7055 
7056   { //========================================================
7057     // We don't know where these strings are located
7058     // and we can't read beyond them. Load them through stack.
7059     Label BIG_STRINGS, CHECK_STR, COPY_SUBSTR, COPY_STR;
7060 
7061     movptr(tmp, rsp); // save old SP
7062 
7063     if (int_cnt2 > 0) {     // small (< 8 chars) constant substring
7064       if (int_cnt2 == (1>>scale2)) { // One byte
7065         assert((ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL), "Only possible for latin1 encoding");
7066         load_unsigned_byte(result, Address(str2, 0));
7067         movdl(vec, result); // move 32 bits
7068       } else if (ae == StrIntrinsicNode::LL && int_cnt2 == 3) {  // Three bytes
7069         // Not enough header space in 32-bit VM: 12+3 = 15.
7070         movl(result, Address(str2, -1));
7071         shrl(result, 8);
7072         movdl(vec, result); // move 32 bits
7073       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (2>>scale2)) {  // One char
7074         load_unsigned_short(result, Address(str2, 0));
7075         movdl(vec, result); // move 32 bits
7076       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (4>>scale2)) { // Two chars
7077         movdl(vec, Address(str2, 0)); // move 32 bits
7078       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (8>>scale2)) { // Four chars
7079         movq(vec, Address(str2, 0));  // move 64 bits
7080       } else { // cnt2 = { 3, 5, 6, 7 } || (ae == StrIntrinsicNode::UL && cnt2 ={2, ..., 7})
7081         // Array header size is 12 bytes in 32-bit VM
7082         // + 6 bytes for 3 chars == 18 bytes,
7083         // enough space to load vec and shift.
7084         assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity");
7085         if (ae == StrIntrinsicNode::UL) {
7086           int tail_off = int_cnt2-8;
7087           pmovzxbw(vec, Address(str2, tail_off));
7088           psrldq(vec, -2*tail_off);
7089         }
7090         else {
7091           int tail_off = int_cnt2*(1<<scale2);
7092           movdqu(vec, Address(str2, tail_off-16));
7093           psrldq(vec, 16-tail_off);
7094         }
7095       }
7096     } else { // not constant substring
7097       cmpl(cnt2, stride);
7098       jccb(Assembler::aboveEqual, BIG_STRINGS); // Both strings are big enough
7099 
7100       // We can read beyond string if srt+16 does not cross page boundary
7101       // since heaps are aligned and mapped by pages.
7102       assert(os::vm_page_size() < (int)G, "default page should be small");
7103       movl(result, str2); // We need only low 32 bits
7104       andl(result, (os::vm_page_size()-1));
7105       cmpl(result, (os::vm_page_size()-16));
7106       jccb(Assembler::belowEqual, CHECK_STR);
7107 
7108       // Move small strings to stack to allow load 16 bytes into vec.
7109       subptr(rsp, 16);
7110       int stk_offset = wordSize-(1<<scale2);
7111       push(cnt2);
7112 
7113       bind(COPY_SUBSTR);
7114       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL) {
7115         load_unsigned_byte(result, Address(str2, cnt2, scale2, -1));
7116         movb(Address(rsp, cnt2, scale2, stk_offset), result);
7117       } else if (ae == StrIntrinsicNode::UU) {
7118         load_unsigned_short(result, Address(str2, cnt2, scale2, -2));
7119         movw(Address(rsp, cnt2, scale2, stk_offset), result);
7120       }
7121       decrement(cnt2);
7122       jccb(Assembler::notZero, COPY_SUBSTR);
7123 
7124       pop(cnt2);
7125       movptr(str2, rsp);  // New substring address
7126     } // non constant
7127 
7128     bind(CHECK_STR);
7129     cmpl(cnt1, stride);
7130     jccb(Assembler::aboveEqual, BIG_STRINGS);
7131 
7132     // Check cross page boundary.
7133     movl(result, str1); // We need only low 32 bits
7134     andl(result, (os::vm_page_size()-1));
7135     cmpl(result, (os::vm_page_size()-16));
7136     jccb(Assembler::belowEqual, BIG_STRINGS);
7137 
7138     subptr(rsp, 16);
7139     int stk_offset = -(1<<scale1);
7140     if (int_cnt2 < 0) { // not constant
7141       push(cnt2);
7142       stk_offset += wordSize;
7143     }
7144     movl(cnt2, cnt1);
7145 
7146     bind(COPY_STR);
7147     if (ae == StrIntrinsicNode::LL) {
7148       load_unsigned_byte(result, Address(str1, cnt2, scale1, -1));
7149       movb(Address(rsp, cnt2, scale1, stk_offset), result);
7150     } else {
7151       load_unsigned_short(result, Address(str1, cnt2, scale1, -2));
7152       movw(Address(rsp, cnt2, scale1, stk_offset), result);
7153     }
7154     decrement(cnt2);
7155     jccb(Assembler::notZero, COPY_STR);
7156 
7157     if (int_cnt2 < 0) { // not constant
7158       pop(cnt2);
7159     }
7160     movptr(str1, rsp);  // New string address
7161 
7162     bind(BIG_STRINGS);
7163     // Load substring.
7164     if (int_cnt2 < 0) { // -1
7165       if (ae == StrIntrinsicNode::UL) {
7166         pmovzxbw(vec, Address(str2, 0));
7167       } else {
7168         movdqu(vec, Address(str2, 0));
7169       }
7170       push(cnt2);       // substr count
7171       push(str2);       // substr addr
7172       push(str1);       // string addr
7173     } else {
7174       // Small (< 8 chars) constant substrings are loaded already.
7175       movl(cnt2, int_cnt2);
7176     }
7177     push(tmp);  // original SP
7178 
7179   } // Finished loading
7180 
7181   //========================================================
7182   // Start search
7183   //
7184 
7185   movptr(result, str1); // string addr
7186 
7187   if (int_cnt2  < 0) {  // Only for non constant substring
7188     jmpb(SCAN_TO_SUBSTR);
7189 
7190     // SP saved at sp+0
7191     // String saved at sp+1*wordSize
7192     // Substr saved at sp+2*wordSize
7193     // Substr count saved at sp+3*wordSize
7194 
7195     // Reload substr for rescan, this code
7196     // is executed only for large substrings (> 8 chars)
7197     bind(RELOAD_SUBSTR);
7198     movptr(str2, Address(rsp, 2*wordSize));
7199     movl(cnt2, Address(rsp, 3*wordSize));
7200     if (ae == StrIntrinsicNode::UL) {
7201       pmovzxbw(vec, Address(str2, 0));
7202     } else {
7203       movdqu(vec, Address(str2, 0));
7204     }
7205     // We came here after the beginning of the substring was
7206     // matched but the rest of it was not so we need to search
7207     // again. Start from the next element after the previous match.
7208     subptr(str1, result); // Restore counter
7209     if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7210       shrl(str1, 1);
7211     }
7212     addl(cnt1, str1);
7213     decrementl(cnt1);   // Shift to next element
7214     cmpl(cnt1, cnt2);
7215     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7216 
7217     addptr(result, (1<<scale1));
7218   } // non constant
7219 
7220   // Scan string for start of substr in 16-byte vectors
7221   bind(SCAN_TO_SUBSTR);
7222   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
7223   pcmpestri(vec, Address(result, 0), mode);
7224   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
7225   subl(cnt1, stride);
7226   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
7227   cmpl(cnt1, cnt2);
7228   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
7229   addptr(result, 16);
7230 
7231   bind(ADJUST_STR);
7232   cmpl(cnt1, stride); // Do not read beyond string
7233   jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
7234   // Back-up string to avoid reading beyond string.
7235   lea(result, Address(result, cnt1, scale1, -16));
7236   movl(cnt1, stride);
7237   jmpb(SCAN_TO_SUBSTR);
7238 
7239   // Found a potential substr
7240   bind(FOUND_CANDIDATE);
7241   // After pcmpestri tmp(rcx) contains matched element index
7242 
7243   // Make sure string is still long enough
7244   subl(cnt1, tmp);
7245   cmpl(cnt1, cnt2);
7246   jccb(Assembler::greaterEqual, FOUND_SUBSTR);
7247   // Left less then substring.
7248 
7249   bind(RET_NOT_FOUND);
7250   movl(result, -1);
7251   jmpb(CLEANUP);
7252 
7253   bind(FOUND_SUBSTR);
7254   // Compute start addr of substr
7255   lea(result, Address(result, tmp, scale1));
7256   if (int_cnt2 > 0) { // Constant substring
7257     // Repeat search for small substring (< 8 chars)
7258     // from new point without reloading substring.
7259     // Have to check that we don't read beyond string.
7260     cmpl(tmp, stride-int_cnt2);
7261     jccb(Assembler::greater, ADJUST_STR);
7262     // Fall through if matched whole substring.
7263   } else { // non constant
7264     assert(int_cnt2 == -1, "should be != 0");
7265 
7266     addl(tmp, cnt2);
7267     // Found result if we matched whole substring.
7268     cmpl(tmp, stride);
7269     jccb(Assembler::lessEqual, RET_FOUND);
7270 
7271     // Repeat search for small substring (<= 8 chars)
7272     // from new point 'str1' without reloading substring.
7273     cmpl(cnt2, stride);
7274     // Have to check that we don't read beyond string.
7275     jccb(Assembler::lessEqual, ADJUST_STR);
7276 
7277     Label CHECK_NEXT, CONT_SCAN_SUBSTR, RET_FOUND_LONG;
7278     // Compare the rest of substring (> 8 chars).
7279     movptr(str1, result);
7280 
7281     cmpl(tmp, cnt2);
7282     // First 8 chars are already matched.
7283     jccb(Assembler::equal, CHECK_NEXT);
7284 
7285     bind(SCAN_SUBSTR);
7286     pcmpestri(vec, Address(str1, 0), mode);
7287     // Need to reload strings pointers if not matched whole vector
7288     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
7289 
7290     bind(CHECK_NEXT);
7291     subl(cnt2, stride);
7292     jccb(Assembler::lessEqual, RET_FOUND_LONG); // Found full substring
7293     addptr(str1, 16);
7294     if (ae == StrIntrinsicNode::UL) {
7295       addptr(str2, 8);
7296     } else {
7297       addptr(str2, 16);
7298     }
7299     subl(cnt1, stride);
7300     cmpl(cnt2, stride); // Do not read beyond substring
7301     jccb(Assembler::greaterEqual, CONT_SCAN_SUBSTR);
7302     // Back-up strings to avoid reading beyond substring.
7303 
7304     if (ae == StrIntrinsicNode::UL) {
7305       lea(str2, Address(str2, cnt2, scale2, -8));
7306       lea(str1, Address(str1, cnt2, scale1, -16));
7307     } else {
7308       lea(str2, Address(str2, cnt2, scale2, -16));
7309       lea(str1, Address(str1, cnt2, scale1, -16));
7310     }
7311     subl(cnt1, cnt2);
7312     movl(cnt2, stride);
7313     addl(cnt1, stride);
7314     bind(CONT_SCAN_SUBSTR);
7315     if (ae == StrIntrinsicNode::UL) {
7316       pmovzxbw(vec, Address(str2, 0));
7317     } else {
7318       movdqu(vec, Address(str2, 0));
7319     }
7320     jmp(SCAN_SUBSTR);
7321 
7322     bind(RET_FOUND_LONG);
7323     movptr(str1, Address(rsp, wordSize));
7324   } // non constant
7325 
7326   bind(RET_FOUND);
7327   // Compute substr offset
7328   subptr(result, str1);
7329   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7330     shrl(result, 1); // index
7331   }
7332   bind(CLEANUP);
7333   pop(rsp); // restore SP
7334 
7335 } // string_indexof
7336 
7337 void MacroAssembler::string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
7338                                          XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp) {
7339   ShortBranchVerifier sbv(this);
7340   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7341 
7342   int stride = 8;
7343 
7344   Label FOUND_CHAR, SCAN_TO_CHAR, SCAN_TO_CHAR_LOOP,
7345         SCAN_TO_8_CHAR, SCAN_TO_8_CHAR_LOOP, SCAN_TO_16_CHAR_LOOP,
7346         RET_NOT_FOUND, SCAN_TO_8_CHAR_INIT,
7347         FOUND_SEQ_CHAR, DONE_LABEL;
7348 
7349   movptr(result, str1);
7350   if (UseAVX >= 2) {
7351     cmpl(cnt1, stride);
7352     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7353     cmpl(cnt1, 2*stride);
7354     jcc(Assembler::less, SCAN_TO_8_CHAR_INIT);
7355     movdl(vec1, ch);
7356     vpbroadcastw(vec1, vec1);
7357     vpxor(vec2, vec2);
7358     movl(tmp, cnt1);
7359     andl(tmp, 0xFFFFFFF0);  //vector count (in chars)
7360     andl(cnt1,0x0000000F);  //tail count (in chars)
7361 
7362     bind(SCAN_TO_16_CHAR_LOOP);
7363     vmovdqu(vec3, Address(result, 0));
7364     vpcmpeqw(vec3, vec3, vec1, 1);
7365     vptest(vec2, vec3);
7366     jcc(Assembler::carryClear, FOUND_CHAR);
7367     addptr(result, 32);
7368     subl(tmp, 2*stride);
7369     jccb(Assembler::notZero, SCAN_TO_16_CHAR_LOOP);
7370     jmp(SCAN_TO_8_CHAR);
7371     bind(SCAN_TO_8_CHAR_INIT);
7372     movdl(vec1, ch);
7373     pshuflw(vec1, vec1, 0x00);
7374     pshufd(vec1, vec1, 0);
7375     pxor(vec2, vec2);
7376   }
7377   bind(SCAN_TO_8_CHAR);
7378   cmpl(cnt1, stride);
7379   if (UseAVX >= 2) {
7380     jcc(Assembler::less, SCAN_TO_CHAR);
7381   } else {
7382     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7383     movdl(vec1, ch);
7384     pshuflw(vec1, vec1, 0x00);
7385     pshufd(vec1, vec1, 0);
7386     pxor(vec2, vec2);
7387   }
7388   movl(tmp, cnt1);
7389   andl(tmp, 0xFFFFFFF8);  //vector count (in chars)
7390   andl(cnt1,0x00000007);  //tail count (in chars)
7391 
7392   bind(SCAN_TO_8_CHAR_LOOP);
7393   movdqu(vec3, Address(result, 0));
7394   pcmpeqw(vec3, vec1);
7395   ptest(vec2, vec3);
7396   jcc(Assembler::carryClear, FOUND_CHAR);
7397   addptr(result, 16);
7398   subl(tmp, stride);
7399   jccb(Assembler::notZero, SCAN_TO_8_CHAR_LOOP);
7400   bind(SCAN_TO_CHAR);
7401   testl(cnt1, cnt1);
7402   jcc(Assembler::zero, RET_NOT_FOUND);
7403   bind(SCAN_TO_CHAR_LOOP);
7404   load_unsigned_short(tmp, Address(result, 0));
7405   cmpl(ch, tmp);
7406   jccb(Assembler::equal, FOUND_SEQ_CHAR);
7407   addptr(result, 2);
7408   subl(cnt1, 1);
7409   jccb(Assembler::zero, RET_NOT_FOUND);
7410   jmp(SCAN_TO_CHAR_LOOP);
7411 
7412   bind(RET_NOT_FOUND);
7413   movl(result, -1);
7414   jmpb(DONE_LABEL);
7415 
7416   bind(FOUND_CHAR);
7417   if (UseAVX >= 2) {
7418     vpmovmskb(tmp, vec3);
7419   } else {
7420     pmovmskb(tmp, vec3);
7421   }
7422   bsfl(ch, tmp);
7423   addl(result, ch);
7424 
7425   bind(FOUND_SEQ_CHAR);
7426   subptr(result, str1);
7427   shrl(result, 1);
7428 
7429   bind(DONE_LABEL);
7430 } // string_indexof_char
7431 
7432 // helper function for string_compare
7433 void MacroAssembler::load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
7434                                         Address::ScaleFactor scale, Address::ScaleFactor scale1,
7435                                         Address::ScaleFactor scale2, Register index, int ae) {
7436   if (ae == StrIntrinsicNode::LL) {
7437     load_unsigned_byte(elem1, Address(str1, index, scale, 0));
7438     load_unsigned_byte(elem2, Address(str2, index, scale, 0));
7439   } else if (ae == StrIntrinsicNode::UU) {
7440     load_unsigned_short(elem1, Address(str1, index, scale, 0));
7441     load_unsigned_short(elem2, Address(str2, index, scale, 0));
7442   } else {
7443     load_unsigned_byte(elem1, Address(str1, index, scale1, 0));
7444     load_unsigned_short(elem2, Address(str2, index, scale2, 0));
7445   }
7446 }
7447 
7448 // Compare strings, used for char[] and byte[].
7449 void MacroAssembler::string_compare(Register str1, Register str2,
7450                                     Register cnt1, Register cnt2, Register result,
7451                                     XMMRegister vec1, int ae) {
7452   ShortBranchVerifier sbv(this);
7453   Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
7454   Label COMPARE_WIDE_VECTORS_LOOP_FAILED;  // used only _LP64 && AVX3
7455   int stride, stride2, adr_stride, adr_stride1, adr_stride2;
7456   int stride2x2 = 0x40;
7457   Address::ScaleFactor scale = Address::no_scale;
7458   Address::ScaleFactor scale1 = Address::no_scale;
7459   Address::ScaleFactor scale2 = Address::no_scale;
7460 
7461   if (ae != StrIntrinsicNode::LL) {
7462     stride2x2 = 0x20;
7463   }
7464 
7465   if (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL) {
7466     shrl(cnt2, 1);
7467   }
7468   // Compute the minimum of the string lengths and the
7469   // difference of the string lengths (stack).
7470   // Do the conditional move stuff
7471   movl(result, cnt1);
7472   subl(cnt1, cnt2);
7473   push(cnt1);
7474   cmov32(Assembler::lessEqual, cnt2, result);    // cnt2 = min(cnt1, cnt2)
7475 
7476   // Is the minimum length zero?
7477   testl(cnt2, cnt2);
7478   jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7479   if (ae == StrIntrinsicNode::LL) {
7480     // Load first bytes
7481     load_unsigned_byte(result, Address(str1, 0));  // result = str1[0]
7482     load_unsigned_byte(cnt1, Address(str2, 0));    // cnt1   = str2[0]
7483   } else if (ae == StrIntrinsicNode::UU) {
7484     // Load first characters
7485     load_unsigned_short(result, Address(str1, 0));
7486     load_unsigned_short(cnt1, Address(str2, 0));
7487   } else {
7488     load_unsigned_byte(result, Address(str1, 0));
7489     load_unsigned_short(cnt1, Address(str2, 0));
7490   }
7491   subl(result, cnt1);
7492   jcc(Assembler::notZero,  POP_LABEL);
7493 
7494   if (ae == StrIntrinsicNode::UU) {
7495     // Divide length by 2 to get number of chars
7496     shrl(cnt2, 1);
7497   }
7498   cmpl(cnt2, 1);
7499   jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7500 
7501   // Check if the strings start at the same location and setup scale and stride
7502   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7503     cmpptr(str1, str2);
7504     jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7505     if (ae == StrIntrinsicNode::LL) {
7506       scale = Address::times_1;
7507       stride = 16;
7508     } else {
7509       scale = Address::times_2;
7510       stride = 8;
7511     }
7512   } else {
7513     scale1 = Address::times_1;
7514     scale2 = Address::times_2;
7515     // scale not used
7516     stride = 8;
7517   }
7518 
7519   if (UseAVX >= 2 && UseSSE42Intrinsics) {
7520     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_WIDE_TAIL, COMPARE_SMALL_STR;
7521     Label COMPARE_WIDE_VECTORS_LOOP, COMPARE_16_CHARS, COMPARE_INDEX_CHAR;
7522     Label COMPARE_WIDE_VECTORS_LOOP_AVX2;
7523     Label COMPARE_TAIL_LONG;
7524     Label COMPARE_WIDE_VECTORS_LOOP_AVX3;  // used only _LP64 && AVX3
7525 
7526     int pcmpmask = 0x19;
7527     if (ae == StrIntrinsicNode::LL) {
7528       pcmpmask &= ~0x01;
7529     }
7530 
7531     // Setup to compare 16-chars (32-bytes) vectors,
7532     // start from first character again because it has aligned address.
7533     if (ae == StrIntrinsicNode::LL) {
7534       stride2 = 32;
7535     } else {
7536       stride2 = 16;
7537     }
7538     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7539       adr_stride = stride << scale;
7540     } else {
7541       adr_stride1 = 8;  //stride << scale1;
7542       adr_stride2 = 16; //stride << scale2;
7543     }
7544 
7545     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
7546     // rax and rdx are used by pcmpestri as elements counters
7547     movl(result, cnt2);
7548     andl(cnt2, ~(stride2-1));   // cnt2 holds the vector count
7549     jcc(Assembler::zero, COMPARE_TAIL_LONG);
7550 
7551     // fast path : compare first 2 8-char vectors.
7552     bind(COMPARE_16_CHARS);
7553     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7554       movdqu(vec1, Address(str1, 0));
7555     } else {
7556       pmovzxbw(vec1, Address(str1, 0));
7557     }
7558     pcmpestri(vec1, Address(str2, 0), pcmpmask);
7559     jccb(Assembler::below, COMPARE_INDEX_CHAR);
7560 
7561     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7562       movdqu(vec1, Address(str1, adr_stride));
7563       pcmpestri(vec1, Address(str2, adr_stride), pcmpmask);
7564     } else {
7565       pmovzxbw(vec1, Address(str1, adr_stride1));
7566       pcmpestri(vec1, Address(str2, adr_stride2), pcmpmask);
7567     }
7568     jccb(Assembler::aboveEqual, COMPARE_WIDE_VECTORS);
7569     addl(cnt1, stride);
7570 
7571     // Compare the characters at index in cnt1
7572     bind(COMPARE_INDEX_CHAR); // cnt1 has the offset of the mismatching character
7573     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
7574     subl(result, cnt2);
7575     jmp(POP_LABEL);
7576 
7577     // Setup the registers to start vector comparison loop
7578     bind(COMPARE_WIDE_VECTORS);
7579     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7580       lea(str1, Address(str1, result, scale));
7581       lea(str2, Address(str2, result, scale));
7582     } else {
7583       lea(str1, Address(str1, result, scale1));
7584       lea(str2, Address(str2, result, scale2));
7585     }
7586     subl(result, stride2);
7587     subl(cnt2, stride2);
7588     jcc(Assembler::zero, COMPARE_WIDE_TAIL);
7589     negptr(result);
7590 
7591     //  In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest)
7592     bind(COMPARE_WIDE_VECTORS_LOOP);
7593 
7594 #ifdef _LP64
7595     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
7596       cmpl(cnt2, stride2x2);
7597       jccb(Assembler::below, COMPARE_WIDE_VECTORS_LOOP_AVX2);
7598       testl(cnt2, stride2x2-1);   // cnt2 holds the vector count
7599       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX2);   // means we cannot subtract by 0x40
7600 
7601       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
7602       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7603         evmovdquq(vec1, Address(str1, result, scale), Assembler::AVX_512bit);
7604         evpcmpeqb(k7, vec1, Address(str2, result, scale), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7605       } else {
7606         vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_512bit);
7607         evpcmpeqb(k7, vec1, Address(str2, result, scale2), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7608       }
7609       kortestql(k7, k7);
7610       jcc(Assembler::aboveEqual, COMPARE_WIDE_VECTORS_LOOP_FAILED);     // miscompare
7611       addptr(result, stride2x2);  // update since we already compared at this addr
7612       subl(cnt2, stride2x2);      // and sub the size too
7613       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX3);
7614 
7615       vpxor(vec1, vec1);
7616       jmpb(COMPARE_WIDE_TAIL);
7617     }//if (VM_Version::supports_avx512vlbw())
7618 #endif // _LP64
7619 
7620 
7621     bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7622     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7623       vmovdqu(vec1, Address(str1, result, scale));
7624       vpxor(vec1, Address(str2, result, scale));
7625     } else {
7626       vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_256bit);
7627       vpxor(vec1, Address(str2, result, scale2));
7628     }
7629     vptest(vec1, vec1);
7630     jcc(Assembler::notZero, VECTOR_NOT_EQUAL);
7631     addptr(result, stride2);
7632     subl(cnt2, stride2);
7633     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP);
7634     // clean upper bits of YMM registers
7635     vpxor(vec1, vec1);
7636 
7637     // compare wide vectors tail
7638     bind(COMPARE_WIDE_TAIL);
7639     testptr(result, result);
7640     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7641 
7642     movl(result, stride2);
7643     movl(cnt2, result);
7644     negptr(result);
7645     jmp(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7646 
7647     // Identifies the mismatching (higher or lower)16-bytes in the 32-byte vectors.
7648     bind(VECTOR_NOT_EQUAL);
7649     // clean upper bits of YMM registers
7650     vpxor(vec1, vec1);
7651     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7652       lea(str1, Address(str1, result, scale));
7653       lea(str2, Address(str2, result, scale));
7654     } else {
7655       lea(str1, Address(str1, result, scale1));
7656       lea(str2, Address(str2, result, scale2));
7657     }
7658     jmp(COMPARE_16_CHARS);
7659 
7660     // Compare tail chars, length between 1 to 15 chars
7661     bind(COMPARE_TAIL_LONG);
7662     movl(cnt2, result);
7663     cmpl(cnt2, stride);
7664     jcc(Assembler::less, COMPARE_SMALL_STR);
7665 
7666     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7667       movdqu(vec1, Address(str1, 0));
7668     } else {
7669       pmovzxbw(vec1, Address(str1, 0));
7670     }
7671     pcmpestri(vec1, Address(str2, 0), pcmpmask);
7672     jcc(Assembler::below, COMPARE_INDEX_CHAR);
7673     subptr(cnt2, stride);
7674     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7675     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7676       lea(str1, Address(str1, result, scale));
7677       lea(str2, Address(str2, result, scale));
7678     } else {
7679       lea(str1, Address(str1, result, scale1));
7680       lea(str2, Address(str2, result, scale2));
7681     }
7682     negptr(cnt2);
7683     jmpb(WHILE_HEAD_LABEL);
7684 
7685     bind(COMPARE_SMALL_STR);
7686   } else if (UseSSE42Intrinsics) {
7687     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
7688     int pcmpmask = 0x19;
7689     // Setup to compare 8-char (16-byte) vectors,
7690     // start from first character again because it has aligned address.
7691     movl(result, cnt2);
7692     andl(cnt2, ~(stride - 1));   // cnt2 holds the vector count
7693     if (ae == StrIntrinsicNode::LL) {
7694       pcmpmask &= ~0x01;
7695     }
7696     jcc(Assembler::zero, COMPARE_TAIL);
7697     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7698       lea(str1, Address(str1, result, scale));
7699       lea(str2, Address(str2, result, scale));
7700     } else {
7701       lea(str1, Address(str1, result, scale1));
7702       lea(str2, Address(str2, result, scale2));
7703     }
7704     negptr(result);
7705 
7706     // pcmpestri
7707     //   inputs:
7708     //     vec1- substring
7709     //     rax - negative string length (elements count)
7710     //     mem - scanned string
7711     //     rdx - string length (elements count)
7712     //     pcmpmask - cmp mode: 11000 (string compare with negated result)
7713     //               + 00 (unsigned bytes) or  + 01 (unsigned shorts)
7714     //   outputs:
7715     //     rcx - first mismatched element index
7716     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
7717 
7718     bind(COMPARE_WIDE_VECTORS);
7719     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7720       movdqu(vec1, Address(str1, result, scale));
7721       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
7722     } else {
7723       pmovzxbw(vec1, Address(str1, result, scale1));
7724       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
7725     }
7726     // After pcmpestri cnt1(rcx) contains mismatched element index
7727 
7728     jccb(Assembler::below, VECTOR_NOT_EQUAL);  // CF==1
7729     addptr(result, stride);
7730     subptr(cnt2, stride);
7731     jccb(Assembler::notZero, COMPARE_WIDE_VECTORS);
7732 
7733     // compare wide vectors tail
7734     testptr(result, result);
7735     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7736 
7737     movl(cnt2, stride);
7738     movl(result, stride);
7739     negptr(result);
7740     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7741       movdqu(vec1, Address(str1, result, scale));
7742       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
7743     } else {
7744       pmovzxbw(vec1, Address(str1, result, scale1));
7745       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
7746     }
7747     jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL);
7748 
7749     // Mismatched characters in the vectors
7750     bind(VECTOR_NOT_EQUAL);
7751     addptr(cnt1, result);
7752     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
7753     subl(result, cnt2);
7754     jmpb(POP_LABEL);
7755 
7756     bind(COMPARE_TAIL); // limit is zero
7757     movl(cnt2, result);
7758     // Fallthru to tail compare
7759   }
7760   // Shift str2 and str1 to the end of the arrays, negate min
7761   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7762     lea(str1, Address(str1, cnt2, scale));
7763     lea(str2, Address(str2, cnt2, scale));
7764   } else {
7765     lea(str1, Address(str1, cnt2, scale1));
7766     lea(str2, Address(str2, cnt2, scale2));
7767   }
7768   decrementl(cnt2);  // first character was compared already
7769   negptr(cnt2);
7770 
7771   // Compare the rest of the elements
7772   bind(WHILE_HEAD_LABEL);
7773   load_next_elements(result, cnt1, str1, str2, scale, scale1, scale2, cnt2, ae);
7774   subl(result, cnt1);
7775   jccb(Assembler::notZero, POP_LABEL);
7776   increment(cnt2);
7777   jccb(Assembler::notZero, WHILE_HEAD_LABEL);
7778 
7779   // Strings are equal up to min length.  Return the length difference.
7780   bind(LENGTH_DIFF_LABEL);
7781   pop(result);
7782   if (ae == StrIntrinsicNode::UU) {
7783     // Divide diff by 2 to get number of chars
7784     sarl(result, 1);
7785   }
7786   jmpb(DONE_LABEL);
7787 
7788 #ifdef _LP64
7789   if (VM_Version::supports_avx512vlbw()) {
7790 
7791     bind(COMPARE_WIDE_VECTORS_LOOP_FAILED);
7792 
7793     kmovql(cnt1, k7);
7794     notq(cnt1);
7795     bsfq(cnt2, cnt1);
7796     if (ae != StrIntrinsicNode::LL) {
7797       // Divide diff by 2 to get number of chars
7798       sarl(cnt2, 1);
7799     }
7800     addq(result, cnt2);
7801     if (ae == StrIntrinsicNode::LL) {
7802       load_unsigned_byte(cnt1, Address(str2, result));
7803       load_unsigned_byte(result, Address(str1, result));
7804     } else if (ae == StrIntrinsicNode::UU) {
7805       load_unsigned_short(cnt1, Address(str2, result, scale));
7806       load_unsigned_short(result, Address(str1, result, scale));
7807     } else {
7808       load_unsigned_short(cnt1, Address(str2, result, scale2));
7809       load_unsigned_byte(result, Address(str1, result, scale1));
7810     }
7811     subl(result, cnt1);
7812     jmpb(POP_LABEL);
7813   }//if (VM_Version::supports_avx512vlbw())
7814 #endif // _LP64
7815 
7816   // Discard the stored length difference
7817   bind(POP_LABEL);
7818   pop(cnt1);
7819 
7820   // That's it
7821   bind(DONE_LABEL);
7822   if(ae == StrIntrinsicNode::UL) {
7823     negl(result);
7824   }
7825 
7826 }
7827 
7828 // Search for Non-ASCII character (Negative byte value) in a byte array,
7829 // return true if it has any and false otherwise.
7830 //   ..\jdk\src\java.base\share\classes\java\lang\StringCoding.java
7831 //   @HotSpotIntrinsicCandidate
7832 //   private static boolean hasNegatives(byte[] ba, int off, int len) {
7833 //     for (int i = off; i < off + len; i++) {
7834 //       if (ba[i] < 0) {
7835 //         return true;
7836 //       }
7837 //     }
7838 //     return false;
7839 //   }
7840 void MacroAssembler::has_negatives(Register ary1, Register len,
7841   Register result, Register tmp1,
7842   XMMRegister vec1, XMMRegister vec2) {
7843   // rsi: byte array
7844   // rcx: len
7845   // rax: result
7846   ShortBranchVerifier sbv(this);
7847   assert_different_registers(ary1, len, result, tmp1);
7848   assert_different_registers(vec1, vec2);
7849   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_CHAR, COMPARE_VECTORS, COMPARE_BYTE;
7850 
7851   // len == 0
7852   testl(len, len);
7853   jcc(Assembler::zero, FALSE_LABEL);
7854 
7855   if ((UseAVX > 2) && // AVX512
7856     VM_Version::supports_avx512vlbw() &&
7857     VM_Version::supports_bmi2()) {
7858 
7859     set_vector_masking();  // opening of the stub context for programming mask registers
7860 
7861     Label test_64_loop, test_tail;
7862     Register tmp3_aliased = len;
7863 
7864     movl(tmp1, len);
7865     vpxor(vec2, vec2, vec2, Assembler::AVX_512bit);
7866 
7867     andl(tmp1, 64 - 1);   // tail count (in chars) 0x3F
7868     andl(len, ~(64 - 1));    // vector count (in chars)
7869     jccb(Assembler::zero, test_tail);
7870 
7871     lea(ary1, Address(ary1, len, Address::times_1));
7872     negptr(len);
7873 
7874     bind(test_64_loop);
7875     // Check whether our 64 elements of size byte contain negatives
7876     evpcmpgtb(k2, vec2, Address(ary1, len, Address::times_1), Assembler::AVX_512bit);
7877     kortestql(k2, k2);
7878     jcc(Assembler::notZero, TRUE_LABEL);
7879 
7880     addptr(len, 64);
7881     jccb(Assembler::notZero, test_64_loop);
7882 
7883 
7884     bind(test_tail);
7885     // bail out when there is nothing to be done
7886     testl(tmp1, -1);
7887     jcc(Assembler::zero, FALSE_LABEL);
7888 
7889     // Save k1
7890     kmovql(k3, k1);
7891 
7892     // ~(~0 << len) applied up to two times (for 32-bit scenario)
7893 #ifdef _LP64
7894     mov64(tmp3_aliased, 0xFFFFFFFFFFFFFFFF);
7895     shlxq(tmp3_aliased, tmp3_aliased, tmp1);
7896     notq(tmp3_aliased);
7897     kmovql(k1, tmp3_aliased);
7898 #else
7899     Label k_init;
7900     jmp(k_init);
7901 
7902     // We could not read 64-bits from a general purpose register thus we move
7903     // data required to compose 64 1's to the instruction stream
7904     // We emit 64 byte wide series of elements from 0..63 which later on would
7905     // be used as a compare targets with tail count contained in tmp1 register.
7906     // Result would be a k1 register having tmp1 consecutive number or 1
7907     // counting from least significant bit.
7908     address tmp = pc();
7909     emit_int64(0x0706050403020100);
7910     emit_int64(0x0F0E0D0C0B0A0908);
7911     emit_int64(0x1716151413121110);
7912     emit_int64(0x1F1E1D1C1B1A1918);
7913     emit_int64(0x2726252423222120);
7914     emit_int64(0x2F2E2D2C2B2A2928);
7915     emit_int64(0x3736353433323130);
7916     emit_int64(0x3F3E3D3C3B3A3938);
7917 
7918     bind(k_init);
7919     lea(len, InternalAddress(tmp));
7920     // create mask to test for negative byte inside a vector
7921     evpbroadcastb(vec1, tmp1, Assembler::AVX_512bit);
7922     evpcmpgtb(k1, vec1, Address(len, 0), Assembler::AVX_512bit);
7923 
7924 #endif
7925     evpcmpgtb(k2, k1, vec2, Address(ary1, 0), Assembler::AVX_512bit);
7926     ktestq(k2, k1);
7927     // Restore k1
7928     kmovql(k1, k3);
7929     jcc(Assembler::notZero, TRUE_LABEL);
7930 
7931     jmp(FALSE_LABEL);
7932 
7933     clear_vector_masking();   // closing of the stub context for programming mask registers
7934   } else {
7935     movl(result, len); // copy
7936 
7937     if (UseAVX == 2 && UseSSE >= 2) {
7938       // With AVX2, use 32-byte vector compare
7939       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7940 
7941       // Compare 32-byte vectors
7942       andl(result, 0x0000001f);  //   tail count (in bytes)
7943       andl(len, 0xffffffe0);   // vector count (in bytes)
7944       jccb(Assembler::zero, COMPARE_TAIL);
7945 
7946       lea(ary1, Address(ary1, len, Address::times_1));
7947       negptr(len);
7948 
7949       movl(tmp1, 0x80808080);   // create mask to test for Unicode chars in vector
7950       movdl(vec2, tmp1);
7951       vpbroadcastd(vec2, vec2);
7952 
7953       bind(COMPARE_WIDE_VECTORS);
7954       vmovdqu(vec1, Address(ary1, len, Address::times_1));
7955       vptest(vec1, vec2);
7956       jccb(Assembler::notZero, TRUE_LABEL);
7957       addptr(len, 32);
7958       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7959 
7960       testl(result, result);
7961       jccb(Assembler::zero, FALSE_LABEL);
7962 
7963       vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
7964       vptest(vec1, vec2);
7965       jccb(Assembler::notZero, TRUE_LABEL);
7966       jmpb(FALSE_LABEL);
7967 
7968       bind(COMPARE_TAIL); // len is zero
7969       movl(len, result);
7970       // Fallthru to tail compare
7971     } else if (UseSSE42Intrinsics) {
7972       // With SSE4.2, use double quad vector compare
7973       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7974 
7975       // Compare 16-byte vectors
7976       andl(result, 0x0000000f);  //   tail count (in bytes)
7977       andl(len, 0xfffffff0);   // vector count (in bytes)
7978       jccb(Assembler::zero, COMPARE_TAIL);
7979 
7980       lea(ary1, Address(ary1, len, Address::times_1));
7981       negptr(len);
7982 
7983       movl(tmp1, 0x80808080);
7984       movdl(vec2, tmp1);
7985       pshufd(vec2, vec2, 0);
7986 
7987       bind(COMPARE_WIDE_VECTORS);
7988       movdqu(vec1, Address(ary1, len, Address::times_1));
7989       ptest(vec1, vec2);
7990       jccb(Assembler::notZero, TRUE_LABEL);
7991       addptr(len, 16);
7992       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7993 
7994       testl(result, result);
7995       jccb(Assembler::zero, FALSE_LABEL);
7996 
7997       movdqu(vec1, Address(ary1, result, Address::times_1, -16));
7998       ptest(vec1, vec2);
7999       jccb(Assembler::notZero, TRUE_LABEL);
8000       jmpb(FALSE_LABEL);
8001 
8002       bind(COMPARE_TAIL); // len is zero
8003       movl(len, result);
8004       // Fallthru to tail compare
8005     }
8006   }
8007   // Compare 4-byte vectors
8008   andl(len, 0xfffffffc); // vector count (in bytes)
8009   jccb(Assembler::zero, COMPARE_CHAR);
8010 
8011   lea(ary1, Address(ary1, len, Address::times_1));
8012   negptr(len);
8013 
8014   bind(COMPARE_VECTORS);
8015   movl(tmp1, Address(ary1, len, Address::times_1));
8016   andl(tmp1, 0x80808080);
8017   jccb(Assembler::notZero, TRUE_LABEL);
8018   addptr(len, 4);
8019   jcc(Assembler::notZero, COMPARE_VECTORS);
8020 
8021   // Compare trailing char (final 2 bytes), if any
8022   bind(COMPARE_CHAR);
8023   testl(result, 0x2);   // tail  char
8024   jccb(Assembler::zero, COMPARE_BYTE);
8025   load_unsigned_short(tmp1, Address(ary1, 0));
8026   andl(tmp1, 0x00008080);
8027   jccb(Assembler::notZero, TRUE_LABEL);
8028   subptr(result, 2);
8029   lea(ary1, Address(ary1, 2));
8030 
8031   bind(COMPARE_BYTE);
8032   testl(result, 0x1);   // tail  byte
8033   jccb(Assembler::zero, FALSE_LABEL);
8034   load_unsigned_byte(tmp1, Address(ary1, 0));
8035   andl(tmp1, 0x00000080);
8036   jccb(Assembler::notEqual, TRUE_LABEL);
8037   jmpb(FALSE_LABEL);
8038 
8039   bind(TRUE_LABEL);
8040   movl(result, 1);   // return true
8041   jmpb(DONE);
8042 
8043   bind(FALSE_LABEL);
8044   xorl(result, result); // return false
8045 
8046   // That's it
8047   bind(DONE);
8048   if (UseAVX >= 2 && UseSSE >= 2) {
8049     // clean upper bits of YMM registers
8050     vpxor(vec1, vec1);
8051     vpxor(vec2, vec2);
8052   }
8053 }
8054 // Compare char[] or byte[] arrays aligned to 4 bytes or substrings.
8055 void MacroAssembler::arrays_equals(bool is_array_equ, Register ary1, Register ary2,
8056                                    Register limit, Register result, Register chr,
8057                                    XMMRegister vec1, XMMRegister vec2, bool is_char) {
8058   ShortBranchVerifier sbv(this);
8059   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR, COMPARE_BYTE;
8060 
8061   int length_offset  = arrayOopDesc::length_offset_in_bytes();
8062   int base_offset    = arrayOopDesc::base_offset_in_bytes(is_char ? T_CHAR : T_BYTE);
8063 
8064   if (is_array_equ) {
8065     // Check the input args
8066     cmpoop(ary1, ary2);
8067     jcc(Assembler::equal, TRUE_LABEL);
8068 
8069     // Need additional checks for arrays_equals.
8070     testptr(ary1, ary1);
8071     jcc(Assembler::zero, FALSE_LABEL);
8072     testptr(ary2, ary2);
8073     jcc(Assembler::zero, FALSE_LABEL);
8074 
8075     // Check the lengths
8076     movl(limit, Address(ary1, length_offset));
8077     cmpl(limit, Address(ary2, length_offset));
8078     jcc(Assembler::notEqual, FALSE_LABEL);
8079   }
8080 
8081   // count == 0
8082   testl(limit, limit);
8083   jcc(Assembler::zero, TRUE_LABEL);
8084 
8085   if (is_array_equ) {
8086     // Load array address
8087     lea(ary1, Address(ary1, base_offset));
8088     lea(ary2, Address(ary2, base_offset));
8089   }
8090 
8091   if (is_array_equ && is_char) {
8092     // arrays_equals when used for char[].
8093     shll(limit, 1);      // byte count != 0
8094   }
8095   movl(result, limit); // copy
8096 
8097   if (UseAVX >= 2) {
8098     // With AVX2, use 32-byte vector compare
8099     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8100 
8101     // Compare 32-byte vectors
8102     andl(result, 0x0000001f);  //   tail count (in bytes)
8103     andl(limit, 0xffffffe0);   // vector count (in bytes)
8104     jcc(Assembler::zero, COMPARE_TAIL);
8105 
8106     lea(ary1, Address(ary1, limit, Address::times_1));
8107     lea(ary2, Address(ary2, limit, Address::times_1));
8108     negptr(limit);
8109 
8110     bind(COMPARE_WIDE_VECTORS);
8111 
8112 #ifdef _LP64
8113     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
8114       Label COMPARE_WIDE_VECTORS_LOOP_AVX2, COMPARE_WIDE_VECTORS_LOOP_AVX3;
8115 
8116       cmpl(limit, -64);
8117       jccb(Assembler::greater, COMPARE_WIDE_VECTORS_LOOP_AVX2);
8118 
8119       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
8120 
8121       evmovdquq(vec1, Address(ary1, limit, Address::times_1), Assembler::AVX_512bit);
8122       evpcmpeqb(k7, vec1, Address(ary2, limit, Address::times_1), Assembler::AVX_512bit);
8123       kortestql(k7, k7);
8124       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
8125       addptr(limit, 64);  // update since we already compared at this addr
8126       cmpl(limit, -64);
8127       jccb(Assembler::lessEqual, COMPARE_WIDE_VECTORS_LOOP_AVX3);
8128 
8129       // At this point we may still need to compare -limit+result bytes.
8130       // We could execute the next two instruction and just continue via non-wide path:
8131       //  cmpl(limit, 0);
8132       //  jcc(Assembler::equal, COMPARE_TAIL);  // true
8133       // But since we stopped at the points ary{1,2}+limit which are
8134       // not farther than 64 bytes from the ends of arrays ary{1,2}+result
8135       // (|limit| <= 32 and result < 32),
8136       // we may just compare the last 64 bytes.
8137       //
8138       addptr(result, -64);   // it is safe, bc we just came from this area
8139       evmovdquq(vec1, Address(ary1, result, Address::times_1), Assembler::AVX_512bit);
8140       evpcmpeqb(k7, vec1, Address(ary2, result, Address::times_1), Assembler::AVX_512bit);
8141       kortestql(k7, k7);
8142       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
8143 
8144       jmp(TRUE_LABEL);
8145 
8146       bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
8147 
8148     }//if (VM_Version::supports_avx512vlbw())
8149 #endif //_LP64
8150 
8151     vmovdqu(vec1, Address(ary1, limit, Address::times_1));
8152     vmovdqu(vec2, Address(ary2, limit, Address::times_1));
8153     vpxor(vec1, vec2);
8154 
8155     vptest(vec1, vec1);
8156     jcc(Assembler::notZero, FALSE_LABEL);
8157     addptr(limit, 32);
8158     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8159 
8160     testl(result, result);
8161     jcc(Assembler::zero, TRUE_LABEL);
8162 
8163     vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
8164     vmovdqu(vec2, Address(ary2, result, Address::times_1, -32));
8165     vpxor(vec1, vec2);
8166 
8167     vptest(vec1, vec1);
8168     jccb(Assembler::notZero, FALSE_LABEL);
8169     jmpb(TRUE_LABEL);
8170 
8171     bind(COMPARE_TAIL); // limit is zero
8172     movl(limit, result);
8173     // Fallthru to tail compare
8174   } else if (UseSSE42Intrinsics) {
8175     // With SSE4.2, use double quad vector compare
8176     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
8177 
8178     // Compare 16-byte vectors
8179     andl(result, 0x0000000f);  //   tail count (in bytes)
8180     andl(limit, 0xfffffff0);   // vector count (in bytes)
8181     jcc(Assembler::zero, COMPARE_TAIL);
8182 
8183     lea(ary1, Address(ary1, limit, Address::times_1));
8184     lea(ary2, Address(ary2, limit, Address::times_1));
8185     negptr(limit);
8186 
8187     bind(COMPARE_WIDE_VECTORS);
8188     movdqu(vec1, Address(ary1, limit, Address::times_1));
8189     movdqu(vec2, Address(ary2, limit, Address::times_1));
8190     pxor(vec1, vec2);
8191 
8192     ptest(vec1, vec1);
8193     jcc(Assembler::notZero, FALSE_LABEL);
8194     addptr(limit, 16);
8195     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
8196 
8197     testl(result, result);
8198     jcc(Assembler::zero, TRUE_LABEL);
8199 
8200     movdqu(vec1, Address(ary1, result, Address::times_1, -16));
8201     movdqu(vec2, Address(ary2, result, Address::times_1, -16));
8202     pxor(vec1, vec2);
8203 
8204     ptest(vec1, vec1);
8205     jccb(Assembler::notZero, FALSE_LABEL);
8206     jmpb(TRUE_LABEL);
8207 
8208     bind(COMPARE_TAIL); // limit is zero
8209     movl(limit, result);
8210     // Fallthru to tail compare
8211   }
8212 
8213   // Compare 4-byte vectors
8214   andl(limit, 0xfffffffc); // vector count (in bytes)
8215   jccb(Assembler::zero, COMPARE_CHAR);
8216 
8217   lea(ary1, Address(ary1, limit, Address::times_1));
8218   lea(ary2, Address(ary2, limit, Address::times_1));
8219   negptr(limit);
8220 
8221   bind(COMPARE_VECTORS);
8222   movl(chr, Address(ary1, limit, Address::times_1));
8223   cmpl(chr, Address(ary2, limit, Address::times_1));
8224   jccb(Assembler::notEqual, FALSE_LABEL);
8225   addptr(limit, 4);
8226   jcc(Assembler::notZero, COMPARE_VECTORS);
8227 
8228   // Compare trailing char (final 2 bytes), if any
8229   bind(COMPARE_CHAR);
8230   testl(result, 0x2);   // tail  char
8231   jccb(Assembler::zero, COMPARE_BYTE);
8232   load_unsigned_short(chr, Address(ary1, 0));
8233   load_unsigned_short(limit, Address(ary2, 0));
8234   cmpl(chr, limit);
8235   jccb(Assembler::notEqual, FALSE_LABEL);
8236 
8237   if (is_array_equ && is_char) {
8238     bind(COMPARE_BYTE);
8239   } else {
8240     lea(ary1, Address(ary1, 2));
8241     lea(ary2, Address(ary2, 2));
8242 
8243     bind(COMPARE_BYTE);
8244     testl(result, 0x1);   // tail  byte
8245     jccb(Assembler::zero, TRUE_LABEL);
8246     load_unsigned_byte(chr, Address(ary1, 0));
8247     load_unsigned_byte(limit, Address(ary2, 0));
8248     cmpl(chr, limit);
8249     jccb(Assembler::notEqual, FALSE_LABEL);
8250   }
8251   bind(TRUE_LABEL);
8252   movl(result, 1);   // return true
8253   jmpb(DONE);
8254 
8255   bind(FALSE_LABEL);
8256   xorl(result, result); // return false
8257 
8258   // That's it
8259   bind(DONE);
8260   if (UseAVX >= 2) {
8261     // clean upper bits of YMM registers
8262     vpxor(vec1, vec1);
8263     vpxor(vec2, vec2);
8264   }
8265 }
8266 
8267 #endif
8268 
8269 void MacroAssembler::generate_fill(BasicType t, bool aligned,
8270                                    Register to, Register value, Register count,
8271                                    Register rtmp, XMMRegister xtmp) {
8272   ShortBranchVerifier sbv(this);
8273   assert_different_registers(to, value, count, rtmp);
8274   Label L_exit, L_skip_align1, L_skip_align2, L_fill_byte;
8275   Label L_fill_2_bytes, L_fill_4_bytes;
8276 
8277   int shift = -1;
8278   switch (t) {
8279     case T_BYTE:
8280       shift = 2;
8281       break;
8282     case T_SHORT:
8283       shift = 1;
8284       break;
8285     case T_INT:
8286       shift = 0;
8287       break;
8288     default: ShouldNotReachHere();
8289   }
8290 
8291   if (t == T_BYTE) {
8292     andl(value, 0xff);
8293     movl(rtmp, value);
8294     shll(rtmp, 8);
8295     orl(value, rtmp);
8296   }
8297   if (t == T_SHORT) {
8298     andl(value, 0xffff);
8299   }
8300   if (t == T_BYTE || t == T_SHORT) {
8301     movl(rtmp, value);
8302     shll(rtmp, 16);
8303     orl(value, rtmp);
8304   }
8305 
8306   cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
8307   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
8308   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
8309     // align source address at 4 bytes address boundary
8310     if (t == T_BYTE) {
8311       // One byte misalignment happens only for byte arrays
8312       testptr(to, 1);
8313       jccb(Assembler::zero, L_skip_align1);
8314       movb(Address(to, 0), value);
8315       increment(to);
8316       decrement(count);
8317       BIND(L_skip_align1);
8318     }
8319     // Two bytes misalignment happens only for byte and short (char) arrays
8320     testptr(to, 2);
8321     jccb(Assembler::zero, L_skip_align2);
8322     movw(Address(to, 0), value);
8323     addptr(to, 2);
8324     subl(count, 1<<(shift-1));
8325     BIND(L_skip_align2);
8326   }
8327   if (UseSSE < 2) {
8328     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8329     // Fill 32-byte chunks
8330     subl(count, 8 << shift);
8331     jcc(Assembler::less, L_check_fill_8_bytes);
8332     align(16);
8333 
8334     BIND(L_fill_32_bytes_loop);
8335 
8336     for (int i = 0; i < 32; i += 4) {
8337       movl(Address(to, i), value);
8338     }
8339 
8340     addptr(to, 32);
8341     subl(count, 8 << shift);
8342     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8343     BIND(L_check_fill_8_bytes);
8344     addl(count, 8 << shift);
8345     jccb(Assembler::zero, L_exit);
8346     jmpb(L_fill_8_bytes);
8347 
8348     //
8349     // length is too short, just fill qwords
8350     //
8351     BIND(L_fill_8_bytes_loop);
8352     movl(Address(to, 0), value);
8353     movl(Address(to, 4), value);
8354     addptr(to, 8);
8355     BIND(L_fill_8_bytes);
8356     subl(count, 1 << (shift + 1));
8357     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8358     // fall through to fill 4 bytes
8359   } else {
8360     Label L_fill_32_bytes;
8361     if (!UseUnalignedLoadStores) {
8362       // align to 8 bytes, we know we are 4 byte aligned to start
8363       testptr(to, 4);
8364       jccb(Assembler::zero, L_fill_32_bytes);
8365       movl(Address(to, 0), value);
8366       addptr(to, 4);
8367       subl(count, 1<<shift);
8368     }
8369     BIND(L_fill_32_bytes);
8370     {
8371       assert( UseSSE >= 2, "supported cpu only" );
8372       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8373       if (UseAVX > 2) {
8374         movl(rtmp, 0xffff);
8375         kmovwl(k1, rtmp);
8376       }
8377       movdl(xtmp, value);
8378       if (UseAVX > 2 && UseUnalignedLoadStores) {
8379         // Fill 64-byte chunks
8380         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8381         evpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
8382 
8383         subl(count, 16 << shift);
8384         jcc(Assembler::less, L_check_fill_32_bytes);
8385         align(16);
8386 
8387         BIND(L_fill_64_bytes_loop);
8388         evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
8389         addptr(to, 64);
8390         subl(count, 16 << shift);
8391         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8392 
8393         BIND(L_check_fill_32_bytes);
8394         addl(count, 8 << shift);
8395         jccb(Assembler::less, L_check_fill_8_bytes);
8396         vmovdqu(Address(to, 0), xtmp);
8397         addptr(to, 32);
8398         subl(count, 8 << shift);
8399 
8400         BIND(L_check_fill_8_bytes);
8401       } else if (UseAVX == 2 && UseUnalignedLoadStores) {
8402         // Fill 64-byte chunks
8403         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8404         vpbroadcastd(xtmp, xtmp);
8405 
8406         subl(count, 16 << shift);
8407         jcc(Assembler::less, L_check_fill_32_bytes);
8408         align(16);
8409 
8410         BIND(L_fill_64_bytes_loop);
8411         vmovdqu(Address(to, 0), xtmp);
8412         vmovdqu(Address(to, 32), xtmp);
8413         addptr(to, 64);
8414         subl(count, 16 << shift);
8415         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8416 
8417         BIND(L_check_fill_32_bytes);
8418         addl(count, 8 << shift);
8419         jccb(Assembler::less, L_check_fill_8_bytes);
8420         vmovdqu(Address(to, 0), xtmp);
8421         addptr(to, 32);
8422         subl(count, 8 << shift);
8423 
8424         BIND(L_check_fill_8_bytes);
8425         // clean upper bits of YMM registers
8426         movdl(xtmp, value);
8427         pshufd(xtmp, xtmp, 0);
8428       } else {
8429         // Fill 32-byte chunks
8430         pshufd(xtmp, xtmp, 0);
8431 
8432         subl(count, 8 << shift);
8433         jcc(Assembler::less, L_check_fill_8_bytes);
8434         align(16);
8435 
8436         BIND(L_fill_32_bytes_loop);
8437 
8438         if (UseUnalignedLoadStores) {
8439           movdqu(Address(to, 0), xtmp);
8440           movdqu(Address(to, 16), xtmp);
8441         } else {
8442           movq(Address(to, 0), xtmp);
8443           movq(Address(to, 8), xtmp);
8444           movq(Address(to, 16), xtmp);
8445           movq(Address(to, 24), xtmp);
8446         }
8447 
8448         addptr(to, 32);
8449         subl(count, 8 << shift);
8450         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8451 
8452         BIND(L_check_fill_8_bytes);
8453       }
8454       addl(count, 8 << shift);
8455       jccb(Assembler::zero, L_exit);
8456       jmpb(L_fill_8_bytes);
8457 
8458       //
8459       // length is too short, just fill qwords
8460       //
8461       BIND(L_fill_8_bytes_loop);
8462       movq(Address(to, 0), xtmp);
8463       addptr(to, 8);
8464       BIND(L_fill_8_bytes);
8465       subl(count, 1 << (shift + 1));
8466       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8467     }
8468   }
8469   // fill trailing 4 bytes
8470   BIND(L_fill_4_bytes);
8471   testl(count, 1<<shift);
8472   jccb(Assembler::zero, L_fill_2_bytes);
8473   movl(Address(to, 0), value);
8474   if (t == T_BYTE || t == T_SHORT) {
8475     addptr(to, 4);
8476     BIND(L_fill_2_bytes);
8477     // fill trailing 2 bytes
8478     testl(count, 1<<(shift-1));
8479     jccb(Assembler::zero, L_fill_byte);
8480     movw(Address(to, 0), value);
8481     if (t == T_BYTE) {
8482       addptr(to, 2);
8483       BIND(L_fill_byte);
8484       // fill trailing byte
8485       testl(count, 1);
8486       jccb(Assembler::zero, L_exit);
8487       movb(Address(to, 0), value);
8488     } else {
8489       BIND(L_fill_byte);
8490     }
8491   } else {
8492     BIND(L_fill_2_bytes);
8493   }
8494   BIND(L_exit);
8495 }
8496 
8497 // encode char[] to byte[] in ISO_8859_1
8498    //@HotSpotIntrinsicCandidate
8499    //private static int implEncodeISOArray(byte[] sa, int sp,
8500    //byte[] da, int dp, int len) {
8501    //  int i = 0;
8502    //  for (; i < len; i++) {
8503    //    char c = StringUTF16.getChar(sa, sp++);
8504    //    if (c > '\u00FF')
8505    //      break;
8506    //    da[dp++] = (byte)c;
8507    //  }
8508    //  return i;
8509    //}
8510 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
8511   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
8512   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
8513   Register tmp5, Register result) {
8514 
8515   // rsi: src
8516   // rdi: dst
8517   // rdx: len
8518   // rcx: tmp5
8519   // rax: result
8520   ShortBranchVerifier sbv(this);
8521   assert_different_registers(src, dst, len, tmp5, result);
8522   Label L_done, L_copy_1_char, L_copy_1_char_exit;
8523 
8524   // set result
8525   xorl(result, result);
8526   // check for zero length
8527   testl(len, len);
8528   jcc(Assembler::zero, L_done);
8529 
8530   movl(result, len);
8531 
8532   // Setup pointers
8533   lea(src, Address(src, len, Address::times_2)); // char[]
8534   lea(dst, Address(dst, len, Address::times_1)); // byte[]
8535   negptr(len);
8536 
8537   if (UseSSE42Intrinsics || UseAVX >= 2) {
8538     Label L_chars_8_check, L_copy_8_chars, L_copy_8_chars_exit;
8539     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
8540 
8541     if (UseAVX >= 2) {
8542       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
8543       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8544       movdl(tmp1Reg, tmp5);
8545       vpbroadcastd(tmp1Reg, tmp1Reg);
8546       jmp(L_chars_32_check);
8547 
8548       bind(L_copy_32_chars);
8549       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
8550       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
8551       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8552       vptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8553       jccb(Assembler::notZero, L_copy_32_chars_exit);
8554       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8555       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
8556       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
8557 
8558       bind(L_chars_32_check);
8559       addptr(len, 32);
8560       jcc(Assembler::lessEqual, L_copy_32_chars);
8561 
8562       bind(L_copy_32_chars_exit);
8563       subptr(len, 16);
8564       jccb(Assembler::greater, L_copy_16_chars_exit);
8565 
8566     } else if (UseSSE42Intrinsics) {
8567       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8568       movdl(tmp1Reg, tmp5);
8569       pshufd(tmp1Reg, tmp1Reg, 0);
8570       jmpb(L_chars_16_check);
8571     }
8572 
8573     bind(L_copy_16_chars);
8574     if (UseAVX >= 2) {
8575       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
8576       vptest(tmp2Reg, tmp1Reg);
8577       jcc(Assembler::notZero, L_copy_16_chars_exit);
8578       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
8579       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
8580     } else {
8581       if (UseAVX > 0) {
8582         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8583         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8584         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
8585       } else {
8586         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8587         por(tmp2Reg, tmp3Reg);
8588         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8589         por(tmp2Reg, tmp4Reg);
8590       }
8591       ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8592       jccb(Assembler::notZero, L_copy_16_chars_exit);
8593       packuswb(tmp3Reg, tmp4Reg);
8594     }
8595     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
8596 
8597     bind(L_chars_16_check);
8598     addptr(len, 16);
8599     jcc(Assembler::lessEqual, L_copy_16_chars);
8600 
8601     bind(L_copy_16_chars_exit);
8602     if (UseAVX >= 2) {
8603       // clean upper bits of YMM registers
8604       vpxor(tmp2Reg, tmp2Reg);
8605       vpxor(tmp3Reg, tmp3Reg);
8606       vpxor(tmp4Reg, tmp4Reg);
8607       movdl(tmp1Reg, tmp5);
8608       pshufd(tmp1Reg, tmp1Reg, 0);
8609     }
8610     subptr(len, 8);
8611     jccb(Assembler::greater, L_copy_8_chars_exit);
8612 
8613     bind(L_copy_8_chars);
8614     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
8615     ptest(tmp3Reg, tmp1Reg);
8616     jccb(Assembler::notZero, L_copy_8_chars_exit);
8617     packuswb(tmp3Reg, tmp1Reg);
8618     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
8619     addptr(len, 8);
8620     jccb(Assembler::lessEqual, L_copy_8_chars);
8621 
8622     bind(L_copy_8_chars_exit);
8623     subptr(len, 8);
8624     jccb(Assembler::zero, L_done);
8625   }
8626 
8627   bind(L_copy_1_char);
8628   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
8629   testl(tmp5, 0xff00);      // check if Unicode char
8630   jccb(Assembler::notZero, L_copy_1_char_exit);
8631   movb(Address(dst, len, Address::times_1, 0), tmp5);
8632   addptr(len, 1);
8633   jccb(Assembler::less, L_copy_1_char);
8634 
8635   bind(L_copy_1_char_exit);
8636   addptr(result, len); // len is negative count of not processed elements
8637 
8638   bind(L_done);
8639 }
8640 
8641 #ifdef _LP64
8642 /**
8643  * Helper for multiply_to_len().
8644  */
8645 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
8646   addq(dest_lo, src1);
8647   adcq(dest_hi, 0);
8648   addq(dest_lo, src2);
8649   adcq(dest_hi, 0);
8650 }
8651 
8652 /**
8653  * Multiply 64 bit by 64 bit first loop.
8654  */
8655 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
8656                                            Register y, Register y_idx, Register z,
8657                                            Register carry, Register product,
8658                                            Register idx, Register kdx) {
8659   //
8660   //  jlong carry, x[], y[], z[];
8661   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
8662   //    huge_128 product = y[idx] * x[xstart] + carry;
8663   //    z[kdx] = (jlong)product;
8664   //    carry  = (jlong)(product >>> 64);
8665   //  }
8666   //  z[xstart] = carry;
8667   //
8668 
8669   Label L_first_loop, L_first_loop_exit;
8670   Label L_one_x, L_one_y, L_multiply;
8671 
8672   decrementl(xstart);
8673   jcc(Assembler::negative, L_one_x);
8674 
8675   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
8676   rorq(x_xstart, 32); // convert big-endian to little-endian
8677 
8678   bind(L_first_loop);
8679   decrementl(idx);
8680   jcc(Assembler::negative, L_first_loop_exit);
8681   decrementl(idx);
8682   jcc(Assembler::negative, L_one_y);
8683   movq(y_idx, Address(y, idx, Address::times_4,  0));
8684   rorq(y_idx, 32); // convert big-endian to little-endian
8685   bind(L_multiply);
8686   movq(product, x_xstart);
8687   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
8688   addq(product, carry);
8689   adcq(rdx, 0);
8690   subl(kdx, 2);
8691   movl(Address(z, kdx, Address::times_4,  4), product);
8692   shrq(product, 32);
8693   movl(Address(z, kdx, Address::times_4,  0), product);
8694   movq(carry, rdx);
8695   jmp(L_first_loop);
8696 
8697   bind(L_one_y);
8698   movl(y_idx, Address(y,  0));
8699   jmp(L_multiply);
8700 
8701   bind(L_one_x);
8702   movl(x_xstart, Address(x,  0));
8703   jmp(L_first_loop);
8704 
8705   bind(L_first_loop_exit);
8706 }
8707 
8708 /**
8709  * Multiply 64 bit by 64 bit and add 128 bit.
8710  */
8711 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
8712                                             Register yz_idx, Register idx,
8713                                             Register carry, Register product, int offset) {
8714   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
8715   //     z[kdx] = (jlong)product;
8716 
8717   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
8718   rorq(yz_idx, 32); // convert big-endian to little-endian
8719   movq(product, x_xstart);
8720   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
8721   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
8722   rorq(yz_idx, 32); // convert big-endian to little-endian
8723 
8724   add2_with_carry(rdx, product, carry, yz_idx);
8725 
8726   movl(Address(z, idx, Address::times_4,  offset+4), product);
8727   shrq(product, 32);
8728   movl(Address(z, idx, Address::times_4,  offset), product);
8729 
8730 }
8731 
8732 /**
8733  * Multiply 128 bit by 128 bit. Unrolled inner loop.
8734  */
8735 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
8736                                              Register yz_idx, Register idx, Register jdx,
8737                                              Register carry, Register product,
8738                                              Register carry2) {
8739   //   jlong carry, x[], y[], z[];
8740   //   int kdx = ystart+1;
8741   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
8742   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
8743   //     z[kdx+idx+1] = (jlong)product;
8744   //     jlong carry2  = (jlong)(product >>> 64);
8745   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
8746   //     z[kdx+idx] = (jlong)product;
8747   //     carry  = (jlong)(product >>> 64);
8748   //   }
8749   //   idx += 2;
8750   //   if (idx > 0) {
8751   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
8752   //     z[kdx+idx] = (jlong)product;
8753   //     carry  = (jlong)(product >>> 64);
8754   //   }
8755   //
8756 
8757   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
8758 
8759   movl(jdx, idx);
8760   andl(jdx, 0xFFFFFFFC);
8761   shrl(jdx, 2);
8762 
8763   bind(L_third_loop);
8764   subl(jdx, 1);
8765   jcc(Assembler::negative, L_third_loop_exit);
8766   subl(idx, 4);
8767 
8768   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
8769   movq(carry2, rdx);
8770 
8771   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
8772   movq(carry, rdx);
8773   jmp(L_third_loop);
8774 
8775   bind (L_third_loop_exit);
8776 
8777   andl (idx, 0x3);
8778   jcc(Assembler::zero, L_post_third_loop_done);
8779 
8780   Label L_check_1;
8781   subl(idx, 2);
8782   jcc(Assembler::negative, L_check_1);
8783 
8784   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
8785   movq(carry, rdx);
8786 
8787   bind (L_check_1);
8788   addl (idx, 0x2);
8789   andl (idx, 0x1);
8790   subl(idx, 1);
8791   jcc(Assembler::negative, L_post_third_loop_done);
8792 
8793   movl(yz_idx, Address(y, idx, Address::times_4,  0));
8794   movq(product, x_xstart);
8795   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
8796   movl(yz_idx, Address(z, idx, Address::times_4,  0));
8797 
8798   add2_with_carry(rdx, product, yz_idx, carry);
8799 
8800   movl(Address(z, idx, Address::times_4,  0), product);
8801   shrq(product, 32);
8802 
8803   shlq(rdx, 32);
8804   orq(product, rdx);
8805   movq(carry, product);
8806 
8807   bind(L_post_third_loop_done);
8808 }
8809 
8810 /**
8811  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
8812  *
8813  */
8814 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
8815                                                   Register carry, Register carry2,
8816                                                   Register idx, Register jdx,
8817                                                   Register yz_idx1, Register yz_idx2,
8818                                                   Register tmp, Register tmp3, Register tmp4) {
8819   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
8820 
8821   //   jlong carry, x[], y[], z[];
8822   //   int kdx = ystart+1;
8823   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
8824   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
8825   //     jlong carry2  = (jlong)(tmp3 >>> 64);
8826   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
8827   //     carry  = (jlong)(tmp4 >>> 64);
8828   //     z[kdx+idx+1] = (jlong)tmp3;
8829   //     z[kdx+idx] = (jlong)tmp4;
8830   //   }
8831   //   idx += 2;
8832   //   if (idx > 0) {
8833   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
8834   //     z[kdx+idx] = (jlong)yz_idx1;
8835   //     carry  = (jlong)(yz_idx1 >>> 64);
8836   //   }
8837   //
8838 
8839   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
8840 
8841   movl(jdx, idx);
8842   andl(jdx, 0xFFFFFFFC);
8843   shrl(jdx, 2);
8844 
8845   bind(L_third_loop);
8846   subl(jdx, 1);
8847   jcc(Assembler::negative, L_third_loop_exit);
8848   subl(idx, 4);
8849 
8850   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
8851   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
8852   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
8853   rorxq(yz_idx2, yz_idx2, 32);
8854 
8855   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
8856   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
8857 
8858   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
8859   rorxq(yz_idx1, yz_idx1, 32);
8860   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
8861   rorxq(yz_idx2, yz_idx2, 32);
8862 
8863   if (VM_Version::supports_adx()) {
8864     adcxq(tmp3, carry);
8865     adoxq(tmp3, yz_idx1);
8866 
8867     adcxq(tmp4, tmp);
8868     adoxq(tmp4, yz_idx2);
8869 
8870     movl(carry, 0); // does not affect flags
8871     adcxq(carry2, carry);
8872     adoxq(carry2, carry);
8873   } else {
8874     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
8875     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
8876   }
8877   movq(carry, carry2);
8878 
8879   movl(Address(z, idx, Address::times_4, 12), tmp3);
8880   shrq(tmp3, 32);
8881   movl(Address(z, idx, Address::times_4,  8), tmp3);
8882 
8883   movl(Address(z, idx, Address::times_4,  4), tmp4);
8884   shrq(tmp4, 32);
8885   movl(Address(z, idx, Address::times_4,  0), tmp4);
8886 
8887   jmp(L_third_loop);
8888 
8889   bind (L_third_loop_exit);
8890 
8891   andl (idx, 0x3);
8892   jcc(Assembler::zero, L_post_third_loop_done);
8893 
8894   Label L_check_1;
8895   subl(idx, 2);
8896   jcc(Assembler::negative, L_check_1);
8897 
8898   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
8899   rorxq(yz_idx1, yz_idx1, 32);
8900   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
8901   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
8902   rorxq(yz_idx2, yz_idx2, 32);
8903 
8904   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
8905 
8906   movl(Address(z, idx, Address::times_4,  4), tmp3);
8907   shrq(tmp3, 32);
8908   movl(Address(z, idx, Address::times_4,  0), tmp3);
8909   movq(carry, tmp4);
8910 
8911   bind (L_check_1);
8912   addl (idx, 0x2);
8913   andl (idx, 0x1);
8914   subl(idx, 1);
8915   jcc(Assembler::negative, L_post_third_loop_done);
8916   movl(tmp4, Address(y, idx, Address::times_4,  0));
8917   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
8918   movl(tmp4, Address(z, idx, Address::times_4,  0));
8919 
8920   add2_with_carry(carry2, tmp3, tmp4, carry);
8921 
8922   movl(Address(z, idx, Address::times_4,  0), tmp3);
8923   shrq(tmp3, 32);
8924 
8925   shlq(carry2, 32);
8926   orq(tmp3, carry2);
8927   movq(carry, tmp3);
8928 
8929   bind(L_post_third_loop_done);
8930 }
8931 
8932 /**
8933  * Code for BigInteger::multiplyToLen() instrinsic.
8934  *
8935  * rdi: x
8936  * rax: xlen
8937  * rsi: y
8938  * rcx: ylen
8939  * r8:  z
8940  * r11: zlen
8941  * r12: tmp1
8942  * r13: tmp2
8943  * r14: tmp3
8944  * r15: tmp4
8945  * rbx: tmp5
8946  *
8947  */
8948 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
8949                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
8950   ShortBranchVerifier sbv(this);
8951   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
8952 
8953   push(tmp1);
8954   push(tmp2);
8955   push(tmp3);
8956   push(tmp4);
8957   push(tmp5);
8958 
8959   push(xlen);
8960   push(zlen);
8961 
8962   const Register idx = tmp1;
8963   const Register kdx = tmp2;
8964   const Register xstart = tmp3;
8965 
8966   const Register y_idx = tmp4;
8967   const Register carry = tmp5;
8968   const Register product  = xlen;
8969   const Register x_xstart = zlen;  // reuse register
8970 
8971   // First Loop.
8972   //
8973   //  final static long LONG_MASK = 0xffffffffL;
8974   //  int xstart = xlen - 1;
8975   //  int ystart = ylen - 1;
8976   //  long carry = 0;
8977   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
8978   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
8979   //    z[kdx] = (int)product;
8980   //    carry = product >>> 32;
8981   //  }
8982   //  z[xstart] = (int)carry;
8983   //
8984 
8985   movl(idx, ylen);      // idx = ylen;
8986   movl(kdx, zlen);      // kdx = xlen+ylen;
8987   xorq(carry, carry);   // carry = 0;
8988 
8989   Label L_done;
8990 
8991   movl(xstart, xlen);
8992   decrementl(xstart);
8993   jcc(Assembler::negative, L_done);
8994 
8995   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
8996 
8997   Label L_second_loop;
8998   testl(kdx, kdx);
8999   jcc(Assembler::zero, L_second_loop);
9000 
9001   Label L_carry;
9002   subl(kdx, 1);
9003   jcc(Assembler::zero, L_carry);
9004 
9005   movl(Address(z, kdx, Address::times_4,  0), carry);
9006   shrq(carry, 32);
9007   subl(kdx, 1);
9008 
9009   bind(L_carry);
9010   movl(Address(z, kdx, Address::times_4,  0), carry);
9011 
9012   // Second and third (nested) loops.
9013   //
9014   // for (int i = xstart-1; i >= 0; i--) { // Second loop
9015   //   carry = 0;
9016   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
9017   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
9018   //                    (z[k] & LONG_MASK) + carry;
9019   //     z[k] = (int)product;
9020   //     carry = product >>> 32;
9021   //   }
9022   //   z[i] = (int)carry;
9023   // }
9024   //
9025   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
9026 
9027   const Register jdx = tmp1;
9028 
9029   bind(L_second_loop);
9030   xorl(carry, carry);    // carry = 0;
9031   movl(jdx, ylen);       // j = ystart+1
9032 
9033   subl(xstart, 1);       // i = xstart-1;
9034   jcc(Assembler::negative, L_done);
9035 
9036   push (z);
9037 
9038   Label L_last_x;
9039   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
9040   subl(xstart, 1);       // i = xstart-1;
9041   jcc(Assembler::negative, L_last_x);
9042 
9043   if (UseBMI2Instructions) {
9044     movq(rdx,  Address(x, xstart, Address::times_4,  0));
9045     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
9046   } else {
9047     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
9048     rorq(x_xstart, 32);  // convert big-endian to little-endian
9049   }
9050 
9051   Label L_third_loop_prologue;
9052   bind(L_third_loop_prologue);
9053 
9054   push (x);
9055   push (xstart);
9056   push (ylen);
9057 
9058 
9059   if (UseBMI2Instructions) {
9060     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
9061   } else { // !UseBMI2Instructions
9062     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
9063   }
9064 
9065   pop(ylen);
9066   pop(xlen);
9067   pop(x);
9068   pop(z);
9069 
9070   movl(tmp3, xlen);
9071   addl(tmp3, 1);
9072   movl(Address(z, tmp3, Address::times_4,  0), carry);
9073   subl(tmp3, 1);
9074   jccb(Assembler::negative, L_done);
9075 
9076   shrq(carry, 32);
9077   movl(Address(z, tmp3, Address::times_4,  0), carry);
9078   jmp(L_second_loop);
9079 
9080   // Next infrequent code is moved outside loops.
9081   bind(L_last_x);
9082   if (UseBMI2Instructions) {
9083     movl(rdx, Address(x,  0));
9084   } else {
9085     movl(x_xstart, Address(x,  0));
9086   }
9087   jmp(L_third_loop_prologue);
9088 
9089   bind(L_done);
9090 
9091   pop(zlen);
9092   pop(xlen);
9093 
9094   pop(tmp5);
9095   pop(tmp4);
9096   pop(tmp3);
9097   pop(tmp2);
9098   pop(tmp1);
9099 }
9100 
9101 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
9102   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
9103   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
9104   Label VECTOR64_LOOP, VECTOR64_TAIL, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
9105   Label VECTOR32_LOOP, VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
9106   Label VECTOR16_TAIL, VECTOR8_TAIL, VECTOR4_TAIL;
9107   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
9108   Label SAME_TILL_END, DONE;
9109   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
9110 
9111   //scale is in rcx in both Win64 and Unix
9112   ShortBranchVerifier sbv(this);
9113 
9114   shlq(length);
9115   xorq(result, result);
9116 
9117   if ((UseAVX > 2) &&
9118       VM_Version::supports_avx512vlbw()) {
9119     set_vector_masking();  // opening of the stub context for programming mask registers
9120     cmpq(length, 64);
9121     jcc(Assembler::less, VECTOR32_TAIL);
9122     movq(tmp1, length);
9123     andq(tmp1, 0x3F);      // tail count
9124     andq(length, ~(0x3F)); //vector count
9125 
9126     bind(VECTOR64_LOOP);
9127     // AVX512 code to compare 64 byte vectors.
9128     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
9129     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
9130     kortestql(k7, k7);
9131     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
9132     addq(result, 64);
9133     subq(length, 64);
9134     jccb(Assembler::notZero, VECTOR64_LOOP);
9135 
9136     //bind(VECTOR64_TAIL);
9137     testq(tmp1, tmp1);
9138     jcc(Assembler::zero, SAME_TILL_END);
9139 
9140     bind(VECTOR64_TAIL);
9141     // AVX512 code to compare upto 63 byte vectors.
9142     // Save k1
9143     kmovql(k3, k1);
9144     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
9145     shlxq(tmp2, tmp2, tmp1);
9146     notq(tmp2);
9147     kmovql(k1, tmp2);
9148 
9149     evmovdqub(rymm0, k1, Address(obja, result), Assembler::AVX_512bit);
9150     evpcmpeqb(k7, k1, rymm0, Address(objb, result), Assembler::AVX_512bit);
9151 
9152     ktestql(k7, k1);
9153     // Restore k1
9154     kmovql(k1, k3);
9155     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
9156 
9157     bind(VECTOR64_NOT_EQUAL);
9158     kmovql(tmp1, k7);
9159     notq(tmp1);
9160     tzcntq(tmp1, tmp1);
9161     addq(result, tmp1);
9162     shrq(result);
9163     jmp(DONE);
9164     bind(VECTOR32_TAIL);
9165     clear_vector_masking();   // closing of the stub context for programming mask registers
9166   }
9167 
9168   cmpq(length, 8);
9169   jcc(Assembler::equal, VECTOR8_LOOP);
9170   jcc(Assembler::less, VECTOR4_TAIL);
9171 
9172   if (UseAVX >= 2) {
9173 
9174     cmpq(length, 16);
9175     jcc(Assembler::equal, VECTOR16_LOOP);
9176     jcc(Assembler::less, VECTOR8_LOOP);
9177 
9178     cmpq(length, 32);
9179     jccb(Assembler::less, VECTOR16_TAIL);
9180 
9181     subq(length, 32);
9182     bind(VECTOR32_LOOP);
9183     vmovdqu(rymm0, Address(obja, result));
9184     vmovdqu(rymm1, Address(objb, result));
9185     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
9186     vptest(rymm2, rymm2);
9187     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
9188     addq(result, 32);
9189     subq(length, 32);
9190     jccb(Assembler::greaterEqual, VECTOR32_LOOP);
9191     addq(length, 32);
9192     jcc(Assembler::equal, SAME_TILL_END);
9193     //falling through if less than 32 bytes left //close the branch here.
9194 
9195     bind(VECTOR16_TAIL);
9196     cmpq(length, 16);
9197     jccb(Assembler::less, VECTOR8_TAIL);
9198     bind(VECTOR16_LOOP);
9199     movdqu(rymm0, Address(obja, result));
9200     movdqu(rymm1, Address(objb, result));
9201     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
9202     ptest(rymm2, rymm2);
9203     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
9204     addq(result, 16);
9205     subq(length, 16);
9206     jcc(Assembler::equal, SAME_TILL_END);
9207     //falling through if less than 16 bytes left
9208   } else {//regular intrinsics
9209 
9210     cmpq(length, 16);
9211     jccb(Assembler::less, VECTOR8_TAIL);
9212 
9213     subq(length, 16);
9214     bind(VECTOR16_LOOP);
9215     movdqu(rymm0, Address(obja, result));
9216     movdqu(rymm1, Address(objb, result));
9217     pxor(rymm0, rymm1);
9218     ptest(rymm0, rymm0);
9219     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
9220     addq(result, 16);
9221     subq(length, 16);
9222     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
9223     addq(length, 16);
9224     jcc(Assembler::equal, SAME_TILL_END);
9225     //falling through if less than 16 bytes left
9226   }
9227 
9228   bind(VECTOR8_TAIL);
9229   cmpq(length, 8);
9230   jccb(Assembler::less, VECTOR4_TAIL);
9231   bind(VECTOR8_LOOP);
9232   movq(tmp1, Address(obja, result));
9233   movq(tmp2, Address(objb, result));
9234   xorq(tmp1, tmp2);
9235   testq(tmp1, tmp1);
9236   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
9237   addq(result, 8);
9238   subq(length, 8);
9239   jcc(Assembler::equal, SAME_TILL_END);
9240   //falling through if less than 8 bytes left
9241 
9242   bind(VECTOR4_TAIL);
9243   cmpq(length, 4);
9244   jccb(Assembler::less, BYTES_TAIL);
9245   bind(VECTOR4_LOOP);
9246   movl(tmp1, Address(obja, result));
9247   xorl(tmp1, Address(objb, result));
9248   testl(tmp1, tmp1);
9249   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
9250   addq(result, 4);
9251   subq(length, 4);
9252   jcc(Assembler::equal, SAME_TILL_END);
9253   //falling through if less than 4 bytes left
9254 
9255   bind(BYTES_TAIL);
9256   bind(BYTES_LOOP);
9257   load_unsigned_byte(tmp1, Address(obja, result));
9258   load_unsigned_byte(tmp2, Address(objb, result));
9259   xorl(tmp1, tmp2);
9260   testl(tmp1, tmp1);
9261   jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9262   decq(length);
9263   jccb(Assembler::zero, SAME_TILL_END);
9264   incq(result);
9265   load_unsigned_byte(tmp1, Address(obja, result));
9266   load_unsigned_byte(tmp2, Address(objb, result));
9267   xorl(tmp1, tmp2);
9268   testl(tmp1, tmp1);
9269   jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9270   decq(length);
9271   jccb(Assembler::zero, SAME_TILL_END);
9272   incq(result);
9273   load_unsigned_byte(tmp1, Address(obja, result));
9274   load_unsigned_byte(tmp2, Address(objb, result));
9275   xorl(tmp1, tmp2);
9276   testl(tmp1, tmp1);
9277   jccb(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
9278   jmpb(SAME_TILL_END);
9279 
9280   if (UseAVX >= 2) {
9281     bind(VECTOR32_NOT_EQUAL);
9282     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
9283     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
9284     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
9285     vpmovmskb(tmp1, rymm0);
9286     bsfq(tmp1, tmp1);
9287     addq(result, tmp1);
9288     shrq(result);
9289     jmpb(DONE);
9290   }
9291 
9292   bind(VECTOR16_NOT_EQUAL);
9293   if (UseAVX >= 2) {
9294     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
9295     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
9296     pxor(rymm0, rymm2);
9297   } else {
9298     pcmpeqb(rymm2, rymm2);
9299     pxor(rymm0, rymm1);
9300     pcmpeqb(rymm0, rymm1);
9301     pxor(rymm0, rymm2);
9302   }
9303   pmovmskb(tmp1, rymm0);
9304   bsfq(tmp1, tmp1);
9305   addq(result, tmp1);
9306   shrq(result);
9307   jmpb(DONE);
9308 
9309   bind(VECTOR8_NOT_EQUAL);
9310   bind(VECTOR4_NOT_EQUAL);
9311   bsfq(tmp1, tmp1);
9312   shrq(tmp1, 3);
9313   addq(result, tmp1);
9314   bind(BYTES_NOT_EQUAL);
9315   shrq(result);
9316   jmpb(DONE);
9317 
9318   bind(SAME_TILL_END);
9319   mov64(result, -1);
9320 
9321   bind(DONE);
9322 }
9323 
9324 //Helper functions for square_to_len()
9325 
9326 /**
9327  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
9328  * Preserves x and z and modifies rest of the registers.
9329  */
9330 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9331   // Perform square and right shift by 1
9332   // Handle odd xlen case first, then for even xlen do the following
9333   // jlong carry = 0;
9334   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
9335   //     huge_128 product = x[j:j+1] * x[j:j+1];
9336   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
9337   //     z[i+2:i+3] = (jlong)(product >>> 1);
9338   //     carry = (jlong)product;
9339   // }
9340 
9341   xorq(tmp5, tmp5);     // carry
9342   xorq(rdxReg, rdxReg);
9343   xorl(tmp1, tmp1);     // index for x
9344   xorl(tmp4, tmp4);     // index for z
9345 
9346   Label L_first_loop, L_first_loop_exit;
9347 
9348   testl(xlen, 1);
9349   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
9350 
9351   // Square and right shift by 1 the odd element using 32 bit multiply
9352   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
9353   imulq(raxReg, raxReg);
9354   shrq(raxReg, 1);
9355   adcq(tmp5, 0);
9356   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
9357   incrementl(tmp1);
9358   addl(tmp4, 2);
9359 
9360   // Square and  right shift by 1 the rest using 64 bit multiply
9361   bind(L_first_loop);
9362   cmpptr(tmp1, xlen);
9363   jccb(Assembler::equal, L_first_loop_exit);
9364 
9365   // Square
9366   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
9367   rorq(raxReg, 32);    // convert big-endian to little-endian
9368   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
9369 
9370   // Right shift by 1 and save carry
9371   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
9372   rcrq(rdxReg, 1);
9373   rcrq(raxReg, 1);
9374   adcq(tmp5, 0);
9375 
9376   // Store result in z
9377   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
9378   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
9379 
9380   // Update indices for x and z
9381   addl(tmp1, 2);
9382   addl(tmp4, 4);
9383   jmp(L_first_loop);
9384 
9385   bind(L_first_loop_exit);
9386 }
9387 
9388 
9389 /**
9390  * Perform the following multiply add operation using BMI2 instructions
9391  * carry:sum = sum + op1*op2 + carry
9392  * op2 should be in rdx
9393  * op2 is preserved, all other registers are modified
9394  */
9395 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
9396   // assert op2 is rdx
9397   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
9398   addq(sum, carry);
9399   adcq(tmp2, 0);
9400   addq(sum, op1);
9401   adcq(tmp2, 0);
9402   movq(carry, tmp2);
9403 }
9404 
9405 /**
9406  * Perform the following multiply add operation:
9407  * carry:sum = sum + op1*op2 + carry
9408  * Preserves op1, op2 and modifies rest of registers
9409  */
9410 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
9411   // rdx:rax = op1 * op2
9412   movq(raxReg, op2);
9413   mulq(op1);
9414 
9415   //  rdx:rax = sum + carry + rdx:rax
9416   addq(sum, carry);
9417   adcq(rdxReg, 0);
9418   addq(sum, raxReg);
9419   adcq(rdxReg, 0);
9420 
9421   // carry:sum = rdx:sum
9422   movq(carry, rdxReg);
9423 }
9424 
9425 /**
9426  * Add 64 bit long carry into z[] with carry propogation.
9427  * Preserves z and carry register values and modifies rest of registers.
9428  *
9429  */
9430 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
9431   Label L_fourth_loop, L_fourth_loop_exit;
9432 
9433   movl(tmp1, 1);
9434   subl(zlen, 2);
9435   addq(Address(z, zlen, Address::times_4, 0), carry);
9436 
9437   bind(L_fourth_loop);
9438   jccb(Assembler::carryClear, L_fourth_loop_exit);
9439   subl(zlen, 2);
9440   jccb(Assembler::negative, L_fourth_loop_exit);
9441   addq(Address(z, zlen, Address::times_4, 0), tmp1);
9442   jmp(L_fourth_loop);
9443   bind(L_fourth_loop_exit);
9444 }
9445 
9446 /**
9447  * Shift z[] left by 1 bit.
9448  * Preserves x, len, z and zlen registers and modifies rest of the registers.
9449  *
9450  */
9451 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
9452 
9453   Label L_fifth_loop, L_fifth_loop_exit;
9454 
9455   // Fifth loop
9456   // Perform primitiveLeftShift(z, zlen, 1)
9457 
9458   const Register prev_carry = tmp1;
9459   const Register new_carry = tmp4;
9460   const Register value = tmp2;
9461   const Register zidx = tmp3;
9462 
9463   // int zidx, carry;
9464   // long value;
9465   // carry = 0;
9466   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
9467   //    (carry:value)  = (z[i] << 1) | carry ;
9468   //    z[i] = value;
9469   // }
9470 
9471   movl(zidx, zlen);
9472   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
9473 
9474   bind(L_fifth_loop);
9475   decl(zidx);  // Use decl to preserve carry flag
9476   decl(zidx);
9477   jccb(Assembler::negative, L_fifth_loop_exit);
9478 
9479   if (UseBMI2Instructions) {
9480      movq(value, Address(z, zidx, Address::times_4, 0));
9481      rclq(value, 1);
9482      rorxq(value, value, 32);
9483      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9484   }
9485   else {
9486     // clear new_carry
9487     xorl(new_carry, new_carry);
9488 
9489     // Shift z[i] by 1, or in previous carry and save new carry
9490     movq(value, Address(z, zidx, Address::times_4, 0));
9491     shlq(value, 1);
9492     adcl(new_carry, 0);
9493 
9494     orq(value, prev_carry);
9495     rorq(value, 0x20);
9496     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9497 
9498     // Set previous carry = new carry
9499     movl(prev_carry, new_carry);
9500   }
9501   jmp(L_fifth_loop);
9502 
9503   bind(L_fifth_loop_exit);
9504 }
9505 
9506 
9507 /**
9508  * Code for BigInteger::squareToLen() intrinsic
9509  *
9510  * rdi: x
9511  * rsi: len
9512  * r8:  z
9513  * rcx: zlen
9514  * r12: tmp1
9515  * r13: tmp2
9516  * r14: tmp3
9517  * r15: tmp4
9518  * rbx: tmp5
9519  *
9520  */
9521 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9522 
9523   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, fifth_loop, fifth_loop_exit, L_last_x, L_multiply;
9524   push(tmp1);
9525   push(tmp2);
9526   push(tmp3);
9527   push(tmp4);
9528   push(tmp5);
9529 
9530   // First loop
9531   // Store the squares, right shifted one bit (i.e., divided by 2).
9532   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
9533 
9534   // Add in off-diagonal sums.
9535   //
9536   // Second, third (nested) and fourth loops.
9537   // zlen +=2;
9538   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
9539   //    carry = 0;
9540   //    long op2 = x[xidx:xidx+1];
9541   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
9542   //       k -= 2;
9543   //       long op1 = x[j:j+1];
9544   //       long sum = z[k:k+1];
9545   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
9546   //       z[k:k+1] = sum;
9547   //    }
9548   //    add_one_64(z, k, carry, tmp_regs);
9549   // }
9550 
9551   const Register carry = tmp5;
9552   const Register sum = tmp3;
9553   const Register op1 = tmp4;
9554   Register op2 = tmp2;
9555 
9556   push(zlen);
9557   push(len);
9558   addl(zlen,2);
9559   bind(L_second_loop);
9560   xorq(carry, carry);
9561   subl(zlen, 4);
9562   subl(len, 2);
9563   push(zlen);
9564   push(len);
9565   cmpl(len, 0);
9566   jccb(Assembler::lessEqual, L_second_loop_exit);
9567 
9568   // Multiply an array by one 64 bit long.
9569   if (UseBMI2Instructions) {
9570     op2 = rdxReg;
9571     movq(op2, Address(x, len, Address::times_4,  0));
9572     rorxq(op2, op2, 32);
9573   }
9574   else {
9575     movq(op2, Address(x, len, Address::times_4,  0));
9576     rorq(op2, 32);
9577   }
9578 
9579   bind(L_third_loop);
9580   decrementl(len);
9581   jccb(Assembler::negative, L_third_loop_exit);
9582   decrementl(len);
9583   jccb(Assembler::negative, L_last_x);
9584 
9585   movq(op1, Address(x, len, Address::times_4,  0));
9586   rorq(op1, 32);
9587 
9588   bind(L_multiply);
9589   subl(zlen, 2);
9590   movq(sum, Address(z, zlen, Address::times_4,  0));
9591 
9592   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
9593   if (UseBMI2Instructions) {
9594     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
9595   }
9596   else {
9597     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9598   }
9599 
9600   movq(Address(z, zlen, Address::times_4, 0), sum);
9601 
9602   jmp(L_third_loop);
9603   bind(L_third_loop_exit);
9604 
9605   // Fourth loop
9606   // Add 64 bit long carry into z with carry propogation.
9607   // Uses offsetted zlen.
9608   add_one_64(z, zlen, carry, tmp1);
9609 
9610   pop(len);
9611   pop(zlen);
9612   jmp(L_second_loop);
9613 
9614   // Next infrequent code is moved outside loops.
9615   bind(L_last_x);
9616   movl(op1, Address(x, 0));
9617   jmp(L_multiply);
9618 
9619   bind(L_second_loop_exit);
9620   pop(len);
9621   pop(zlen);
9622   pop(len);
9623   pop(zlen);
9624 
9625   // Fifth loop
9626   // Shift z left 1 bit.
9627   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
9628 
9629   // z[zlen-1] |= x[len-1] & 1;
9630   movl(tmp3, Address(x, len, Address::times_4, -4));
9631   andl(tmp3, 1);
9632   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
9633 
9634   pop(tmp5);
9635   pop(tmp4);
9636   pop(tmp3);
9637   pop(tmp2);
9638   pop(tmp1);
9639 }
9640 
9641 /**
9642  * Helper function for mul_add()
9643  * Multiply the in[] by int k and add to out[] starting at offset offs using
9644  * 128 bit by 32 bit multiply and return the carry in tmp5.
9645  * Only quad int aligned length of in[] is operated on in this function.
9646  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
9647  * This function preserves out, in and k registers.
9648  * len and offset point to the appropriate index in "in" & "out" correspondingly
9649  * tmp5 has the carry.
9650  * other registers are temporary and are modified.
9651  *
9652  */
9653 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
9654   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
9655   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9656 
9657   Label L_first_loop, L_first_loop_exit;
9658 
9659   movl(tmp1, len);
9660   shrl(tmp1, 2);
9661 
9662   bind(L_first_loop);
9663   subl(tmp1, 1);
9664   jccb(Assembler::negative, L_first_loop_exit);
9665 
9666   subl(len, 4);
9667   subl(offset, 4);
9668 
9669   Register op2 = tmp2;
9670   const Register sum = tmp3;
9671   const Register op1 = tmp4;
9672   const Register carry = tmp5;
9673 
9674   if (UseBMI2Instructions) {
9675     op2 = rdxReg;
9676   }
9677 
9678   movq(op1, Address(in, len, Address::times_4,  8));
9679   rorq(op1, 32);
9680   movq(sum, Address(out, offset, Address::times_4,  8));
9681   rorq(sum, 32);
9682   if (UseBMI2Instructions) {
9683     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9684   }
9685   else {
9686     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9687   }
9688   // Store back in big endian from little endian
9689   rorq(sum, 0x20);
9690   movq(Address(out, offset, Address::times_4,  8), sum);
9691 
9692   movq(op1, Address(in, len, Address::times_4,  0));
9693   rorq(op1, 32);
9694   movq(sum, Address(out, offset, Address::times_4,  0));
9695   rorq(sum, 32);
9696   if (UseBMI2Instructions) {
9697     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9698   }
9699   else {
9700     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9701   }
9702   // Store back in big endian from little endian
9703   rorq(sum, 0x20);
9704   movq(Address(out, offset, Address::times_4,  0), sum);
9705 
9706   jmp(L_first_loop);
9707   bind(L_first_loop_exit);
9708 }
9709 
9710 /**
9711  * Code for BigInteger::mulAdd() intrinsic
9712  *
9713  * rdi: out
9714  * rsi: in
9715  * r11: offs (out.length - offset)
9716  * rcx: len
9717  * r8:  k
9718  * r12: tmp1
9719  * r13: tmp2
9720  * r14: tmp3
9721  * r15: tmp4
9722  * rbx: tmp5
9723  * Multiply the in[] by word k and add to out[], return the carry in rax
9724  */
9725 void MacroAssembler::mul_add(Register out, Register in, Register offs,
9726    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
9727    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9728 
9729   Label L_carry, L_last_in, L_done;
9730 
9731 // carry = 0;
9732 // for (int j=len-1; j >= 0; j--) {
9733 //    long product = (in[j] & LONG_MASK) * kLong +
9734 //                   (out[offs] & LONG_MASK) + carry;
9735 //    out[offs--] = (int)product;
9736 //    carry = product >>> 32;
9737 // }
9738 //
9739   push(tmp1);
9740   push(tmp2);
9741   push(tmp3);
9742   push(tmp4);
9743   push(tmp5);
9744 
9745   Register op2 = tmp2;
9746   const Register sum = tmp3;
9747   const Register op1 = tmp4;
9748   const Register carry =  tmp5;
9749 
9750   if (UseBMI2Instructions) {
9751     op2 = rdxReg;
9752     movl(op2, k);
9753   }
9754   else {
9755     movl(op2, k);
9756   }
9757 
9758   xorq(carry, carry);
9759 
9760   //First loop
9761 
9762   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
9763   //The carry is in tmp5
9764   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
9765 
9766   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
9767   decrementl(len);
9768   jccb(Assembler::negative, L_carry);
9769   decrementl(len);
9770   jccb(Assembler::negative, L_last_in);
9771 
9772   movq(op1, Address(in, len, Address::times_4,  0));
9773   rorq(op1, 32);
9774 
9775   subl(offs, 2);
9776   movq(sum, Address(out, offs, Address::times_4,  0));
9777   rorq(sum, 32);
9778 
9779   if (UseBMI2Instructions) {
9780     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9781   }
9782   else {
9783     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9784   }
9785 
9786   // Store back in big endian from little endian
9787   rorq(sum, 0x20);
9788   movq(Address(out, offs, Address::times_4,  0), sum);
9789 
9790   testl(len, len);
9791   jccb(Assembler::zero, L_carry);
9792 
9793   //Multiply the last in[] entry, if any
9794   bind(L_last_in);
9795   movl(op1, Address(in, 0));
9796   movl(sum, Address(out, offs, Address::times_4,  -4));
9797 
9798   movl(raxReg, k);
9799   mull(op1); //tmp4 * eax -> edx:eax
9800   addl(sum, carry);
9801   adcl(rdxReg, 0);
9802   addl(sum, raxReg);
9803   adcl(rdxReg, 0);
9804   movl(carry, rdxReg);
9805 
9806   movl(Address(out, offs, Address::times_4,  -4), sum);
9807 
9808   bind(L_carry);
9809   //return tmp5/carry as carry in rax
9810   movl(rax, carry);
9811 
9812   bind(L_done);
9813   pop(tmp5);
9814   pop(tmp4);
9815   pop(tmp3);
9816   pop(tmp2);
9817   pop(tmp1);
9818 }
9819 #endif
9820 
9821 /**
9822  * Emits code to update CRC-32 with a byte value according to constants in table
9823  *
9824  * @param [in,out]crc   Register containing the crc.
9825  * @param [in]val       Register containing the byte to fold into the CRC.
9826  * @param [in]table     Register containing the table of crc constants.
9827  *
9828  * uint32_t crc;
9829  * val = crc_table[(val ^ crc) & 0xFF];
9830  * crc = val ^ (crc >> 8);
9831  *
9832  */
9833 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
9834   xorl(val, crc);
9835   andl(val, 0xFF);
9836   shrl(crc, 8); // unsigned shift
9837   xorl(crc, Address(table, val, Address::times_4, 0));
9838 }
9839 
9840 /**
9841 * Fold four 128-bit data chunks
9842 */
9843 void MacroAssembler::fold_128bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
9844   evpclmulhdq(xtmp, xK, xcrc, Assembler::AVX_512bit); // [123:64]
9845   evpclmulldq(xcrc, xK, xcrc, Assembler::AVX_512bit); // [63:0]
9846   evpxorq(xcrc, xcrc, Address(buf, offset), Assembler::AVX_512bit /* vector_len */);
9847   evpxorq(xcrc, xcrc, xtmp, Assembler::AVX_512bit /* vector_len */);
9848 }
9849 
9850 /**
9851  * Fold 128-bit data chunk
9852  */
9853 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
9854   if (UseAVX > 0) {
9855     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
9856     vpclmulldq(xcrc, xK, xcrc); // [63:0]
9857     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
9858     pxor(xcrc, xtmp);
9859   } else {
9860     movdqa(xtmp, xcrc);
9861     pclmulhdq(xtmp, xK);   // [123:64]
9862     pclmulldq(xcrc, xK);   // [63:0]
9863     pxor(xcrc, xtmp);
9864     movdqu(xtmp, Address(buf, offset));
9865     pxor(xcrc, xtmp);
9866   }
9867 }
9868 
9869 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
9870   if (UseAVX > 0) {
9871     vpclmulhdq(xtmp, xK, xcrc);
9872     vpclmulldq(xcrc, xK, xcrc);
9873     pxor(xcrc, xbuf);
9874     pxor(xcrc, xtmp);
9875   } else {
9876     movdqa(xtmp, xcrc);
9877     pclmulhdq(xtmp, xK);
9878     pclmulldq(xcrc, xK);
9879     pxor(xcrc, xbuf);
9880     pxor(xcrc, xtmp);
9881   }
9882 }
9883 
9884 /**
9885  * 8-bit folds to compute 32-bit CRC
9886  *
9887  * uint64_t xcrc;
9888  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
9889  */
9890 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
9891   movdl(tmp, xcrc);
9892   andl(tmp, 0xFF);
9893   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
9894   psrldq(xcrc, 1); // unsigned shift one byte
9895   pxor(xcrc, xtmp);
9896 }
9897 
9898 /**
9899  * uint32_t crc;
9900  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
9901  */
9902 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
9903   movl(tmp, crc);
9904   andl(tmp, 0xFF);
9905   shrl(crc, 8);
9906   xorl(crc, Address(table, tmp, Address::times_4, 0));
9907 }
9908 
9909 /**
9910  * @param crc   register containing existing CRC (32-bit)
9911  * @param buf   register pointing to input byte buffer (byte*)
9912  * @param len   register containing number of bytes
9913  * @param table register that will contain address of CRC table
9914  * @param tmp   scratch register
9915  */
9916 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
9917   assert_different_registers(crc, buf, len, table, tmp, rax);
9918 
9919   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
9920   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
9921 
9922   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
9923   // context for the registers used, where all instructions below are using 128-bit mode
9924   // On EVEX without VL and BW, these instructions will all be AVX.
9925   if (VM_Version::supports_avx512vlbw()) {
9926     movl(tmp, 0xffff);
9927     kmovwl(k1, tmp);
9928   }
9929 
9930   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
9931   notl(crc); // ~crc
9932   cmpl(len, 16);
9933   jcc(Assembler::less, L_tail);
9934 
9935   // Align buffer to 16 bytes
9936   movl(tmp, buf);
9937   andl(tmp, 0xF);
9938   jccb(Assembler::zero, L_aligned);
9939   subl(tmp,  16);
9940   addl(len, tmp);
9941 
9942   align(4);
9943   BIND(L_align_loop);
9944   movsbl(rax, Address(buf, 0)); // load byte with sign extension
9945   update_byte_crc32(crc, rax, table);
9946   increment(buf);
9947   incrementl(tmp);
9948   jccb(Assembler::less, L_align_loop);
9949 
9950   BIND(L_aligned);
9951   movl(tmp, len); // save
9952   shrl(len, 4);
9953   jcc(Assembler::zero, L_tail_restore);
9954 
9955   // Fold total 512 bits of polynomial on each iteration
9956   if (VM_Version::supports_vpclmulqdq()) {
9957     Label Parallel_loop, L_No_Parallel;
9958 
9959     cmpl(len, 8);
9960     jccb(Assembler::less, L_No_Parallel);
9961 
9962     movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
9963     evmovdquq(xmm1, Address(buf, 0), Assembler::AVX_512bit);
9964     movdl(xmm5, crc);
9965     evpxorq(xmm1, xmm1, xmm5, Assembler::AVX_512bit);
9966     addptr(buf, 64);
9967     subl(len, 7);
9968     evshufi64x2(xmm0, xmm0, xmm0, 0x00, Assembler::AVX_512bit); //propagate the mask from 128 bits to 512 bits
9969 
9970     BIND(Parallel_loop);
9971     fold_128bit_crc32_avx512(xmm1, xmm0, xmm5, buf, 0);
9972     addptr(buf, 64);
9973     subl(len, 4);
9974     jcc(Assembler::greater, Parallel_loop);
9975 
9976     vextracti64x2(xmm2, xmm1, 0x01);
9977     vextracti64x2(xmm3, xmm1, 0x02);
9978     vextracti64x2(xmm4, xmm1, 0x03);
9979     jmp(L_fold_512b);
9980 
9981     BIND(L_No_Parallel);
9982   }
9983   // Fold crc into first bytes of vector
9984   movdqa(xmm1, Address(buf, 0));
9985   movdl(rax, xmm1);
9986   xorl(crc, rax);
9987   if (VM_Version::supports_sse4_1()) {
9988     pinsrd(xmm1, crc, 0);
9989   } else {
9990     pinsrw(xmm1, crc, 0);
9991     shrl(crc, 16);
9992     pinsrw(xmm1, crc, 1);
9993   }
9994   addptr(buf, 16);
9995   subl(len, 4); // len > 0
9996   jcc(Assembler::less, L_fold_tail);
9997 
9998   movdqa(xmm2, Address(buf,  0));
9999   movdqa(xmm3, Address(buf, 16));
10000   movdqa(xmm4, Address(buf, 32));
10001   addptr(buf, 48);
10002   subl(len, 3);
10003   jcc(Assembler::lessEqual, L_fold_512b);
10004 
10005   // Fold total 512 bits of polynomial on each iteration,
10006   // 128 bits per each of 4 parallel streams.
10007   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
10008 
10009   align(32);
10010   BIND(L_fold_512b_loop);
10011   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
10012   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
10013   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
10014   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
10015   addptr(buf, 64);
10016   subl(len, 4);
10017   jcc(Assembler::greater, L_fold_512b_loop);
10018 
10019   // Fold 512 bits to 128 bits.
10020   BIND(L_fold_512b);
10021   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
10022   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
10023   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
10024   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
10025 
10026   // Fold the rest of 128 bits data chunks
10027   BIND(L_fold_tail);
10028   addl(len, 3);
10029   jccb(Assembler::lessEqual, L_fold_128b);
10030   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
10031 
10032   BIND(L_fold_tail_loop);
10033   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
10034   addptr(buf, 16);
10035   decrementl(len);
10036   jccb(Assembler::greater, L_fold_tail_loop);
10037 
10038   // Fold 128 bits in xmm1 down into 32 bits in crc register.
10039   BIND(L_fold_128b);
10040   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()));
10041   if (UseAVX > 0) {
10042     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
10043     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
10044     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
10045   } else {
10046     movdqa(xmm2, xmm0);
10047     pclmulqdq(xmm2, xmm1, 0x1);
10048     movdqa(xmm3, xmm0);
10049     pand(xmm3, xmm2);
10050     pclmulqdq(xmm0, xmm3, 0x1);
10051   }
10052   psrldq(xmm1, 8);
10053   psrldq(xmm2, 4);
10054   pxor(xmm0, xmm1);
10055   pxor(xmm0, xmm2);
10056 
10057   // 8 8-bit folds to compute 32-bit CRC.
10058   for (int j = 0; j < 4; j++) {
10059     fold_8bit_crc32(xmm0, table, xmm1, rax);
10060   }
10061   movdl(crc, xmm0); // mov 32 bits to general register
10062   for (int j = 0; j < 4; j++) {
10063     fold_8bit_crc32(crc, table, rax);
10064   }
10065 
10066   BIND(L_tail_restore);
10067   movl(len, tmp); // restore
10068   BIND(L_tail);
10069   andl(len, 0xf);
10070   jccb(Assembler::zero, L_exit);
10071 
10072   // Fold the rest of bytes
10073   align(4);
10074   BIND(L_tail_loop);
10075   movsbl(rax, Address(buf, 0)); // load byte with sign extension
10076   update_byte_crc32(crc, rax, table);
10077   increment(buf);
10078   decrementl(len);
10079   jccb(Assembler::greater, L_tail_loop);
10080 
10081   BIND(L_exit);
10082   notl(crc); // ~c
10083 }
10084 
10085 #ifdef _LP64
10086 // S. Gueron / Information Processing Letters 112 (2012) 184
10087 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
10088 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
10089 // Output: the 64-bit carry-less product of B * CONST
10090 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
10091                                      Register tmp1, Register tmp2, Register tmp3) {
10092   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
10093   if (n > 0) {
10094     addq(tmp3, n * 256 * 8);
10095   }
10096   //    Q1 = TABLEExt[n][B & 0xFF];
10097   movl(tmp1, in);
10098   andl(tmp1, 0x000000FF);
10099   shll(tmp1, 3);
10100   addq(tmp1, tmp3);
10101   movq(tmp1, Address(tmp1, 0));
10102 
10103   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
10104   movl(tmp2, in);
10105   shrl(tmp2, 8);
10106   andl(tmp2, 0x000000FF);
10107   shll(tmp2, 3);
10108   addq(tmp2, tmp3);
10109   movq(tmp2, Address(tmp2, 0));
10110 
10111   shlq(tmp2, 8);
10112   xorq(tmp1, tmp2);
10113 
10114   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
10115   movl(tmp2, in);
10116   shrl(tmp2, 16);
10117   andl(tmp2, 0x000000FF);
10118   shll(tmp2, 3);
10119   addq(tmp2, tmp3);
10120   movq(tmp2, Address(tmp2, 0));
10121 
10122   shlq(tmp2, 16);
10123   xorq(tmp1, tmp2);
10124 
10125   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
10126   shrl(in, 24);
10127   andl(in, 0x000000FF);
10128   shll(in, 3);
10129   addq(in, tmp3);
10130   movq(in, Address(in, 0));
10131 
10132   shlq(in, 24);
10133   xorq(in, tmp1);
10134   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
10135 }
10136 
10137 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
10138                                       Register in_out,
10139                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
10140                                       XMMRegister w_xtmp2,
10141                                       Register tmp1,
10142                                       Register n_tmp2, Register n_tmp3) {
10143   if (is_pclmulqdq_supported) {
10144     movdl(w_xtmp1, in_out); // modified blindly
10145 
10146     movl(tmp1, const_or_pre_comp_const_index);
10147     movdl(w_xtmp2, tmp1);
10148     pclmulqdq(w_xtmp1, w_xtmp2, 0);
10149 
10150     movdq(in_out, w_xtmp1);
10151   } else {
10152     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
10153   }
10154 }
10155 
10156 // Recombination Alternative 2: No bit-reflections
10157 // T1 = (CRC_A * U1) << 1
10158 // T2 = (CRC_B * U2) << 1
10159 // C1 = T1 >> 32
10160 // C2 = T2 >> 32
10161 // T1 = T1 & 0xFFFFFFFF
10162 // T2 = T2 & 0xFFFFFFFF
10163 // T1 = CRC32(0, T1)
10164 // T2 = CRC32(0, T2)
10165 // C1 = C1 ^ T1
10166 // C2 = C2 ^ T2
10167 // CRC = C1 ^ C2 ^ CRC_C
10168 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
10169                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10170                                      Register tmp1, Register tmp2,
10171                                      Register n_tmp3) {
10172   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10173   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10174   shlq(in_out, 1);
10175   movl(tmp1, in_out);
10176   shrq(in_out, 32);
10177   xorl(tmp2, tmp2);
10178   crc32(tmp2, tmp1, 4);
10179   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
10180   shlq(in1, 1);
10181   movl(tmp1, in1);
10182   shrq(in1, 32);
10183   xorl(tmp2, tmp2);
10184   crc32(tmp2, tmp1, 4);
10185   xorl(in1, tmp2);
10186   xorl(in_out, in1);
10187   xorl(in_out, in2);
10188 }
10189 
10190 // Set N to predefined value
10191 // Subtract from a lenght of a buffer
10192 // execute in a loop:
10193 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
10194 // for i = 1 to N do
10195 //  CRC_A = CRC32(CRC_A, A[i])
10196 //  CRC_B = CRC32(CRC_B, B[i])
10197 //  CRC_C = CRC32(CRC_C, C[i])
10198 // end for
10199 // Recombine
10200 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
10201                                        Register in_out1, Register in_out2, Register in_out3,
10202                                        Register tmp1, Register tmp2, Register tmp3,
10203                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10204                                        Register tmp4, Register tmp5,
10205                                        Register n_tmp6) {
10206   Label L_processPartitions;
10207   Label L_processPartition;
10208   Label L_exit;
10209 
10210   bind(L_processPartitions);
10211   cmpl(in_out1, 3 * size);
10212   jcc(Assembler::less, L_exit);
10213     xorl(tmp1, tmp1);
10214     xorl(tmp2, tmp2);
10215     movq(tmp3, in_out2);
10216     addq(tmp3, size);
10217 
10218     bind(L_processPartition);
10219       crc32(in_out3, Address(in_out2, 0), 8);
10220       crc32(tmp1, Address(in_out2, size), 8);
10221       crc32(tmp2, Address(in_out2, size * 2), 8);
10222       addq(in_out2, 8);
10223       cmpq(in_out2, tmp3);
10224       jcc(Assembler::less, L_processPartition);
10225     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
10226             w_xtmp1, w_xtmp2, w_xtmp3,
10227             tmp4, tmp5,
10228             n_tmp6);
10229     addq(in_out2, 2 * size);
10230     subl(in_out1, 3 * size);
10231     jmp(L_processPartitions);
10232 
10233   bind(L_exit);
10234 }
10235 #else
10236 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
10237                                      Register tmp1, Register tmp2, Register tmp3,
10238                                      XMMRegister xtmp1, XMMRegister xtmp2) {
10239   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
10240   if (n > 0) {
10241     addl(tmp3, n * 256 * 8);
10242   }
10243   //    Q1 = TABLEExt[n][B & 0xFF];
10244   movl(tmp1, in_out);
10245   andl(tmp1, 0x000000FF);
10246   shll(tmp1, 3);
10247   addl(tmp1, tmp3);
10248   movq(xtmp1, Address(tmp1, 0));
10249 
10250   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
10251   movl(tmp2, in_out);
10252   shrl(tmp2, 8);
10253   andl(tmp2, 0x000000FF);
10254   shll(tmp2, 3);
10255   addl(tmp2, tmp3);
10256   movq(xtmp2, Address(tmp2, 0));
10257 
10258   psllq(xtmp2, 8);
10259   pxor(xtmp1, xtmp2);
10260 
10261   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
10262   movl(tmp2, in_out);
10263   shrl(tmp2, 16);
10264   andl(tmp2, 0x000000FF);
10265   shll(tmp2, 3);
10266   addl(tmp2, tmp3);
10267   movq(xtmp2, Address(tmp2, 0));
10268 
10269   psllq(xtmp2, 16);
10270   pxor(xtmp1, xtmp2);
10271 
10272   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
10273   shrl(in_out, 24);
10274   andl(in_out, 0x000000FF);
10275   shll(in_out, 3);
10276   addl(in_out, tmp3);
10277   movq(xtmp2, Address(in_out, 0));
10278 
10279   psllq(xtmp2, 24);
10280   pxor(xtmp1, xtmp2); // Result in CXMM
10281   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
10282 }
10283 
10284 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
10285                                       Register in_out,
10286                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
10287                                       XMMRegister w_xtmp2,
10288                                       Register tmp1,
10289                                       Register n_tmp2, Register n_tmp3) {
10290   if (is_pclmulqdq_supported) {
10291     movdl(w_xtmp1, in_out);
10292 
10293     movl(tmp1, const_or_pre_comp_const_index);
10294     movdl(w_xtmp2, tmp1);
10295     pclmulqdq(w_xtmp1, w_xtmp2, 0);
10296     // Keep result in XMM since GPR is 32 bit in length
10297   } else {
10298     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
10299   }
10300 }
10301 
10302 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
10303                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10304                                      Register tmp1, Register tmp2,
10305                                      Register n_tmp3) {
10306   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10307   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10308 
10309   psllq(w_xtmp1, 1);
10310   movdl(tmp1, w_xtmp1);
10311   psrlq(w_xtmp1, 32);
10312   movdl(in_out, w_xtmp1);
10313 
10314   xorl(tmp2, tmp2);
10315   crc32(tmp2, tmp1, 4);
10316   xorl(in_out, tmp2);
10317 
10318   psllq(w_xtmp2, 1);
10319   movdl(tmp1, w_xtmp2);
10320   psrlq(w_xtmp2, 32);
10321   movdl(in1, w_xtmp2);
10322 
10323   xorl(tmp2, tmp2);
10324   crc32(tmp2, tmp1, 4);
10325   xorl(in1, tmp2);
10326   xorl(in_out, in1);
10327   xorl(in_out, in2);
10328 }
10329 
10330 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
10331                                        Register in_out1, Register in_out2, Register in_out3,
10332                                        Register tmp1, Register tmp2, Register tmp3,
10333                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10334                                        Register tmp4, Register tmp5,
10335                                        Register n_tmp6) {
10336   Label L_processPartitions;
10337   Label L_processPartition;
10338   Label L_exit;
10339 
10340   bind(L_processPartitions);
10341   cmpl(in_out1, 3 * size);
10342   jcc(Assembler::less, L_exit);
10343     xorl(tmp1, tmp1);
10344     xorl(tmp2, tmp2);
10345     movl(tmp3, in_out2);
10346     addl(tmp3, size);
10347 
10348     bind(L_processPartition);
10349       crc32(in_out3, Address(in_out2, 0), 4);
10350       crc32(tmp1, Address(in_out2, size), 4);
10351       crc32(tmp2, Address(in_out2, size*2), 4);
10352       crc32(in_out3, Address(in_out2, 0+4), 4);
10353       crc32(tmp1, Address(in_out2, size+4), 4);
10354       crc32(tmp2, Address(in_out2, size*2+4), 4);
10355       addl(in_out2, 8);
10356       cmpl(in_out2, tmp3);
10357       jcc(Assembler::less, L_processPartition);
10358 
10359         push(tmp3);
10360         push(in_out1);
10361         push(in_out2);
10362         tmp4 = tmp3;
10363         tmp5 = in_out1;
10364         n_tmp6 = in_out2;
10365 
10366       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
10367             w_xtmp1, w_xtmp2, w_xtmp3,
10368             tmp4, tmp5,
10369             n_tmp6);
10370 
10371         pop(in_out2);
10372         pop(in_out1);
10373         pop(tmp3);
10374 
10375     addl(in_out2, 2 * size);
10376     subl(in_out1, 3 * size);
10377     jmp(L_processPartitions);
10378 
10379   bind(L_exit);
10380 }
10381 #endif //LP64
10382 
10383 #ifdef _LP64
10384 // Algorithm 2: Pipelined usage of the CRC32 instruction.
10385 // Input: A buffer I of L bytes.
10386 // Output: the CRC32C value of the buffer.
10387 // Notations:
10388 // Write L = 24N + r, with N = floor (L/24).
10389 // r = L mod 24 (0 <= r < 24).
10390 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
10391 // N quadwords, and R consists of r bytes.
10392 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
10393 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
10394 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
10395 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
10396 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10397                                           Register tmp1, Register tmp2, Register tmp3,
10398                                           Register tmp4, Register tmp5, Register tmp6,
10399                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10400                                           bool is_pclmulqdq_supported) {
10401   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10402   Label L_wordByWord;
10403   Label L_byteByByteProlog;
10404   Label L_byteByByte;
10405   Label L_exit;
10406 
10407   if (is_pclmulqdq_supported ) {
10408     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10409     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
10410 
10411     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10412     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10413 
10414     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10415     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10416     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
10417   } else {
10418     const_or_pre_comp_const_index[0] = 1;
10419     const_or_pre_comp_const_index[1] = 0;
10420 
10421     const_or_pre_comp_const_index[2] = 3;
10422     const_or_pre_comp_const_index[3] = 2;
10423 
10424     const_or_pre_comp_const_index[4] = 5;
10425     const_or_pre_comp_const_index[5] = 4;
10426    }
10427   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10428                     in2, in1, in_out,
10429                     tmp1, tmp2, tmp3,
10430                     w_xtmp1, w_xtmp2, w_xtmp3,
10431                     tmp4, tmp5,
10432                     tmp6);
10433   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10434                     in2, in1, in_out,
10435                     tmp1, tmp2, tmp3,
10436                     w_xtmp1, w_xtmp2, w_xtmp3,
10437                     tmp4, tmp5,
10438                     tmp6);
10439   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10440                     in2, in1, in_out,
10441                     tmp1, tmp2, tmp3,
10442                     w_xtmp1, w_xtmp2, w_xtmp3,
10443                     tmp4, tmp5,
10444                     tmp6);
10445   movl(tmp1, in2);
10446   andl(tmp1, 0x00000007);
10447   negl(tmp1);
10448   addl(tmp1, in2);
10449   addq(tmp1, in1);
10450 
10451   BIND(L_wordByWord);
10452   cmpq(in1, tmp1);
10453   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10454     crc32(in_out, Address(in1, 0), 4);
10455     addq(in1, 4);
10456     jmp(L_wordByWord);
10457 
10458   BIND(L_byteByByteProlog);
10459   andl(in2, 0x00000007);
10460   movl(tmp2, 1);
10461 
10462   BIND(L_byteByByte);
10463   cmpl(tmp2, in2);
10464   jccb(Assembler::greater, L_exit);
10465     crc32(in_out, Address(in1, 0), 1);
10466     incq(in1);
10467     incl(tmp2);
10468     jmp(L_byteByByte);
10469 
10470   BIND(L_exit);
10471 }
10472 #else
10473 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10474                                           Register tmp1, Register  tmp2, Register tmp3,
10475                                           Register tmp4, Register  tmp5, Register tmp6,
10476                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10477                                           bool is_pclmulqdq_supported) {
10478   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10479   Label L_wordByWord;
10480   Label L_byteByByteProlog;
10481   Label L_byteByByte;
10482   Label L_exit;
10483 
10484   if (is_pclmulqdq_supported) {
10485     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10486     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
10487 
10488     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10489     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10490 
10491     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10492     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10493   } else {
10494     const_or_pre_comp_const_index[0] = 1;
10495     const_or_pre_comp_const_index[1] = 0;
10496 
10497     const_or_pre_comp_const_index[2] = 3;
10498     const_or_pre_comp_const_index[3] = 2;
10499 
10500     const_or_pre_comp_const_index[4] = 5;
10501     const_or_pre_comp_const_index[5] = 4;
10502   }
10503   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10504                     in2, in1, in_out,
10505                     tmp1, tmp2, tmp3,
10506                     w_xtmp1, w_xtmp2, w_xtmp3,
10507                     tmp4, tmp5,
10508                     tmp6);
10509   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10510                     in2, in1, in_out,
10511                     tmp1, tmp2, tmp3,
10512                     w_xtmp1, w_xtmp2, w_xtmp3,
10513                     tmp4, tmp5,
10514                     tmp6);
10515   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10516                     in2, in1, in_out,
10517                     tmp1, tmp2, tmp3,
10518                     w_xtmp1, w_xtmp2, w_xtmp3,
10519                     tmp4, tmp5,
10520                     tmp6);
10521   movl(tmp1, in2);
10522   andl(tmp1, 0x00000007);
10523   negl(tmp1);
10524   addl(tmp1, in2);
10525   addl(tmp1, in1);
10526 
10527   BIND(L_wordByWord);
10528   cmpl(in1, tmp1);
10529   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10530     crc32(in_out, Address(in1,0), 4);
10531     addl(in1, 4);
10532     jmp(L_wordByWord);
10533 
10534   BIND(L_byteByByteProlog);
10535   andl(in2, 0x00000007);
10536   movl(tmp2, 1);
10537 
10538   BIND(L_byteByByte);
10539   cmpl(tmp2, in2);
10540   jccb(Assembler::greater, L_exit);
10541     movb(tmp1, Address(in1, 0));
10542     crc32(in_out, tmp1, 1);
10543     incl(in1);
10544     incl(tmp2);
10545     jmp(L_byteByByte);
10546 
10547   BIND(L_exit);
10548 }
10549 #endif // LP64
10550 #undef BIND
10551 #undef BLOCK_COMMENT
10552 
10553 // Compress char[] array to byte[].
10554 //   ..\jdk\src\java.base\share\classes\java\lang\StringUTF16.java
10555 //   @HotSpotIntrinsicCandidate
10556 //   private static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
10557 //     for (int i = 0; i < len; i++) {
10558 //       int c = src[srcOff++];
10559 //       if (c >>> 8 != 0) {
10560 //         return 0;
10561 //       }
10562 //       dst[dstOff++] = (byte)c;
10563 //     }
10564 //     return len;
10565 //   }
10566 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
10567   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
10568   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
10569   Register tmp5, Register result) {
10570   Label copy_chars_loop, return_length, return_zero, done, below_threshold;
10571 
10572   // rsi: src
10573   // rdi: dst
10574   // rdx: len
10575   // rcx: tmp5
10576   // rax: result
10577 
10578   // rsi holds start addr of source char[] to be compressed
10579   // rdi holds start addr of destination byte[]
10580   // rdx holds length
10581 
10582   assert(len != result, "");
10583 
10584   // save length for return
10585   push(len);
10586 
10587   if ((UseAVX > 2) && // AVX512
10588     VM_Version::supports_avx512vlbw() &&
10589     VM_Version::supports_bmi2()) {
10590 
10591     set_vector_masking();  // opening of the stub context for programming mask registers
10592 
10593     Label copy_32_loop, copy_loop_tail, restore_k1_return_zero;
10594 
10595     // alignement
10596     Label post_alignement;
10597 
10598     // if length of the string is less than 16, handle it in an old fashioned
10599     // way
10600     testl(len, -32);
10601     jcc(Assembler::zero, below_threshold);
10602 
10603     // First check whether a character is compressable ( <= 0xFF).
10604     // Create mask to test for Unicode chars inside zmm vector
10605     movl(result, 0x00FF);
10606     evpbroadcastw(tmp2Reg, result, Assembler::AVX_512bit);
10607 
10608     // Save k1
10609     kmovql(k3, k1);
10610 
10611     testl(len, -64);
10612     jcc(Assembler::zero, post_alignement);
10613 
10614     movl(tmp5, dst);
10615     andl(tmp5, (32 - 1));
10616     negl(tmp5);
10617     andl(tmp5, (32 - 1));
10618 
10619     // bail out when there is nothing to be done
10620     testl(tmp5, 0xFFFFFFFF);
10621     jcc(Assembler::zero, post_alignement);
10622 
10623     // ~(~0 << len), where len is the # of remaining elements to process
10624     movl(result, 0xFFFFFFFF);
10625     shlxl(result, result, tmp5);
10626     notl(result);
10627     kmovdl(k1, result);
10628 
10629     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
10630     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10631     ktestd(k2, k1);
10632     jcc(Assembler::carryClear, restore_k1_return_zero);
10633 
10634     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
10635 
10636     addptr(src, tmp5);
10637     addptr(src, tmp5);
10638     addptr(dst, tmp5);
10639     subl(len, tmp5);
10640 
10641     bind(post_alignement);
10642     // end of alignement
10643 
10644     movl(tmp5, len);
10645     andl(tmp5, (32 - 1));    // tail count (in chars)
10646     andl(len, ~(32 - 1));    // vector count (in chars)
10647     jcc(Assembler::zero, copy_loop_tail);
10648 
10649     lea(src, Address(src, len, Address::times_2));
10650     lea(dst, Address(dst, len, Address::times_1));
10651     negptr(len);
10652 
10653     bind(copy_32_loop);
10654     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
10655     evpcmpuw(k2, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10656     kortestdl(k2, k2);
10657     jcc(Assembler::carryClear, restore_k1_return_zero);
10658 
10659     // All elements in current processed chunk are valid candidates for
10660     // compression. Write a truncated byte elements to the memory.
10661     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
10662     addptr(len, 32);
10663     jcc(Assembler::notZero, copy_32_loop);
10664 
10665     bind(copy_loop_tail);
10666     // bail out when there is nothing to be done
10667     testl(tmp5, 0xFFFFFFFF);
10668     // Restore k1
10669     kmovql(k1, k3);
10670     jcc(Assembler::zero, return_length);
10671 
10672     movl(len, tmp5);
10673 
10674     // ~(~0 << len), where len is the # of remaining elements to process
10675     movl(result, 0xFFFFFFFF);
10676     shlxl(result, result, len);
10677     notl(result);
10678 
10679     kmovdl(k1, result);
10680 
10681     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
10682     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10683     ktestd(k2, k1);
10684     jcc(Assembler::carryClear, restore_k1_return_zero);
10685 
10686     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
10687     // Restore k1
10688     kmovql(k1, k3);
10689     jmp(return_length);
10690 
10691     bind(restore_k1_return_zero);
10692     // Restore k1
10693     kmovql(k1, k3);
10694     jmp(return_zero);
10695 
10696     clear_vector_masking();   // closing of the stub context for programming mask registers
10697   }
10698   if (UseSSE42Intrinsics) {
10699     Label copy_32_loop, copy_16, copy_tail;
10700 
10701     bind(below_threshold);
10702 
10703     movl(result, len);
10704 
10705     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
10706 
10707     // vectored compression
10708     andl(len, 0xfffffff0);    // vector count (in chars)
10709     andl(result, 0x0000000f);    // tail count (in chars)
10710     testl(len, len);
10711     jccb(Assembler::zero, copy_16);
10712 
10713     // compress 16 chars per iter
10714     movdl(tmp1Reg, tmp5);
10715     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
10716     pxor(tmp4Reg, tmp4Reg);
10717 
10718     lea(src, Address(src, len, Address::times_2));
10719     lea(dst, Address(dst, len, Address::times_1));
10720     negptr(len);
10721 
10722     bind(copy_32_loop);
10723     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
10724     por(tmp4Reg, tmp2Reg);
10725     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
10726     por(tmp4Reg, tmp3Reg);
10727     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
10728     jcc(Assembler::notZero, return_zero);
10729     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
10730     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
10731     addptr(len, 16);
10732     jcc(Assembler::notZero, copy_32_loop);
10733 
10734     // compress next vector of 8 chars (if any)
10735     bind(copy_16);
10736     movl(len, result);
10737     andl(len, 0xfffffff8);    // vector count (in chars)
10738     andl(result, 0x00000007);    // tail count (in chars)
10739     testl(len, len);
10740     jccb(Assembler::zero, copy_tail);
10741 
10742     movdl(tmp1Reg, tmp5);
10743     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
10744     pxor(tmp3Reg, tmp3Reg);
10745 
10746     movdqu(tmp2Reg, Address(src, 0));
10747     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
10748     jccb(Assembler::notZero, return_zero);
10749     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
10750     movq(Address(dst, 0), tmp2Reg);
10751     addptr(src, 16);
10752     addptr(dst, 8);
10753 
10754     bind(copy_tail);
10755     movl(len, result);
10756   }
10757   // compress 1 char per iter
10758   testl(len, len);
10759   jccb(Assembler::zero, return_length);
10760   lea(src, Address(src, len, Address::times_2));
10761   lea(dst, Address(dst, len, Address::times_1));
10762   negptr(len);
10763 
10764   bind(copy_chars_loop);
10765   load_unsigned_short(result, Address(src, len, Address::times_2));
10766   testl(result, 0xff00);      // check if Unicode char
10767   jccb(Assembler::notZero, return_zero);
10768   movb(Address(dst, len, Address::times_1), result);  // ASCII char; compress to 1 byte
10769   increment(len);
10770   jcc(Assembler::notZero, copy_chars_loop);
10771 
10772   // if compression succeeded, return length
10773   bind(return_length);
10774   pop(result);
10775   jmpb(done);
10776 
10777   // if compression failed, return 0
10778   bind(return_zero);
10779   xorl(result, result);
10780   addptr(rsp, wordSize);
10781 
10782   bind(done);
10783 }
10784 
10785 // Inflate byte[] array to char[].
10786 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
10787 //   @HotSpotIntrinsicCandidate
10788 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
10789 //     for (int i = 0; i < len; i++) {
10790 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
10791 //     }
10792 //   }
10793 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
10794   XMMRegister tmp1, Register tmp2) {
10795   Label copy_chars_loop, done, below_threshold;
10796   // rsi: src
10797   // rdi: dst
10798   // rdx: len
10799   // rcx: tmp2
10800 
10801   // rsi holds start addr of source byte[] to be inflated
10802   // rdi holds start addr of destination char[]
10803   // rdx holds length
10804   assert_different_registers(src, dst, len, tmp2);
10805 
10806   if ((UseAVX > 2) && // AVX512
10807     VM_Version::supports_avx512vlbw() &&
10808     VM_Version::supports_bmi2()) {
10809 
10810     set_vector_masking();  // opening of the stub context for programming mask registers
10811 
10812     Label copy_32_loop, copy_tail;
10813     Register tmp3_aliased = len;
10814 
10815     // if length of the string is less than 16, handle it in an old fashioned
10816     // way
10817     testl(len, -16);
10818     jcc(Assembler::zero, below_threshold);
10819 
10820     // In order to use only one arithmetic operation for the main loop we use
10821     // this pre-calculation
10822     movl(tmp2, len);
10823     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
10824     andl(len, -32);     // vector count
10825     jccb(Assembler::zero, copy_tail);
10826 
10827     lea(src, Address(src, len, Address::times_1));
10828     lea(dst, Address(dst, len, Address::times_2));
10829     negptr(len);
10830 
10831 
10832     // inflate 32 chars per iter
10833     bind(copy_32_loop);
10834     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
10835     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
10836     addptr(len, 32);
10837     jcc(Assembler::notZero, copy_32_loop);
10838 
10839     bind(copy_tail);
10840     // bail out when there is nothing to be done
10841     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
10842     jcc(Assembler::zero, done);
10843 
10844     // Save k1
10845     kmovql(k2, k1);
10846 
10847     // ~(~0 << length), where length is the # of remaining elements to process
10848     movl(tmp3_aliased, -1);
10849     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
10850     notl(tmp3_aliased);
10851     kmovdl(k1, tmp3_aliased);
10852     evpmovzxbw(tmp1, k1, Address(src, 0), Assembler::AVX_512bit);
10853     evmovdquw(Address(dst, 0), k1, tmp1, Assembler::AVX_512bit);
10854 
10855     // Restore k1
10856     kmovql(k1, k2);
10857     jmp(done);
10858 
10859     clear_vector_masking();   // closing of the stub context for programming mask registers
10860   }
10861   if (UseSSE42Intrinsics) {
10862     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
10863 
10864     movl(tmp2, len);
10865 
10866     if (UseAVX > 1) {
10867       andl(tmp2, (16 - 1));
10868       andl(len, -16);
10869       jccb(Assembler::zero, copy_new_tail);
10870     } else {
10871       andl(tmp2, 0x00000007);   // tail count (in chars)
10872       andl(len, 0xfffffff8);    // vector count (in chars)
10873       jccb(Assembler::zero, copy_tail);
10874     }
10875 
10876     // vectored inflation
10877     lea(src, Address(src, len, Address::times_1));
10878     lea(dst, Address(dst, len, Address::times_2));
10879     negptr(len);
10880 
10881     if (UseAVX > 1) {
10882       bind(copy_16_loop);
10883       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
10884       vmovdqu(Address(dst, len, Address::times_2), tmp1);
10885       addptr(len, 16);
10886       jcc(Assembler::notZero, copy_16_loop);
10887 
10888       bind(below_threshold);
10889       bind(copy_new_tail);
10890       if ((UseAVX > 2) &&
10891         VM_Version::supports_avx512vlbw() &&
10892         VM_Version::supports_bmi2()) {
10893         movl(tmp2, len);
10894       } else {
10895         movl(len, tmp2);
10896       }
10897       andl(tmp2, 0x00000007);
10898       andl(len, 0xFFFFFFF8);
10899       jccb(Assembler::zero, copy_tail);
10900 
10901       pmovzxbw(tmp1, Address(src, 0));
10902       movdqu(Address(dst, 0), tmp1);
10903       addptr(src, 8);
10904       addptr(dst, 2 * 8);
10905 
10906       jmp(copy_tail, true);
10907     }
10908 
10909     // inflate 8 chars per iter
10910     bind(copy_8_loop);
10911     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
10912     movdqu(Address(dst, len, Address::times_2), tmp1);
10913     addptr(len, 8);
10914     jcc(Assembler::notZero, copy_8_loop);
10915 
10916     bind(copy_tail);
10917     movl(len, tmp2);
10918 
10919     cmpl(len, 4);
10920     jccb(Assembler::less, copy_bytes);
10921 
10922     movdl(tmp1, Address(src, 0));  // load 4 byte chars
10923     pmovzxbw(tmp1, tmp1);
10924     movq(Address(dst, 0), tmp1);
10925     subptr(len, 4);
10926     addptr(src, 4);
10927     addptr(dst, 8);
10928 
10929     bind(copy_bytes);
10930   }
10931   testl(len, len);
10932   jccb(Assembler::zero, done);
10933   lea(src, Address(src, len, Address::times_1));
10934   lea(dst, Address(dst, len, Address::times_2));
10935   negptr(len);
10936 
10937   // inflate 1 char per iter
10938   bind(copy_chars_loop);
10939   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
10940   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
10941   increment(len);
10942   jcc(Assembler::notZero, copy_chars_loop);
10943 
10944   bind(done);
10945 }
10946 
10947 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
10948   switch (cond) {
10949     // Note some conditions are synonyms for others
10950     case Assembler::zero:         return Assembler::notZero;
10951     case Assembler::notZero:      return Assembler::zero;
10952     case Assembler::less:         return Assembler::greaterEqual;
10953     case Assembler::lessEqual:    return Assembler::greater;
10954     case Assembler::greater:      return Assembler::lessEqual;
10955     case Assembler::greaterEqual: return Assembler::less;
10956     case Assembler::below:        return Assembler::aboveEqual;
10957     case Assembler::belowEqual:   return Assembler::above;
10958     case Assembler::above:        return Assembler::belowEqual;
10959     case Assembler::aboveEqual:   return Assembler::below;
10960     case Assembler::overflow:     return Assembler::noOverflow;
10961     case Assembler::noOverflow:   return Assembler::overflow;
10962     case Assembler::negative:     return Assembler::positive;
10963     case Assembler::positive:     return Assembler::negative;
10964     case Assembler::parity:       return Assembler::noParity;
10965     case Assembler::noParity:     return Assembler::parity;
10966   }
10967   ShouldNotReachHere(); return Assembler::overflow;
10968 }
10969 
10970 SkipIfEqual::SkipIfEqual(
10971     MacroAssembler* masm, const bool* flag_addr, bool value) {
10972   _masm = masm;
10973   _masm->cmp8(ExternalAddress((address)flag_addr), value);
10974   _masm->jcc(Assembler::equal, _label);
10975 }
10976 
10977 SkipIfEqual::~SkipIfEqual() {
10978   _masm->bind(_label);
10979 }
10980 
10981 // 32-bit Windows has its own fast-path implementation
10982 // of get_thread
10983 #if !defined(WIN32) || defined(_LP64)
10984 
10985 // This is simply a call to Thread::current()
10986 void MacroAssembler::get_thread(Register thread) {
10987   if (thread != rax) {
10988     push(rax);
10989   }
10990   LP64_ONLY(push(rdi);)
10991   LP64_ONLY(push(rsi);)
10992   push(rdx);
10993   push(rcx);
10994 #ifdef _LP64
10995   push(r8);
10996   push(r9);
10997   push(r10);
10998   push(r11);
10999 #endif
11000 
11001   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
11002 
11003 #ifdef _LP64
11004   pop(r11);
11005   pop(r10);
11006   pop(r9);
11007   pop(r8);
11008 #endif
11009   pop(rcx);
11010   pop(rdx);
11011   LP64_ONLY(pop(rsi);)
11012   LP64_ONLY(pop(rdi);)
11013   if (thread != rax) {
11014     mov(thread, rax);
11015     pop(rax);
11016   }
11017 }
11018 
11019 #endif