1 /*
   2  * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "asm/assembler.hpp"
  28 #include "asm/assembler.inline.hpp"
  29 #include "compiler/disassembler.hpp"
  30 #include "gc/shared/barrierSet.hpp"
  31 #include "gc/shared/barrierSetAssembler.hpp"
  32 #include "gc/shared/collectedHeap.inline.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "memory/universe.hpp"
  36 #include "oops/accessDecorators.hpp"
  37 #include "oops/klass.inline.hpp"
  38 #include "prims/methodHandles.hpp"
  39 #include "runtime/biasedLocking.hpp"
  40 #include "runtime/flags/flagSetting.hpp"
  41 #include "runtime/interfaceSupport.inline.hpp"
  42 #include "runtime/objectMonitor.hpp"
  43 #include "runtime/os.hpp"
  44 #include "runtime/safepoint.hpp"
  45 #include "runtime/safepointMechanism.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/stubRoutines.hpp"
  48 #include "runtime/thread.hpp"
  49 #include "utilities/macros.hpp"
  50 #include "crc32c.h"
  51 #ifdef COMPILER2
  52 #include "opto/intrinsicnode.hpp"
  53 #endif
  54 
  55 #ifdef PRODUCT
  56 #define BLOCK_COMMENT(str) /* nothing */
  57 #define STOP(error) stop(error)
  58 #else
  59 #define BLOCK_COMMENT(str) block_comment(str)
  60 #define STOP(error) block_comment(error); stop(error)
  61 #endif
  62 
  63 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  64 
  65 #ifdef ASSERT
  66 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  67 #endif
  68 
  69 static Assembler::Condition reverse[] = {
  70     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  71     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  72     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  73     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  74     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
  75     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
  76     Assembler::above          /* belowEqual    = 0x6 */ ,
  77     Assembler::belowEqual     /* above         = 0x7 */ ,
  78     Assembler::positive       /* negative      = 0x8 */ ,
  79     Assembler::negative       /* positive      = 0x9 */ ,
  80     Assembler::noParity       /* parity        = 0xa */ ,
  81     Assembler::parity         /* noParity      = 0xb */ ,
  82     Assembler::greaterEqual   /* less          = 0xc */ ,
  83     Assembler::less           /* greaterEqual  = 0xd */ ,
  84     Assembler::greater        /* lessEqual     = 0xe */ ,
  85     Assembler::lessEqual      /* greater       = 0xf, */
  86 
  87 };
  88 
  89 
  90 // Implementation of MacroAssembler
  91 
  92 // First all the versions that have distinct versions depending on 32/64 bit
  93 // Unless the difference is trivial (1 line or so).
  94 
  95 #ifndef _LP64
  96 
  97 // 32bit versions
  98 
  99 Address MacroAssembler::as_Address(AddressLiteral adr) {
 100   return Address(adr.target(), adr.rspec());
 101 }
 102 
 103 Address MacroAssembler::as_Address(ArrayAddress adr) {
 104   return Address::make_array(adr);
 105 }
 106 
 107 void MacroAssembler::call_VM_leaf_base(address entry_point,
 108                                        int number_of_arguments) {
 109   call(RuntimeAddress(entry_point));
 110   increment(rsp, number_of_arguments * wordSize);
 111 }
 112 
 113 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
 114   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 115 }
 116 
 117 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
 118   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 119 }
 120 
 121 void MacroAssembler::cmpoop_raw(Address src1, jobject obj) {
 122   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 123 }
 124 
 125 void MacroAssembler::cmpoop_raw(Register src1, jobject obj) {
 126   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 127 }
 128 
 129 void MacroAssembler::cmpoop(Address src1, jobject obj) {
 130   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 131   bs->obj_equals(this, src1, obj);
 132 }
 133 
 134 void MacroAssembler::cmpoop(Register src1, jobject obj) {
 135   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 136   bs->obj_equals(this, src1, obj);
 137 }
 138 
 139 void MacroAssembler::extend_sign(Register hi, Register lo) {
 140   // According to Intel Doc. AP-526, "Integer Divide", p.18.
 141   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
 142     cdql();
 143   } else {
 144     movl(hi, lo);
 145     sarl(hi, 31);
 146   }
 147 }
 148 
 149 void MacroAssembler::jC2(Register tmp, Label& L) {
 150   // set parity bit if FPU flag C2 is set (via rax)
 151   save_rax(tmp);
 152   fwait(); fnstsw_ax();
 153   sahf();
 154   restore_rax(tmp);
 155   // branch
 156   jcc(Assembler::parity, L);
 157 }
 158 
 159 void MacroAssembler::jnC2(Register tmp, Label& L) {
 160   // set parity bit if FPU flag C2 is set (via rax)
 161   save_rax(tmp);
 162   fwait(); fnstsw_ax();
 163   sahf();
 164   restore_rax(tmp);
 165   // branch
 166   jcc(Assembler::noParity, L);
 167 }
 168 
 169 // 32bit can do a case table jump in one instruction but we no longer allow the base
 170 // to be installed in the Address class
 171 void MacroAssembler::jump(ArrayAddress entry) {
 172   jmp(as_Address(entry));
 173 }
 174 
 175 // Note: y_lo will be destroyed
 176 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 177   // Long compare for Java (semantics as described in JVM spec.)
 178   Label high, low, done;
 179 
 180   cmpl(x_hi, y_hi);
 181   jcc(Assembler::less, low);
 182   jcc(Assembler::greater, high);
 183   // x_hi is the return register
 184   xorl(x_hi, x_hi);
 185   cmpl(x_lo, y_lo);
 186   jcc(Assembler::below, low);
 187   jcc(Assembler::equal, done);
 188 
 189   bind(high);
 190   xorl(x_hi, x_hi);
 191   increment(x_hi);
 192   jmp(done);
 193 
 194   bind(low);
 195   xorl(x_hi, x_hi);
 196   decrementl(x_hi);
 197 
 198   bind(done);
 199 }
 200 
 201 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 202     mov_literal32(dst, (int32_t)src.target(), src.rspec());
 203 }
 204 
 205 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 206   // leal(dst, as_Address(adr));
 207   // see note in movl as to why we must use a move
 208   mov_literal32(dst, (int32_t) adr.target(), adr.rspec());
 209 }
 210 
 211 void MacroAssembler::leave() {
 212   mov(rsp, rbp);
 213   pop(rbp);
 214 }
 215 
 216 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
 217   // Multiplication of two Java long values stored on the stack
 218   // as illustrated below. Result is in rdx:rax.
 219   //
 220   // rsp ---> [  ??  ] \               \
 221   //            ....    | y_rsp_offset  |
 222   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
 223   //          [ y_hi ]                  | (in bytes)
 224   //            ....                    |
 225   //          [ x_lo ]                 /
 226   //          [ x_hi ]
 227   //            ....
 228   //
 229   // Basic idea: lo(result) = lo(x_lo * y_lo)
 230   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
 231   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
 232   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
 233   Label quick;
 234   // load x_hi, y_hi and check if quick
 235   // multiplication is possible
 236   movl(rbx, x_hi);
 237   movl(rcx, y_hi);
 238   movl(rax, rbx);
 239   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
 240   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
 241   // do full multiplication
 242   // 1st step
 243   mull(y_lo);                                    // x_hi * y_lo
 244   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
 245   // 2nd step
 246   movl(rax, x_lo);
 247   mull(rcx);                                     // x_lo * y_hi
 248   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
 249   // 3rd step
 250   bind(quick);                                   // note: rbx, = 0 if quick multiply!
 251   movl(rax, x_lo);
 252   mull(y_lo);                                    // x_lo * y_lo
 253   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
 254 }
 255 
 256 void MacroAssembler::lneg(Register hi, Register lo) {
 257   negl(lo);
 258   adcl(hi, 0);
 259   negl(hi);
 260 }
 261 
 262 void MacroAssembler::lshl(Register hi, Register lo) {
 263   // Java shift left long support (semantics as described in JVM spec., p.305)
 264   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
 265   // shift value is in rcx !
 266   assert(hi != rcx, "must not use rcx");
 267   assert(lo != rcx, "must not use rcx");
 268   const Register s = rcx;                        // shift count
 269   const int      n = BitsPerWord;
 270   Label L;
 271   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 272   cmpl(s, n);                                    // if (s < n)
 273   jcc(Assembler::less, L);                       // else (s >= n)
 274   movl(hi, lo);                                  // x := x << n
 275   xorl(lo, lo);
 276   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 277   bind(L);                                       // s (mod n) < n
 278   shldl(hi, lo);                                 // x := x << s
 279   shll(lo);
 280 }
 281 
 282 
 283 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
 284   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
 285   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
 286   assert(hi != rcx, "must not use rcx");
 287   assert(lo != rcx, "must not use rcx");
 288   const Register s = rcx;                        // shift count
 289   const int      n = BitsPerWord;
 290   Label L;
 291   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 292   cmpl(s, n);                                    // if (s < n)
 293   jcc(Assembler::less, L);                       // else (s >= n)
 294   movl(lo, hi);                                  // x := x >> n
 295   if (sign_extension) sarl(hi, 31);
 296   else                xorl(hi, hi);
 297   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 298   bind(L);                                       // s (mod n) < n
 299   shrdl(lo, hi);                                 // x := x >> s
 300   if (sign_extension) sarl(hi);
 301   else                shrl(hi);
 302 }
 303 
 304 void MacroAssembler::movoop(Register dst, jobject obj) {
 305   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 306 }
 307 
 308 void MacroAssembler::movoop(Address dst, jobject obj) {
 309   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 310 }
 311 
 312 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 313   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 314 }
 315 
 316 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 317   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 318 }
 319 
 320 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 321   // scratch register is not used,
 322   // it is defined to match parameters of 64-bit version of this method.
 323   if (src.is_lval()) {
 324     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
 325   } else {
 326     movl(dst, as_Address(src));
 327   }
 328 }
 329 
 330 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 331   movl(as_Address(dst), src);
 332 }
 333 
 334 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 335   movl(dst, as_Address(src));
 336 }
 337 
 338 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 339 void MacroAssembler::movptr(Address dst, intptr_t src) {
 340   movl(dst, src);
 341 }
 342 
 343 
 344 void MacroAssembler::pop_callee_saved_registers() {
 345   pop(rcx);
 346   pop(rdx);
 347   pop(rdi);
 348   pop(rsi);
 349 }
 350 
 351 void MacroAssembler::pop_fTOS() {
 352   fld_d(Address(rsp, 0));
 353   addl(rsp, 2 * wordSize);
 354 }
 355 
 356 void MacroAssembler::push_callee_saved_registers() {
 357   push(rsi);
 358   push(rdi);
 359   push(rdx);
 360   push(rcx);
 361 }
 362 
 363 void MacroAssembler::push_fTOS() {
 364   subl(rsp, 2 * wordSize);
 365   fstp_d(Address(rsp, 0));
 366 }
 367 
 368 
 369 void MacroAssembler::pushoop(jobject obj) {
 370   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
 371 }
 372 
 373 void MacroAssembler::pushklass(Metadata* obj) {
 374   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
 375 }
 376 
 377 void MacroAssembler::pushptr(AddressLiteral src) {
 378   if (src.is_lval()) {
 379     push_literal32((int32_t)src.target(), src.rspec());
 380   } else {
 381     pushl(as_Address(src));
 382   }
 383 }
 384 
 385 void MacroAssembler::set_word_if_not_zero(Register dst) {
 386   xorl(dst, dst);
 387   set_byte_if_not_zero(dst);
 388 }
 389 
 390 static void pass_arg0(MacroAssembler* masm, Register arg) {
 391   masm->push(arg);
 392 }
 393 
 394 static void pass_arg1(MacroAssembler* masm, Register arg) {
 395   masm->push(arg);
 396 }
 397 
 398 static void pass_arg2(MacroAssembler* masm, Register arg) {
 399   masm->push(arg);
 400 }
 401 
 402 static void pass_arg3(MacroAssembler* masm, Register arg) {
 403   masm->push(arg);
 404 }
 405 
 406 #ifndef PRODUCT
 407 extern "C" void findpc(intptr_t x);
 408 #endif
 409 
 410 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
 411   // In order to get locks to work, we need to fake a in_VM state
 412   JavaThread* thread = JavaThread::current();
 413   JavaThreadState saved_state = thread->thread_state();
 414   thread->set_thread_state(_thread_in_vm);
 415   if (ShowMessageBoxOnError) {
 416     JavaThread* thread = JavaThread::current();
 417     JavaThreadState saved_state = thread->thread_state();
 418     thread->set_thread_state(_thread_in_vm);
 419     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 420       ttyLocker ttyl;
 421       BytecodeCounter::print();
 422     }
 423     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 424     // This is the value of eip which points to where verify_oop will return.
 425     if (os::message_box(msg, "Execution stopped, print registers?")) {
 426       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
 427       BREAKPOINT;
 428     }
 429   } else {
 430     ttyLocker ttyl;
 431     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
 432   }
 433   // Don't assert holding the ttyLock
 434     assert(false, "DEBUG MESSAGE: %s", msg);
 435   ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 436 }
 437 
 438 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
 439   ttyLocker ttyl;
 440   FlagSetting fs(Debugging, true);
 441   tty->print_cr("eip = 0x%08x", eip);
 442 #ifndef PRODUCT
 443   if ((WizardMode || Verbose) && PrintMiscellaneous) {
 444     tty->cr();
 445     findpc(eip);
 446     tty->cr();
 447   }
 448 #endif
 449 #define PRINT_REG(rax) \
 450   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
 451   PRINT_REG(rax);
 452   PRINT_REG(rbx);
 453   PRINT_REG(rcx);
 454   PRINT_REG(rdx);
 455   PRINT_REG(rdi);
 456   PRINT_REG(rsi);
 457   PRINT_REG(rbp);
 458   PRINT_REG(rsp);
 459 #undef PRINT_REG
 460   // Print some words near top of staack.
 461   int* dump_sp = (int*) rsp;
 462   for (int col1 = 0; col1 < 8; col1++) {
 463     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 464     os::print_location(tty, *dump_sp++);
 465   }
 466   for (int row = 0; row < 16; row++) {
 467     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 468     for (int col = 0; col < 8; col++) {
 469       tty->print(" 0x%08x", *dump_sp++);
 470     }
 471     tty->cr();
 472   }
 473   // Print some instructions around pc:
 474   Disassembler::decode((address)eip-64, (address)eip);
 475   tty->print_cr("--------");
 476   Disassembler::decode((address)eip, (address)eip+32);
 477 }
 478 
 479 void MacroAssembler::stop(const char* msg) {
 480   ExternalAddress message((address)msg);
 481   // push address of message
 482   pushptr(message.addr());
 483   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 484   pusha();                                            // push registers
 485   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 486   hlt();
 487 }
 488 
 489 void MacroAssembler::warn(const char* msg) {
 490   push_CPU_state();
 491 
 492   ExternalAddress message((address) msg);
 493   // push address of message
 494   pushptr(message.addr());
 495 
 496   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
 497   addl(rsp, wordSize);       // discard argument
 498   pop_CPU_state();
 499 }
 500 
 501 void MacroAssembler::print_state() {
 502   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 503   pusha();                                            // push registers
 504 
 505   push_CPU_state();
 506   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
 507   pop_CPU_state();
 508 
 509   popa();
 510   addl(rsp, wordSize);
 511 }
 512 
 513 #else // _LP64
 514 
 515 // 64 bit versions
 516 
 517 Address MacroAssembler::as_Address(AddressLiteral adr) {
 518   // amd64 always does this as a pc-rel
 519   // we can be absolute or disp based on the instruction type
 520   // jmp/call are displacements others are absolute
 521   assert(!adr.is_lval(), "must be rval");
 522   assert(reachable(adr), "must be");
 523   return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
 524 
 525 }
 526 
 527 Address MacroAssembler::as_Address(ArrayAddress adr) {
 528   AddressLiteral base = adr.base();
 529   lea(rscratch1, base);
 530   Address index = adr.index();
 531   assert(index._disp == 0, "must not have disp"); // maybe it can?
 532   Address array(rscratch1, index._index, index._scale, index._disp);
 533   return array;
 534 }
 535 
 536 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
 537   Label L, E;
 538 
 539 #ifdef _WIN64
 540   // Windows always allocates space for it's register args
 541   assert(num_args <= 4, "only register arguments supported");
 542   subq(rsp,  frame::arg_reg_save_area_bytes);
 543 #endif
 544 
 545   // Align stack if necessary
 546   testl(rsp, 15);
 547   jcc(Assembler::zero, L);
 548 
 549   subq(rsp, 8);
 550   {
 551     call(RuntimeAddress(entry_point));
 552   }
 553   addq(rsp, 8);
 554   jmp(E);
 555 
 556   bind(L);
 557   {
 558     call(RuntimeAddress(entry_point));
 559   }
 560 
 561   bind(E);
 562 
 563 #ifdef _WIN64
 564   // restore stack pointer
 565   addq(rsp, frame::arg_reg_save_area_bytes);
 566 #endif
 567 
 568 }
 569 
 570 void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
 571   assert(!src2.is_lval(), "should use cmpptr");
 572 
 573   if (reachable(src2)) {
 574     cmpq(src1, as_Address(src2));
 575   } else {
 576     lea(rscratch1, src2);
 577     Assembler::cmpq(src1, Address(rscratch1, 0));
 578   }
 579 }
 580 
 581 int MacroAssembler::corrected_idivq(Register reg) {
 582   // Full implementation of Java ldiv and lrem; checks for special
 583   // case as described in JVM spec., p.243 & p.271.  The function
 584   // returns the (pc) offset of the idivl instruction - may be needed
 585   // for implicit exceptions.
 586   //
 587   //         normal case                           special case
 588   //
 589   // input : rax: dividend                         min_long
 590   //         reg: divisor   (may not be eax/edx)   -1
 591   //
 592   // output: rax: quotient  (= rax idiv reg)       min_long
 593   //         rdx: remainder (= rax irem reg)       0
 594   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
 595   static const int64_t min_long = 0x8000000000000000;
 596   Label normal_case, special_case;
 597 
 598   // check for special case
 599   cmp64(rax, ExternalAddress((address) &min_long));
 600   jcc(Assembler::notEqual, normal_case);
 601   xorl(rdx, rdx); // prepare rdx for possible special case (where
 602                   // remainder = 0)
 603   cmpq(reg, -1);
 604   jcc(Assembler::equal, special_case);
 605 
 606   // handle normal case
 607   bind(normal_case);
 608   cdqq();
 609   int idivq_offset = offset();
 610   idivq(reg);
 611 
 612   // normal and special case exit
 613   bind(special_case);
 614 
 615   return idivq_offset;
 616 }
 617 
 618 void MacroAssembler::decrementq(Register reg, int value) {
 619   if (value == min_jint) { subq(reg, value); return; }
 620   if (value <  0) { incrementq(reg, -value); return; }
 621   if (value == 0) {                        ; return; }
 622   if (value == 1 && UseIncDec) { decq(reg) ; return; }
 623   /* else */      { subq(reg, value)       ; return; }
 624 }
 625 
 626 void MacroAssembler::decrementq(Address dst, int value) {
 627   if (value == min_jint) { subq(dst, value); return; }
 628   if (value <  0) { incrementq(dst, -value); return; }
 629   if (value == 0) {                        ; return; }
 630   if (value == 1 && UseIncDec) { decq(dst) ; return; }
 631   /* else */      { subq(dst, value)       ; return; }
 632 }
 633 
 634 void MacroAssembler::incrementq(AddressLiteral dst) {
 635   if (reachable(dst)) {
 636     incrementq(as_Address(dst));
 637   } else {
 638     lea(rscratch1, dst);
 639     incrementq(Address(rscratch1, 0));
 640   }
 641 }
 642 
 643 void MacroAssembler::incrementq(Register reg, int value) {
 644   if (value == min_jint) { addq(reg, value); return; }
 645   if (value <  0) { decrementq(reg, -value); return; }
 646   if (value == 0) {                        ; return; }
 647   if (value == 1 && UseIncDec) { incq(reg) ; return; }
 648   /* else */      { addq(reg, value)       ; return; }
 649 }
 650 
 651 void MacroAssembler::incrementq(Address dst, int value) {
 652   if (value == min_jint) { addq(dst, value); return; }
 653   if (value <  0) { decrementq(dst, -value); return; }
 654   if (value == 0) {                        ; return; }
 655   if (value == 1 && UseIncDec) { incq(dst) ; return; }
 656   /* else */      { addq(dst, value)       ; return; }
 657 }
 658 
 659 // 32bit can do a case table jump in one instruction but we no longer allow the base
 660 // to be installed in the Address class
 661 void MacroAssembler::jump(ArrayAddress entry) {
 662   lea(rscratch1, entry.base());
 663   Address dispatch = entry.index();
 664   assert(dispatch._base == noreg, "must be");
 665   dispatch._base = rscratch1;
 666   jmp(dispatch);
 667 }
 668 
 669 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 670   ShouldNotReachHere(); // 64bit doesn't use two regs
 671   cmpq(x_lo, y_lo);
 672 }
 673 
 674 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 675     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 676 }
 677 
 678 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 679   mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());
 680   movptr(dst, rscratch1);
 681 }
 682 
 683 void MacroAssembler::leave() {
 684   // %%% is this really better? Why not on 32bit too?
 685   emit_int8((unsigned char)0xC9); // LEAVE
 686 }
 687 
 688 void MacroAssembler::lneg(Register hi, Register lo) {
 689   ShouldNotReachHere(); // 64bit doesn't use two regs
 690   negq(lo);
 691 }
 692 
 693 void MacroAssembler::movoop(Register dst, jobject obj) {
 694   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 695 }
 696 
 697 void MacroAssembler::movoop(Address dst, jobject obj) {
 698   mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 699   movq(dst, rscratch1);
 700 }
 701 
 702 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 703   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 704 }
 705 
 706 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 707   mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 708   movq(dst, rscratch1);
 709 }
 710 
 711 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 712   if (src.is_lval()) {
 713     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 714   } else {
 715     if (reachable(src)) {
 716       movq(dst, as_Address(src));
 717     } else {
 718       lea(scratch, src);
 719       movq(dst, Address(scratch, 0));
 720     }
 721   }
 722 }
 723 
 724 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 725   movq(as_Address(dst), src);
 726 }
 727 
 728 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 729   movq(dst, as_Address(src));
 730 }
 731 
 732 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 733 void MacroAssembler::movptr(Address dst, intptr_t src) {
 734   mov64(rscratch1, src);
 735   movq(dst, rscratch1);
 736 }
 737 
 738 // These are mostly for initializing NULL
 739 void MacroAssembler::movptr(Address dst, int32_t src) {
 740   movslq(dst, src);
 741 }
 742 
 743 void MacroAssembler::movptr(Register dst, int32_t src) {
 744   mov64(dst, (intptr_t)src);
 745 }
 746 
 747 void MacroAssembler::pushoop(jobject obj) {
 748   movoop(rscratch1, obj);
 749   push(rscratch1);
 750 }
 751 
 752 void MacroAssembler::pushklass(Metadata* obj) {
 753   mov_metadata(rscratch1, obj);
 754   push(rscratch1);
 755 }
 756 
 757 void MacroAssembler::pushptr(AddressLiteral src) {
 758   lea(rscratch1, src);
 759   if (src.is_lval()) {
 760     push(rscratch1);
 761   } else {
 762     pushq(Address(rscratch1, 0));
 763   }
 764 }
 765 
 766 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
 767   // we must set sp to zero to clear frame
 768   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 769   // must clear fp, so that compiled frames are not confused; it is
 770   // possible that we need it only for debugging
 771   if (clear_fp) {
 772     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 773   }
 774 
 775   // Always clear the pc because it could have been set by make_walkable()
 776   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 777   vzeroupper();
 778 }
 779 
 780 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 781                                          Register last_java_fp,
 782                                          address  last_java_pc) {
 783   vzeroupper();
 784   // determine last_java_sp register
 785   if (!last_java_sp->is_valid()) {
 786     last_java_sp = rsp;
 787   }
 788 
 789   // last_java_fp is optional
 790   if (last_java_fp->is_valid()) {
 791     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),
 792            last_java_fp);
 793   }
 794 
 795   // last_java_pc is optional
 796   if (last_java_pc != NULL) {
 797     Address java_pc(r15_thread,
 798                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 799     lea(rscratch1, InternalAddress(last_java_pc));
 800     movptr(java_pc, rscratch1);
 801   }
 802 
 803   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 804 }
 805 
 806 static void pass_arg0(MacroAssembler* masm, Register arg) {
 807   if (c_rarg0 != arg ) {
 808     masm->mov(c_rarg0, arg);
 809   }
 810 }
 811 
 812 static void pass_arg1(MacroAssembler* masm, Register arg) {
 813   if (c_rarg1 != arg ) {
 814     masm->mov(c_rarg1, arg);
 815   }
 816 }
 817 
 818 static void pass_arg2(MacroAssembler* masm, Register arg) {
 819   if (c_rarg2 != arg ) {
 820     masm->mov(c_rarg2, arg);
 821   }
 822 }
 823 
 824 static void pass_arg3(MacroAssembler* masm, Register arg) {
 825   if (c_rarg3 != arg ) {
 826     masm->mov(c_rarg3, arg);
 827   }
 828 }
 829 
 830 void MacroAssembler::stop(const char* msg) {
 831   address rip = pc();
 832   pusha(); // get regs on stack
 833   lea(c_rarg0, ExternalAddress((address) msg));
 834   lea(c_rarg1, InternalAddress(rip));
 835   movq(c_rarg2, rsp); // pass pointer to regs array
 836   andq(rsp, -16); // align stack as required by ABI
 837   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
 838   hlt();
 839 }
 840 
 841 void MacroAssembler::warn(const char* msg) {
 842   push(rbp);
 843   movq(rbp, rsp);
 844   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 845   push_CPU_state();   // keeps alignment at 16 bytes
 846   lea(c_rarg0, ExternalAddress((address) msg));
 847   lea(rax, ExternalAddress(CAST_FROM_FN_PTR(address, warning)));
 848   call(rax);
 849   pop_CPU_state();
 850   mov(rsp, rbp);
 851   pop(rbp);
 852 }
 853 
 854 void MacroAssembler::print_state() {
 855   address rip = pc();
 856   pusha();            // get regs on stack
 857   push(rbp);
 858   movq(rbp, rsp);
 859   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 860   push_CPU_state();   // keeps alignment at 16 bytes
 861 
 862   lea(c_rarg0, InternalAddress(rip));
 863   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
 864   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
 865 
 866   pop_CPU_state();
 867   mov(rsp, rbp);
 868   pop(rbp);
 869   popa();
 870 }
 871 
 872 #ifndef PRODUCT
 873 extern "C" void findpc(intptr_t x);
 874 #endif
 875 
 876 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
 877   // In order to get locks to work, we need to fake a in_VM state
 878   if (ShowMessageBoxOnError) {
 879     JavaThread* thread = JavaThread::current();
 880     JavaThreadState saved_state = thread->thread_state();
 881     thread->set_thread_state(_thread_in_vm);
 882 #ifndef PRODUCT
 883     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 884       ttyLocker ttyl;
 885       BytecodeCounter::print();
 886     }
 887 #endif
 888     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 889     // XXX correct this offset for amd64
 890     // This is the value of eip which points to where verify_oop will return.
 891     if (os::message_box(msg, "Execution stopped, print registers?")) {
 892       print_state64(pc, regs);
 893       BREAKPOINT;
 894       assert(false, "start up GDB");
 895     }
 896     ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 897   } else {
 898     ttyLocker ttyl;
 899     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
 900                     msg);
 901     assert(false, "DEBUG MESSAGE: %s", msg);
 902   }
 903 }
 904 
 905 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
 906   ttyLocker ttyl;
 907   FlagSetting fs(Debugging, true);
 908   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
 909 #ifndef PRODUCT
 910   tty->cr();
 911   findpc(pc);
 912   tty->cr();
 913 #endif
 914 #define PRINT_REG(rax, value) \
 915   { tty->print("%s = ", #rax); os::print_location(tty, value); }
 916   PRINT_REG(rax, regs[15]);
 917   PRINT_REG(rbx, regs[12]);
 918   PRINT_REG(rcx, regs[14]);
 919   PRINT_REG(rdx, regs[13]);
 920   PRINT_REG(rdi, regs[8]);
 921   PRINT_REG(rsi, regs[9]);
 922   PRINT_REG(rbp, regs[10]);
 923   PRINT_REG(rsp, regs[11]);
 924   PRINT_REG(r8 , regs[7]);
 925   PRINT_REG(r9 , regs[6]);
 926   PRINT_REG(r10, regs[5]);
 927   PRINT_REG(r11, regs[4]);
 928   PRINT_REG(r12, regs[3]);
 929   PRINT_REG(r13, regs[2]);
 930   PRINT_REG(r14, regs[1]);
 931   PRINT_REG(r15, regs[0]);
 932 #undef PRINT_REG
 933   // Print some words near top of staack.
 934   int64_t* rsp = (int64_t*) regs[11];
 935   int64_t* dump_sp = rsp;
 936   for (int col1 = 0; col1 < 8; col1++) {
 937     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 938     os::print_location(tty, *dump_sp++);
 939   }
 940   for (int row = 0; row < 25; row++) {
 941     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 942     for (int col = 0; col < 4; col++) {
 943       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
 944     }
 945     tty->cr();
 946   }
 947   // Print some instructions around pc:
 948   Disassembler::decode((address)pc-64, (address)pc);
 949   tty->print_cr("--------");
 950   Disassembler::decode((address)pc, (address)pc+32);
 951 }
 952 
 953 #endif // _LP64
 954 
 955 // Now versions that are common to 32/64 bit
 956 
 957 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 958   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 959 }
 960 
 961 void MacroAssembler::addptr(Register dst, Register src) {
 962   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 963 }
 964 
 965 void MacroAssembler::addptr(Address dst, Register src) {
 966   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 967 }
 968 
 969 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {
 970   if (reachable(src)) {
 971     Assembler::addsd(dst, as_Address(src));
 972   } else {
 973     lea(rscratch1, src);
 974     Assembler::addsd(dst, Address(rscratch1, 0));
 975   }
 976 }
 977 
 978 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {
 979   if (reachable(src)) {
 980     addss(dst, as_Address(src));
 981   } else {
 982     lea(rscratch1, src);
 983     addss(dst, Address(rscratch1, 0));
 984   }
 985 }
 986 
 987 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) {
 988   if (reachable(src)) {
 989     Assembler::addpd(dst, as_Address(src));
 990   } else {
 991     lea(rscratch1, src);
 992     Assembler::addpd(dst, Address(rscratch1, 0));
 993   }
 994 }
 995 
 996 void MacroAssembler::align(int modulus) {
 997   align(modulus, offset());
 998 }
 999 
1000 void MacroAssembler::align(int modulus, int target) {
1001   if (target % modulus != 0) {
1002     nop(modulus - (target % modulus));
1003   }
1004 }
1005 
1006 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {
1007   // Used in sign-masking with aligned address.
1008   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1009   if (reachable(src)) {
1010     Assembler::andpd(dst, as_Address(src));
1011   } else {
1012     lea(rscratch1, src);
1013     Assembler::andpd(dst, Address(rscratch1, 0));
1014   }
1015 }
1016 
1017 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {
1018   // Used in sign-masking with aligned address.
1019   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1020   if (reachable(src)) {
1021     Assembler::andps(dst, as_Address(src));
1022   } else {
1023     lea(rscratch1, src);
1024     Assembler::andps(dst, Address(rscratch1, 0));
1025   }
1026 }
1027 
1028 void MacroAssembler::andptr(Register dst, int32_t imm32) {
1029   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
1030 }
1031 
1032 void MacroAssembler::atomic_incl(Address counter_addr) {

1033   lock();
1034   incrementl(counter_addr);
1035 }
1036 
1037 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
1038   if (reachable(counter_addr)) {
1039     atomic_incl(as_Address(counter_addr));
1040   } else {
1041     lea(scr, counter_addr);
1042     atomic_incl(Address(scr, 0));
1043   }
1044 }
1045 
1046 #ifdef _LP64
1047 void MacroAssembler::atomic_incq(Address counter_addr) {

1048   lock();
1049   incrementq(counter_addr);
1050 }
1051 
1052 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) {
1053   if (reachable(counter_addr)) {
1054     atomic_incq(as_Address(counter_addr));
1055   } else {
1056     lea(scr, counter_addr);
1057     atomic_incq(Address(scr, 0));
1058   }
1059 }
1060 #endif
1061 
1062 // Writes to stack successive pages until offset reached to check for
1063 // stack overflow + shadow pages.  This clobbers tmp.
1064 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
1065   movptr(tmp, rsp);
1066   // Bang stack for total size given plus shadow page size.
1067   // Bang one page at a time because large size can bang beyond yellow and
1068   // red zones.
1069   Label loop;
1070   bind(loop);
1071   movl(Address(tmp, (-os::vm_page_size())), size );
1072   subptr(tmp, os::vm_page_size());
1073   subl(size, os::vm_page_size());
1074   jcc(Assembler::greater, loop);
1075 
1076   // Bang down shadow pages too.
1077   // At this point, (tmp-0) is the last address touched, so don't
1078   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
1079   // was post-decremented.)  Skip this address by starting at i=1, and
1080   // touch a few more pages below.  N.B.  It is important to touch all
1081   // the way down including all pages in the shadow zone.
1082   for (int i = 1; i < ((int)JavaThread::stack_shadow_zone_size() / os::vm_page_size()); i++) {
1083     // this could be any sized move but this is can be a debugging crumb
1084     // so the bigger the better.
1085     movptr(Address(tmp, (-i*os::vm_page_size())), size );
1086   }
1087 }
1088 
1089 void MacroAssembler::reserved_stack_check() {
1090     // testing if reserved zone needs to be enabled
1091     Label no_reserved_zone_enabling;
1092     Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
1093     NOT_LP64(get_thread(rsi);)
1094 
1095     cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
1096     jcc(Assembler::below, no_reserved_zone_enabling);
1097 
1098     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
1099     jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
1100     should_not_reach_here();
1101 
1102     bind(no_reserved_zone_enabling);
1103 }
1104 
1105 int MacroAssembler::biased_locking_enter(Register lock_reg,
1106                                          Register obj_reg,
1107                                          Register swap_reg,
1108                                          Register tmp_reg,
1109                                          bool swap_reg_contains_mark,
1110                                          Label& done,
1111                                          Label* slow_case,
1112                                          BiasedLockingCounters* counters) {
1113   assert(UseBiasedLocking, "why call this otherwise?");
1114   assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
1115   assert(tmp_reg != noreg, "tmp_reg must be supplied");
1116   assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
1117   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
1118   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
1119   NOT_LP64( Address saved_mark_addr(lock_reg, 0); )
1120 
1121   if (PrintBiasedLockingStatistics && counters == NULL) {
1122     counters = BiasedLocking::counters();
1123   }
1124   // Biased locking
1125   // See whether the lock is currently biased toward our thread and
1126   // whether the epoch is still valid
1127   // Note that the runtime guarantees sufficient alignment of JavaThread
1128   // pointers to allow age to be placed into low bits
1129   // First check to see whether biasing is even enabled for this object
1130   Label cas_label;
1131   int null_check_offset = -1;
1132   if (!swap_reg_contains_mark) {
1133     null_check_offset = offset();
1134     movptr(swap_reg, mark_addr);
1135   }
1136   movptr(tmp_reg, swap_reg);
1137   andptr(tmp_reg, markOopDesc::biased_lock_mask_in_place);
1138   cmpptr(tmp_reg, markOopDesc::biased_lock_pattern);
1139   jcc(Assembler::notEqual, cas_label);
1140   // The bias pattern is present in the object's header. Need to check
1141   // whether the bias owner and the epoch are both still current.
1142 #ifndef _LP64
1143   // Note that because there is no current thread register on x86_32 we
1144   // need to store off the mark word we read out of the object to
1145   // avoid reloading it and needing to recheck invariants below. This
1146   // store is unfortunate but it makes the overall code shorter and
1147   // simpler.
1148   movptr(saved_mark_addr, swap_reg);
1149 #endif
1150   if (swap_reg_contains_mark) {
1151     null_check_offset = offset();
1152   }
1153   load_prototype_header(tmp_reg, obj_reg);
1154 #ifdef _LP64
1155   orptr(tmp_reg, r15_thread);
1156   xorptr(tmp_reg, swap_reg);
1157   Register header_reg = tmp_reg;
1158 #else
1159   xorptr(tmp_reg, swap_reg);
1160   get_thread(swap_reg);
1161   xorptr(swap_reg, tmp_reg);
1162   Register header_reg = swap_reg;
1163 #endif
1164   andptr(header_reg, ~((int) markOopDesc::age_mask_in_place));
1165   if (counters != NULL) {
1166     cond_inc32(Assembler::zero,
1167                ExternalAddress((address) counters->biased_lock_entry_count_addr()));
1168   }
1169   jcc(Assembler::equal, done);
1170 
1171   Label try_revoke_bias;
1172   Label try_rebias;
1173 
1174   // At this point we know that the header has the bias pattern and
1175   // that we are not the bias owner in the current epoch. We need to
1176   // figure out more details about the state of the header in order to
1177   // know what operations can be legally performed on the object's
1178   // header.
1179 
1180   // If the low three bits in the xor result aren't clear, that means
1181   // the prototype header is no longer biased and we have to revoke
1182   // the bias on this object.
1183   testptr(header_reg, markOopDesc::biased_lock_mask_in_place);
1184   jccb(Assembler::notZero, try_revoke_bias);
1185 
1186   // Biasing is still enabled for this data type. See whether the
1187   // epoch of the current bias is still valid, meaning that the epoch
1188   // bits of the mark word are equal to the epoch bits of the
1189   // prototype header. (Note that the prototype header's epoch bits
1190   // only change at a safepoint.) If not, attempt to rebias the object
1191   // toward the current thread. Note that we must be absolutely sure
1192   // that the current epoch is invalid in order to do this because
1193   // otherwise the manipulations it performs on the mark word are
1194   // illegal.
1195   testptr(header_reg, markOopDesc::epoch_mask_in_place);
1196   jccb(Assembler::notZero, try_rebias);
1197 
1198   // The epoch of the current bias is still valid but we know nothing
1199   // about the owner; it might be set or it might be clear. Try to
1200   // acquire the bias of the object using an atomic operation. If this
1201   // fails we will go in to the runtime to revoke the object's bias.
1202   // Note that we first construct the presumed unbiased header so we
1203   // don't accidentally blow away another thread's valid bias.
1204   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1205   andptr(swap_reg,
1206          markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
1207 #ifdef _LP64
1208   movptr(tmp_reg, swap_reg);
1209   orptr(tmp_reg, r15_thread);
1210 #else
1211   get_thread(tmp_reg);
1212   orptr(tmp_reg, swap_reg);
1213 #endif

1214   lock();

1215   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1216   // If the biasing toward our thread failed, this means that
1217   // another thread succeeded in biasing it toward itself and we
1218   // need to revoke that bias. The revocation will occur in the
1219   // interpreter runtime in the slow case.
1220   if (counters != NULL) {
1221     cond_inc32(Assembler::zero,
1222                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
1223   }
1224   if (slow_case != NULL) {
1225     jcc(Assembler::notZero, *slow_case);
1226   }
1227   jmp(done);
1228 
1229   bind(try_rebias);
1230   // At this point we know the epoch has expired, meaning that the
1231   // current "bias owner", if any, is actually invalid. Under these
1232   // circumstances _only_, we are allowed to use the current header's
1233   // value as the comparison value when doing the cas to acquire the
1234   // bias in the current epoch. In other words, we allow transfer of
1235   // the bias from one thread to another directly in this situation.
1236   //
1237   // FIXME: due to a lack of registers we currently blow away the age
1238   // bits in this situation. Should attempt to preserve them.
1239   load_prototype_header(tmp_reg, obj_reg);
1240 #ifdef _LP64
1241   orptr(tmp_reg, r15_thread);
1242 #else
1243   get_thread(swap_reg);
1244   orptr(tmp_reg, swap_reg);
1245   movptr(swap_reg, saved_mark_addr);
1246 #endif

1247   lock();

1248   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1249   // If the biasing toward our thread failed, then another thread
1250   // succeeded in biasing it toward itself and we need to revoke that
1251   // bias. The revocation will occur in the runtime in the slow case.
1252   if (counters != NULL) {
1253     cond_inc32(Assembler::zero,
1254                ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
1255   }
1256   if (slow_case != NULL) {
1257     jcc(Assembler::notZero, *slow_case);
1258   }
1259   jmp(done);
1260 
1261   bind(try_revoke_bias);
1262   // The prototype mark in the klass doesn't have the bias bit set any
1263   // more, indicating that objects of this data type are not supposed
1264   // to be biased any more. We are going to try to reset the mark of
1265   // this object to the prototype value and fall through to the
1266   // CAS-based locking scheme. Note that if our CAS fails, it means
1267   // that another thread raced us for the privilege of revoking the
1268   // bias of this particular object, so it's okay to continue in the
1269   // normal locking code.
1270   //
1271   // FIXME: due to a lack of registers we currently blow away the age
1272   // bits in this situation. Should attempt to preserve them.
1273   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1274   load_prototype_header(tmp_reg, obj_reg);

1275   lock();

1276   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1277   // Fall through to the normal CAS-based lock, because no matter what
1278   // the result of the above CAS, some thread must have succeeded in
1279   // removing the bias bit from the object's header.
1280   if (counters != NULL) {
1281     cond_inc32(Assembler::zero,
1282                ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
1283   }
1284 
1285   bind(cas_label);
1286 
1287   return null_check_offset;
1288 }
1289 
1290 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
1291   assert(UseBiasedLocking, "why call this otherwise?");
1292 
1293   // Check for biased locking unlock case, which is a no-op
1294   // Note: we do not have to check the thread ID for two reasons.
1295   // First, the interpreter checks for IllegalMonitorStateException at
1296   // a higher level. Second, if the bias was revoked while we held the
1297   // lock, the object could not be rebiased toward another thread, so
1298   // the bias bit would be clear.
1299   movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1300   andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);
1301   cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
1302   jcc(Assembler::equal, done);
1303 }
1304 
1305 #ifdef COMPILER2
1306 
1307 #if INCLUDE_RTM_OPT
1308 
1309 // Update rtm_counters based on abort status
1310 // input: abort_status
1311 //        rtm_counters (RTMLockingCounters*)
1312 // flags are killed
1313 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) {
1314 
1315   atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset()));
1316   if (PrintPreciseRTMLockingStatistics) {
1317     for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
1318       Label check_abort;
1319       testl(abort_status, (1<<i));
1320       jccb(Assembler::equal, check_abort);
1321       atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx))));
1322       bind(check_abort);
1323     }
1324   }
1325 }
1326 
1327 // Branch if (random & (count-1) != 0), count is 2^n
1328 // tmp, scr and flags are killed
1329 void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) {
1330   assert(tmp == rax, "");
1331   assert(scr == rdx, "");
1332   rdtsc(); // modifies EDX:EAX
1333   andptr(tmp, count-1);
1334   jccb(Assembler::notZero, brLabel);
1335 }
1336 
1337 // Perform abort ratio calculation, set no_rtm bit if high ratio
1338 // input:  rtm_counters_Reg (RTMLockingCounters* address)
1339 // tmpReg, rtm_counters_Reg and flags are killed
1340 void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg,
1341                                                  Register rtm_counters_Reg,
1342                                                  RTMLockingCounters* rtm_counters,
1343                                                  Metadata* method_data) {
1344   Label L_done, L_check_always_rtm1, L_check_always_rtm2;
1345 
1346   if (RTMLockingCalculationDelay > 0) {
1347     // Delay calculation
1348     movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg);
1349     testptr(tmpReg, tmpReg);
1350     jccb(Assembler::equal, L_done);
1351   }
1352   // Abort ratio calculation only if abort_count > RTMAbortThreshold
1353   //   Aborted transactions = abort_count * 100
1354   //   All transactions = total_count *  RTMTotalCountIncrRate
1355   //   Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio)
1356 
1357   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset()));
1358   cmpptr(tmpReg, RTMAbortThreshold);
1359   jccb(Assembler::below, L_check_always_rtm2);
1360   imulptr(tmpReg, tmpReg, 100);
1361 
1362   Register scrReg = rtm_counters_Reg;
1363   movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1364   imulptr(scrReg, scrReg, RTMTotalCountIncrRate);
1365   imulptr(scrReg, scrReg, RTMAbortRatio);
1366   cmpptr(tmpReg, scrReg);
1367   jccb(Assembler::below, L_check_always_rtm1);
1368   if (method_data != NULL) {
1369     // set rtm_state to "no rtm" in MDO
1370     mov_metadata(tmpReg, method_data);

1371     lock();

1372     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM);
1373   }
1374   jmpb(L_done);
1375   bind(L_check_always_rtm1);
1376   // Reload RTMLockingCounters* address
1377   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1378   bind(L_check_always_rtm2);
1379   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1380   cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate);
1381   jccb(Assembler::below, L_done);
1382   if (method_data != NULL) {
1383     // set rtm_state to "always rtm" in MDO
1384     mov_metadata(tmpReg, method_data);

1385     lock();

1386     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM);
1387   }
1388   bind(L_done);
1389 }
1390 
1391 // Update counters and perform abort ratio calculation
1392 // input:  abort_status_Reg
1393 // rtm_counters_Reg, flags are killed
1394 void MacroAssembler::rtm_profiling(Register abort_status_Reg,
1395                                    Register rtm_counters_Reg,
1396                                    RTMLockingCounters* rtm_counters,
1397                                    Metadata* method_data,
1398                                    bool profile_rtm) {
1399 
1400   assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1401   // update rtm counters based on rax value at abort
1402   // reads abort_status_Reg, updates flags
1403   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1404   rtm_counters_update(abort_status_Reg, rtm_counters_Reg);
1405   if (profile_rtm) {
1406     // Save abort status because abort_status_Reg is used by following code.
1407     if (RTMRetryCount > 0) {
1408       push(abort_status_Reg);
1409     }
1410     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1411     rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data);
1412     // restore abort status
1413     if (RTMRetryCount > 0) {
1414       pop(abort_status_Reg);
1415     }
1416   }
1417 }
1418 
1419 // Retry on abort if abort's status is 0x6: can retry (0x2) | memory conflict (0x4)
1420 // inputs: retry_count_Reg
1421 //       : abort_status_Reg
1422 // output: retry_count_Reg decremented by 1
1423 // flags are killed
1424 void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) {
1425   Label doneRetry;
1426   assert(abort_status_Reg == rax, "");
1427   // The abort reason bits are in eax (see all states in rtmLocking.hpp)
1428   // 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4)
1429   // if reason is in 0x6 and retry count != 0 then retry
1430   andptr(abort_status_Reg, 0x6);
1431   jccb(Assembler::zero, doneRetry);
1432   testl(retry_count_Reg, retry_count_Reg);
1433   jccb(Assembler::zero, doneRetry);
1434   pause();
1435   decrementl(retry_count_Reg);
1436   jmp(retryLabel);
1437   bind(doneRetry);
1438 }
1439 
1440 // Spin and retry if lock is busy,
1441 // inputs: box_Reg (monitor address)
1442 //       : retry_count_Reg
1443 // output: retry_count_Reg decremented by 1
1444 //       : clear z flag if retry count exceeded
1445 // tmp_Reg, scr_Reg, flags are killed
1446 void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg,
1447                                             Register tmp_Reg, Register scr_Reg, Label& retryLabel) {
1448   Label SpinLoop, SpinExit, doneRetry;
1449   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1450 
1451   testl(retry_count_Reg, retry_count_Reg);
1452   jccb(Assembler::zero, doneRetry);
1453   decrementl(retry_count_Reg);
1454   movptr(scr_Reg, RTMSpinLoopCount);
1455 
1456   bind(SpinLoop);
1457   pause();
1458   decrementl(scr_Reg);
1459   jccb(Assembler::lessEqual, SpinExit);
1460   movptr(tmp_Reg, Address(box_Reg, owner_offset));
1461   testptr(tmp_Reg, tmp_Reg);
1462   jccb(Assembler::notZero, SpinLoop);
1463 
1464   bind(SpinExit);
1465   jmp(retryLabel);
1466   bind(doneRetry);
1467   incrementl(retry_count_Reg); // clear z flag
1468 }
1469 
1470 // Use RTM for normal stack locks
1471 // Input: objReg (object to lock)
1472 void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg,
1473                                        Register retry_on_abort_count_Reg,
1474                                        RTMLockingCounters* stack_rtm_counters,
1475                                        Metadata* method_data, bool profile_rtm,
1476                                        Label& DONE_LABEL, Label& IsInflated) {
1477   assert(UseRTMForStackLocks, "why call this otherwise?");
1478   assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1479   assert(tmpReg == rax, "");
1480   assert(scrReg == rdx, "");
1481   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1482 
1483   if (RTMRetryCount > 0) {
1484     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1485     bind(L_rtm_retry);
1486   }
1487   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1488   testptr(tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased
1489   jcc(Assembler::notZero, IsInflated);
1490 
1491   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1492     Label L_noincrement;
1493     if (RTMTotalCountIncrRate > 1) {
1494       // tmpReg, scrReg and flags are killed
1495       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1496     }
1497     assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM");
1498     atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg);
1499     bind(L_noincrement);
1500   }
1501   xbegin(L_on_abort);
1502   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));       // fetch markword
1503   andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
1504   cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
1505   jcc(Assembler::equal, DONE_LABEL);        // all done if unlocked
1506 
1507   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1508   if (UseRTMXendForLockBusy) {
1509     xend();
1510     movptr(abort_status_Reg, 0x2);   // Set the abort status to 2 (so we can retry)
1511     jmp(L_decrement_retry);
1512   }
1513   else {
1514     xabort(0);
1515   }
1516   bind(L_on_abort);
1517   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1518     rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm);
1519   }
1520   bind(L_decrement_retry);
1521   if (RTMRetryCount > 0) {
1522     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1523     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1524   }
1525 }
1526 
1527 // Use RTM for inflating locks
1528 // inputs: objReg (object to lock)
1529 //         boxReg (on-stack box address (displaced header location) - KILLED)
1530 //         tmpReg (ObjectMonitor address + markOopDesc::monitor_value)
1531 void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg,
1532                                           Register scrReg, Register retry_on_busy_count_Reg,
1533                                           Register retry_on_abort_count_Reg,
1534                                           RTMLockingCounters* rtm_counters,
1535                                           Metadata* method_data, bool profile_rtm,
1536                                           Label& DONE_LABEL) {
1537   assert(UseRTMLocking, "why call this otherwise?");
1538   assert(tmpReg == rax, "");
1539   assert(scrReg == rdx, "");
1540   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1541   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1542 
1543   // Without cast to int32_t a movptr will destroy r10 which is typically obj
1544   movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1545   movptr(boxReg, tmpReg); // Save ObjectMonitor address
1546 
1547   if (RTMRetryCount > 0) {
1548     movl(retry_on_busy_count_Reg, RTMRetryCount);  // Retry on lock busy
1549     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1550     bind(L_rtm_retry);
1551   }
1552   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1553     Label L_noincrement;
1554     if (RTMTotalCountIncrRate > 1) {
1555       // tmpReg, scrReg and flags are killed
1556       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1557     }
1558     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1559     atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);
1560     bind(L_noincrement);
1561   }
1562   xbegin(L_on_abort);
1563   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1564   movptr(tmpReg, Address(tmpReg, owner_offset));
1565   testptr(tmpReg, tmpReg);
1566   jcc(Assembler::zero, DONE_LABEL);
1567   if (UseRTMXendForLockBusy) {
1568     xend();
1569     jmp(L_decrement_retry);
1570   }
1571   else {
1572     xabort(0);
1573   }
1574   bind(L_on_abort);
1575   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1576   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1577     rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm);
1578   }
1579   if (RTMRetryCount > 0) {
1580     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1581     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1582   }
1583 
1584   movptr(tmpReg, Address(boxReg, owner_offset)) ;
1585   testptr(tmpReg, tmpReg) ;
1586   jccb(Assembler::notZero, L_decrement_retry) ;
1587 
1588   // Appears unlocked - try to swing _owner from null to non-null.
1589   // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1590 #ifdef _LP64
1591   Register threadReg = r15_thread;
1592 #else
1593   get_thread(scrReg);
1594   Register threadReg = scrReg;
1595 #endif

1596   lock();

1597   cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg
1598 
1599   if (RTMRetryCount > 0) {
1600     // success done else retry
1601     jccb(Assembler::equal, DONE_LABEL) ;
1602     bind(L_decrement_retry);
1603     // Spin and retry if lock is busy.
1604     rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
1605   }
1606   else {
1607     bind(L_decrement_retry);
1608   }
1609 }
1610 
1611 #endif //  INCLUDE_RTM_OPT
1612 
1613 // Fast_Lock and Fast_Unlock used by C2
1614 
1615 // Because the transitions from emitted code to the runtime
1616 // monitorenter/exit helper stubs are so slow it's critical that
1617 // we inline both the stack-locking fast-path and the inflated fast path.
1618 //
1619 // See also: cmpFastLock and cmpFastUnlock.
1620 //
1621 // What follows is a specialized inline transliteration of the code
1622 // in slow_enter() and slow_exit().  If we're concerned about I$ bloat
1623 // another option would be to emit TrySlowEnter and TrySlowExit methods
1624 // at startup-time.  These methods would accept arguments as
1625 // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure
1626 // indications in the icc.ZFlag.  Fast_Lock and Fast_Unlock would simply
1627 // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.
1628 // In practice, however, the # of lock sites is bounded and is usually small.
1629 // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer
1630 // if the processor uses simple bimodal branch predictors keyed by EIP
1631 // Since the helper routines would be called from multiple synchronization
1632 // sites.
1633 //
1634 // An even better approach would be write "MonitorEnter()" and "MonitorExit()"
1635 // in java - using j.u.c and unsafe - and just bind the lock and unlock sites
1636 // to those specialized methods.  That'd give us a mostly platform-independent
1637 // implementation that the JITs could optimize and inline at their pleasure.
1638 // Done correctly, the only time we'd need to cross to native could would be
1639 // to park() or unpark() threads.  We'd also need a few more unsafe operators
1640 // to (a) prevent compiler-JIT reordering of non-volatile accesses, and
1641 // (b) explicit barriers or fence operations.
1642 //
1643 // TODO:
1644 //
1645 // *  Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr).
1646 //    This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals.
1647 //    Given TLAB allocation, Self is usually manifested in a register, so passing it into
1648 //    the lock operators would typically be faster than reifying Self.
1649 //
1650 // *  Ideally I'd define the primitives as:
1651 //       fast_lock   (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.
1652 //       fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED
1653 //    Unfortunately ADLC bugs prevent us from expressing the ideal form.
1654 //    Instead, we're stuck with a rather awkward and brittle register assignments below.
1655 //    Furthermore the register assignments are overconstrained, possibly resulting in
1656 //    sub-optimal code near the synchronization site.
1657 //
1658 // *  Eliminate the sp-proximity tests and just use "== Self" tests instead.
1659 //    Alternately, use a better sp-proximity test.
1660 //
1661 // *  Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.
1662 //    Either one is sufficient to uniquely identify a thread.
1663 //    TODO: eliminate use of sp in _owner and use get_thread(tr) instead.
1664 //
1665 // *  Intrinsify notify() and notifyAll() for the common cases where the
1666 //    object is locked by the calling thread but the waitlist is empty.
1667 //    avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().
1668 //
1669 // *  use jccb and jmpb instead of jcc and jmp to improve code density.
1670 //    But beware of excessive branch density on AMD Opterons.
1671 //
1672 // *  Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success
1673 //    or failure of the fast-path.  If the fast-path fails then we pass
1674 //    control to the slow-path, typically in C.  In Fast_Lock and
1675 //    Fast_Unlock we often branch to DONE_LABEL, just to find that C2
1676 //    will emit a conditional branch immediately after the node.
1677 //    So we have branches to branches and lots of ICC.ZF games.
1678 //    Instead, it might be better to have C2 pass a "FailureLabel"
1679 //    into Fast_Lock and Fast_Unlock.  In the case of success, control
1680 //    will drop through the node.  ICC.ZF is undefined at exit.
1681 //    In the case of failure, the node will branch directly to the
1682 //    FailureLabel
1683 
1684 
1685 // obj: object to lock
1686 // box: on-stack box address (displaced header location) - KILLED
1687 // rax,: tmp -- KILLED
1688 // scr: tmp -- KILLED
1689 void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
1690                                Register scrReg, Register cx1Reg, Register cx2Reg,
1691                                BiasedLockingCounters* counters,
1692                                RTMLockingCounters* rtm_counters,
1693                                RTMLockingCounters* stack_rtm_counters,
1694                                Metadata* method_data,
1695                                bool use_rtm, bool profile_rtm) {
1696   // Ensure the register assignments are disjoint
1697   assert(tmpReg == rax, "");
1698 
1699   if (use_rtm) {
1700     assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg);
1701   } else {
1702     assert(cx1Reg == noreg, "");
1703     assert(cx2Reg == noreg, "");
1704     assert_different_registers(objReg, boxReg, tmpReg, scrReg);
1705   }
1706 
1707   if (counters != NULL) {
1708     atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);
1709   }
1710 
1711   // Possible cases that we'll encounter in fast_lock
1712   // ------------------------------------------------
1713   // * Inflated
1714   //    -- unlocked
1715   //    -- Locked
1716   //       = by self
1717   //       = by other
1718   // * biased
1719   //    -- by Self
1720   //    -- by other
1721   // * neutral
1722   // * stack-locked
1723   //    -- by self
1724   //       = sp-proximity test hits
1725   //       = sp-proximity test generates false-negative
1726   //    -- by other
1727   //
1728 
1729   Label IsInflated, DONE_LABEL;
1730 
1731   // it's stack-locked, biased or neutral
1732   // TODO: optimize away redundant LDs of obj->mark and improve the markword triage
1733   // order to reduce the number of conditional branches in the most common cases.
1734   // Beware -- there's a subtle invariant that fetch of the markword
1735   // at [FETCH], below, will never observe a biased encoding (*101b).
1736   // If this invariant is not held we risk exclusion (safety) failure.
1737   if (UseBiasedLocking && !UseOptoBiasInlining) {
1738     biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters);
1739   }
1740 
1741 #if INCLUDE_RTM_OPT
1742   if (UseRTMForStackLocks && use_rtm) {
1743     rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
1744                       stack_rtm_counters, method_data, profile_rtm,
1745                       DONE_LABEL, IsInflated);
1746   }
1747 #endif // INCLUDE_RTM_OPT
1748 
1749   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));          // [FETCH]
1750   testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
1751   jccb(Assembler::notZero, IsInflated);
1752 
1753   // Attempt stack-locking ...
1754   orptr (tmpReg, markOopDesc::unlocked_value);
1755   movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS

1756   lock();

1757   cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes()));      // Updates tmpReg
1758   if (counters != NULL) {
1759     cond_inc32(Assembler::equal,
1760                ExternalAddress((address)counters->fast_path_entry_count_addr()));
1761   }
1762   jcc(Assembler::equal, DONE_LABEL);           // Success
1763 
1764   // Recursive locking.
1765   // The object is stack-locked: markword contains stack pointer to BasicLock.
1766   // Locked by current thread if difference with current SP is less than one page.
1767   subptr(tmpReg, rsp);
1768   // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
1769   andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
1770   movptr(Address(boxReg, 0), tmpReg);
1771   if (counters != NULL) {
1772     cond_inc32(Assembler::equal,
1773                ExternalAddress((address)counters->fast_path_entry_count_addr()));
1774   }
1775   jmp(DONE_LABEL);
1776 
1777   bind(IsInflated);
1778   // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markOopDesc::monitor_value
1779 
1780 #if INCLUDE_RTM_OPT
1781   // Use the same RTM locking code in 32- and 64-bit VM.
1782   if (use_rtm) {
1783     rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg,
1784                          rtm_counters, method_data, profile_rtm, DONE_LABEL);
1785   } else {
1786 #endif // INCLUDE_RTM_OPT
1787 
1788 #ifndef _LP64
1789   // The object is inflated.
1790 
1791   // boxReg refers to the on-stack BasicLock in the current frame.
1792   // We'd like to write:
1793   //   set box->_displaced_header = markOopDesc::unused_mark().  Any non-0 value suffices.
1794   // This is convenient but results a ST-before-CAS penalty.  The following CAS suffers
1795   // additional latency as we have another ST in the store buffer that must drain.
1796 
1797   // avoid ST-before-CAS
1798   // register juggle because we need tmpReg for cmpxchgptr below
1799   movptr(scrReg, boxReg);
1800   movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2]
1801 
1802   // Optimistic form: consider XORL tmpReg,tmpReg
1803   movptr(tmpReg, NULL_WORD);
1804 
1805   // Appears unlocked - try to swing _owner from null to non-null.
1806   // Ideally, I'd manifest "Self" with get_thread and then attempt
1807   // to CAS the register containing Self into m->Owner.
1808   // But we don't have enough registers, so instead we can either try to CAS
1809   // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
1810   // we later store "Self" into m->Owner.  Transiently storing a stack address
1811   // (rsp or the address of the box) into  m->owner is harmless.
1812   // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.

1813   lock();

1814   cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1815   movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3
1816   // If we weren't able to swing _owner from NULL to the BasicLock
1817   // then take the slow path.
1818   jccb  (Assembler::notZero, DONE_LABEL);
1819   // update _owner from BasicLock to thread
1820   get_thread (scrReg);                    // beware: clobbers ICCs
1821   movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
1822   xorptr(boxReg, boxReg);                 // set icc.ZFlag = 1 to indicate success
1823 
1824   // If the CAS fails we can either retry or pass control to the slow-path.
1825   // We use the latter tactic.
1826   // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1827   // If the CAS was successful ...
1828   //   Self has acquired the lock
1829   //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1830   // Intentional fall-through into DONE_LABEL ...
1831 #else // _LP64
1832   // It's inflated
1833   movq(scrReg, tmpReg);
1834   xorq(tmpReg, tmpReg);
1835 

1836   lock();

1837   cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1838   // Unconditionally set box->_displaced_header = markOopDesc::unused_mark().
1839   // Without cast to int32_t movptr will destroy r10 which is typically obj.
1840   movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1841   // Intentional fall-through into DONE_LABEL ...
1842   // Propagate ICC.ZF from CAS above into DONE_LABEL.
1843 #endif // _LP64
1844 #if INCLUDE_RTM_OPT
1845   } // use_rtm()
1846 #endif
1847   // DONE_LABEL is a hot target - we'd really like to place it at the
1848   // start of cache line by padding with NOPs.
1849   // See the AMD and Intel software optimization manuals for the
1850   // most efficient "long" NOP encodings.
1851   // Unfortunately none of our alignment mechanisms suffice.
1852   bind(DONE_LABEL);
1853 
1854   // At DONE_LABEL the icc ZFlag is set as follows ...
1855   // Fast_Unlock uses the same protocol.
1856   // ZFlag == 1 -> Success
1857   // ZFlag == 0 -> Failure - force control through the slow-path
1858 }
1859 
1860 // obj: object to unlock
1861 // box: box address (displaced header location), killed.  Must be EAX.
1862 // tmp: killed, cannot be obj nor box.
1863 //
1864 // Some commentary on balanced locking:
1865 //
1866 // Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites.
1867 // Methods that don't have provably balanced locking are forced to run in the
1868 // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
1869 // The interpreter provides two properties:
1870 // I1:  At return-time the interpreter automatically and quietly unlocks any
1871 //      objects acquired the current activation (frame).  Recall that the
1872 //      interpreter maintains an on-stack list of locks currently held by
1873 //      a frame.
1874 // I2:  If a method attempts to unlock an object that is not held by the
1875 //      the frame the interpreter throws IMSX.
1876 //
1877 // Lets say A(), which has provably balanced locking, acquires O and then calls B().
1878 // B() doesn't have provably balanced locking so it runs in the interpreter.
1879 // Control returns to A() and A() unlocks O.  By I1 and I2, above, we know that O
1880 // is still locked by A().
1881 //
1882 // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:
1883 // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
1884 // should not be unlocked by "normal" java-level locking and vice-versa.  The specification
1885 // doesn't specify what will occur if a program engages in such mixed-mode locking, however.
1886 // Arguably given that the spec legislates the JNI case as undefined our implementation
1887 // could reasonably *avoid* checking owner in Fast_Unlock().
1888 // In the interest of performance we elide m->Owner==Self check in unlock.
1889 // A perfectly viable alternative is to elide the owner check except when
1890 // Xcheck:jni is enabled.
1891 
1892 void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
1893   assert(boxReg == rax, "");
1894   assert_different_registers(objReg, boxReg, tmpReg);
1895 
1896   Label DONE_LABEL, Stacked, CheckSucc;
1897 
1898   // Critically, the biased locking test must have precedence over
1899   // and appear before the (box->dhw == 0) recursive stack-lock test.
1900   if (UseBiasedLocking && !UseOptoBiasInlining) {
1901     biased_locking_exit(objReg, tmpReg, DONE_LABEL);
1902   }
1903 
1904 #if INCLUDE_RTM_OPT
1905   if (UseRTMForStackLocks && use_rtm) {
1906     assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1907     Label L_regular_unlock;
1908     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));           // fetch markword
1909     andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
1910     cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
1911     jccb(Assembler::notEqual, L_regular_unlock);  // if !HLE RegularLock
1912     xend();                                       // otherwise end...
1913     jmp(DONE_LABEL);                              // ... and we're done
1914     bind(L_regular_unlock);
1915   }
1916 #endif
1917 
1918   cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
1919   jcc   (Assembler::zero, DONE_LABEL);            // 0 indicates recursive stack-lock
1920   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));             // Examine the object's markword
1921   testptr(tmpReg, markOopDesc::monitor_value);    // Inflated?
1922   jccb  (Assembler::zero, Stacked);
1923 
1924   // It's inflated.
1925 #if INCLUDE_RTM_OPT
1926   if (use_rtm) {
1927     Label L_regular_inflated_unlock;
1928     int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1929     movptr(boxReg, Address(tmpReg, owner_offset));
1930     testptr(boxReg, boxReg);
1931     jccb(Assembler::notZero, L_regular_inflated_unlock);
1932     xend();
1933     jmpb(DONE_LABEL);
1934     bind(L_regular_inflated_unlock);
1935   }
1936 #endif
1937 
1938   // Despite our balanced locking property we still check that m->_owner == Self
1939   // as java routines or native JNI code called by this thread might
1940   // have released the lock.
1941   // Refer to the comments in synchronizer.cpp for how we might encode extra
1942   // state in _succ so we can avoid fetching EntryList|cxq.
1943   //
1944   // I'd like to add more cases in fast_lock() and fast_unlock() --
1945   // such as recursive enter and exit -- but we have to be wary of
1946   // I$ bloat, T$ effects and BP$ effects.
1947   //
1948   // If there's no contention try a 1-0 exit.  That is, exit without
1949   // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how
1950   // we detect and recover from the race that the 1-0 exit admits.
1951   //
1952   // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier
1953   // before it STs null into _owner, releasing the lock.  Updates
1954   // to data protected by the critical section must be visible before
1955   // we drop the lock (and thus before any other thread could acquire
1956   // the lock and observe the fields protected by the lock).
1957   // IA32's memory-model is SPO, so STs are ordered with respect to
1958   // each other and there's no need for an explicit barrier (fence).
1959   // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
1960 #ifndef _LP64
1961   get_thread (boxReg);
1962 
1963   // Note that we could employ various encoding schemes to reduce
1964   // the number of loads below (currently 4) to just 2 or 3.
1965   // Refer to the comments in synchronizer.cpp.
1966   // In practice the chain of fetches doesn't seem to impact performance, however.
1967   xorptr(boxReg, boxReg);
1968   orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
1969   jccb  (Assembler::notZero, DONE_LABEL);
1970   movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
1971   orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
1972   jccb  (Assembler::notZero, CheckSucc);
1973   movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
1974   jmpb  (DONE_LABEL);
1975 
1976   bind (Stacked);
1977   // It's not inflated and it's not recursively stack-locked and it's not biased.
1978   // It must be stack-locked.
1979   // Try to reset the header to displaced header.
1980   // The "box" value on the stack is stable, so we can reload
1981   // and be assured we observe the same value as above.
1982   movptr(tmpReg, Address(boxReg, 0));

1983   lock();

1984   cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
1985   // Intention fall-thru into DONE_LABEL
1986 
1987   // DONE_LABEL is a hot target - we'd really like to place it at the
1988   // start of cache line by padding with NOPs.
1989   // See the AMD and Intel software optimization manuals for the
1990   // most efficient "long" NOP encodings.
1991   // Unfortunately none of our alignment mechanisms suffice.
1992   bind (CheckSucc);
1993 #else // _LP64
1994   // It's inflated
1995   xorptr(boxReg, boxReg);
1996   orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
1997   jccb  (Assembler::notZero, DONE_LABEL);
1998   movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
1999   orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2000   jccb  (Assembler::notZero, CheckSucc);
2001   movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2002   jmpb  (DONE_LABEL);
2003 
2004   // Try to avoid passing control into the slow_path ...
2005   Label LSuccess, LGoSlowPath ;
2006   bind  (CheckSucc);
2007 
2008   // The following optional optimization can be elided if necessary
2009   // Effectively: if (succ == null) goto SlowPath
2010   // The code reduces the window for a race, however,
2011   // and thus benefits performance.
2012   cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2013   jccb  (Assembler::zero, LGoSlowPath);
2014 
2015   xorptr(boxReg, boxReg);
2016   movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2017 
2018   // Memory barrier/fence
2019   // Dekker pivot point -- fulcrum : ST Owner; MEMBAR; LD Succ
2020   // Instead of MFENCE we use a dummy locked add of 0 to the top-of-stack.
2021   // This is faster on Nehalem and AMD Shanghai/Barcelona.
2022   // See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences
2023   // We might also restructure (ST Owner=0;barrier;LD _Succ) to
2024   // (mov box,0; xchgq box, &m->Owner; LD _succ) .
2025   lock(); addl(Address(rsp, 0), 0);
2026 
2027   cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2028   jccb  (Assembler::notZero, LSuccess);
2029 
2030   // Rare inopportune interleaving - race.
2031   // The successor vanished in the small window above.
2032   // The lock is contended -- (cxq|EntryList) != null -- and there's no apparent successor.
2033   // We need to ensure progress and succession.
2034   // Try to reacquire the lock.
2035   // If that fails then the new owner is responsible for succession and this
2036   // thread needs to take no further action and can exit via the fast path (success).
2037   // If the re-acquire succeeds then pass control into the slow path.
2038   // As implemented, this latter mode is horrible because we generated more
2039   // coherence traffic on the lock *and* artifically extended the critical section
2040   // length while by virtue of passing control into the slow path.
2041 
2042   // box is really RAX -- the following CMPXCHG depends on that binding
2043   // cmpxchg R,[M] is equivalent to rax = CAS(M,rax,R)
2044   lock();
2045   cmpxchgptr(r15_thread, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2046   // There's no successor so we tried to regrab the lock.
2047   // If that didn't work, then another thread grabbed the
2048   // lock so we're done (and exit was a success).
2049   jccb  (Assembler::notEqual, LSuccess);
2050   // Intentional fall-through into slow-path
2051 
2052   bind  (LGoSlowPath);
2053   orl   (boxReg, 1);                      // set ICC.ZF=0 to indicate failure
2054   jmpb  (DONE_LABEL);
2055 
2056   bind  (LSuccess);
2057   testl (boxReg, 0);                      // set ICC.ZF=1 to indicate success
2058   jmpb  (DONE_LABEL);
2059 
2060   bind  (Stacked);
2061   movptr(tmpReg, Address (boxReg, 0));      // re-fetch
2062   lock();
2063   cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
2064 
2065 #endif
2066   bind(DONE_LABEL);
2067 }
2068 #endif // COMPILER2
2069 
2070 void MacroAssembler::c2bool(Register x) {
2071   // implements x == 0 ? 0 : 1
2072   // note: must only look at least-significant byte of x
2073   //       since C-style booleans are stored in one byte
2074   //       only! (was bug)
2075   andl(x, 0xFF);
2076   setb(Assembler::notZero, x);
2077 }
2078 
2079 // Wouldn't need if AddressLiteral version had new name
2080 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
2081   Assembler::call(L, rtype);
2082 }
2083 
2084 void MacroAssembler::call(Register entry) {
2085   Assembler::call(entry);
2086 }
2087 
2088 void MacroAssembler::call(AddressLiteral entry) {
2089   if (reachable(entry)) {
2090     Assembler::call_literal(entry.target(), entry.rspec());
2091   } else {
2092     lea(rscratch1, entry);
2093     Assembler::call(rscratch1);
2094   }
2095 }
2096 
2097 void MacroAssembler::ic_call(address entry, jint method_index) {
2098   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
2099   movptr(rax, (intptr_t)Universe::non_oop_word());
2100   call(AddressLiteral(entry, rh));
2101 }
2102 
2103 // Implementation of call_VM versions
2104 
2105 void MacroAssembler::call_VM(Register oop_result,
2106                              address entry_point,
2107                              bool check_exceptions) {
2108   Label C, E;
2109   call(C, relocInfo::none);
2110   jmp(E);
2111 
2112   bind(C);
2113   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
2114   ret(0);
2115 
2116   bind(E);
2117 }
2118 
2119 void MacroAssembler::call_VM(Register oop_result,
2120                              address entry_point,
2121                              Register arg_1,
2122                              bool check_exceptions) {
2123   Label C, E;
2124   call(C, relocInfo::none);
2125   jmp(E);
2126 
2127   bind(C);
2128   pass_arg1(this, arg_1);
2129   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
2130   ret(0);
2131 
2132   bind(E);
2133 }
2134 
2135 void MacroAssembler::call_VM(Register oop_result,
2136                              address entry_point,
2137                              Register arg_1,
2138                              Register arg_2,
2139                              bool check_exceptions) {
2140   Label C, E;
2141   call(C, relocInfo::none);
2142   jmp(E);
2143 
2144   bind(C);
2145 
2146   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2147 
2148   pass_arg2(this, arg_2);
2149   pass_arg1(this, arg_1);
2150   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
2151   ret(0);
2152 
2153   bind(E);
2154 }
2155 
2156 void MacroAssembler::call_VM(Register oop_result,
2157                              address entry_point,
2158                              Register arg_1,
2159                              Register arg_2,
2160                              Register arg_3,
2161                              bool check_exceptions) {
2162   Label C, E;
2163   call(C, relocInfo::none);
2164   jmp(E);
2165 
2166   bind(C);
2167 
2168   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2169   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2170   pass_arg3(this, arg_3);
2171 
2172   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2173   pass_arg2(this, arg_2);
2174 
2175   pass_arg1(this, arg_1);
2176   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
2177   ret(0);
2178 
2179   bind(E);
2180 }
2181 
2182 void MacroAssembler::call_VM(Register oop_result,
2183                              Register last_java_sp,
2184                              address entry_point,
2185                              int number_of_arguments,
2186                              bool check_exceptions) {
2187   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2188   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2189 }
2190 
2191 void MacroAssembler::call_VM(Register oop_result,
2192                              Register last_java_sp,
2193                              address entry_point,
2194                              Register arg_1,
2195                              bool check_exceptions) {
2196   pass_arg1(this, arg_1);
2197   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2198 }
2199 
2200 void MacroAssembler::call_VM(Register oop_result,
2201                              Register last_java_sp,
2202                              address entry_point,
2203                              Register arg_1,
2204                              Register arg_2,
2205                              bool check_exceptions) {
2206 
2207   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2208   pass_arg2(this, arg_2);
2209   pass_arg1(this, arg_1);
2210   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2211 }
2212 
2213 void MacroAssembler::call_VM(Register oop_result,
2214                              Register last_java_sp,
2215                              address entry_point,
2216                              Register arg_1,
2217                              Register arg_2,
2218                              Register arg_3,
2219                              bool check_exceptions) {
2220   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2221   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2222   pass_arg3(this, arg_3);
2223   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2224   pass_arg2(this, arg_2);
2225   pass_arg1(this, arg_1);
2226   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2227 }
2228 
2229 void MacroAssembler::super_call_VM(Register oop_result,
2230                                    Register last_java_sp,
2231                                    address entry_point,
2232                                    int number_of_arguments,
2233                                    bool check_exceptions) {
2234   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2235   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2236 }
2237 
2238 void MacroAssembler::super_call_VM(Register oop_result,
2239                                    Register last_java_sp,
2240                                    address entry_point,
2241                                    Register arg_1,
2242                                    bool check_exceptions) {
2243   pass_arg1(this, arg_1);
2244   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2245 }
2246 
2247 void MacroAssembler::super_call_VM(Register oop_result,
2248                                    Register last_java_sp,
2249                                    address entry_point,
2250                                    Register arg_1,
2251                                    Register arg_2,
2252                                    bool check_exceptions) {
2253 
2254   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2255   pass_arg2(this, arg_2);
2256   pass_arg1(this, arg_1);
2257   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2258 }
2259 
2260 void MacroAssembler::super_call_VM(Register oop_result,
2261                                    Register last_java_sp,
2262                                    address entry_point,
2263                                    Register arg_1,
2264                                    Register arg_2,
2265                                    Register arg_3,
2266                                    bool check_exceptions) {
2267   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2268   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2269   pass_arg3(this, arg_3);
2270   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2271   pass_arg2(this, arg_2);
2272   pass_arg1(this, arg_1);
2273   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2274 }
2275 
2276 void MacroAssembler::call_VM_base(Register oop_result,
2277                                   Register java_thread,
2278                                   Register last_java_sp,
2279                                   address  entry_point,
2280                                   int      number_of_arguments,
2281                                   bool     check_exceptions) {
2282   // determine java_thread register
2283   if (!java_thread->is_valid()) {
2284 #ifdef _LP64
2285     java_thread = r15_thread;
2286 #else
2287     java_thread = rdi;
2288     get_thread(java_thread);
2289 #endif // LP64
2290   }
2291   // determine last_java_sp register
2292   if (!last_java_sp->is_valid()) {
2293     last_java_sp = rsp;
2294   }
2295   // debugging support
2296   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
2297   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
2298 #ifdef ASSERT
2299   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
2300   // r12 is the heapbase.
2301   LP64_ONLY(if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
2302 #endif // ASSERT
2303 
2304   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
2305   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
2306 
2307   // push java thread (becomes first argument of C function)
2308 
2309   NOT_LP64(push(java_thread); number_of_arguments++);
2310   LP64_ONLY(mov(c_rarg0, r15_thread));
2311 
2312   // set last Java frame before call
2313   assert(last_java_sp != rbp, "can't use ebp/rbp");
2314 
2315   // Only interpreter should have to set fp
2316   set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);
2317 
2318   // do the call, remove parameters
2319   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
2320 
2321   // restore the thread (cannot use the pushed argument since arguments
2322   // may be overwritten by C code generated by an optimizing compiler);
2323   // however can use the register value directly if it is callee saved.
2324   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
2325     // rdi & rsi (also r15) are callee saved -> nothing to do
2326 #ifdef ASSERT
2327     guarantee(java_thread != rax, "change this code");
2328     push(rax);
2329     { Label L;
2330       get_thread(rax);
2331       cmpptr(java_thread, rax);
2332       jcc(Assembler::equal, L);
2333       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
2334       bind(L);
2335     }
2336     pop(rax);
2337 #endif
2338   } else {
2339     get_thread(java_thread);
2340   }
2341   // reset last Java frame
2342   // Only interpreter should have to clear fp
2343   reset_last_Java_frame(java_thread, true);
2344 
2345    // C++ interp handles this in the interpreter
2346   check_and_handle_popframe(java_thread);
2347   check_and_handle_earlyret(java_thread);
2348 
2349   if (check_exceptions) {
2350     // check for pending exceptions (java_thread is set upon return)
2351     cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
2352 #ifndef _LP64
2353     jump_cc(Assembler::notEqual,
2354             RuntimeAddress(StubRoutines::forward_exception_entry()));
2355 #else
2356     // This used to conditionally jump to forward_exception however it is
2357     // possible if we relocate that the branch will not reach. So we must jump
2358     // around so we can always reach
2359 
2360     Label ok;
2361     jcc(Assembler::equal, ok);
2362     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2363     bind(ok);
2364 #endif // LP64
2365   }
2366 
2367   // get oop result if there is one and reset the value in the thread
2368   if (oop_result->is_valid()) {
2369     get_vm_result(oop_result, java_thread);
2370   }
2371 }
2372 
2373 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
2374 
2375   // Calculate the value for last_Java_sp
2376   // somewhat subtle. call_VM does an intermediate call
2377   // which places a return address on the stack just under the
2378   // stack pointer as the user finsihed with it. This allows
2379   // use to retrieve last_Java_pc from last_Java_sp[-1].
2380   // On 32bit we then have to push additional args on the stack to accomplish
2381   // the actual requested call. On 64bit call_VM only can use register args
2382   // so the only extra space is the return address that call_VM created.
2383   // This hopefully explains the calculations here.
2384 
2385 #ifdef _LP64
2386   // We've pushed one address, correct last_Java_sp
2387   lea(rax, Address(rsp, wordSize));
2388 #else
2389   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
2390 #endif // LP64
2391 
2392   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
2393 
2394 }
2395 
2396 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
2397 void MacroAssembler::call_VM_leaf0(address entry_point) {
2398   MacroAssembler::call_VM_leaf_base(entry_point, 0);
2399 }
2400 
2401 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
2402   call_VM_leaf_base(entry_point, number_of_arguments);
2403 }
2404 
2405 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
2406   pass_arg0(this, arg_0);
2407   call_VM_leaf(entry_point, 1);
2408 }
2409 
2410 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2411 
2412   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2413   pass_arg1(this, arg_1);
2414   pass_arg0(this, arg_0);
2415   call_VM_leaf(entry_point, 2);
2416 }
2417 
2418 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2419   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2420   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2421   pass_arg2(this, arg_2);
2422   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2423   pass_arg1(this, arg_1);
2424   pass_arg0(this, arg_0);
2425   call_VM_leaf(entry_point, 3);
2426 }
2427 
2428 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
2429   pass_arg0(this, arg_0);
2430   MacroAssembler::call_VM_leaf_base(entry_point, 1);
2431 }
2432 
2433 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2434 
2435   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2436   pass_arg1(this, arg_1);
2437   pass_arg0(this, arg_0);
2438   MacroAssembler::call_VM_leaf_base(entry_point, 2);
2439 }
2440 
2441 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2442   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2443   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2444   pass_arg2(this, arg_2);
2445   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2446   pass_arg1(this, arg_1);
2447   pass_arg0(this, arg_0);
2448   MacroAssembler::call_VM_leaf_base(entry_point, 3);
2449 }
2450 
2451 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
2452   LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg"));
2453   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2454   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2455   pass_arg3(this, arg_3);
2456   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2457   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2458   pass_arg2(this, arg_2);
2459   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2460   pass_arg1(this, arg_1);
2461   pass_arg0(this, arg_0);
2462   MacroAssembler::call_VM_leaf_base(entry_point, 4);
2463 }
2464 
2465 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
2466   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
2467   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
2468   verify_oop(oop_result, "broken oop in call_VM_base");
2469 }
2470 
2471 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
2472   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
2473   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
2474 }
2475 
2476 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
2477 }
2478 
2479 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
2480 }
2481 
2482 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) {
2483   if (reachable(src1)) {
2484     cmpl(as_Address(src1), imm);
2485   } else {
2486     lea(rscratch1, src1);
2487     cmpl(Address(rscratch1, 0), imm);
2488   }
2489 }
2490 
2491 void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
2492   assert(!src2.is_lval(), "use cmpptr");
2493   if (reachable(src2)) {
2494     cmpl(src1, as_Address(src2));
2495   } else {
2496     lea(rscratch1, src2);
2497     cmpl(src1, Address(rscratch1, 0));
2498   }
2499 }
2500 
2501 void MacroAssembler::cmp32(Register src1, int32_t imm) {
2502   Assembler::cmpl(src1, imm);
2503 }
2504 
2505 void MacroAssembler::cmp32(Register src1, Address src2) {
2506   Assembler::cmpl(src1, src2);
2507 }
2508 
2509 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2510   ucomisd(opr1, opr2);
2511 
2512   Label L;
2513   if (unordered_is_less) {
2514     movl(dst, -1);
2515     jcc(Assembler::parity, L);
2516     jcc(Assembler::below , L);
2517     movl(dst, 0);
2518     jcc(Assembler::equal , L);
2519     increment(dst);
2520   } else { // unordered is greater
2521     movl(dst, 1);
2522     jcc(Assembler::parity, L);
2523     jcc(Assembler::above , L);
2524     movl(dst, 0);
2525     jcc(Assembler::equal , L);
2526     decrementl(dst);
2527   }
2528   bind(L);
2529 }
2530 
2531 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2532   ucomiss(opr1, opr2);
2533 
2534   Label L;
2535   if (unordered_is_less) {
2536     movl(dst, -1);
2537     jcc(Assembler::parity, L);
2538     jcc(Assembler::below , L);
2539     movl(dst, 0);
2540     jcc(Assembler::equal , L);
2541     increment(dst);
2542   } else { // unordered is greater
2543     movl(dst, 1);
2544     jcc(Assembler::parity, L);
2545     jcc(Assembler::above , L);
2546     movl(dst, 0);
2547     jcc(Assembler::equal , L);
2548     decrementl(dst);
2549   }
2550   bind(L);
2551 }
2552 
2553 
2554 void MacroAssembler::cmp8(AddressLiteral src1, int imm) {
2555   if (reachable(src1)) {
2556     cmpb(as_Address(src1), imm);
2557   } else {
2558     lea(rscratch1, src1);
2559     cmpb(Address(rscratch1, 0), imm);
2560   }
2561 }
2562 
2563 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
2564 #ifdef _LP64
2565   if (src2.is_lval()) {
2566     movptr(rscratch1, src2);
2567     Assembler::cmpq(src1, rscratch1);
2568   } else if (reachable(src2)) {
2569     cmpq(src1, as_Address(src2));
2570   } else {
2571     lea(rscratch1, src2);
2572     Assembler::cmpq(src1, Address(rscratch1, 0));
2573   }
2574 #else
2575   if (src2.is_lval()) {
2576     cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2577   } else {
2578     cmpl(src1, as_Address(src2));
2579   }
2580 #endif // _LP64
2581 }
2582 
2583 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
2584   assert(src2.is_lval(), "not a mem-mem compare");
2585 #ifdef _LP64
2586   // moves src2's literal address
2587   movptr(rscratch1, src2);
2588   Assembler::cmpq(src1, rscratch1);
2589 #else
2590   cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2591 #endif // _LP64
2592 }
2593 
2594 void MacroAssembler::cmpoop(Register src1, Register src2) {
2595   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2596   bs->obj_equals(this, src1, src2);
2597 }
2598 
2599 void MacroAssembler::cmpoop(Register src1, Address src2) {
2600   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2601   bs->obj_equals(this, src1, src2);
2602 }
2603 
2604 #ifdef _LP64
2605 void MacroAssembler::cmpoop(Register src1, jobject src2) {
2606   movoop(rscratch1, src2);
2607   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2608   bs->obj_equals(this, src1, rscratch1);
2609 }
2610 #endif
2611 
2612 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) {
2613   if (reachable(adr)) {

2614     lock();
2615     cmpxchgptr(reg, as_Address(adr));
2616   } else {
2617     lea(rscratch1, adr);

2618     lock();
2619     cmpxchgptr(reg, Address(rscratch1, 0));
2620   }
2621 }
2622 
2623 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
2624   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
2625 }
2626 
2627 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {
2628   if (reachable(src)) {
2629     Assembler::comisd(dst, as_Address(src));
2630   } else {
2631     lea(rscratch1, src);
2632     Assembler::comisd(dst, Address(rscratch1, 0));
2633   }
2634 }
2635 
2636 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {
2637   if (reachable(src)) {
2638     Assembler::comiss(dst, as_Address(src));
2639   } else {
2640     lea(rscratch1, src);
2641     Assembler::comiss(dst, Address(rscratch1, 0));
2642   }
2643 }
2644 
2645 
2646 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
2647   Condition negated_cond = negate_condition(cond);
2648   Label L;
2649   jcc(negated_cond, L);
2650   pushf(); // Preserve flags
2651   atomic_incl(counter_addr);
2652   popf();
2653   bind(L);
2654 }
2655 
2656 int MacroAssembler::corrected_idivl(Register reg) {
2657   // Full implementation of Java idiv and irem; checks for
2658   // special case as described in JVM spec., p.243 & p.271.
2659   // The function returns the (pc) offset of the idivl
2660   // instruction - may be needed for implicit exceptions.
2661   //
2662   //         normal case                           special case
2663   //
2664   // input : rax,: dividend                         min_int
2665   //         reg: divisor   (may not be rax,/rdx)   -1
2666   //
2667   // output: rax,: quotient  (= rax, idiv reg)       min_int
2668   //         rdx: remainder (= rax, irem reg)       0
2669   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
2670   const int min_int = 0x80000000;
2671   Label normal_case, special_case;
2672 
2673   // check for special case
2674   cmpl(rax, min_int);
2675   jcc(Assembler::notEqual, normal_case);
2676   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
2677   cmpl(reg, -1);
2678   jcc(Assembler::equal, special_case);
2679 
2680   // handle normal case
2681   bind(normal_case);
2682   cdql();
2683   int idivl_offset = offset();
2684   idivl(reg);
2685 
2686   // normal and special case exit
2687   bind(special_case);
2688 
2689   return idivl_offset;
2690 }
2691 
2692 
2693 
2694 void MacroAssembler::decrementl(Register reg, int value) {
2695   if (value == min_jint) {subl(reg, value) ; return; }
2696   if (value <  0) { incrementl(reg, -value); return; }
2697   if (value == 0) {                        ; return; }
2698   if (value == 1 && UseIncDec) { decl(reg) ; return; }
2699   /* else */      { subl(reg, value)       ; return; }
2700 }
2701 
2702 void MacroAssembler::decrementl(Address dst, int value) {
2703   if (value == min_jint) {subl(dst, value) ; return; }
2704   if (value <  0) { incrementl(dst, -value); return; }
2705   if (value == 0) {                        ; return; }
2706   if (value == 1 && UseIncDec) { decl(dst) ; return; }
2707   /* else */      { subl(dst, value)       ; return; }
2708 }
2709 
2710 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
2711   assert (shift_value > 0, "illegal shift value");
2712   Label _is_positive;
2713   testl (reg, reg);
2714   jcc (Assembler::positive, _is_positive);
2715   int offset = (1 << shift_value) - 1 ;
2716 
2717   if (offset == 1) {
2718     incrementl(reg);
2719   } else {
2720     addl(reg, offset);
2721   }
2722 
2723   bind (_is_positive);
2724   sarl(reg, shift_value);
2725 }
2726 
2727 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
2728   if (reachable(src)) {
2729     Assembler::divsd(dst, as_Address(src));
2730   } else {
2731     lea(rscratch1, src);
2732     Assembler::divsd(dst, Address(rscratch1, 0));
2733   }
2734 }
2735 
2736 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
2737   if (reachable(src)) {
2738     Assembler::divss(dst, as_Address(src));
2739   } else {
2740     lea(rscratch1, src);
2741     Assembler::divss(dst, Address(rscratch1, 0));
2742   }
2743 }
2744 
2745 // !defined(COMPILER2) is because of stupid core builds
2746 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) || INCLUDE_JVMCI
2747 void MacroAssembler::empty_FPU_stack() {
2748   if (VM_Version::supports_mmx()) {
2749     emms();
2750   } else {
2751     for (int i = 8; i-- > 0; ) ffree(i);
2752   }
2753 }
2754 #endif // !LP64 || C1 || !C2 || INCLUDE_JVMCI
2755 
2756 
2757 void MacroAssembler::enter() {
2758   push(rbp);
2759   mov(rbp, rsp);
2760 }
2761 
2762 // A 5 byte nop that is safe for patching (see patch_verified_entry)
2763 void MacroAssembler::fat_nop() {
2764   if (UseAddressNop) {
2765     addr_nop_5();
2766   } else {
2767     emit_int8(0x26); // es:
2768     emit_int8(0x2e); // cs:
2769     emit_int8(0x64); // fs:
2770     emit_int8(0x65); // gs:
2771     emit_int8((unsigned char)0x90);
2772   }
2773 }
2774 
2775 void MacroAssembler::fcmp(Register tmp) {
2776   fcmp(tmp, 1, true, true);
2777 }
2778 
2779 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
2780   assert(!pop_right || pop_left, "usage error");
2781   if (VM_Version::supports_cmov()) {
2782     assert(tmp == noreg, "unneeded temp");
2783     if (pop_left) {
2784       fucomip(index);
2785     } else {
2786       fucomi(index);
2787     }
2788     if (pop_right) {
2789       fpop();
2790     }
2791   } else {
2792     assert(tmp != noreg, "need temp");
2793     if (pop_left) {
2794       if (pop_right) {
2795         fcompp();
2796       } else {
2797         fcomp(index);
2798       }
2799     } else {
2800       fcom(index);
2801     }
2802     // convert FPU condition into eflags condition via rax,
2803     save_rax(tmp);
2804     fwait(); fnstsw_ax();
2805     sahf();
2806     restore_rax(tmp);
2807   }
2808   // condition codes set as follows:
2809   //
2810   // CF (corresponds to C0) if x < y
2811   // PF (corresponds to C2) if unordered
2812   // ZF (corresponds to C3) if x = y
2813 }
2814 
2815 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
2816   fcmp2int(dst, unordered_is_less, 1, true, true);
2817 }
2818 
2819 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
2820   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
2821   Label L;
2822   if (unordered_is_less) {
2823     movl(dst, -1);
2824     jcc(Assembler::parity, L);
2825     jcc(Assembler::below , L);
2826     movl(dst, 0);
2827     jcc(Assembler::equal , L);
2828     increment(dst);
2829   } else { // unordered is greater
2830     movl(dst, 1);
2831     jcc(Assembler::parity, L);
2832     jcc(Assembler::above , L);
2833     movl(dst, 0);
2834     jcc(Assembler::equal , L);
2835     decrementl(dst);
2836   }
2837   bind(L);
2838 }
2839 
2840 void MacroAssembler::fld_d(AddressLiteral src) {
2841   fld_d(as_Address(src));
2842 }
2843 
2844 void MacroAssembler::fld_s(AddressLiteral src) {
2845   fld_s(as_Address(src));
2846 }
2847 
2848 void MacroAssembler::fld_x(AddressLiteral src) {
2849   Assembler::fld_x(as_Address(src));
2850 }
2851 
2852 void MacroAssembler::fldcw(AddressLiteral src) {
2853   Assembler::fldcw(as_Address(src));
2854 }
2855 
2856 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src) {
2857   if (reachable(src)) {
2858     Assembler::mulpd(dst, as_Address(src));
2859   } else {
2860     lea(rscratch1, src);
2861     Assembler::mulpd(dst, Address(rscratch1, 0));
2862   }
2863 }
2864 
2865 void MacroAssembler::increase_precision() {
2866   subptr(rsp, BytesPerWord);
2867   fnstcw(Address(rsp, 0));
2868   movl(rax, Address(rsp, 0));
2869   orl(rax, 0x300);
2870   push(rax);
2871   fldcw(Address(rsp, 0));
2872   pop(rax);
2873 }
2874 
2875 void MacroAssembler::restore_precision() {
2876   fldcw(Address(rsp, 0));
2877   addptr(rsp, BytesPerWord);
2878 }
2879 
2880 void MacroAssembler::fpop() {
2881   ffree();
2882   fincstp();
2883 }
2884 
2885 void MacroAssembler::load_float(Address src) {
2886   if (UseSSE >= 1) {
2887     movflt(xmm0, src);
2888   } else {
2889     LP64_ONLY(ShouldNotReachHere());
2890     NOT_LP64(fld_s(src));
2891   }
2892 }
2893 
2894 void MacroAssembler::store_float(Address dst) {
2895   if (UseSSE >= 1) {
2896     movflt(dst, xmm0);
2897   } else {
2898     LP64_ONLY(ShouldNotReachHere());
2899     NOT_LP64(fstp_s(dst));
2900   }
2901 }
2902 
2903 void MacroAssembler::load_double(Address src) {
2904   if (UseSSE >= 2) {
2905     movdbl(xmm0, src);
2906   } else {
2907     LP64_ONLY(ShouldNotReachHere());
2908     NOT_LP64(fld_d(src));
2909   }
2910 }
2911 
2912 void MacroAssembler::store_double(Address dst) {
2913   if (UseSSE >= 2) {
2914     movdbl(dst, xmm0);
2915   } else {
2916     LP64_ONLY(ShouldNotReachHere());
2917     NOT_LP64(fstp_d(dst));
2918   }
2919 }
2920 
2921 void MacroAssembler::push_zmm(XMMRegister reg) {
2922   lea(rsp, Address(rsp, -64)); // Use lea to not affect flags
2923   evmovdqul(Address(rsp, 0), reg, Assembler::AVX_512bit);
2924 }
2925 
2926 void MacroAssembler::pop_zmm(XMMRegister reg) {
2927   evmovdqul(reg, Address(rsp, 0), Assembler::AVX_512bit);
2928   lea(rsp, Address(rsp, 64)); // Use lea to not affect flags
2929 }
2930 
2931 void MacroAssembler::fremr(Register tmp) {
2932   save_rax(tmp);
2933   { Label L;
2934     bind(L);
2935     fprem();
2936     fwait(); fnstsw_ax();
2937 #ifdef _LP64
2938     testl(rax, 0x400);
2939     jcc(Assembler::notEqual, L);
2940 #else
2941     sahf();
2942     jcc(Assembler::parity, L);
2943 #endif // _LP64
2944   }
2945   restore_rax(tmp);
2946   // Result is in ST0.
2947   // Note: fxch & fpop to get rid of ST1
2948   // (otherwise FPU stack could overflow eventually)
2949   fxch(1);
2950   fpop();
2951 }
2952 
2953 // dst = c = a * b + c
2954 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
2955   Assembler::vfmadd231sd(c, a, b);
2956   if (dst != c) {
2957     movdbl(dst, c);
2958   }
2959 }
2960 
2961 // dst = c = a * b + c
2962 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
2963   Assembler::vfmadd231ss(c, a, b);
2964   if (dst != c) {
2965     movflt(dst, c);
2966   }
2967 }
2968 
2969 // dst = c = a * b + c
2970 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
2971   Assembler::vfmadd231pd(c, a, b, vector_len);
2972   if (dst != c) {
2973     vmovdqu(dst, c);
2974   }
2975 }
2976 
2977 // dst = c = a * b + c
2978 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
2979   Assembler::vfmadd231ps(c, a, b, vector_len);
2980   if (dst != c) {
2981     vmovdqu(dst, c);
2982   }
2983 }
2984 
2985 // dst = c = a * b + c
2986 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
2987   Assembler::vfmadd231pd(c, a, b, vector_len);
2988   if (dst != c) {
2989     vmovdqu(dst, c);
2990   }
2991 }
2992 
2993 // dst = c = a * b + c
2994 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
2995   Assembler::vfmadd231ps(c, a, b, vector_len);
2996   if (dst != c) {
2997     vmovdqu(dst, c);
2998   }
2999 }
3000 
3001 void MacroAssembler::incrementl(AddressLiteral dst) {
3002   if (reachable(dst)) {
3003     incrementl(as_Address(dst));
3004   } else {
3005     lea(rscratch1, dst);
3006     incrementl(Address(rscratch1, 0));
3007   }
3008 }
3009 
3010 void MacroAssembler::incrementl(ArrayAddress dst) {
3011   incrementl(as_Address(dst));
3012 }
3013 
3014 void MacroAssembler::incrementl(Register reg, int value) {
3015   if (value == min_jint) {addl(reg, value) ; return; }
3016   if (value <  0) { decrementl(reg, -value); return; }
3017   if (value == 0) {                        ; return; }
3018   if (value == 1 && UseIncDec) { incl(reg) ; return; }
3019   /* else */      { addl(reg, value)       ; return; }
3020 }
3021 
3022 void MacroAssembler::incrementl(Address dst, int value) {
3023   if (value == min_jint) {addl(dst, value) ; return; }
3024   if (value <  0) { decrementl(dst, -value); return; }
3025   if (value == 0) {                        ; return; }
3026   if (value == 1 && UseIncDec) { incl(dst) ; return; }
3027   /* else */      { addl(dst, value)       ; return; }
3028 }
3029 
3030 void MacroAssembler::jump(AddressLiteral dst) {
3031   if (reachable(dst)) {
3032     jmp_literal(dst.target(), dst.rspec());
3033   } else {
3034     lea(rscratch1, dst);
3035     jmp(rscratch1);
3036   }
3037 }
3038 
3039 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
3040   if (reachable(dst)) {
3041     InstructionMark im(this);
3042     relocate(dst.reloc());
3043     const int short_size = 2;
3044     const int long_size = 6;
3045     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
3046     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
3047       // 0111 tttn #8-bit disp
3048       emit_int8(0x70 | cc);
3049       emit_int8((offs - short_size) & 0xFF);
3050     } else {
3051       // 0000 1111 1000 tttn #32-bit disp
3052       emit_int8(0x0F);
3053       emit_int8((unsigned char)(0x80 | cc));
3054       emit_int32(offs - long_size);
3055     }
3056   } else {
3057 #ifdef ASSERT
3058     warning("reversing conditional branch");
3059 #endif /* ASSERT */
3060     Label skip;
3061     jccb(reverse[cc], skip);
3062     lea(rscratch1, dst);
3063     Assembler::jmp(rscratch1);
3064     bind(skip);
3065   }
3066 }
3067 
3068 void MacroAssembler::ldmxcsr(AddressLiteral src) {
3069   if (reachable(src)) {
3070     Assembler::ldmxcsr(as_Address(src));
3071   } else {
3072     lea(rscratch1, src);
3073     Assembler::ldmxcsr(Address(rscratch1, 0));
3074   }
3075 }
3076 
3077 int MacroAssembler::load_signed_byte(Register dst, Address src) {
3078   int off;
3079   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3080     off = offset();
3081     movsbl(dst, src); // movsxb
3082   } else {
3083     off = load_unsigned_byte(dst, src);
3084     shll(dst, 24);
3085     sarl(dst, 24);
3086   }
3087   return off;
3088 }
3089 
3090 // Note: load_signed_short used to be called load_signed_word.
3091 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
3092 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
3093 // The term "word" in HotSpot means a 32- or 64-bit machine word.
3094 int MacroAssembler::load_signed_short(Register dst, Address src) {
3095   int off;
3096   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3097     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
3098     // version but this is what 64bit has always done. This seems to imply
3099     // that users are only using 32bits worth.
3100     off = offset();
3101     movswl(dst, src); // movsxw
3102   } else {
3103     off = load_unsigned_short(dst, src);
3104     shll(dst, 16);
3105     sarl(dst, 16);
3106   }
3107   return off;
3108 }
3109 
3110 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
3111   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3112   // and "3.9 Partial Register Penalties", p. 22).
3113   int off;
3114   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
3115     off = offset();
3116     movzbl(dst, src); // movzxb
3117   } else {
3118     xorl(dst, dst);
3119     off = offset();
3120     movb(dst, src);
3121   }
3122   return off;
3123 }
3124 
3125 // Note: load_unsigned_short used to be called load_unsigned_word.
3126 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
3127   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3128   // and "3.9 Partial Register Penalties", p. 22).
3129   int off;
3130   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
3131     off = offset();
3132     movzwl(dst, src); // movzxw
3133   } else {
3134     xorl(dst, dst);
3135     off = offset();
3136     movw(dst, src);
3137   }
3138   return off;
3139 }
3140 
3141 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
3142   switch (size_in_bytes) {
3143 #ifndef _LP64
3144   case  8:
3145     assert(dst2 != noreg, "second dest register required");
3146     movl(dst,  src);
3147     movl(dst2, src.plus_disp(BytesPerInt));
3148     break;
3149 #else
3150   case  8:  movq(dst, src); break;
3151 #endif
3152   case  4:  movl(dst, src); break;
3153   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
3154   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
3155   default:  ShouldNotReachHere();
3156   }
3157 }
3158 
3159 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
3160   switch (size_in_bytes) {
3161 #ifndef _LP64
3162   case  8:
3163     assert(src2 != noreg, "second source register required");
3164     movl(dst,                        src);
3165     movl(dst.plus_disp(BytesPerInt), src2);
3166     break;
3167 #else
3168   case  8:  movq(dst, src); break;
3169 #endif
3170   case  4:  movl(dst, src); break;
3171   case  2:  movw(dst, src); break;
3172   case  1:  movb(dst, src); break;
3173   default:  ShouldNotReachHere();
3174   }
3175 }
3176 
3177 void MacroAssembler::mov32(AddressLiteral dst, Register src) {
3178   if (reachable(dst)) {
3179     movl(as_Address(dst), src);
3180   } else {
3181     lea(rscratch1, dst);
3182     movl(Address(rscratch1, 0), src);
3183   }
3184 }
3185 
3186 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
3187   if (reachable(src)) {
3188     movl(dst, as_Address(src));
3189   } else {
3190     lea(rscratch1, src);
3191     movl(dst, Address(rscratch1, 0));
3192   }
3193 }
3194 
3195 // C++ bool manipulation
3196 
3197 void MacroAssembler::movbool(Register dst, Address src) {
3198   if(sizeof(bool) == 1)
3199     movb(dst, src);
3200   else if(sizeof(bool) == 2)
3201     movw(dst, src);
3202   else if(sizeof(bool) == 4)
3203     movl(dst, src);
3204   else
3205     // unsupported
3206     ShouldNotReachHere();
3207 }
3208 
3209 void MacroAssembler::movbool(Address dst, bool boolconst) {
3210   if(sizeof(bool) == 1)
3211     movb(dst, (int) boolconst);
3212   else if(sizeof(bool) == 2)
3213     movw(dst, (int) boolconst);
3214   else if(sizeof(bool) == 4)
3215     movl(dst, (int) boolconst);
3216   else
3217     // unsupported
3218     ShouldNotReachHere();
3219 }
3220 
3221 void MacroAssembler::movbool(Address dst, Register src) {
3222   if(sizeof(bool) == 1)
3223     movb(dst, src);
3224   else if(sizeof(bool) == 2)
3225     movw(dst, src);
3226   else if(sizeof(bool) == 4)
3227     movl(dst, src);
3228   else
3229     // unsupported
3230     ShouldNotReachHere();
3231 }
3232 
3233 void MacroAssembler::movbyte(ArrayAddress dst, int src) {
3234   movb(as_Address(dst), src);
3235 }
3236 
3237 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src) {
3238   if (reachable(src)) {
3239     movdl(dst, as_Address(src));
3240   } else {
3241     lea(rscratch1, src);
3242     movdl(dst, Address(rscratch1, 0));
3243   }
3244 }
3245 
3246 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src) {
3247   if (reachable(src)) {
3248     movq(dst, as_Address(src));
3249   } else {
3250     lea(rscratch1, src);
3251     movq(dst, Address(rscratch1, 0));
3252   }
3253 }
3254 
3255 void MacroAssembler::setvectmask(Register dst, Register src) {
3256   Assembler::movl(dst, 1);
3257   Assembler::shlxl(dst, dst, src);
3258   Assembler::decl(dst);
3259   Assembler::kmovdl(k1, dst);
3260   Assembler::movl(dst, src);
3261 }
3262 
3263 void MacroAssembler::restorevectmask() {
3264   Assembler::knotwl(k1, k0);
3265 }
3266 
3267 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
3268   if (reachable(src)) {
3269     if (UseXmmLoadAndClearUpper) {
3270       movsd (dst, as_Address(src));
3271     } else {
3272       movlpd(dst, as_Address(src));
3273     }
3274   } else {
3275     lea(rscratch1, src);
3276     if (UseXmmLoadAndClearUpper) {
3277       movsd (dst, Address(rscratch1, 0));
3278     } else {
3279       movlpd(dst, Address(rscratch1, 0));
3280     }
3281   }
3282 }
3283 
3284 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
3285   if (reachable(src)) {
3286     movss(dst, as_Address(src));
3287   } else {
3288     lea(rscratch1, src);
3289     movss(dst, Address(rscratch1, 0));
3290   }
3291 }
3292 
3293 void MacroAssembler::movptr(Register dst, Register src) {
3294   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3295 }
3296 
3297 void MacroAssembler::movptr(Register dst, Address src) {
3298   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3299 }
3300 
3301 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
3302 void MacroAssembler::movptr(Register dst, intptr_t src) {
3303   LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src));
3304 }
3305 
3306 void MacroAssembler::movptr(Address dst, Register src) {
3307   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3308 }
3309 
3310 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
3311   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3312     Assembler::vextractf32x4(dst, src, 0);
3313   } else {
3314     Assembler::movdqu(dst, src);
3315   }
3316 }
3317 
3318 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
3319   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3320     Assembler::vinsertf32x4(dst, dst, src, 0);
3321   } else {
3322     Assembler::movdqu(dst, src);
3323   }
3324 }
3325 
3326 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
3327   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3328     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3329   } else {
3330     Assembler::movdqu(dst, src);
3331   }
3332 }
3333 
3334 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register scratchReg) {
3335   if (reachable(src)) {
3336     movdqu(dst, as_Address(src));
3337   } else {
3338     lea(scratchReg, src);
3339     movdqu(dst, Address(scratchReg, 0));
3340   }
3341 }
3342 
3343 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
3344   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3345     vextractf64x4_low(dst, src);
3346   } else {
3347     Assembler::vmovdqu(dst, src);
3348   }
3349 }
3350 
3351 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
3352   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3353     vinsertf64x4_low(dst, src);
3354   } else {
3355     Assembler::vmovdqu(dst, src);
3356   }
3357 }
3358 
3359 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3360   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3361     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3362   }
3363   else {
3364     Assembler::vmovdqu(dst, src);
3365   }
3366 }
3367 
3368 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src) {
3369   if (reachable(src)) {
3370     vmovdqu(dst, as_Address(src));
3371   }
3372   else {
3373     lea(rscratch1, src);
3374     vmovdqu(dst, Address(rscratch1, 0));
3375   }
3376 }
3377 
3378 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
3379   if (reachable(src)) {
3380     Assembler::evmovdquq(dst, as_Address(src), vector_len);
3381   } else {
3382     lea(rscratch, src);
3383     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
3384   }
3385 }
3386 
3387 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) {
3388   if (reachable(src)) {
3389     Assembler::movdqa(dst, as_Address(src));
3390   } else {
3391     lea(rscratch1, src);
3392     Assembler::movdqa(dst, Address(rscratch1, 0));
3393   }
3394 }
3395 
3396 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
3397   if (reachable(src)) {
3398     Assembler::movsd(dst, as_Address(src));
3399   } else {
3400     lea(rscratch1, src);
3401     Assembler::movsd(dst, Address(rscratch1, 0));
3402   }
3403 }
3404 
3405 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
3406   if (reachable(src)) {
3407     Assembler::movss(dst, as_Address(src));
3408   } else {
3409     lea(rscratch1, src);
3410     Assembler::movss(dst, Address(rscratch1, 0));
3411   }
3412 }
3413 
3414 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) {
3415   if (reachable(src)) {
3416     Assembler::mulsd(dst, as_Address(src));
3417   } else {
3418     lea(rscratch1, src);
3419     Assembler::mulsd(dst, Address(rscratch1, 0));
3420   }
3421 }
3422 
3423 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) {
3424   if (reachable(src)) {
3425     Assembler::mulss(dst, as_Address(src));
3426   } else {
3427     lea(rscratch1, src);
3428     Assembler::mulss(dst, Address(rscratch1, 0));
3429   }
3430 }
3431 
3432 void MacroAssembler::null_check(Register reg, int offset) {
3433   if (needs_explicit_null_check(offset)) {
3434     // provoke OS NULL exception if reg = NULL by
3435     // accessing M[reg] w/o changing any (non-CC) registers
3436     // NOTE: cmpl is plenty here to provoke a segv
3437     cmpptr(rax, Address(reg, 0));
3438     // Note: should probably use testl(rax, Address(reg, 0));
3439     //       may be shorter code (however, this version of
3440     //       testl needs to be implemented first)
3441   } else {
3442     // nothing to do, (later) access of M[reg + offset]
3443     // will provoke OS NULL exception if reg = NULL
3444   }
3445 }
3446 
3447 void MacroAssembler::os_breakpoint() {
3448   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
3449   // (e.g., MSVC can't call ps() otherwise)
3450   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
3451 }
3452 
3453 void MacroAssembler::unimplemented(const char* what) {
3454   const char* buf = NULL;
3455   {
3456     ResourceMark rm;
3457     stringStream ss;
3458     ss.print("unimplemented: %s", what);
3459     buf = code_string(ss.as_string());
3460   }
3461   stop(buf);
3462 }
3463 
3464 #ifdef _LP64
3465 #define XSTATE_BV 0x200
3466 #endif
3467 
3468 void MacroAssembler::pop_CPU_state() {
3469   pop_FPU_state();
3470   pop_IU_state();
3471 }
3472 
3473 void MacroAssembler::pop_FPU_state() {
3474 #ifndef _LP64
3475   frstor(Address(rsp, 0));
3476 #else
3477   fxrstor(Address(rsp, 0));
3478 #endif
3479   addptr(rsp, FPUStateSizeInWords * wordSize);
3480 }
3481 
3482 void MacroAssembler::pop_IU_state() {
3483   popa();
3484   LP64_ONLY(addq(rsp, 8));
3485   popf();
3486 }
3487 
3488 // Save Integer and Float state
3489 // Warning: Stack must be 16 byte aligned (64bit)
3490 void MacroAssembler::push_CPU_state() {
3491   push_IU_state();
3492   push_FPU_state();
3493 }
3494 
3495 void MacroAssembler::push_FPU_state() {
3496   subptr(rsp, FPUStateSizeInWords * wordSize);
3497 #ifndef _LP64
3498   fnsave(Address(rsp, 0));
3499   fwait();
3500 #else
3501   fxsave(Address(rsp, 0));
3502 #endif // LP64
3503 }
3504 
3505 void MacroAssembler::push_IU_state() {
3506   // Push flags first because pusha kills them
3507   pushf();
3508   // Make sure rsp stays 16-byte aligned
3509   LP64_ONLY(subq(rsp, 8));
3510   pusha();
3511 }
3512 
3513 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
3514   if (!java_thread->is_valid()) {
3515     java_thread = rdi;
3516     get_thread(java_thread);
3517   }
3518   // we must set sp to zero to clear frame
3519   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
3520   if (clear_fp) {
3521     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
3522   }
3523 
3524   // Always clear the pc because it could have been set by make_walkable()
3525   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
3526 
3527   vzeroupper();
3528 }
3529 
3530 void MacroAssembler::restore_rax(Register tmp) {
3531   if (tmp == noreg) pop(rax);
3532   else if (tmp != rax) mov(rax, tmp);
3533 }
3534 
3535 void MacroAssembler::round_to(Register reg, int modulus) {
3536   addptr(reg, modulus - 1);
3537   andptr(reg, -modulus);
3538 }
3539 
3540 void MacroAssembler::save_rax(Register tmp) {
3541   if (tmp == noreg) push(rax);
3542   else if (tmp != rax) mov(tmp, rax);
3543 }
3544 
3545 // Write serialization page so VM thread can do a pseudo remote membar.
3546 // We use the current thread pointer to calculate a thread specific
3547 // offset to write to within the page. This minimizes bus traffic
3548 // due to cache line collision.
3549 void MacroAssembler::serialize_memory(Register thread, Register tmp) {
3550   movl(tmp, thread);
3551   shrl(tmp, os::get_serialize_page_shift_count());
3552   andl(tmp, (os::vm_page_size() - sizeof(int)));
3553 
3554   Address index(noreg, tmp, Address::times_1);
3555   ExternalAddress page(os::get_memory_serialize_page());
3556 
3557   // Size of store must match masking code above
3558   movl(as_Address(ArrayAddress(page, index)), tmp);
3559 }
3560 
3561 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, Register temp_reg) {
3562   if (SafepointMechanism::uses_thread_local_poll()) {
3563 #ifdef _LP64
3564     assert(thread_reg == r15_thread, "should be");
3565 #else
3566     if (thread_reg == noreg) {
3567       thread_reg = temp_reg;
3568       get_thread(thread_reg);
3569     }
3570 #endif
3571     testb(Address(thread_reg, Thread::polling_page_offset()), SafepointMechanism::poll_bit());
3572     jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
3573   } else {
3574     cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
3575         SafepointSynchronize::_not_synchronized);
3576     jcc(Assembler::notEqual, slow_path);
3577   }
3578 }
3579 
3580 // Calls to C land
3581 //
3582 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
3583 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
3584 // has to be reset to 0. This is required to allow proper stack traversal.
3585 void MacroAssembler::set_last_Java_frame(Register java_thread,
3586                                          Register last_java_sp,
3587                                          Register last_java_fp,
3588                                          address  last_java_pc) {
3589   vzeroupper();
3590   // determine java_thread register
3591   if (!java_thread->is_valid()) {
3592     java_thread = rdi;
3593     get_thread(java_thread);
3594   }
3595   // determine last_java_sp register
3596   if (!last_java_sp->is_valid()) {
3597     last_java_sp = rsp;
3598   }
3599 
3600   // last_java_fp is optional
3601 
3602   if (last_java_fp->is_valid()) {
3603     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
3604   }
3605 
3606   // last_java_pc is optional
3607 
3608   if (last_java_pc != NULL) {
3609     lea(Address(java_thread,
3610                  JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()),
3611         InternalAddress(last_java_pc));
3612 
3613   }
3614   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
3615 }
3616 
3617 void MacroAssembler::shlptr(Register dst, int imm8) {
3618   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
3619 }
3620 
3621 void MacroAssembler::shrptr(Register dst, int imm8) {
3622   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
3623 }
3624 
3625 void MacroAssembler::sign_extend_byte(Register reg) {
3626   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
3627     movsbl(reg, reg); // movsxb
3628   } else {
3629     shll(reg, 24);
3630     sarl(reg, 24);
3631   }
3632 }
3633 
3634 void MacroAssembler::sign_extend_short(Register reg) {
3635   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3636     movswl(reg, reg); // movsxw
3637   } else {
3638     shll(reg, 16);
3639     sarl(reg, 16);
3640   }
3641 }
3642 
3643 void MacroAssembler::testl(Register dst, AddressLiteral src) {
3644   assert(reachable(src), "Address should be reachable");
3645   testl(dst, as_Address(src));
3646 }
3647 
3648 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3649   int dst_enc = dst->encoding();
3650   int src_enc = src->encoding();
3651   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3652     Assembler::pcmpeqb(dst, src);
3653   } else if ((dst_enc < 16) && (src_enc < 16)) {
3654     Assembler::pcmpeqb(dst, src);
3655   } else if (src_enc < 16) {
3656     push_zmm(xmm0);
3657     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3658     Assembler::pcmpeqb(xmm0, src);
3659     movdqu(dst, xmm0);
3660     pop_zmm(xmm0);
3661   } else if (dst_enc < 16) {
3662     push_zmm(xmm0);
3663     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3664     Assembler::pcmpeqb(dst, xmm0);
3665     pop_zmm(xmm0);
3666   } else {
3667     push_zmm(xmm0);
3668     push_zmm(xmm1);
3669     movdqu(xmm0, src);
3670     movdqu(xmm1, dst);
3671     Assembler::pcmpeqb(xmm1, xmm0);
3672     movdqu(dst, xmm1);
3673     pop_zmm(xmm1);
3674     pop_zmm(xmm0);
3675   }
3676 }
3677 
3678 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3679   int dst_enc = dst->encoding();
3680   int src_enc = src->encoding();
3681   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3682     Assembler::pcmpeqw(dst, src);
3683   } else if ((dst_enc < 16) && (src_enc < 16)) {
3684     Assembler::pcmpeqw(dst, src);
3685   } else if (src_enc < 16) {
3686     push_zmm(xmm0);
3687     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3688     Assembler::pcmpeqw(xmm0, src);
3689     movdqu(dst, xmm0);
3690     pop_zmm(xmm0);
3691   } else if (dst_enc < 16) {
3692     push_zmm(xmm0);
3693     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3694     Assembler::pcmpeqw(dst, xmm0);
3695     pop_zmm(xmm0);
3696   } else {
3697     push_zmm(xmm0);
3698     push_zmm(xmm1);
3699     movdqu(xmm0, src);
3700     movdqu(xmm1, dst);
3701     Assembler::pcmpeqw(xmm1, xmm0);
3702     movdqu(dst, xmm1);
3703     pop_zmm(xmm1);
3704     pop_zmm(xmm0);
3705   }
3706 }
3707 
3708 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3709   int dst_enc = dst->encoding();
3710   if (dst_enc < 16) {
3711     Assembler::pcmpestri(dst, src, imm8);
3712   } else {
3713     push_zmm(xmm0);
3714     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3715     Assembler::pcmpestri(xmm0, src, imm8);
3716     movdqu(dst, xmm0);
3717     pop_zmm(xmm0);
3718   }
3719 }
3720 
3721 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3722   int dst_enc = dst->encoding();
3723   int src_enc = src->encoding();
3724   if ((dst_enc < 16) && (src_enc < 16)) {
3725     Assembler::pcmpestri(dst, src, imm8);
3726   } else if (src_enc < 16) {
3727     push_zmm(xmm0);
3728     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3729     Assembler::pcmpestri(xmm0, src, imm8);
3730     movdqu(dst, xmm0);
3731     pop_zmm(xmm0);
3732   } else if (dst_enc < 16) {
3733     push_zmm(xmm0);
3734     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3735     Assembler::pcmpestri(dst, xmm0, imm8);
3736     pop_zmm(xmm0);
3737   } else {
3738     push_zmm(xmm0);
3739     push_zmm(xmm1);
3740     movdqu(xmm0, src);
3741     movdqu(xmm1, dst);
3742     Assembler::pcmpestri(xmm1, xmm0, imm8);
3743     movdqu(dst, xmm1);
3744     pop_zmm(xmm1);
3745     pop_zmm(xmm0);
3746   }
3747 }
3748 
3749 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3750   int dst_enc = dst->encoding();
3751   int src_enc = src->encoding();
3752   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3753     Assembler::pmovzxbw(dst, src);
3754   } else if ((dst_enc < 16) && (src_enc < 16)) {
3755     Assembler::pmovzxbw(dst, src);
3756   } else if (src_enc < 16) {
3757     push_zmm(xmm0);
3758     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3759     Assembler::pmovzxbw(xmm0, src);
3760     movdqu(dst, xmm0);
3761     pop_zmm(xmm0);
3762   } else if (dst_enc < 16) {
3763     push_zmm(xmm0);
3764     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3765     Assembler::pmovzxbw(dst, xmm0);
3766     pop_zmm(xmm0);
3767   } else {
3768     push_zmm(xmm0);
3769     push_zmm(xmm1);
3770     movdqu(xmm0, src);
3771     movdqu(xmm1, dst);
3772     Assembler::pmovzxbw(xmm1, xmm0);
3773     movdqu(dst, xmm1);
3774     pop_zmm(xmm1);
3775     pop_zmm(xmm0);
3776   }
3777 }
3778 
3779 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
3780   int dst_enc = dst->encoding();
3781   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3782     Assembler::pmovzxbw(dst, src);
3783   } else if (dst_enc < 16) {
3784     Assembler::pmovzxbw(dst, src);
3785   } else {
3786     push_zmm(xmm0);
3787     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3788     Assembler::pmovzxbw(xmm0, src);
3789     movdqu(dst, xmm0);
3790     pop_zmm(xmm0);
3791   }
3792 }
3793 
3794 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
3795   int src_enc = src->encoding();
3796   if (src_enc < 16) {
3797     Assembler::pmovmskb(dst, src);
3798   } else {
3799     push_zmm(xmm0);
3800     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3801     Assembler::pmovmskb(dst, xmm0);
3802     pop_zmm(xmm0);
3803   }
3804 }
3805 
3806 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
3807   int dst_enc = dst->encoding();
3808   int src_enc = src->encoding();
3809   if ((dst_enc < 16) && (src_enc < 16)) {
3810     Assembler::ptest(dst, src);
3811   } else if (src_enc < 16) {
3812     push_zmm(xmm0);
3813     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3814     Assembler::ptest(xmm0, src);
3815     pop_zmm(xmm0);
3816   } else if (dst_enc < 16) {
3817     push_zmm(xmm0);
3818     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3819     Assembler::ptest(dst, xmm0);
3820     pop_zmm(xmm0);
3821   } else {
3822     push_zmm(xmm0);
3823     push_zmm(xmm1);
3824     movdqu(xmm0, src);
3825     movdqu(xmm1, dst);
3826     Assembler::ptest(xmm1, xmm0);
3827     pop_zmm(xmm1);
3828     pop_zmm(xmm0);
3829   }
3830 }
3831 
3832 void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) {
3833   if (reachable(src)) {
3834     Assembler::sqrtsd(dst, as_Address(src));
3835   } else {
3836     lea(rscratch1, src);
3837     Assembler::sqrtsd(dst, Address(rscratch1, 0));
3838   }
3839 }
3840 
3841 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) {
3842   if (reachable(src)) {
3843     Assembler::sqrtss(dst, as_Address(src));
3844   } else {
3845     lea(rscratch1, src);
3846     Assembler::sqrtss(dst, Address(rscratch1, 0));
3847   }
3848 }
3849 
3850 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) {
3851   if (reachable(src)) {
3852     Assembler::subsd(dst, as_Address(src));
3853   } else {
3854     lea(rscratch1, src);
3855     Assembler::subsd(dst, Address(rscratch1, 0));
3856   }
3857 }
3858 
3859 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) {
3860   if (reachable(src)) {
3861     Assembler::subss(dst, as_Address(src));
3862   } else {
3863     lea(rscratch1, src);
3864     Assembler::subss(dst, Address(rscratch1, 0));
3865   }
3866 }
3867 
3868 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
3869   if (reachable(src)) {
3870     Assembler::ucomisd(dst, as_Address(src));
3871   } else {
3872     lea(rscratch1, src);
3873     Assembler::ucomisd(dst, Address(rscratch1, 0));
3874   }
3875 }
3876 
3877 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
3878   if (reachable(src)) {
3879     Assembler::ucomiss(dst, as_Address(src));
3880   } else {
3881     lea(rscratch1, src);
3882     Assembler::ucomiss(dst, Address(rscratch1, 0));
3883   }
3884 }
3885 
3886 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
3887   // Used in sign-bit flipping with aligned address.
3888   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3889   if (reachable(src)) {
3890     Assembler::xorpd(dst, as_Address(src));
3891   } else {
3892     lea(rscratch1, src);
3893     Assembler::xorpd(dst, Address(rscratch1, 0));
3894   }
3895 }
3896 
3897 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
3898   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
3899     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
3900   }
3901   else {
3902     Assembler::xorpd(dst, src);
3903   }
3904 }
3905 
3906 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
3907   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
3908     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
3909   } else {
3910     Assembler::xorps(dst, src);
3911   }
3912 }
3913 
3914 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
3915   // Used in sign-bit flipping with aligned address.
3916   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3917   if (reachable(src)) {
3918     Assembler::xorps(dst, as_Address(src));
3919   } else {
3920     lea(rscratch1, src);
3921     Assembler::xorps(dst, Address(rscratch1, 0));
3922   }
3923 }
3924 
3925 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) {
3926   // Used in sign-bit flipping with aligned address.
3927   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
3928   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
3929   if (reachable(src)) {
3930     Assembler::pshufb(dst, as_Address(src));
3931   } else {
3932     lea(rscratch1, src);
3933     Assembler::pshufb(dst, Address(rscratch1, 0));
3934   }
3935 }
3936 
3937 // AVX 3-operands instructions
3938 
3939 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3940   if (reachable(src)) {
3941     vaddsd(dst, nds, as_Address(src));
3942   } else {
3943     lea(rscratch1, src);
3944     vaddsd(dst, nds, Address(rscratch1, 0));
3945   }
3946 }
3947 
3948 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3949   if (reachable(src)) {
3950     vaddss(dst, nds, as_Address(src));
3951   } else {
3952     lea(rscratch1, src);
3953     vaddss(dst, nds, Address(rscratch1, 0));
3954   }
3955 }
3956 
3957 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
3958   int dst_enc = dst->encoding();
3959   int nds_enc = nds->encoding();
3960   int src_enc = src->encoding();
3961   if ((dst_enc < 16) && (nds_enc < 16)) {
3962     vandps(dst, nds, negate_field, vector_len);
3963   } else if ((src_enc < 16) && (dst_enc < 16)) {
3964     // Use src scratch register
3965     evmovdqul(src, nds, Assembler::AVX_512bit);
3966     vandps(dst, src, negate_field, vector_len);
3967   } else if (dst_enc < 16) {
3968     evmovdqul(dst, nds, Assembler::AVX_512bit);
3969     vandps(dst, dst, negate_field, vector_len);
3970   } else if (nds_enc < 16) {
3971     vandps(nds, nds, negate_field, vector_len);
3972     evmovdqul(dst, nds, Assembler::AVX_512bit);
3973   } else if (src_enc < 16) {
3974     evmovdqul(src, nds, Assembler::AVX_512bit);
3975     vandps(src, src, negate_field, vector_len);
3976     evmovdqul(dst, src, Assembler::AVX_512bit);
3977   } else {
3978     if (src_enc != dst_enc) {
3979       // Use src scratch register
3980       evmovdqul(src, xmm0, Assembler::AVX_512bit);
3981       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
3982       vandps(xmm0, xmm0, negate_field, vector_len);
3983       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
3984       evmovdqul(xmm0, src, Assembler::AVX_512bit);
3985     } else {
3986       push_zmm(xmm0);
3987       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
3988       vandps(xmm0, xmm0, negate_field, vector_len);
3989       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
3990       pop_zmm(xmm0);
3991     }
3992   }
3993 }
3994 
3995 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
3996   int dst_enc = dst->encoding();
3997   int nds_enc = nds->encoding();
3998   int src_enc = src->encoding();
3999   if ((dst_enc < 16) && (nds_enc < 16)) {
4000     vandpd(dst, nds, negate_field, vector_len);
4001   } else if ((src_enc < 16) && (dst_enc < 16)) {
4002     // Use src scratch register
4003     evmovdqul(src, nds, Assembler::AVX_512bit);
4004     vandpd(dst, src, negate_field, vector_len);
4005   } else if (dst_enc < 16) {
4006     evmovdqul(dst, nds, Assembler::AVX_512bit);
4007     vandpd(dst, dst, negate_field, vector_len);
4008   } else if (nds_enc < 16) {
4009     vandpd(nds, nds, negate_field, vector_len);
4010     evmovdqul(dst, nds, Assembler::AVX_512bit);
4011   } else if (src_enc < 16) {
4012     evmovdqul(src, nds, Assembler::AVX_512bit);
4013     vandpd(src, src, negate_field, vector_len);
4014     evmovdqul(dst, src, Assembler::AVX_512bit);
4015   } else {
4016     if (src_enc != dst_enc) {
4017       evmovdqul(src, xmm0, Assembler::AVX_512bit);
4018       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4019       vandpd(xmm0, xmm0, negate_field, vector_len);
4020       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4021       evmovdqul(xmm0, src, Assembler::AVX_512bit);
4022     } else {
4023       push_zmm(xmm0);
4024       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4025       vandpd(xmm0, xmm0, negate_field, vector_len);
4026       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4027       pop_zmm(xmm0);
4028     }
4029   }
4030 }
4031 
4032 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4033   int dst_enc = dst->encoding();
4034   int nds_enc = nds->encoding();
4035   int src_enc = src->encoding();
4036   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4037     Assembler::vpaddb(dst, nds, src, vector_len);
4038   } else if ((dst_enc < 16) && (src_enc < 16)) {
4039     Assembler::vpaddb(dst, dst, src, vector_len);
4040   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4041     // use nds as scratch for src
4042     evmovdqul(nds, src, Assembler::AVX_512bit);
4043     Assembler::vpaddb(dst, dst, nds, vector_len);
4044   } else if ((src_enc < 16) && (nds_enc < 16)) {
4045     // use nds as scratch for dst
4046     evmovdqul(nds, dst, Assembler::AVX_512bit);
4047     Assembler::vpaddb(nds, nds, src, vector_len);
4048     evmovdqul(dst, nds, Assembler::AVX_512bit);
4049   } else if (dst_enc < 16) {
4050     // use nds as scatch for xmm0 to hold src
4051     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4052     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4053     Assembler::vpaddb(dst, dst, xmm0, vector_len);
4054     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4055   } else {
4056     // worse case scenario, all regs are in the upper bank
4057     push_zmm(xmm1);
4058     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4059     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4060     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4061     Assembler::vpaddb(xmm0, xmm0, xmm1, vector_len);
4062     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4063     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4064     pop_zmm(xmm1);
4065   }
4066 }
4067 
4068 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4069   int dst_enc = dst->encoding();
4070   int nds_enc = nds->encoding();
4071   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4072     Assembler::vpaddb(dst, nds, src, vector_len);
4073   } else if (dst_enc < 16) {
4074     Assembler::vpaddb(dst, dst, src, vector_len);
4075   } else if (nds_enc < 16) {
4076     // implies dst_enc in upper bank with src as scratch
4077     evmovdqul(nds, dst, Assembler::AVX_512bit);
4078     Assembler::vpaddb(nds, nds, src, vector_len);
4079     evmovdqul(dst, nds, Assembler::AVX_512bit);
4080   } else {
4081     // worse case scenario, all regs in upper bank
4082     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4083     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4084     Assembler::vpaddb(xmm0, xmm0, src, vector_len);
4085     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4086     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4087   }
4088 }
4089 
4090 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4091   int dst_enc = dst->encoding();
4092   int nds_enc = nds->encoding();
4093   int src_enc = src->encoding();
4094   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4095     Assembler::vpaddw(dst, nds, src, vector_len);
4096   } else if ((dst_enc < 16) && (src_enc < 16)) {
4097     Assembler::vpaddw(dst, dst, src, vector_len);
4098   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4099     // use nds as scratch for src
4100     evmovdqul(nds, src, Assembler::AVX_512bit);
4101     Assembler::vpaddw(dst, dst, nds, vector_len);
4102   } else if ((src_enc < 16) && (nds_enc < 16)) {
4103     // use nds as scratch for dst
4104     evmovdqul(nds, dst, Assembler::AVX_512bit);
4105     Assembler::vpaddw(nds, nds, src, vector_len);
4106     evmovdqul(dst, nds, Assembler::AVX_512bit);
4107   } else if (dst_enc < 16) {
4108     // use nds as scatch for xmm0 to hold src
4109     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4110     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4111     Assembler::vpaddw(dst, dst, xmm0, vector_len);
4112     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4113   } else {
4114     // worse case scenario, all regs are in the upper bank
4115     push_zmm(xmm1);
4116     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4117     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4118     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4119     Assembler::vpaddw(xmm0, xmm0, xmm1, vector_len);
4120     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4121     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4122     pop_zmm(xmm1);
4123   }
4124 }
4125 
4126 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4127   int dst_enc = dst->encoding();
4128   int nds_enc = nds->encoding();
4129   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4130     Assembler::vpaddw(dst, nds, src, vector_len);
4131   } else if (dst_enc < 16) {
4132     Assembler::vpaddw(dst, dst, src, vector_len);
4133   } else if (nds_enc < 16) {
4134     // implies dst_enc in upper bank with nds as scratch
4135     evmovdqul(nds, dst, Assembler::AVX_512bit);
4136     Assembler::vpaddw(nds, nds, src, vector_len);
4137     evmovdqul(dst, nds, Assembler::AVX_512bit);
4138   } else {
4139     // worse case scenario, all regs in upper bank
4140     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4141     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4142     Assembler::vpaddw(xmm0, xmm0, src, vector_len);
4143     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4144     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4145   }
4146 }
4147 
4148 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4149   if (reachable(src)) {
4150     Assembler::vpand(dst, nds, as_Address(src), vector_len);
4151   } else {
4152     lea(rscratch1, src);
4153     Assembler::vpand(dst, nds, Address(rscratch1, 0), vector_len);
4154   }
4155 }
4156 
4157 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
4158   int dst_enc = dst->encoding();
4159   int src_enc = src->encoding();
4160   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4161     Assembler::vpbroadcastw(dst, src);
4162   } else if ((dst_enc < 16) && (src_enc < 16)) {
4163     Assembler::vpbroadcastw(dst, src);
4164   } else if (src_enc < 16) {
4165     push_zmm(xmm0);
4166     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4167     Assembler::vpbroadcastw(xmm0, src);
4168     movdqu(dst, xmm0);
4169     pop_zmm(xmm0);
4170   } else if (dst_enc < 16) {
4171     push_zmm(xmm0);
4172     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4173     Assembler::vpbroadcastw(dst, xmm0);
4174     pop_zmm(xmm0);
4175   } else {
4176     push_zmm(xmm0);
4177     push_zmm(xmm1);
4178     movdqu(xmm0, src);
4179     movdqu(xmm1, dst);
4180     Assembler::vpbroadcastw(xmm1, xmm0);
4181     movdqu(dst, xmm1);
4182     pop_zmm(xmm1);
4183     pop_zmm(xmm0);
4184   }
4185 }
4186 
4187 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4188   int dst_enc = dst->encoding();
4189   int nds_enc = nds->encoding();
4190   int src_enc = src->encoding();
4191   assert(dst_enc == nds_enc, "");
4192   if ((dst_enc < 16) && (src_enc < 16)) {
4193     Assembler::vpcmpeqb(dst, nds, src, vector_len);
4194   } else if (src_enc < 16) {
4195     push_zmm(xmm0);
4196     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4197     Assembler::vpcmpeqb(xmm0, xmm0, src, vector_len);
4198     movdqu(dst, xmm0);
4199     pop_zmm(xmm0);
4200   } else if (dst_enc < 16) {
4201     push_zmm(xmm0);
4202     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4203     Assembler::vpcmpeqb(dst, dst, xmm0, vector_len);
4204     pop_zmm(xmm0);
4205   } else {
4206     push_zmm(xmm0);
4207     push_zmm(xmm1);
4208     movdqu(xmm0, src);
4209     movdqu(xmm1, dst);
4210     Assembler::vpcmpeqb(xmm1, xmm1, xmm0, vector_len);
4211     movdqu(dst, xmm1);
4212     pop_zmm(xmm1);
4213     pop_zmm(xmm0);
4214   }
4215 }
4216 
4217 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4218   int dst_enc = dst->encoding();
4219   int nds_enc = nds->encoding();
4220   int src_enc = src->encoding();
4221   assert(dst_enc == nds_enc, "");
4222   if ((dst_enc < 16) && (src_enc < 16)) {
4223     Assembler::vpcmpeqw(dst, nds, src, vector_len);
4224   } else if (src_enc < 16) {
4225     push_zmm(xmm0);
4226     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4227     Assembler::vpcmpeqw(xmm0, xmm0, src, vector_len);
4228     movdqu(dst, xmm0);
4229     pop_zmm(xmm0);
4230   } else if (dst_enc < 16) {
4231     push_zmm(xmm0);
4232     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4233     Assembler::vpcmpeqw(dst, dst, xmm0, vector_len);
4234     pop_zmm(xmm0);
4235   } else {
4236     push_zmm(xmm0);
4237     push_zmm(xmm1);
4238     movdqu(xmm0, src);
4239     movdqu(xmm1, dst);
4240     Assembler::vpcmpeqw(xmm1, xmm1, xmm0, vector_len);
4241     movdqu(dst, xmm1);
4242     pop_zmm(xmm1);
4243     pop_zmm(xmm0);
4244   }
4245 }
4246 
4247 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
4248   int dst_enc = dst->encoding();
4249   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4250     Assembler::vpmovzxbw(dst, src, vector_len);
4251   } else if (dst_enc < 16) {
4252     Assembler::vpmovzxbw(dst, src, vector_len);
4253   } else {
4254     push_zmm(xmm0);
4255     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4256     Assembler::vpmovzxbw(xmm0, src, vector_len);
4257     movdqu(dst, xmm0);
4258     pop_zmm(xmm0);
4259   }
4260 }
4261 
4262 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src) {
4263   int src_enc = src->encoding();
4264   if (src_enc < 16) {
4265     Assembler::vpmovmskb(dst, src);
4266   } else {
4267     push_zmm(xmm0);
4268     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4269     Assembler::vpmovmskb(dst, xmm0);
4270     pop_zmm(xmm0);
4271   }
4272 }
4273 
4274 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4275   int dst_enc = dst->encoding();
4276   int nds_enc = nds->encoding();
4277   int src_enc = src->encoding();
4278   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4279     Assembler::vpmullw(dst, nds, src, vector_len);
4280   } else if ((dst_enc < 16) && (src_enc < 16)) {
4281     Assembler::vpmullw(dst, dst, src, vector_len);
4282   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4283     // use nds as scratch for src
4284     evmovdqul(nds, src, Assembler::AVX_512bit);
4285     Assembler::vpmullw(dst, dst, nds, vector_len);
4286   } else if ((src_enc < 16) && (nds_enc < 16)) {
4287     // use nds as scratch for dst
4288     evmovdqul(nds, dst, Assembler::AVX_512bit);
4289     Assembler::vpmullw(nds, nds, src, vector_len);
4290     evmovdqul(dst, nds, Assembler::AVX_512bit);
4291   } else if (dst_enc < 16) {
4292     // use nds as scatch for xmm0 to hold src
4293     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4294     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4295     Assembler::vpmullw(dst, dst, xmm0, vector_len);
4296     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4297   } else {
4298     // worse case scenario, all regs are in the upper bank
4299     push_zmm(xmm1);
4300     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4301     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4302     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4303     Assembler::vpmullw(xmm0, xmm0, xmm1, vector_len);
4304     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4305     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4306     pop_zmm(xmm1);
4307   }
4308 }
4309 
4310 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4311   int dst_enc = dst->encoding();
4312   int nds_enc = nds->encoding();
4313   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4314     Assembler::vpmullw(dst, nds, src, vector_len);
4315   } else if (dst_enc < 16) {
4316     Assembler::vpmullw(dst, dst, src, vector_len);
4317   } else if (nds_enc < 16) {
4318     // implies dst_enc in upper bank with src as scratch
4319     evmovdqul(nds, dst, Assembler::AVX_512bit);
4320     Assembler::vpmullw(nds, nds, src, vector_len);
4321     evmovdqul(dst, nds, Assembler::AVX_512bit);
4322   } else {
4323     // worse case scenario, all regs in upper bank
4324     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4325     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4326     Assembler::vpmullw(xmm0, xmm0, src, vector_len);
4327     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4328     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4329   }
4330 }
4331 
4332 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4333   int dst_enc = dst->encoding();
4334   int nds_enc = nds->encoding();
4335   int src_enc = src->encoding();
4336   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4337     Assembler::vpsubb(dst, nds, src, vector_len);
4338   } else if ((dst_enc < 16) && (src_enc < 16)) {
4339     Assembler::vpsubb(dst, dst, src, vector_len);
4340   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4341     // use nds as scratch for src
4342     evmovdqul(nds, src, Assembler::AVX_512bit);
4343     Assembler::vpsubb(dst, dst, nds, vector_len);
4344   } else if ((src_enc < 16) && (nds_enc < 16)) {
4345     // use nds as scratch for dst
4346     evmovdqul(nds, dst, Assembler::AVX_512bit);
4347     Assembler::vpsubb(nds, nds, src, vector_len);
4348     evmovdqul(dst, nds, Assembler::AVX_512bit);
4349   } else if (dst_enc < 16) {
4350     // use nds as scatch for xmm0 to hold src
4351     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4352     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4353     Assembler::vpsubb(dst, dst, xmm0, vector_len);
4354     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4355   } else {
4356     // worse case scenario, all regs are in the upper bank
4357     push_zmm(xmm1);
4358     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4359     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4360     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4361     Assembler::vpsubb(xmm0, xmm0, xmm1, vector_len);
4362     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4363     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4364     pop_zmm(xmm1);
4365   }
4366 }
4367 
4368 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4369   int dst_enc = dst->encoding();
4370   int nds_enc = nds->encoding();
4371   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4372     Assembler::vpsubb(dst, nds, src, vector_len);
4373   } else if (dst_enc < 16) {
4374     Assembler::vpsubb(dst, dst, src, vector_len);
4375   } else if (nds_enc < 16) {
4376     // implies dst_enc in upper bank with src as scratch
4377     evmovdqul(nds, dst, Assembler::AVX_512bit);
4378     Assembler::vpsubb(nds, nds, src, vector_len);
4379     evmovdqul(dst, nds, Assembler::AVX_512bit);
4380   } else {
4381     // worse case scenario, all regs in upper bank
4382     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4383     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4384     Assembler::vpsubb(xmm0, xmm0, src, vector_len);
4385     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4386     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4387   }
4388 }
4389 
4390 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4391   int dst_enc = dst->encoding();
4392   int nds_enc = nds->encoding();
4393   int src_enc = src->encoding();
4394   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4395     Assembler::vpsubw(dst, nds, src, vector_len);
4396   } else if ((dst_enc < 16) && (src_enc < 16)) {
4397     Assembler::vpsubw(dst, dst, src, vector_len);
4398   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4399     // use nds as scratch for src
4400     evmovdqul(nds, src, Assembler::AVX_512bit);
4401     Assembler::vpsubw(dst, dst, nds, vector_len);
4402   } else if ((src_enc < 16) && (nds_enc < 16)) {
4403     // use nds as scratch for dst
4404     evmovdqul(nds, dst, Assembler::AVX_512bit);
4405     Assembler::vpsubw(nds, nds, src, vector_len);
4406     evmovdqul(dst, nds, Assembler::AVX_512bit);
4407   } else if (dst_enc < 16) {
4408     // use nds as scatch for xmm0 to hold src
4409     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4410     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4411     Assembler::vpsubw(dst, dst, xmm0, vector_len);
4412     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4413   } else {
4414     // worse case scenario, all regs are in the upper bank
4415     push_zmm(xmm1);
4416     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4417     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4418     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4419     Assembler::vpsubw(xmm0, xmm0, xmm1, vector_len);
4420     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4421     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4422     pop_zmm(xmm1);
4423   }
4424 }
4425 
4426 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4427   int dst_enc = dst->encoding();
4428   int nds_enc = nds->encoding();
4429   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4430     Assembler::vpsubw(dst, nds, src, vector_len);
4431   } else if (dst_enc < 16) {
4432     Assembler::vpsubw(dst, dst, src, vector_len);
4433   } else if (nds_enc < 16) {
4434     // implies dst_enc in upper bank with src as scratch
4435     evmovdqul(nds, dst, Assembler::AVX_512bit);
4436     Assembler::vpsubw(nds, nds, src, vector_len);
4437     evmovdqul(dst, nds, Assembler::AVX_512bit);
4438   } else {
4439     // worse case scenario, all regs in upper bank
4440     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4441     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4442     Assembler::vpsubw(xmm0, xmm0, src, vector_len);
4443     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4444     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4445   }
4446 }
4447 
4448 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4449   int dst_enc = dst->encoding();
4450   int nds_enc = nds->encoding();
4451   int shift_enc = shift->encoding();
4452   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4453     Assembler::vpsraw(dst, nds, shift, vector_len);
4454   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4455     Assembler::vpsraw(dst, dst, shift, vector_len);
4456   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4457     // use nds as scratch with shift
4458     evmovdqul(nds, shift, Assembler::AVX_512bit);
4459     Assembler::vpsraw(dst, dst, nds, vector_len);
4460   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4461     // use nds as scratch with dst
4462     evmovdqul(nds, dst, Assembler::AVX_512bit);
4463     Assembler::vpsraw(nds, nds, shift, vector_len);
4464     evmovdqul(dst, nds, Assembler::AVX_512bit);
4465   } else if (dst_enc < 16) {
4466     // use nds to save a copy of xmm0 and hold shift
4467     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4468     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4469     Assembler::vpsraw(dst, dst, xmm0, vector_len);
4470     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4471   } else if (nds_enc < 16) {
4472     // use nds and dst as temps
4473     evmovdqul(nds, dst, Assembler::AVX_512bit);
4474     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4475     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4476     Assembler::vpsraw(nds, nds, xmm0, vector_len);
4477     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4478     evmovdqul(dst, nds, Assembler::AVX_512bit);
4479   } else {
4480     // worse case scenario, all regs are in the upper bank
4481     push_zmm(xmm1);
4482     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4483     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4484     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4485     Assembler::vpsraw(xmm0, xmm0, xmm1, vector_len);
4486     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4487     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4488     pop_zmm(xmm1);
4489   }
4490 }
4491 
4492 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4493   int dst_enc = dst->encoding();
4494   int nds_enc = nds->encoding();
4495   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4496     Assembler::vpsraw(dst, nds, shift, vector_len);
4497   } else if (dst_enc < 16) {
4498     Assembler::vpsraw(dst, dst, shift, vector_len);
4499   } else if (nds_enc < 16) {
4500     // use nds as scratch
4501     evmovdqul(nds, dst, Assembler::AVX_512bit);
4502     Assembler::vpsraw(nds, nds, shift, vector_len);
4503     evmovdqul(dst, nds, Assembler::AVX_512bit);
4504   } else {
4505     // use nds as scratch for xmm0
4506     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4507     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4508     Assembler::vpsraw(xmm0, xmm0, shift, vector_len);
4509     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4510     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4511   }
4512 }
4513 
4514 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4515   int dst_enc = dst->encoding();
4516   int nds_enc = nds->encoding();
4517   int shift_enc = shift->encoding();
4518   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4519     Assembler::vpsrlw(dst, nds, shift, vector_len);
4520   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4521     Assembler::vpsrlw(dst, dst, shift, vector_len);
4522   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4523     // use nds as scratch with shift
4524     evmovdqul(nds, shift, Assembler::AVX_512bit);
4525     Assembler::vpsrlw(dst, dst, nds, vector_len);
4526   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4527     // use nds as scratch with dst
4528     evmovdqul(nds, dst, Assembler::AVX_512bit);
4529     Assembler::vpsrlw(nds, nds, shift, vector_len);
4530     evmovdqul(dst, nds, Assembler::AVX_512bit);
4531   } else if (dst_enc < 16) {
4532     // use nds to save a copy of xmm0 and hold shift
4533     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4534     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4535     Assembler::vpsrlw(dst, dst, xmm0, vector_len);
4536     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4537   } else if (nds_enc < 16) {
4538     // use nds and dst as temps
4539     evmovdqul(nds, dst, Assembler::AVX_512bit);
4540     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4541     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4542     Assembler::vpsrlw(nds, nds, xmm0, vector_len);
4543     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4544     evmovdqul(dst, nds, Assembler::AVX_512bit);
4545   } else {
4546     // worse case scenario, all regs are in the upper bank
4547     push_zmm(xmm1);
4548     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4549     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4550     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4551     Assembler::vpsrlw(xmm0, xmm0, xmm1, vector_len);
4552     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4553     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4554     pop_zmm(xmm1);
4555   }
4556 }
4557 
4558 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4559   int dst_enc = dst->encoding();
4560   int nds_enc = nds->encoding();
4561   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4562     Assembler::vpsrlw(dst, nds, shift, vector_len);
4563   } else if (dst_enc < 16) {
4564     Assembler::vpsrlw(dst, dst, shift, vector_len);
4565   } else if (nds_enc < 16) {
4566     // use nds as scratch
4567     evmovdqul(nds, dst, Assembler::AVX_512bit);
4568     Assembler::vpsrlw(nds, nds, shift, vector_len);
4569     evmovdqul(dst, nds, Assembler::AVX_512bit);
4570   } else {
4571     // use nds as scratch for xmm0
4572     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4573     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4574     Assembler::vpsrlw(xmm0, xmm0, shift, vector_len);
4575     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4576     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4577   }
4578 }
4579 
4580 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4581   int dst_enc = dst->encoding();
4582   int nds_enc = nds->encoding();
4583   int shift_enc = shift->encoding();
4584   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4585     Assembler::vpsllw(dst, nds, shift, vector_len);
4586   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4587     Assembler::vpsllw(dst, dst, shift, vector_len);
4588   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4589     // use nds as scratch with shift
4590     evmovdqul(nds, shift, Assembler::AVX_512bit);
4591     Assembler::vpsllw(dst, dst, nds, vector_len);
4592   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4593     // use nds as scratch with dst
4594     evmovdqul(nds, dst, Assembler::AVX_512bit);
4595     Assembler::vpsllw(nds, nds, shift, vector_len);
4596     evmovdqul(dst, nds, Assembler::AVX_512bit);
4597   } else if (dst_enc < 16) {
4598     // use nds to save a copy of xmm0 and hold shift
4599     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4600     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4601     Assembler::vpsllw(dst, dst, xmm0, vector_len);
4602     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4603   } else if (nds_enc < 16) {
4604     // use nds and dst as temps
4605     evmovdqul(nds, dst, Assembler::AVX_512bit);
4606     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4607     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4608     Assembler::vpsllw(nds, nds, xmm0, vector_len);
4609     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4610     evmovdqul(dst, nds, Assembler::AVX_512bit);
4611   } else {
4612     // worse case scenario, all regs are in the upper bank
4613     push_zmm(xmm1);
4614     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4615     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4616     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4617     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4618     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4619     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4620     pop_zmm(xmm1);
4621   }
4622 }
4623 
4624 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4625   int dst_enc = dst->encoding();
4626   int nds_enc = nds->encoding();
4627   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4628     Assembler::vpsllw(dst, nds, shift, vector_len);
4629   } else if (dst_enc < 16) {
4630     Assembler::vpsllw(dst, dst, shift, vector_len);
4631   } else if (nds_enc < 16) {
4632     // use nds as scratch
4633     evmovdqul(nds, dst, Assembler::AVX_512bit);
4634     Assembler::vpsllw(nds, nds, shift, vector_len);
4635     evmovdqul(dst, nds, Assembler::AVX_512bit);
4636   } else {
4637     // use nds as scratch for xmm0
4638     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4639     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4640     Assembler::vpsllw(xmm0, xmm0, shift, vector_len);
4641     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4642     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4643   }
4644 }
4645 
4646 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
4647   int dst_enc = dst->encoding();
4648   int src_enc = src->encoding();
4649   if ((dst_enc < 16) && (src_enc < 16)) {
4650     Assembler::vptest(dst, src);
4651   } else if (src_enc < 16) {
4652     push_zmm(xmm0);
4653     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4654     Assembler::vptest(xmm0, src);
4655     pop_zmm(xmm0);
4656   } else if (dst_enc < 16) {
4657     push_zmm(xmm0);
4658     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4659     Assembler::vptest(dst, xmm0);
4660     pop_zmm(xmm0);
4661   } else {
4662     push_zmm(xmm0);
4663     push_zmm(xmm1);
4664     movdqu(xmm0, src);
4665     movdqu(xmm1, dst);
4666     Assembler::vptest(xmm1, xmm0);
4667     pop_zmm(xmm1);
4668     pop_zmm(xmm0);
4669   }
4670 }
4671 
4672 // This instruction exists within macros, ergo we cannot control its input
4673 // when emitted through those patterns.
4674 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4675   if (VM_Version::supports_avx512nobw()) {
4676     int dst_enc = dst->encoding();
4677     int src_enc = src->encoding();
4678     if (dst_enc == src_enc) {
4679       if (dst_enc < 16) {
4680         Assembler::punpcklbw(dst, src);
4681       } else {
4682         push_zmm(xmm0);
4683         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4684         Assembler::punpcklbw(xmm0, xmm0);
4685         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4686         pop_zmm(xmm0);
4687       }
4688     } else {
4689       if ((src_enc < 16) && (dst_enc < 16)) {
4690         Assembler::punpcklbw(dst, src);
4691       } else if (src_enc < 16) {
4692         push_zmm(xmm0);
4693         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4694         Assembler::punpcklbw(xmm0, src);
4695         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4696         pop_zmm(xmm0);
4697       } else if (dst_enc < 16) {
4698         push_zmm(xmm0);
4699         evmovdqul(xmm0, src, Assembler::AVX_512bit);
4700         Assembler::punpcklbw(dst, xmm0);
4701         pop_zmm(xmm0);
4702       } else {
4703         push_zmm(xmm0);
4704         push_zmm(xmm1);
4705         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4706         evmovdqul(xmm1, src, Assembler::AVX_512bit);
4707         Assembler::punpcklbw(xmm0, xmm1);
4708         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4709         pop_zmm(xmm1);
4710         pop_zmm(xmm0);
4711       }
4712     }
4713   } else {
4714     Assembler::punpcklbw(dst, src);
4715   }
4716 }
4717 
4718 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
4719   if (VM_Version::supports_avx512vl()) {
4720     Assembler::pshufd(dst, src, mode);
4721   } else {
4722     int dst_enc = dst->encoding();
4723     if (dst_enc < 16) {
4724       Assembler::pshufd(dst, src, mode);
4725     } else {
4726       push_zmm(xmm0);
4727       Assembler::pshufd(xmm0, src, mode);
4728       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4729       pop_zmm(xmm0);
4730     }
4731   }
4732 }
4733 
4734 // This instruction exists within macros, ergo we cannot control its input
4735 // when emitted through those patterns.
4736 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
4737   if (VM_Version::supports_avx512nobw()) {
4738     int dst_enc = dst->encoding();
4739     int src_enc = src->encoding();
4740     if (dst_enc == src_enc) {
4741       if (dst_enc < 16) {
4742         Assembler::pshuflw(dst, src, mode);
4743       } else {
4744         push_zmm(xmm0);
4745         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4746         Assembler::pshuflw(xmm0, xmm0, mode);
4747         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4748         pop_zmm(xmm0);
4749       }
4750     } else {
4751       if ((src_enc < 16) && (dst_enc < 16)) {
4752         Assembler::pshuflw(dst, src, mode);
4753       } else if (src_enc < 16) {
4754         push_zmm(xmm0);
4755         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4756         Assembler::pshuflw(xmm0, src, mode);
4757         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4758         pop_zmm(xmm0);
4759       } else if (dst_enc < 16) {
4760         push_zmm(xmm0);
4761         evmovdqul(xmm0, src, Assembler::AVX_512bit);
4762         Assembler::pshuflw(dst, xmm0, mode);
4763         pop_zmm(xmm0);
4764       } else {
4765         push_zmm(xmm0);
4766         push_zmm(xmm1);
4767         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4768         evmovdqul(xmm1, src, Assembler::AVX_512bit);
4769         Assembler::pshuflw(xmm0, xmm1, mode);
4770         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4771         pop_zmm(xmm1);
4772         pop_zmm(xmm0);
4773       }
4774     }
4775   } else {
4776     Assembler::pshuflw(dst, src, mode);
4777   }
4778 }
4779 
4780 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4781   if (reachable(src)) {
4782     vandpd(dst, nds, as_Address(src), vector_len);
4783   } else {
4784     lea(rscratch1, src);
4785     vandpd(dst, nds, Address(rscratch1, 0), vector_len);
4786   }
4787 }
4788 
4789 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4790   if (reachable(src)) {
4791     vandps(dst, nds, as_Address(src), vector_len);
4792   } else {
4793     lea(rscratch1, src);
4794     vandps(dst, nds, Address(rscratch1, 0), vector_len);
4795   }
4796 }
4797 
4798 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4799   if (reachable(src)) {
4800     vdivsd(dst, nds, as_Address(src));
4801   } else {
4802     lea(rscratch1, src);
4803     vdivsd(dst, nds, Address(rscratch1, 0));
4804   }
4805 }
4806 
4807 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4808   if (reachable(src)) {
4809     vdivss(dst, nds, as_Address(src));
4810   } else {
4811     lea(rscratch1, src);
4812     vdivss(dst, nds, Address(rscratch1, 0));
4813   }
4814 }
4815 
4816 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4817   if (reachable(src)) {
4818     vmulsd(dst, nds, as_Address(src));
4819   } else {
4820     lea(rscratch1, src);
4821     vmulsd(dst, nds, Address(rscratch1, 0));
4822   }
4823 }
4824 
4825 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4826   if (reachable(src)) {
4827     vmulss(dst, nds, as_Address(src));
4828   } else {
4829     lea(rscratch1, src);
4830     vmulss(dst, nds, Address(rscratch1, 0));
4831   }
4832 }
4833 
4834 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4835   if (reachable(src)) {
4836     vsubsd(dst, nds, as_Address(src));
4837   } else {
4838     lea(rscratch1, src);
4839     vsubsd(dst, nds, Address(rscratch1, 0));
4840   }
4841 }
4842 
4843 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4844   if (reachable(src)) {
4845     vsubss(dst, nds, as_Address(src));
4846   } else {
4847     lea(rscratch1, src);
4848     vsubss(dst, nds, Address(rscratch1, 0));
4849   }
4850 }
4851 
4852 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4853   int nds_enc = nds->encoding();
4854   int dst_enc = dst->encoding();
4855   bool dst_upper_bank = (dst_enc > 15);
4856   bool nds_upper_bank = (nds_enc > 15);
4857   if (VM_Version::supports_avx512novl() &&
4858       (nds_upper_bank || dst_upper_bank)) {
4859     if (dst_upper_bank) {
4860       push_zmm(xmm0);
4861       movflt(xmm0, nds);
4862       vxorps(xmm0, xmm0, src, Assembler::AVX_128bit);
4863       movflt(dst, xmm0);
4864       pop_zmm(xmm0);
4865     } else {
4866       movflt(dst, nds);
4867       vxorps(dst, dst, src, Assembler::AVX_128bit);
4868     }
4869   } else {
4870     vxorps(dst, nds, src, Assembler::AVX_128bit);
4871   }
4872 }
4873 
4874 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4875   int nds_enc = nds->encoding();
4876   int dst_enc = dst->encoding();
4877   bool dst_upper_bank = (dst_enc > 15);
4878   bool nds_upper_bank = (nds_enc > 15);
4879   if (VM_Version::supports_avx512novl() &&
4880       (nds_upper_bank || dst_upper_bank)) {
4881     if (dst_upper_bank) {
4882       push_zmm(xmm0);
4883       movdbl(xmm0, nds);
4884       vxorpd(xmm0, xmm0, src, Assembler::AVX_128bit);
4885       movdbl(dst, xmm0);
4886       pop_zmm(xmm0);
4887     } else {
4888       movdbl(dst, nds);
4889       vxorpd(dst, dst, src, Assembler::AVX_128bit);
4890     }
4891   } else {
4892     vxorpd(dst, nds, src, Assembler::AVX_128bit);
4893   }
4894 }
4895 
4896 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4897   if (reachable(src)) {
4898     vxorpd(dst, nds, as_Address(src), vector_len);
4899   } else {
4900     lea(rscratch1, src);
4901     vxorpd(dst, nds, Address(rscratch1, 0), vector_len);
4902   }
4903 }
4904 
4905 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4906   if (reachable(src)) {
4907     vxorps(dst, nds, as_Address(src), vector_len);
4908   } else {
4909     lea(rscratch1, src);
4910     vxorps(dst, nds, Address(rscratch1, 0), vector_len);
4911   }
4912 }
4913 
4914 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) {
4915   const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask);
4916   STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code
4917   // The inverted mask is sign-extended
4918   andptr(possibly_jweak, inverted_jweak_mask);
4919 }
4920 
4921 void MacroAssembler::resolve_jobject(Register value,
4922                                      Register thread,
4923                                      Register tmp) {
4924   assert_different_registers(value, thread, tmp);
4925   Label done, not_weak;
4926   testptr(value, value);
4927   jcc(Assembler::zero, done);                // Use NULL as-is.
4928   testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag.
4929   jcc(Assembler::zero, not_weak);
4930   // Resolve jweak.
4931   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
4932                  value, Address(value, -JNIHandles::weak_tag_value), tmp, thread);
4933   verify_oop(value);
4934   jmp(done);
4935   bind(not_weak);
4936   // Resolve (untagged) jobject.
4937   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, 0), tmp, thread);
4938   verify_oop(value);
4939   bind(done);
4940 }
4941 
4942 void MacroAssembler::subptr(Register dst, int32_t imm32) {
4943   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
4944 }
4945 
4946 // Force generation of a 4 byte immediate value even if it fits into 8bit
4947 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
4948   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
4949 }
4950 
4951 void MacroAssembler::subptr(Register dst, Register src) {
4952   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
4953 }
4954 
4955 // C++ bool manipulation
4956 void MacroAssembler::testbool(Register dst) {
4957   if(sizeof(bool) == 1)
4958     testb(dst, 0xff);
4959   else if(sizeof(bool) == 2) {
4960     // testw implementation needed for two byte bools
4961     ShouldNotReachHere();
4962   } else if(sizeof(bool) == 4)
4963     testl(dst, dst);
4964   else
4965     // unsupported
4966     ShouldNotReachHere();
4967 }
4968 
4969 void MacroAssembler::testptr(Register dst, Register src) {
4970   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
4971 }
4972 
4973 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4974 void MacroAssembler::tlab_allocate(Register thread, Register obj,
4975                                    Register var_size_in_bytes,
4976                                    int con_size_in_bytes,
4977                                    Register t1,
4978                                    Register t2,
4979                                    Label& slow_case) {
4980   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4981   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
4982 }
4983 
4984 // Defines obj, preserves var_size_in_bytes
4985 void MacroAssembler::eden_allocate(Register thread, Register obj,
4986                                    Register var_size_in_bytes,
4987                                    int con_size_in_bytes,
4988                                    Register t1,
4989                                    Label& slow_case) {
4990   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4991   bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
4992 }
4993 
4994 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
4995 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
4996   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
4997   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
4998   Label done;
4999 
5000   testptr(length_in_bytes, length_in_bytes);
5001   jcc(Assembler::zero, done);
5002 
5003   // initialize topmost word, divide index by 2, check if odd and test if zero
5004   // note: for the remaining code to work, index must be a multiple of BytesPerWord
5005 #ifdef ASSERT
5006   {
5007     Label L;
5008     testptr(length_in_bytes, BytesPerWord - 1);
5009     jcc(Assembler::zero, L);
5010     stop("length must be a multiple of BytesPerWord");
5011     bind(L);
5012   }
5013 #endif
5014   Register index = length_in_bytes;
5015   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
5016   if (UseIncDec) {
5017     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
5018   } else {
5019     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
5020     shrptr(index, 1);
5021   }
5022 #ifndef _LP64
5023   // index could have not been a multiple of 8 (i.e., bit 2 was set)
5024   {
5025     Label even;
5026     // note: if index was a multiple of 8, then it cannot
5027     //       be 0 now otherwise it must have been 0 before
5028     //       => if it is even, we don't need to check for 0 again
5029     jcc(Assembler::carryClear, even);
5030     // clear topmost word (no jump would be needed if conditional assignment worked here)
5031     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
5032     // index could be 0 now, must check again
5033     jcc(Assembler::zero, done);
5034     bind(even);
5035   }
5036 #endif // !_LP64
5037   // initialize remaining object fields: index is a multiple of 2 now
5038   {
5039     Label loop;
5040     bind(loop);
5041     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
5042     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
5043     decrement(index);
5044     jcc(Assembler::notZero, loop);
5045   }
5046 
5047   bind(done);
5048 }
5049 
5050 // Look up the method for a megamorphic invokeinterface call.
5051 // The target method is determined by <intf_klass, itable_index>.
5052 // The receiver klass is in recv_klass.
5053 // On success, the result will be in method_result, and execution falls through.
5054 // On failure, execution transfers to the given label.
5055 void MacroAssembler::lookup_interface_method(Register recv_klass,
5056                                              Register intf_klass,
5057                                              RegisterOrConstant itable_index,
5058                                              Register method_result,
5059                                              Register scan_temp,
5060                                              Label& L_no_such_interface,
5061                                              bool return_method) {
5062   assert_different_registers(recv_klass, intf_klass, scan_temp);
5063   assert_different_registers(method_result, intf_klass, scan_temp);
5064   assert(recv_klass != method_result || !return_method,
5065          "recv_klass can be destroyed when method isn't needed");
5066 
5067   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
5068          "caller must use same register for non-constant itable index as for method");
5069 
5070   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
5071   int vtable_base = in_bytes(Klass::vtable_start_offset());
5072   int itentry_off = itableMethodEntry::method_offset_in_bytes();
5073   int scan_step   = itableOffsetEntry::size() * wordSize;
5074   int vte_size    = vtableEntry::size_in_bytes();
5075   Address::ScaleFactor times_vte_scale = Address::times_ptr;
5076   assert(vte_size == wordSize, "else adjust times_vte_scale");
5077 
5078   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
5079 
5080   // %%% Could store the aligned, prescaled offset in the klassoop.
5081   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
5082 
5083   if (return_method) {
5084     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
5085     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
5086     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
5087   }
5088 
5089   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
5090   //   if (scan->interface() == intf) {
5091   //     result = (klass + scan->offset() + itable_index);
5092   //   }
5093   // }
5094   Label search, found_method;
5095 
5096   for (int peel = 1; peel >= 0; peel--) {
5097     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
5098     cmpptr(intf_klass, method_result);
5099 
5100     if (peel) {
5101       jccb(Assembler::equal, found_method);
5102     } else {
5103       jccb(Assembler::notEqual, search);
5104       // (invert the test to fall through to found_method...)
5105     }
5106 
5107     if (!peel)  break;
5108 
5109     bind(search);
5110 
5111     // Check that the previous entry is non-null.  A null entry means that
5112     // the receiver class doesn't implement the interface, and wasn't the
5113     // same as when the caller was compiled.
5114     testptr(method_result, method_result);
5115     jcc(Assembler::zero, L_no_such_interface);
5116     addptr(scan_temp, scan_step);
5117   }
5118 
5119   bind(found_method);
5120 
5121   if (return_method) {
5122     // Got a hit.
5123     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
5124     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
5125   }
5126 }
5127 
5128 
5129 // virtual method calling
5130 void MacroAssembler::lookup_virtual_method(Register recv_klass,
5131                                            RegisterOrConstant vtable_index,
5132                                            Register method_result) {
5133   const int base = in_bytes(Klass::vtable_start_offset());
5134   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
5135   Address vtable_entry_addr(recv_klass,
5136                             vtable_index, Address::times_ptr,
5137                             base + vtableEntry::method_offset_in_bytes());
5138   movptr(method_result, vtable_entry_addr);
5139 }
5140 
5141 
5142 void MacroAssembler::check_klass_subtype(Register sub_klass,
5143                            Register super_klass,
5144                            Register temp_reg,
5145                            Label& L_success) {
5146   Label L_failure;
5147   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, NULL);
5148   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
5149   bind(L_failure);
5150 }
5151 
5152 
5153 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
5154                                                    Register super_klass,
5155                                                    Register temp_reg,
5156                                                    Label* L_success,
5157                                                    Label* L_failure,
5158                                                    Label* L_slow_path,
5159                                         RegisterOrConstant super_check_offset) {
5160   assert_different_registers(sub_klass, super_klass, temp_reg);
5161   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
5162   if (super_check_offset.is_register()) {
5163     assert_different_registers(sub_klass, super_klass,
5164                                super_check_offset.as_register());
5165   } else if (must_load_sco) {
5166     assert(temp_reg != noreg, "supply either a temp or a register offset");
5167   }
5168 
5169   Label L_fallthrough;
5170   int label_nulls = 0;
5171   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
5172   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
5173   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
5174   assert(label_nulls <= 1, "at most one NULL in the batch");
5175 
5176   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
5177   int sco_offset = in_bytes(Klass::super_check_offset_offset());
5178   Address super_check_offset_addr(super_klass, sco_offset);
5179 
5180   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
5181   // range of a jccb.  If this routine grows larger, reconsider at
5182   // least some of these.
5183 #define local_jcc(assembler_cond, label)                                \
5184   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
5185   else                             jcc( assembler_cond, label) /*omit semi*/
5186 
5187   // Hacked jmp, which may only be used just before L_fallthrough.
5188 #define final_jmp(label)                                                \
5189   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
5190   else                            jmp(label)                /*omit semi*/
5191 
5192   // If the pointers are equal, we are done (e.g., String[] elements).
5193   // This self-check enables sharing of secondary supertype arrays among
5194   // non-primary types such as array-of-interface.  Otherwise, each such
5195   // type would need its own customized SSA.
5196   // We move this check to the front of the fast path because many
5197   // type checks are in fact trivially successful in this manner,
5198   // so we get a nicely predicted branch right at the start of the check.
5199   cmpptr(sub_klass, super_klass);
5200   local_jcc(Assembler::equal, *L_success);
5201 
5202   // Check the supertype display:
5203   if (must_load_sco) {
5204     // Positive movl does right thing on LP64.
5205     movl(temp_reg, super_check_offset_addr);
5206     super_check_offset = RegisterOrConstant(temp_reg);
5207   }
5208   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
5209   cmpptr(super_klass, super_check_addr); // load displayed supertype
5210 
5211   // This check has worked decisively for primary supers.
5212   // Secondary supers are sought in the super_cache ('super_cache_addr').
5213   // (Secondary supers are interfaces and very deeply nested subtypes.)
5214   // This works in the same check above because of a tricky aliasing
5215   // between the super_cache and the primary super display elements.
5216   // (The 'super_check_addr' can address either, as the case requires.)
5217   // Note that the cache is updated below if it does not help us find
5218   // what we need immediately.
5219   // So if it was a primary super, we can just fail immediately.
5220   // Otherwise, it's the slow path for us (no success at this point).
5221 
5222   if (super_check_offset.is_register()) {
5223     local_jcc(Assembler::equal, *L_success);
5224     cmpl(super_check_offset.as_register(), sc_offset);
5225     if (L_failure == &L_fallthrough) {
5226       local_jcc(Assembler::equal, *L_slow_path);
5227     } else {
5228       local_jcc(Assembler::notEqual, *L_failure);
5229       final_jmp(*L_slow_path);
5230     }
5231   } else if (super_check_offset.as_constant() == sc_offset) {
5232     // Need a slow path; fast failure is impossible.
5233     if (L_slow_path == &L_fallthrough) {
5234       local_jcc(Assembler::equal, *L_success);
5235     } else {
5236       local_jcc(Assembler::notEqual, *L_slow_path);
5237       final_jmp(*L_success);
5238     }
5239   } else {
5240     // No slow path; it's a fast decision.
5241     if (L_failure == &L_fallthrough) {
5242       local_jcc(Assembler::equal, *L_success);
5243     } else {
5244       local_jcc(Assembler::notEqual, *L_failure);
5245       final_jmp(*L_success);
5246     }
5247   }
5248 
5249   bind(L_fallthrough);
5250 
5251 #undef local_jcc
5252 #undef final_jmp
5253 }
5254 
5255 
5256 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
5257                                                    Register super_klass,
5258                                                    Register temp_reg,
5259                                                    Register temp2_reg,
5260                                                    Label* L_success,
5261                                                    Label* L_failure,
5262                                                    bool set_cond_codes) {
5263   assert_different_registers(sub_klass, super_klass, temp_reg);
5264   if (temp2_reg != noreg)
5265     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
5266 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
5267 
5268   Label L_fallthrough;
5269   int label_nulls = 0;
5270   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
5271   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
5272   assert(label_nulls <= 1, "at most one NULL in the batch");
5273 
5274   // a couple of useful fields in sub_klass:
5275   int ss_offset = in_bytes(Klass::secondary_supers_offset());
5276   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
5277   Address secondary_supers_addr(sub_klass, ss_offset);
5278   Address super_cache_addr(     sub_klass, sc_offset);
5279 
5280   // Do a linear scan of the secondary super-klass chain.
5281   // This code is rarely used, so simplicity is a virtue here.
5282   // The repne_scan instruction uses fixed registers, which we must spill.
5283   // Don't worry too much about pre-existing connections with the input regs.
5284 
5285   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
5286   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
5287 
5288   // Get super_klass value into rax (even if it was in rdi or rcx).
5289   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
5290   if (super_klass != rax || UseCompressedOops) {
5291     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
5292     mov(rax, super_klass);
5293   }
5294   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
5295   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
5296 
5297 #ifndef PRODUCT
5298   int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
5299   ExternalAddress pst_counter_addr((address) pst_counter);
5300   NOT_LP64(  incrementl(pst_counter_addr) );
5301   LP64_ONLY( lea(rcx, pst_counter_addr) );
5302   LP64_ONLY( incrementl(Address(rcx, 0)) );
5303 #endif //PRODUCT
5304 
5305   // We will consult the secondary-super array.
5306   movptr(rdi, secondary_supers_addr);
5307   // Load the array length.  (Positive movl does right thing on LP64.)
5308   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
5309   // Skip to start of data.
5310   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
5311 
5312   // Scan RCX words at [RDI] for an occurrence of RAX.
5313   // Set NZ/Z based on last compare.
5314   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
5315   // not change flags (only scas instruction which is repeated sets flags).
5316   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
5317 
5318     testptr(rax,rax); // Set Z = 0
5319     repne_scan();
5320 
5321   // Unspill the temp. registers:
5322   if (pushed_rdi)  pop(rdi);
5323   if (pushed_rcx)  pop(rcx);
5324   if (pushed_rax)  pop(rax);
5325 
5326   if (set_cond_codes) {
5327     // Special hack for the AD files:  rdi is guaranteed non-zero.
5328     assert(!pushed_rdi, "rdi must be left non-NULL");
5329     // Also, the condition codes are properly set Z/NZ on succeed/failure.
5330   }
5331 
5332   if (L_failure == &L_fallthrough)
5333         jccb(Assembler::notEqual, *L_failure);
5334   else  jcc(Assembler::notEqual, *L_failure);
5335 
5336   // Success.  Cache the super we found and proceed in triumph.
5337   movptr(super_cache_addr, super_klass);
5338 
5339   if (L_success != &L_fallthrough) {
5340     jmp(*L_success);
5341   }
5342 
5343 #undef IS_A_TEMP
5344 
5345   bind(L_fallthrough);
5346 }
5347 
5348 
5349 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
5350   if (VM_Version::supports_cmov()) {
5351     cmovl(cc, dst, src);
5352   } else {
5353     Label L;
5354     jccb(negate_condition(cc), L);
5355     movl(dst, src);
5356     bind(L);
5357   }
5358 }
5359 
5360 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
5361   if (VM_Version::supports_cmov()) {
5362     cmovl(cc, dst, src);
5363   } else {
5364     Label L;
5365     jccb(negate_condition(cc), L);
5366     movl(dst, src);
5367     bind(L);
5368   }
5369 }
5370 
5371 void MacroAssembler::verify_oop(Register reg, const char* s) {
5372   if (!VerifyOops) return;
5373 
5374   // Pass register number to verify_oop_subroutine
5375   const char* b = NULL;
5376   {
5377     ResourceMark rm;
5378     stringStream ss;
5379     ss.print("verify_oop: %s: %s", reg->name(), s);
5380     b = code_string(ss.as_string());
5381   }
5382   BLOCK_COMMENT("verify_oop {");
5383 #ifdef _LP64
5384   push(rscratch1);                    // save r10, trashed by movptr()
5385 #endif
5386   push(rax);                          // save rax,
5387   push(reg);                          // pass register argument
5388   ExternalAddress buffer((address) b);
5389   // avoid using pushptr, as it modifies scratch registers
5390   // and our contract is not to modify anything
5391   movptr(rax, buffer.addr());
5392   push(rax);
5393   // call indirectly to solve generation ordering problem
5394   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
5395   call(rax);
5396   // Caller pops the arguments (oop, message) and restores rax, r10
5397   BLOCK_COMMENT("} verify_oop");
5398 }
5399 
5400 
5401 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
5402                                                       Register tmp,
5403                                                       int offset) {
5404   intptr_t value = *delayed_value_addr;
5405   if (value != 0)
5406     return RegisterOrConstant(value + offset);
5407 
5408   // load indirectly to solve generation ordering problem
5409   movptr(tmp, ExternalAddress((address) delayed_value_addr));
5410 
5411 #ifdef ASSERT
5412   { Label L;
5413     testptr(tmp, tmp);
5414     if (WizardMode) {
5415       const char* buf = NULL;
5416       {
5417         ResourceMark rm;
5418         stringStream ss;
5419         ss.print("DelayedValue=" INTPTR_FORMAT, delayed_value_addr[1]);
5420         buf = code_string(ss.as_string());
5421       }
5422       jcc(Assembler::notZero, L);
5423       STOP(buf);
5424     } else {
5425       jccb(Assembler::notZero, L);
5426       hlt();
5427     }
5428     bind(L);
5429   }
5430 #endif
5431 
5432   if (offset != 0)
5433     addptr(tmp, offset);
5434 
5435   return RegisterOrConstant(tmp);
5436 }
5437 
5438 
5439 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
5440                                          int extra_slot_offset) {
5441   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
5442   int stackElementSize = Interpreter::stackElementSize;
5443   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
5444 #ifdef ASSERT
5445   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
5446   assert(offset1 - offset == stackElementSize, "correct arithmetic");
5447 #endif
5448   Register             scale_reg    = noreg;
5449   Address::ScaleFactor scale_factor = Address::no_scale;
5450   if (arg_slot.is_constant()) {
5451     offset += arg_slot.as_constant() * stackElementSize;
5452   } else {
5453     scale_reg    = arg_slot.as_register();
5454     scale_factor = Address::times(stackElementSize);
5455   }
5456   offset += wordSize;           // return PC is on stack
5457   return Address(rsp, scale_reg, scale_factor, offset);
5458 }
5459 
5460 
5461 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
5462   if (!VerifyOops) return;
5463 
5464   // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
5465   // Pass register number to verify_oop_subroutine
5466   const char* b = NULL;
5467   {
5468     ResourceMark rm;
5469     stringStream ss;
5470     ss.print("verify_oop_addr: %s", s);
5471     b = code_string(ss.as_string());
5472   }
5473 #ifdef _LP64
5474   push(rscratch1);                    // save r10, trashed by movptr()
5475 #endif
5476   push(rax);                          // save rax,
5477   // addr may contain rsp so we will have to adjust it based on the push
5478   // we just did (and on 64 bit we do two pushes)
5479   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
5480   // stores rax into addr which is backwards of what was intended.
5481   if (addr.uses(rsp)) {
5482     lea(rax, addr);
5483     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
5484   } else {
5485     pushptr(addr);
5486   }
5487 
5488   ExternalAddress buffer((address) b);
5489   // pass msg argument
5490   // avoid using pushptr, as it modifies scratch registers
5491   // and our contract is not to modify anything
5492   movptr(rax, buffer.addr());
5493   push(rax);
5494 
5495   // call indirectly to solve generation ordering problem
5496   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
5497   call(rax);
5498   // Caller pops the arguments (addr, message) and restores rax, r10.
5499 }
5500 
5501 void MacroAssembler::verify_tlab() {
5502 #ifdef ASSERT
5503   if (UseTLAB && VerifyOops) {
5504     Label next, ok;
5505     Register t1 = rsi;
5506     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
5507 
5508     push(t1);
5509     NOT_LP64(push(thread_reg));
5510     NOT_LP64(get_thread(thread_reg));
5511 
5512     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5513     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
5514     jcc(Assembler::aboveEqual, next);
5515     STOP("assert(top >= start)");
5516     should_not_reach_here();
5517 
5518     bind(next);
5519     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
5520     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5521     jcc(Assembler::aboveEqual, ok);
5522     STOP("assert(top <= end)");
5523     should_not_reach_here();
5524 
5525     bind(ok);
5526     NOT_LP64(pop(thread_reg));
5527     pop(t1);
5528   }
5529 #endif
5530 }
5531 
5532 class ControlWord {
5533  public:
5534   int32_t _value;
5535 
5536   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
5537   int  precision_control() const       { return  (_value >>  8) & 3      ; }
5538   bool precision() const               { return ((_value >>  5) & 1) != 0; }
5539   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
5540   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
5541   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
5542   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
5543   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
5544 
5545   void print() const {
5546     // rounding control
5547     const char* rc;
5548     switch (rounding_control()) {
5549       case 0: rc = "round near"; break;
5550       case 1: rc = "round down"; break;
5551       case 2: rc = "round up  "; break;
5552       case 3: rc = "chop      "; break;
5553     };
5554     // precision control
5555     const char* pc;
5556     switch (precision_control()) {
5557       case 0: pc = "24 bits "; break;
5558       case 1: pc = "reserved"; break;
5559       case 2: pc = "53 bits "; break;
5560       case 3: pc = "64 bits "; break;
5561     };
5562     // flags
5563     char f[9];
5564     f[0] = ' ';
5565     f[1] = ' ';
5566     f[2] = (precision   ()) ? 'P' : 'p';
5567     f[3] = (underflow   ()) ? 'U' : 'u';
5568     f[4] = (overflow    ()) ? 'O' : 'o';
5569     f[5] = (zero_divide ()) ? 'Z' : 'z';
5570     f[6] = (denormalized()) ? 'D' : 'd';
5571     f[7] = (invalid     ()) ? 'I' : 'i';
5572     f[8] = '\x0';
5573     // output
5574     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
5575   }
5576 
5577 };
5578 
5579 class StatusWord {
5580  public:
5581   int32_t _value;
5582 
5583   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
5584   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
5585   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
5586   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
5587   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
5588   int  top() const                     { return  (_value >> 11) & 7      ; }
5589   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
5590   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
5591   bool precision() const               { return ((_value >>  5) & 1) != 0; }
5592   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
5593   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
5594   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
5595   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
5596   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
5597 
5598   void print() const {
5599     // condition codes
5600     char c[5];
5601     c[0] = (C3()) ? '3' : '-';
5602     c[1] = (C2()) ? '2' : '-';
5603     c[2] = (C1()) ? '1' : '-';
5604     c[3] = (C0()) ? '0' : '-';
5605     c[4] = '\x0';
5606     // flags
5607     char f[9];
5608     f[0] = (error_status()) ? 'E' : '-';
5609     f[1] = (stack_fault ()) ? 'S' : '-';
5610     f[2] = (precision   ()) ? 'P' : '-';
5611     f[3] = (underflow   ()) ? 'U' : '-';
5612     f[4] = (overflow    ()) ? 'O' : '-';
5613     f[5] = (zero_divide ()) ? 'Z' : '-';
5614     f[6] = (denormalized()) ? 'D' : '-';
5615     f[7] = (invalid     ()) ? 'I' : '-';
5616     f[8] = '\x0';
5617     // output
5618     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
5619   }
5620 
5621 };
5622 
5623 class TagWord {
5624  public:
5625   int32_t _value;
5626 
5627   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
5628 
5629   void print() const {
5630     printf("%04x", _value & 0xFFFF);
5631   }
5632 
5633 };
5634 
5635 class FPU_Register {
5636  public:
5637   int32_t _m0;
5638   int32_t _m1;
5639   int16_t _ex;
5640 
5641   bool is_indefinite() const           {
5642     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
5643   }
5644 
5645   void print() const {
5646     char  sign = (_ex < 0) ? '-' : '+';
5647     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
5648     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
5649   };
5650 
5651 };
5652 
5653 class FPU_State {
5654  public:
5655   enum {
5656     register_size       = 10,
5657     number_of_registers =  8,
5658     register_mask       =  7
5659   };
5660 
5661   ControlWord  _control_word;
5662   StatusWord   _status_word;
5663   TagWord      _tag_word;
5664   int32_t      _error_offset;
5665   int32_t      _error_selector;
5666   int32_t      _data_offset;
5667   int32_t      _data_selector;
5668   int8_t       _register[register_size * number_of_registers];
5669 
5670   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
5671   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
5672 
5673   const char* tag_as_string(int tag) const {
5674     switch (tag) {
5675       case 0: return "valid";
5676       case 1: return "zero";
5677       case 2: return "special";
5678       case 3: return "empty";
5679     }
5680     ShouldNotReachHere();
5681     return NULL;
5682   }
5683 
5684   void print() const {
5685     // print computation registers
5686     { int t = _status_word.top();
5687       for (int i = 0; i < number_of_registers; i++) {
5688         int j = (i - t) & register_mask;
5689         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
5690         st(j)->print();
5691         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
5692       }
5693     }
5694     printf("\n");
5695     // print control registers
5696     printf("ctrl = "); _control_word.print(); printf("\n");
5697     printf("stat = "); _status_word .print(); printf("\n");
5698     printf("tags = "); _tag_word    .print(); printf("\n");
5699   }
5700 
5701 };
5702 
5703 class Flag_Register {
5704  public:
5705   int32_t _value;
5706 
5707   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
5708   bool direction() const               { return ((_value >> 10) & 1) != 0; }
5709   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
5710   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
5711   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
5712   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
5713   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
5714 
5715   void print() const {
5716     // flags
5717     char f[8];
5718     f[0] = (overflow       ()) ? 'O' : '-';
5719     f[1] = (direction      ()) ? 'D' : '-';
5720     f[2] = (sign           ()) ? 'S' : '-';
5721     f[3] = (zero           ()) ? 'Z' : '-';
5722     f[4] = (auxiliary_carry()) ? 'A' : '-';
5723     f[5] = (parity         ()) ? 'P' : '-';
5724     f[6] = (carry          ()) ? 'C' : '-';
5725     f[7] = '\x0';
5726     // output
5727     printf("%08x  flags = %s", _value, f);
5728   }
5729 
5730 };
5731 
5732 class IU_Register {
5733  public:
5734   int32_t _value;
5735 
5736   void print() const {
5737     printf("%08x  %11d", _value, _value);
5738   }
5739 
5740 };
5741 
5742 class IU_State {
5743  public:
5744   Flag_Register _eflags;
5745   IU_Register   _rdi;
5746   IU_Register   _rsi;
5747   IU_Register   _rbp;
5748   IU_Register   _rsp;
5749   IU_Register   _rbx;
5750   IU_Register   _rdx;
5751   IU_Register   _rcx;
5752   IU_Register   _rax;
5753 
5754   void print() const {
5755     // computation registers
5756     printf("rax,  = "); _rax.print(); printf("\n");
5757     printf("rbx,  = "); _rbx.print(); printf("\n");
5758     printf("rcx  = "); _rcx.print(); printf("\n");
5759     printf("rdx  = "); _rdx.print(); printf("\n");
5760     printf("rdi  = "); _rdi.print(); printf("\n");
5761     printf("rsi  = "); _rsi.print(); printf("\n");
5762     printf("rbp,  = "); _rbp.print(); printf("\n");
5763     printf("rsp  = "); _rsp.print(); printf("\n");
5764     printf("\n");
5765     // control registers
5766     printf("flgs = "); _eflags.print(); printf("\n");
5767   }
5768 };
5769 
5770 
5771 class CPU_State {
5772  public:
5773   FPU_State _fpu_state;
5774   IU_State  _iu_state;
5775 
5776   void print() const {
5777     printf("--------------------------------------------------\n");
5778     _iu_state .print();
5779     printf("\n");
5780     _fpu_state.print();
5781     printf("--------------------------------------------------\n");
5782   }
5783 
5784 };
5785 
5786 
5787 static void _print_CPU_state(CPU_State* state) {
5788   state->print();
5789 };
5790 
5791 
5792 void MacroAssembler::print_CPU_state() {
5793   push_CPU_state();
5794   push(rsp);                // pass CPU state
5795   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
5796   addptr(rsp, wordSize);       // discard argument
5797   pop_CPU_state();
5798 }
5799 
5800 
5801 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
5802   static int counter = 0;
5803   FPU_State* fs = &state->_fpu_state;
5804   counter++;
5805   // For leaf calls, only verify that the top few elements remain empty.
5806   // We only need 1 empty at the top for C2 code.
5807   if( stack_depth < 0 ) {
5808     if( fs->tag_for_st(7) != 3 ) {
5809       printf("FPR7 not empty\n");
5810       state->print();
5811       assert(false, "error");
5812       return false;
5813     }
5814     return true;                // All other stack states do not matter
5815   }
5816 
5817   assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std,
5818          "bad FPU control word");
5819 
5820   // compute stack depth
5821   int i = 0;
5822   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
5823   int d = i;
5824   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
5825   // verify findings
5826   if (i != FPU_State::number_of_registers) {
5827     // stack not contiguous
5828     printf("%s: stack not contiguous at ST%d\n", s, i);
5829     state->print();
5830     assert(false, "error");
5831     return false;
5832   }
5833   // check if computed stack depth corresponds to expected stack depth
5834   if (stack_depth < 0) {
5835     // expected stack depth is -stack_depth or less
5836     if (d > -stack_depth) {
5837       // too many elements on the stack
5838       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
5839       state->print();
5840       assert(false, "error");
5841       return false;
5842     }
5843   } else {
5844     // expected stack depth is stack_depth
5845     if (d != stack_depth) {
5846       // wrong stack depth
5847       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
5848       state->print();
5849       assert(false, "error");
5850       return false;
5851     }
5852   }
5853   // everything is cool
5854   return true;
5855 }
5856 
5857 
5858 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
5859   if (!VerifyFPU) return;
5860   push_CPU_state();
5861   push(rsp);                // pass CPU state
5862   ExternalAddress msg((address) s);
5863   // pass message string s
5864   pushptr(msg.addr());
5865   push(stack_depth);        // pass stack depth
5866   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
5867   addptr(rsp, 3 * wordSize);   // discard arguments
5868   // check for error
5869   { Label L;
5870     testl(rax, rax);
5871     jcc(Assembler::notZero, L);
5872     int3();                  // break if error condition
5873     bind(L);
5874   }
5875   pop_CPU_state();
5876 }
5877 
5878 void MacroAssembler::restore_cpu_control_state_after_jni() {
5879   // Either restore the MXCSR register after returning from the JNI Call
5880   // or verify that it wasn't changed (with -Xcheck:jni flag).
5881   if (VM_Version::supports_sse()) {
5882     if (RestoreMXCSROnJNICalls) {
5883       ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std()));
5884     } else if (CheckJNICalls) {
5885       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
5886     }
5887   }
5888   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
5889   vzeroupper();
5890   // Reset k1 to 0xffff.
5891   if (VM_Version::supports_evex()) {
5892     push(rcx);
5893     movl(rcx, 0xffff);
5894     kmovwl(k1, rcx);
5895     pop(rcx);
5896   }
5897 
5898 #ifndef _LP64
5899   // Either restore the x87 floating pointer control word after returning
5900   // from the JNI call or verify that it wasn't changed.
5901   if (CheckJNICalls) {
5902     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
5903   }
5904 #endif // _LP64
5905 }
5906 
5907 // ((OopHandle)result).resolve();
5908 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
5909   assert_different_registers(result, tmp);
5910 
5911   // Only 64 bit platforms support GCs that require a tmp register
5912   // Only IN_HEAP loads require a thread_tmp register
5913   // OopHandle::resolve is an indirection like jobject.
5914   access_load_at(T_OBJECT, IN_NATIVE,
5915                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
5916 }
5917 
5918 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
5919   // get mirror
5920   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
5921   movptr(mirror, Address(method, Method::const_offset()));
5922   movptr(mirror, Address(mirror, ConstMethod::constants_offset()));
5923   movptr(mirror, Address(mirror, ConstantPool::pool_holder_offset_in_bytes()));
5924   movptr(mirror, Address(mirror, mirror_offset));
5925   resolve_oop_handle(mirror, tmp);
5926 }
5927 
5928 void MacroAssembler::load_klass(Register dst, Register src) {
5929 #ifdef _LP64
5930   if (UseCompressedClassPointers) {
5931     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5932     decode_klass_not_null(dst);
5933   } else
5934 #endif
5935     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5936 }
5937 
5938 void MacroAssembler::load_prototype_header(Register dst, Register src) {
5939   load_klass(dst, src);
5940   movptr(dst, Address(dst, Klass::prototype_header_offset()));
5941 }
5942 
5943 void MacroAssembler::store_klass(Register dst, Register src) {
5944 #ifdef _LP64
5945   if (UseCompressedClassPointers) {
5946     encode_klass_not_null(src);
5947     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5948   } else
5949 #endif
5950     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5951 }
5952 
5953 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
5954                                     Register tmp1, Register thread_tmp) {
5955   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5956   decorators = AccessInternal::decorator_fixup(decorators);
5957   bool as_raw = (decorators & AS_RAW) != 0;
5958   if (as_raw) {
5959     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
5960   } else {
5961     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
5962   }
5963 }
5964 
5965 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
5966                                      Register tmp1, Register tmp2) {
5967   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5968   decorators = AccessInternal::decorator_fixup(decorators);
5969   bool as_raw = (decorators & AS_RAW) != 0;
5970   if (as_raw) {
5971     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2);
5972   } else {
5973     bs->store_at(this, decorators, type, dst, src, tmp1, tmp2);
5974   }
5975 }
5976 
5977 void MacroAssembler::resolve(DecoratorSet decorators, Register obj) {
5978   // Use stronger ACCESS_WRITE|ACCESS_READ by default.
5979   if ((decorators & (ACCESS_READ | ACCESS_WRITE)) == 0) {
5980     decorators |= ACCESS_READ | ACCESS_WRITE;
5981   }
5982   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5983   return bs->resolve(this, decorators, obj);
5984 }
5985 
5986 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
5987                                    Register thread_tmp, DecoratorSet decorators) {
5988   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
5989 }
5990 
5991 // Doesn't do verfication, generates fixed size code
5992 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
5993                                             Register thread_tmp, DecoratorSet decorators) {
5994   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
5995 }
5996 
5997 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
5998                                     Register tmp2, DecoratorSet decorators) {
5999   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
6000 }
6001 
6002 // Used for storing NULLs.
6003 void MacroAssembler::store_heap_oop_null(Address dst) {
6004   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg);
6005 }
6006 
6007 #ifdef _LP64
6008 void MacroAssembler::store_klass_gap(Register dst, Register src) {
6009   if (UseCompressedClassPointers) {
6010     // Store to klass gap in destination
6011     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
6012   }
6013 }
6014 
6015 #ifdef ASSERT
6016 void MacroAssembler::verify_heapbase(const char* msg) {
6017   assert (UseCompressedOops, "should be compressed");
6018   assert (Universe::heap() != NULL, "java heap should be initialized");
6019   if (CheckCompressedOops) {
6020     Label ok;
6021     push(rscratch1); // cmpptr trashes rscratch1
6022     cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6023     jcc(Assembler::equal, ok);
6024     STOP(msg);
6025     bind(ok);
6026     pop(rscratch1);
6027   }
6028 }
6029 #endif
6030 
6031 // Algorithm must match oop.inline.hpp encode_heap_oop.
6032 void MacroAssembler::encode_heap_oop(Register r) {
6033 #ifdef ASSERT
6034   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
6035 #endif
6036   verify_oop(r, "broken oop in encode_heap_oop");
6037   if (Universe::narrow_oop_base() == NULL) {
6038     if (Universe::narrow_oop_shift() != 0) {
6039       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6040       shrq(r, LogMinObjAlignmentInBytes);
6041     }
6042     return;
6043   }
6044   testq(r, r);
6045   cmovq(Assembler::equal, r, r12_heapbase);
6046   subq(r, r12_heapbase);
6047   shrq(r, LogMinObjAlignmentInBytes);
6048 }
6049 
6050 void MacroAssembler::encode_heap_oop_not_null(Register r) {
6051 #ifdef ASSERT
6052   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
6053   if (CheckCompressedOops) {
6054     Label ok;
6055     testq(r, r);
6056     jcc(Assembler::notEqual, ok);
6057     STOP("null oop passed to encode_heap_oop_not_null");
6058     bind(ok);
6059   }
6060 #endif
6061   verify_oop(r, "broken oop in encode_heap_oop_not_null");
6062   if (Universe::narrow_oop_base() != NULL) {
6063     subq(r, r12_heapbase);
6064   }
6065   if (Universe::narrow_oop_shift() != 0) {
6066     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6067     shrq(r, LogMinObjAlignmentInBytes);
6068   }
6069 }
6070 
6071 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
6072 #ifdef ASSERT
6073   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
6074   if (CheckCompressedOops) {
6075     Label ok;
6076     testq(src, src);
6077     jcc(Assembler::notEqual, ok);
6078     STOP("null oop passed to encode_heap_oop_not_null2");
6079     bind(ok);
6080   }
6081 #endif
6082   verify_oop(src, "broken oop in encode_heap_oop_not_null2");
6083   if (dst != src) {
6084     movq(dst, src);
6085   }
6086   if (Universe::narrow_oop_base() != NULL) {
6087     subq(dst, r12_heapbase);
6088   }
6089   if (Universe::narrow_oop_shift() != 0) {
6090     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6091     shrq(dst, LogMinObjAlignmentInBytes);
6092   }
6093 }
6094 
6095 void  MacroAssembler::decode_heap_oop(Register r) {
6096 #ifdef ASSERT
6097   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
6098 #endif
6099   if (Universe::narrow_oop_base() == NULL) {
6100     if (Universe::narrow_oop_shift() != 0) {
6101       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6102       shlq(r, LogMinObjAlignmentInBytes);
6103     }
6104   } else {
6105     Label done;
6106     shlq(r, LogMinObjAlignmentInBytes);
6107     jccb(Assembler::equal, done);
6108     addq(r, r12_heapbase);
6109     bind(done);
6110   }
6111   verify_oop(r, "broken oop in decode_heap_oop");
6112 }
6113 
6114 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
6115   // Note: it will change flags
6116   assert (UseCompressedOops, "should only be used for compressed headers");
6117   assert (Universe::heap() != NULL, "java heap should be initialized");
6118   // Cannot assert, unverified entry point counts instructions (see .ad file)
6119   // vtableStubs also counts instructions in pd_code_size_limit.
6120   // Also do not verify_oop as this is called by verify_oop.
6121   if (Universe::narrow_oop_shift() != 0) {
6122     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6123     shlq(r, LogMinObjAlignmentInBytes);
6124     if (Universe::narrow_oop_base() != NULL) {
6125       addq(r, r12_heapbase);
6126     }
6127   } else {
6128     assert (Universe::narrow_oop_base() == NULL, "sanity");
6129   }
6130 }
6131 
6132 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
6133   // Note: it will change flags
6134   assert (UseCompressedOops, "should only be used for compressed headers");
6135   assert (Universe::heap() != NULL, "java heap should be initialized");
6136   // Cannot assert, unverified entry point counts instructions (see .ad file)
6137   // vtableStubs also counts instructions in pd_code_size_limit.
6138   // Also do not verify_oop as this is called by verify_oop.
6139   if (Universe::narrow_oop_shift() != 0) {
6140     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6141     if (LogMinObjAlignmentInBytes == Address::times_8) {
6142       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
6143     } else {
6144       if (dst != src) {
6145         movq(dst, src);
6146       }
6147       shlq(dst, LogMinObjAlignmentInBytes);
6148       if (Universe::narrow_oop_base() != NULL) {
6149         addq(dst, r12_heapbase);
6150       }
6151     }
6152   } else {
6153     assert (Universe::narrow_oop_base() == NULL, "sanity");
6154     if (dst != src) {
6155       movq(dst, src);
6156     }
6157   }
6158 }
6159 
6160 void MacroAssembler::encode_klass_not_null(Register r) {
6161   if (Universe::narrow_klass_base() != NULL) {
6162     // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6163     assert(r != r12_heapbase, "Encoding a klass in r12");
6164     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6165     subq(r, r12_heapbase);
6166   }
6167   if (Universe::narrow_klass_shift() != 0) {
6168     assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6169     shrq(r, LogKlassAlignmentInBytes);
6170   }
6171   if (Universe::narrow_klass_base() != NULL) {
6172     reinit_heapbase();
6173   }
6174 }
6175 
6176 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
6177   if (dst == src) {
6178     encode_klass_not_null(src);
6179   } else {
6180     if (Universe::narrow_klass_base() != NULL) {
6181       mov64(dst, (int64_t)Universe::narrow_klass_base());
6182       negq(dst);
6183       addq(dst, src);
6184     } else {
6185       movptr(dst, src);
6186     }
6187     if (Universe::narrow_klass_shift() != 0) {
6188       assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6189       shrq(dst, LogKlassAlignmentInBytes);
6190     }
6191   }
6192 }
6193 
6194 // Function instr_size_for_decode_klass_not_null() counts the instructions
6195 // generated by decode_klass_not_null(register r) and reinit_heapbase(),
6196 // when (Universe::heap() != NULL).  Hence, if the instructions they
6197 // generate change, then this method needs to be updated.
6198 int MacroAssembler::instr_size_for_decode_klass_not_null() {
6199   assert (UseCompressedClassPointers, "only for compressed klass ptrs");
6200   if (Universe::narrow_klass_base() != NULL) {
6201     // mov64 + addq + shlq? + mov64  (for reinit_heapbase()).
6202     return (Universe::narrow_klass_shift() == 0 ? 20 : 24);
6203   } else {
6204     // longest load decode klass function, mov64, leaq
6205     return 16;
6206   }
6207 }
6208 
6209 // !!! If the instructions that get generated here change then function
6210 // instr_size_for_decode_klass_not_null() needs to get updated.
6211 void  MacroAssembler::decode_klass_not_null(Register r) {
6212   // Note: it will change flags
6213   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6214   assert(r != r12_heapbase, "Decoding a klass in r12");
6215   // Cannot assert, unverified entry point counts instructions (see .ad file)
6216   // vtableStubs also counts instructions in pd_code_size_limit.
6217   // Also do not verify_oop as this is called by verify_oop.
6218   if (Universe::narrow_klass_shift() != 0) {
6219     assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6220     shlq(r, LogKlassAlignmentInBytes);
6221   }
6222   // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6223   if (Universe::narrow_klass_base() != NULL) {
6224     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6225     addq(r, r12_heapbase);
6226     reinit_heapbase();
6227   }
6228 }
6229 
6230 void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
6231   // Note: it will change flags
6232   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6233   if (dst == src) {
6234     decode_klass_not_null(dst);
6235   } else {
6236     // Cannot assert, unverified entry point counts instructions (see .ad file)
6237     // vtableStubs also counts instructions in pd_code_size_limit.
6238     // Also do not verify_oop as this is called by verify_oop.
6239     mov64(dst, (int64_t)Universe::narrow_klass_base());
6240     if (Universe::narrow_klass_shift() != 0) {
6241       assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6242       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
6243       leaq(dst, Address(dst, src, Address::times_8, 0));
6244     } else {
6245       addq(dst, src);
6246     }
6247   }
6248 }
6249 
6250 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
6251   assert (UseCompressedOops, "should only be used for compressed headers");
6252   assert (Universe::heap() != NULL, "java heap should be initialized");
6253   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6254   int oop_index = oop_recorder()->find_index(obj);
6255   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6256   mov_narrow_oop(dst, oop_index, rspec);
6257 }
6258 
6259 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
6260   assert (UseCompressedOops, "should only be used for compressed headers");
6261   assert (Universe::heap() != NULL, "java heap should be initialized");
6262   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6263   int oop_index = oop_recorder()->find_index(obj);
6264   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6265   mov_narrow_oop(dst, oop_index, rspec);
6266 }
6267 
6268 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
6269   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6270   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6271   int klass_index = oop_recorder()->find_index(k);
6272   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6273   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6274 }
6275 
6276 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
6277   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6278   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6279   int klass_index = oop_recorder()->find_index(k);
6280   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6281   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6282 }
6283 
6284 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
6285   assert (UseCompressedOops, "should only be used for compressed headers");
6286   assert (Universe::heap() != NULL, "java heap should be initialized");
6287   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6288   int oop_index = oop_recorder()->find_index(obj);
6289   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6290   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6291 }
6292 
6293 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
6294   assert (UseCompressedOops, "should only be used for compressed headers");
6295   assert (Universe::heap() != NULL, "java heap should be initialized");
6296   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6297   int oop_index = oop_recorder()->find_index(obj);
6298   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6299   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6300 }
6301 
6302 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
6303   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6304   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6305   int klass_index = oop_recorder()->find_index(k);
6306   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6307   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6308 }
6309 
6310 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
6311   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6312   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6313   int klass_index = oop_recorder()->find_index(k);
6314   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6315   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6316 }
6317 
6318 void MacroAssembler::reinit_heapbase() {
6319   if (UseCompressedOops || UseCompressedClassPointers) {
6320     if (Universe::heap() != NULL) {
6321       if (Universe::narrow_oop_base() == NULL) {
6322         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
6323       } else {
6324         mov64(r12_heapbase, (int64_t)Universe::narrow_ptrs_base());
6325       }
6326     } else {
6327       movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6328     }
6329   }
6330 }
6331 
6332 #endif // _LP64
6333 
6334 // C2 compiled method's prolog code.
6335 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b) {
6336 
6337   // WARNING: Initial instruction MUST be 5 bytes or longer so that
6338   // NativeJump::patch_verified_entry will be able to patch out the entry
6339   // code safely. The push to verify stack depth is ok at 5 bytes,
6340   // the frame allocation can be either 3 or 6 bytes. So if we don't do
6341   // stack bang then we must use the 6 byte frame allocation even if
6342   // we have no frame. :-(
6343   assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
6344 
6345   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
6346   // Remove word for return addr
6347   framesize -= wordSize;
6348   stack_bang_size -= wordSize;
6349 
6350   // Calls to C2R adapters often do not accept exceptional returns.
6351   // We require that their callers must bang for them.  But be careful, because
6352   // some VM calls (such as call site linkage) can use several kilobytes of
6353   // stack.  But the stack safety zone should account for that.
6354   // See bugs 4446381, 4468289, 4497237.
6355   if (stack_bang_size > 0) {
6356     generate_stack_overflow_check(stack_bang_size);
6357 
6358     // We always push rbp, so that on return to interpreter rbp, will be
6359     // restored correctly and we can correct the stack.
6360     push(rbp);
6361     // Save caller's stack pointer into RBP if the frame pointer is preserved.
6362     if (PreserveFramePointer) {
6363       mov(rbp, rsp);
6364     }
6365     // Remove word for ebp
6366     framesize -= wordSize;
6367 
6368     // Create frame
6369     if (framesize) {
6370       subptr(rsp, framesize);
6371     }
6372   } else {
6373     // Create frame (force generation of a 4 byte immediate value)
6374     subptr_imm32(rsp, framesize);
6375 
6376     // Save RBP register now.
6377     framesize -= wordSize;
6378     movptr(Address(rsp, framesize), rbp);
6379     // Save caller's stack pointer into RBP if the frame pointer is preserved.
6380     if (PreserveFramePointer) {
6381       movptr(rbp, rsp);
6382       if (framesize > 0) {
6383         addptr(rbp, framesize);
6384       }
6385     }
6386   }
6387 
6388   if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
6389     framesize -= wordSize;
6390     movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
6391   }
6392 
6393 #ifndef _LP64
6394   // If method sets FPU control word do it now
6395   if (fp_mode_24b) {
6396     fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
6397   }
6398   if (UseSSE >= 2 && VerifyFPU) {
6399     verify_FPU(0, "FPU stack must be clean on entry");
6400   }
6401 #endif
6402 
6403 #ifdef ASSERT
6404   if (VerifyStackAtCalls) {
6405     Label L;
6406     push(rax);
6407     mov(rax, rsp);
6408     andptr(rax, StackAlignmentInBytes-1);
6409     cmpptr(rax, StackAlignmentInBytes-wordSize);
6410     pop(rax);
6411     jcc(Assembler::equal, L);
6412     STOP("Stack is not properly aligned!");
6413     bind(L);
6414   }
6415 #endif
6416 
6417 }
6418 
6419 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM registers
6420 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, XMMRegister xtmp) {
6421   // cnt - number of qwords (8-byte words).
6422   // base - start address, qword aligned.
6423   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
6424   if (UseAVX >= 2) {
6425     vpxor(xtmp, xtmp, xtmp, AVX_256bit);
6426   } else {
6427     pxor(xtmp, xtmp);
6428   }
6429   jmp(L_zero_64_bytes);
6430 
6431   BIND(L_loop);
6432   if (UseAVX >= 2) {
6433     vmovdqu(Address(base,  0), xtmp);
6434     vmovdqu(Address(base, 32), xtmp);
6435   } else {
6436     movdqu(Address(base,  0), xtmp);
6437     movdqu(Address(base, 16), xtmp);
6438     movdqu(Address(base, 32), xtmp);
6439     movdqu(Address(base, 48), xtmp);
6440   }
6441   addptr(base, 64);
6442 
6443   BIND(L_zero_64_bytes);
6444   subptr(cnt, 8);
6445   jccb(Assembler::greaterEqual, L_loop);
6446   addptr(cnt, 4);
6447   jccb(Assembler::less, L_tail);
6448   // Copy trailing 32 bytes
6449   if (UseAVX >= 2) {
6450     vmovdqu(Address(base, 0), xtmp);
6451   } else {
6452     movdqu(Address(base,  0), xtmp);
6453     movdqu(Address(base, 16), xtmp);
6454   }
6455   addptr(base, 32);
6456   subptr(cnt, 4);
6457 
6458   BIND(L_tail);
6459   addptr(cnt, 4);
6460   jccb(Assembler::lessEqual, L_end);
6461   decrement(cnt);
6462 
6463   BIND(L_sloop);
6464   movq(Address(base, 0), xtmp);
6465   addptr(base, 8);
6466   decrement(cnt);
6467   jccb(Assembler::greaterEqual, L_sloop);
6468   BIND(L_end);
6469 }
6470 
6471 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp, bool is_large) {
6472   // cnt - number of qwords (8-byte words).
6473   // base - start address, qword aligned.
6474   // is_large - if optimizers know cnt is larger than InitArrayShortSize
6475   assert(base==rdi, "base register must be edi for rep stos");
6476   assert(tmp==rax,   "tmp register must be eax for rep stos");
6477   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
6478   assert(InitArrayShortSize % BytesPerLong == 0,
6479     "InitArrayShortSize should be the multiple of BytesPerLong");
6480 
6481   Label DONE;
6482 
6483   if (!is_large || !UseXMMForObjInit) {
6484     xorptr(tmp, tmp);
6485   }
6486 
6487   if (!is_large) {
6488     Label LOOP, LONG;
6489     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
6490     jccb(Assembler::greater, LONG);
6491 
6492     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
6493 
6494     decrement(cnt);
6495     jccb(Assembler::negative, DONE); // Zero length
6496 
6497     // Use individual pointer-sized stores for small counts:
6498     BIND(LOOP);
6499     movptr(Address(base, cnt, Address::times_ptr), tmp);
6500     decrement(cnt);
6501     jccb(Assembler::greaterEqual, LOOP);
6502     jmpb(DONE);
6503 
6504     BIND(LONG);
6505   }
6506 
6507   // Use longer rep-prefixed ops for non-small counts:
6508   if (UseFastStosb) {
6509     shlptr(cnt, 3); // convert to number of bytes
6510     rep_stosb();
6511   } else if (UseXMMForObjInit) {
6512     movptr(tmp, base);
6513     xmm_clear_mem(tmp, cnt, xtmp);
6514   } else {
6515     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
6516     rep_stos();
6517   }
6518 
6519   BIND(DONE);
6520 }
6521 
6522 #ifdef COMPILER2
6523 
6524 // IndexOf for constant substrings with size >= 8 chars
6525 // which don't need to be loaded through stack.
6526 void MacroAssembler::string_indexofC8(Register str1, Register str2,
6527                                       Register cnt1, Register cnt2,
6528                                       int int_cnt2,  Register result,
6529                                       XMMRegister vec, Register tmp,
6530                                       int ae) {
6531   ShortBranchVerifier sbv(this);
6532   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
6533   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
6534 
6535   // This method uses the pcmpestri instruction with bound registers
6536   //   inputs:
6537   //     xmm - substring
6538   //     rax - substring length (elements count)
6539   //     mem - scanned string
6540   //     rdx - string length (elements count)
6541   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
6542   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
6543   //   outputs:
6544   //     rcx - matched index in string
6545   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
6546   int mode   = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
6547   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
6548   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
6549   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
6550 
6551   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR,
6552         RET_FOUND, RET_NOT_FOUND, EXIT, FOUND_SUBSTR,
6553         MATCH_SUBSTR_HEAD, RELOAD_STR, FOUND_CANDIDATE;
6554 
6555   // Note, inline_string_indexOf() generates checks:
6556   // if (substr.count > string.count) return -1;
6557   // if (substr.count == 0) return 0;
6558   assert(int_cnt2 >= stride, "this code is used only for cnt2 >= 8 chars");
6559 
6560   // Load substring.
6561   if (ae == StrIntrinsicNode::UL) {
6562     pmovzxbw(vec, Address(str2, 0));
6563   } else {
6564     movdqu(vec, Address(str2, 0));
6565   }
6566   movl(cnt2, int_cnt2);
6567   movptr(result, str1); // string addr
6568 
6569   if (int_cnt2 > stride) {
6570     jmpb(SCAN_TO_SUBSTR);
6571 
6572     // Reload substr for rescan, this code
6573     // is executed only for large substrings (> 8 chars)
6574     bind(RELOAD_SUBSTR);
6575     if (ae == StrIntrinsicNode::UL) {
6576       pmovzxbw(vec, Address(str2, 0));
6577     } else {
6578       movdqu(vec, Address(str2, 0));
6579     }
6580     negptr(cnt2); // Jumped here with negative cnt2, convert to positive
6581 
6582     bind(RELOAD_STR);
6583     // We came here after the beginning of the substring was
6584     // matched but the rest of it was not so we need to search
6585     // again. Start from the next element after the previous match.
6586 
6587     // cnt2 is number of substring reminding elements and
6588     // cnt1 is number of string reminding elements when cmp failed.
6589     // Restored cnt1 = cnt1 - cnt2 + int_cnt2
6590     subl(cnt1, cnt2);
6591     addl(cnt1, int_cnt2);
6592     movl(cnt2, int_cnt2); // Now restore cnt2
6593 
6594     decrementl(cnt1);     // Shift to next element
6595     cmpl(cnt1, cnt2);
6596     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6597 
6598     addptr(result, (1<<scale1));
6599 
6600   } // (int_cnt2 > 8)
6601 
6602   // Scan string for start of substr in 16-byte vectors
6603   bind(SCAN_TO_SUBSTR);
6604   pcmpestri(vec, Address(result, 0), mode);
6605   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
6606   subl(cnt1, stride);
6607   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
6608   cmpl(cnt1, cnt2);
6609   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6610   addptr(result, 16);
6611   jmpb(SCAN_TO_SUBSTR);
6612 
6613   // Found a potential substr
6614   bind(FOUND_CANDIDATE);
6615   // Matched whole vector if first element matched (tmp(rcx) == 0).
6616   if (int_cnt2 == stride) {
6617     jccb(Assembler::overflow, RET_FOUND);    // OF == 1
6618   } else { // int_cnt2 > 8
6619     jccb(Assembler::overflow, FOUND_SUBSTR);
6620   }
6621   // After pcmpestri tmp(rcx) contains matched element index
6622   // Compute start addr of substr
6623   lea(result, Address(result, tmp, scale1));
6624 
6625   // Make sure string is still long enough
6626   subl(cnt1, tmp);
6627   cmpl(cnt1, cnt2);
6628   if (int_cnt2 == stride) {
6629     jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
6630   } else { // int_cnt2 > 8
6631     jccb(Assembler::greaterEqual, MATCH_SUBSTR_HEAD);
6632   }
6633   // Left less then substring.
6634 
6635   bind(RET_NOT_FOUND);
6636   movl(result, -1);
6637   jmp(EXIT);
6638 
6639   if (int_cnt2 > stride) {
6640     // This code is optimized for the case when whole substring
6641     // is matched if its head is matched.
6642     bind(MATCH_SUBSTR_HEAD);
6643     pcmpestri(vec, Address(result, 0), mode);
6644     // Reload only string if does not match
6645     jcc(Assembler::noOverflow, RELOAD_STR); // OF == 0
6646 
6647     Label CONT_SCAN_SUBSTR;
6648     // Compare the rest of substring (> 8 chars).
6649     bind(FOUND_SUBSTR);
6650     // First 8 chars are already matched.
6651     negptr(cnt2);
6652     addptr(cnt2, stride);
6653 
6654     bind(SCAN_SUBSTR);
6655     subl(cnt1, stride);
6656     cmpl(cnt2, -stride); // Do not read beyond substring
6657     jccb(Assembler::lessEqual, CONT_SCAN_SUBSTR);
6658     // Back-up strings to avoid reading beyond substring:
6659     // cnt1 = cnt1 - cnt2 + 8
6660     addl(cnt1, cnt2); // cnt2 is negative
6661     addl(cnt1, stride);
6662     movl(cnt2, stride); negptr(cnt2);
6663     bind(CONT_SCAN_SUBSTR);
6664     if (int_cnt2 < (int)G) {
6665       int tail_off1 = int_cnt2<<scale1;
6666       int tail_off2 = int_cnt2<<scale2;
6667       if (ae == StrIntrinsicNode::UL) {
6668         pmovzxbw(vec, Address(str2, cnt2, scale2, tail_off2));
6669       } else {
6670         movdqu(vec, Address(str2, cnt2, scale2, tail_off2));
6671       }
6672       pcmpestri(vec, Address(result, cnt2, scale1, tail_off1), mode);
6673     } else {
6674       // calculate index in register to avoid integer overflow (int_cnt2*2)
6675       movl(tmp, int_cnt2);
6676       addptr(tmp, cnt2);
6677       if (ae == StrIntrinsicNode::UL) {
6678         pmovzxbw(vec, Address(str2, tmp, scale2, 0));
6679       } else {
6680         movdqu(vec, Address(str2, tmp, scale2, 0));
6681       }
6682       pcmpestri(vec, Address(result, tmp, scale1, 0), mode);
6683     }
6684     // Need to reload strings pointers if not matched whole vector
6685     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
6686     addptr(cnt2, stride);
6687     jcc(Assembler::negative, SCAN_SUBSTR);
6688     // Fall through if found full substring
6689 
6690   } // (int_cnt2 > 8)
6691 
6692   bind(RET_FOUND);
6693   // Found result if we matched full small substring.
6694   // Compute substr offset
6695   subptr(result, str1);
6696   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
6697     shrl(result, 1); // index
6698   }
6699   bind(EXIT);
6700 
6701 } // string_indexofC8
6702 
6703 // Small strings are loaded through stack if they cross page boundary.
6704 void MacroAssembler::string_indexof(Register str1, Register str2,
6705                                     Register cnt1, Register cnt2,
6706                                     int int_cnt2,  Register result,
6707                                     XMMRegister vec, Register tmp,
6708                                     int ae) {
6709   ShortBranchVerifier sbv(this);
6710   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
6711   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
6712 
6713   //
6714   // int_cnt2 is length of small (< 8 chars) constant substring
6715   // or (-1) for non constant substring in which case its length
6716   // is in cnt2 register.
6717   //
6718   // Note, inline_string_indexOf() generates checks:
6719   // if (substr.count > string.count) return -1;
6720   // if (substr.count == 0) return 0;
6721   //
6722   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
6723   assert(int_cnt2 == -1 || (0 < int_cnt2 && int_cnt2 < stride), "should be != 0");
6724   // This method uses the pcmpestri instruction with bound registers
6725   //   inputs:
6726   //     xmm - substring
6727   //     rax - substring length (elements count)
6728   //     mem - scanned string
6729   //     rdx - string length (elements count)
6730   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
6731   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
6732   //   outputs:
6733   //     rcx - matched index in string
6734   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
6735   int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
6736   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
6737   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
6738 
6739   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, ADJUST_STR,
6740         RET_FOUND, RET_NOT_FOUND, CLEANUP, FOUND_SUBSTR,
6741         FOUND_CANDIDATE;
6742 
6743   { //========================================================
6744     // We don't know where these strings are located
6745     // and we can't read beyond them. Load them through stack.
6746     Label BIG_STRINGS, CHECK_STR, COPY_SUBSTR, COPY_STR;
6747 
6748     movptr(tmp, rsp); // save old SP
6749 
6750     if (int_cnt2 > 0) {     // small (< 8 chars) constant substring
6751       if (int_cnt2 == (1>>scale2)) { // One byte
6752         assert((ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL), "Only possible for latin1 encoding");
6753         load_unsigned_byte(result, Address(str2, 0));
6754         movdl(vec, result); // move 32 bits
6755       } else if (ae == StrIntrinsicNode::LL && int_cnt2 == 3) {  // Three bytes
6756         // Not enough header space in 32-bit VM: 12+3 = 15.
6757         movl(result, Address(str2, -1));
6758         shrl(result, 8);
6759         movdl(vec, result); // move 32 bits
6760       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (2>>scale2)) {  // One char
6761         load_unsigned_short(result, Address(str2, 0));
6762         movdl(vec, result); // move 32 bits
6763       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (4>>scale2)) { // Two chars
6764         movdl(vec, Address(str2, 0)); // move 32 bits
6765       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (8>>scale2)) { // Four chars
6766         movq(vec, Address(str2, 0));  // move 64 bits
6767       } else { // cnt2 = { 3, 5, 6, 7 } || (ae == StrIntrinsicNode::UL && cnt2 ={2, ..., 7})
6768         // Array header size is 12 bytes in 32-bit VM
6769         // + 6 bytes for 3 chars == 18 bytes,
6770         // enough space to load vec and shift.
6771         assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity");
6772         if (ae == StrIntrinsicNode::UL) {
6773           int tail_off = int_cnt2-8;
6774           pmovzxbw(vec, Address(str2, tail_off));
6775           psrldq(vec, -2*tail_off);
6776         }
6777         else {
6778           int tail_off = int_cnt2*(1<<scale2);
6779           movdqu(vec, Address(str2, tail_off-16));
6780           psrldq(vec, 16-tail_off);
6781         }
6782       }
6783     } else { // not constant substring
6784       cmpl(cnt2, stride);
6785       jccb(Assembler::aboveEqual, BIG_STRINGS); // Both strings are big enough
6786 
6787       // We can read beyond string if srt+16 does not cross page boundary
6788       // since heaps are aligned and mapped by pages.
6789       assert(os::vm_page_size() < (int)G, "default page should be small");
6790       movl(result, str2); // We need only low 32 bits
6791       andl(result, (os::vm_page_size()-1));
6792       cmpl(result, (os::vm_page_size()-16));
6793       jccb(Assembler::belowEqual, CHECK_STR);
6794 
6795       // Move small strings to stack to allow load 16 bytes into vec.
6796       subptr(rsp, 16);
6797       int stk_offset = wordSize-(1<<scale2);
6798       push(cnt2);
6799 
6800       bind(COPY_SUBSTR);
6801       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL) {
6802         load_unsigned_byte(result, Address(str2, cnt2, scale2, -1));
6803         movb(Address(rsp, cnt2, scale2, stk_offset), result);
6804       } else if (ae == StrIntrinsicNode::UU) {
6805         load_unsigned_short(result, Address(str2, cnt2, scale2, -2));
6806         movw(Address(rsp, cnt2, scale2, stk_offset), result);
6807       }
6808       decrement(cnt2);
6809       jccb(Assembler::notZero, COPY_SUBSTR);
6810 
6811       pop(cnt2);
6812       movptr(str2, rsp);  // New substring address
6813     } // non constant
6814 
6815     bind(CHECK_STR);
6816     cmpl(cnt1, stride);
6817     jccb(Assembler::aboveEqual, BIG_STRINGS);
6818 
6819     // Check cross page boundary.
6820     movl(result, str1); // We need only low 32 bits
6821     andl(result, (os::vm_page_size()-1));
6822     cmpl(result, (os::vm_page_size()-16));
6823     jccb(Assembler::belowEqual, BIG_STRINGS);
6824 
6825     subptr(rsp, 16);
6826     int stk_offset = -(1<<scale1);
6827     if (int_cnt2 < 0) { // not constant
6828       push(cnt2);
6829       stk_offset += wordSize;
6830     }
6831     movl(cnt2, cnt1);
6832 
6833     bind(COPY_STR);
6834     if (ae == StrIntrinsicNode::LL) {
6835       load_unsigned_byte(result, Address(str1, cnt2, scale1, -1));
6836       movb(Address(rsp, cnt2, scale1, stk_offset), result);
6837     } else {
6838       load_unsigned_short(result, Address(str1, cnt2, scale1, -2));
6839       movw(Address(rsp, cnt2, scale1, stk_offset), result);
6840     }
6841     decrement(cnt2);
6842     jccb(Assembler::notZero, COPY_STR);
6843 
6844     if (int_cnt2 < 0) { // not constant
6845       pop(cnt2);
6846     }
6847     movptr(str1, rsp);  // New string address
6848 
6849     bind(BIG_STRINGS);
6850     // Load substring.
6851     if (int_cnt2 < 0) { // -1
6852       if (ae == StrIntrinsicNode::UL) {
6853         pmovzxbw(vec, Address(str2, 0));
6854       } else {
6855         movdqu(vec, Address(str2, 0));
6856       }
6857       push(cnt2);       // substr count
6858       push(str2);       // substr addr
6859       push(str1);       // string addr
6860     } else {
6861       // Small (< 8 chars) constant substrings are loaded already.
6862       movl(cnt2, int_cnt2);
6863     }
6864     push(tmp);  // original SP
6865 
6866   } // Finished loading
6867 
6868   //========================================================
6869   // Start search
6870   //
6871 
6872   movptr(result, str1); // string addr
6873 
6874   if (int_cnt2  < 0) {  // Only for non constant substring
6875     jmpb(SCAN_TO_SUBSTR);
6876 
6877     // SP saved at sp+0
6878     // String saved at sp+1*wordSize
6879     // Substr saved at sp+2*wordSize
6880     // Substr count saved at sp+3*wordSize
6881 
6882     // Reload substr for rescan, this code
6883     // is executed only for large substrings (> 8 chars)
6884     bind(RELOAD_SUBSTR);
6885     movptr(str2, Address(rsp, 2*wordSize));
6886     movl(cnt2, Address(rsp, 3*wordSize));
6887     if (ae == StrIntrinsicNode::UL) {
6888       pmovzxbw(vec, Address(str2, 0));
6889     } else {
6890       movdqu(vec, Address(str2, 0));
6891     }
6892     // We came here after the beginning of the substring was
6893     // matched but the rest of it was not so we need to search
6894     // again. Start from the next element after the previous match.
6895     subptr(str1, result); // Restore counter
6896     if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
6897       shrl(str1, 1);
6898     }
6899     addl(cnt1, str1);
6900     decrementl(cnt1);   // Shift to next element
6901     cmpl(cnt1, cnt2);
6902     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6903 
6904     addptr(result, (1<<scale1));
6905   } // non constant
6906 
6907   // Scan string for start of substr in 16-byte vectors
6908   bind(SCAN_TO_SUBSTR);
6909   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
6910   pcmpestri(vec, Address(result, 0), mode);
6911   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
6912   subl(cnt1, stride);
6913   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
6914   cmpl(cnt1, cnt2);
6915   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6916   addptr(result, 16);
6917 
6918   bind(ADJUST_STR);
6919   cmpl(cnt1, stride); // Do not read beyond string
6920   jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
6921   // Back-up string to avoid reading beyond string.
6922   lea(result, Address(result, cnt1, scale1, -16));
6923   movl(cnt1, stride);
6924   jmpb(SCAN_TO_SUBSTR);
6925 
6926   // Found a potential substr
6927   bind(FOUND_CANDIDATE);
6928   // After pcmpestri tmp(rcx) contains matched element index
6929 
6930   // Make sure string is still long enough
6931   subl(cnt1, tmp);
6932   cmpl(cnt1, cnt2);
6933   jccb(Assembler::greaterEqual, FOUND_SUBSTR);
6934   // Left less then substring.
6935 
6936   bind(RET_NOT_FOUND);
6937   movl(result, -1);
6938   jmp(CLEANUP);
6939 
6940   bind(FOUND_SUBSTR);
6941   // Compute start addr of substr
6942   lea(result, Address(result, tmp, scale1));
6943   if (int_cnt2 > 0) { // Constant substring
6944     // Repeat search for small substring (< 8 chars)
6945     // from new point without reloading substring.
6946     // Have to check that we don't read beyond string.
6947     cmpl(tmp, stride-int_cnt2);
6948     jccb(Assembler::greater, ADJUST_STR);
6949     // Fall through if matched whole substring.
6950   } else { // non constant
6951     assert(int_cnt2 == -1, "should be != 0");
6952 
6953     addl(tmp, cnt2);
6954     // Found result if we matched whole substring.
6955     cmpl(tmp, stride);
6956     jcc(Assembler::lessEqual, RET_FOUND);
6957 
6958     // Repeat search for small substring (<= 8 chars)
6959     // from new point 'str1' without reloading substring.
6960     cmpl(cnt2, stride);
6961     // Have to check that we don't read beyond string.
6962     jccb(Assembler::lessEqual, ADJUST_STR);
6963 
6964     Label CHECK_NEXT, CONT_SCAN_SUBSTR, RET_FOUND_LONG;
6965     // Compare the rest of substring (> 8 chars).
6966     movptr(str1, result);
6967 
6968     cmpl(tmp, cnt2);
6969     // First 8 chars are already matched.
6970     jccb(Assembler::equal, CHECK_NEXT);
6971 
6972     bind(SCAN_SUBSTR);
6973     pcmpestri(vec, Address(str1, 0), mode);
6974     // Need to reload strings pointers if not matched whole vector
6975     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
6976 
6977     bind(CHECK_NEXT);
6978     subl(cnt2, stride);
6979     jccb(Assembler::lessEqual, RET_FOUND_LONG); // Found full substring
6980     addptr(str1, 16);
6981     if (ae == StrIntrinsicNode::UL) {
6982       addptr(str2, 8);
6983     } else {
6984       addptr(str2, 16);
6985     }
6986     subl(cnt1, stride);
6987     cmpl(cnt2, stride); // Do not read beyond substring
6988     jccb(Assembler::greaterEqual, CONT_SCAN_SUBSTR);
6989     // Back-up strings to avoid reading beyond substring.
6990 
6991     if (ae == StrIntrinsicNode::UL) {
6992       lea(str2, Address(str2, cnt2, scale2, -8));
6993       lea(str1, Address(str1, cnt2, scale1, -16));
6994     } else {
6995       lea(str2, Address(str2, cnt2, scale2, -16));
6996       lea(str1, Address(str1, cnt2, scale1, -16));
6997     }
6998     subl(cnt1, cnt2);
6999     movl(cnt2, stride);
7000     addl(cnt1, stride);
7001     bind(CONT_SCAN_SUBSTR);
7002     if (ae == StrIntrinsicNode::UL) {
7003       pmovzxbw(vec, Address(str2, 0));
7004     } else {
7005       movdqu(vec, Address(str2, 0));
7006     }
7007     jmp(SCAN_SUBSTR);
7008 
7009     bind(RET_FOUND_LONG);
7010     movptr(str1, Address(rsp, wordSize));
7011   } // non constant
7012 
7013   bind(RET_FOUND);
7014   // Compute substr offset
7015   subptr(result, str1);
7016   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7017     shrl(result, 1); // index
7018   }
7019   bind(CLEANUP);
7020   pop(rsp); // restore SP
7021 
7022 } // string_indexof
7023 
7024 void MacroAssembler::string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
7025                                          XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp) {
7026   ShortBranchVerifier sbv(this);
7027   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7028 
7029   int stride = 8;
7030 
7031   Label FOUND_CHAR, SCAN_TO_CHAR, SCAN_TO_CHAR_LOOP,
7032         SCAN_TO_8_CHAR, SCAN_TO_8_CHAR_LOOP, SCAN_TO_16_CHAR_LOOP,
7033         RET_NOT_FOUND, SCAN_TO_8_CHAR_INIT,
7034         FOUND_SEQ_CHAR, DONE_LABEL;
7035 
7036   movptr(result, str1);
7037   if (UseAVX >= 2) {
7038     cmpl(cnt1, stride);
7039     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7040     cmpl(cnt1, 2*stride);
7041     jcc(Assembler::less, SCAN_TO_8_CHAR_INIT);
7042     movdl(vec1, ch);
7043     vpbroadcastw(vec1, vec1);
7044     vpxor(vec2, vec2);
7045     movl(tmp, cnt1);
7046     andl(tmp, 0xFFFFFFF0);  //vector count (in chars)
7047     andl(cnt1,0x0000000F);  //tail count (in chars)
7048 
7049     bind(SCAN_TO_16_CHAR_LOOP);
7050     vmovdqu(vec3, Address(result, 0));
7051     vpcmpeqw(vec3, vec3, vec1, 1);
7052     vptest(vec2, vec3);
7053     jcc(Assembler::carryClear, FOUND_CHAR);
7054     addptr(result, 32);
7055     subl(tmp, 2*stride);
7056     jcc(Assembler::notZero, SCAN_TO_16_CHAR_LOOP);
7057     jmp(SCAN_TO_8_CHAR);
7058     bind(SCAN_TO_8_CHAR_INIT);
7059     movdl(vec1, ch);
7060     pshuflw(vec1, vec1, 0x00);
7061     pshufd(vec1, vec1, 0);
7062     pxor(vec2, vec2);
7063   }
7064   bind(SCAN_TO_8_CHAR);
7065   cmpl(cnt1, stride);
7066   if (UseAVX >= 2) {
7067     jcc(Assembler::less, SCAN_TO_CHAR);
7068   } else {
7069     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7070     movdl(vec1, ch);
7071     pshuflw(vec1, vec1, 0x00);
7072     pshufd(vec1, vec1, 0);
7073     pxor(vec2, vec2);
7074   }
7075   movl(tmp, cnt1);
7076   andl(tmp, 0xFFFFFFF8);  //vector count (in chars)
7077   andl(cnt1,0x00000007);  //tail count (in chars)
7078 
7079   bind(SCAN_TO_8_CHAR_LOOP);
7080   movdqu(vec3, Address(result, 0));
7081   pcmpeqw(vec3, vec1);
7082   ptest(vec2, vec3);
7083   jcc(Assembler::carryClear, FOUND_CHAR);
7084   addptr(result, 16);
7085   subl(tmp, stride);
7086   jcc(Assembler::notZero, SCAN_TO_8_CHAR_LOOP);
7087   bind(SCAN_TO_CHAR);
7088   testl(cnt1, cnt1);
7089   jcc(Assembler::zero, RET_NOT_FOUND);
7090   bind(SCAN_TO_CHAR_LOOP);
7091   load_unsigned_short(tmp, Address(result, 0));
7092   cmpl(ch, tmp);
7093   jccb(Assembler::equal, FOUND_SEQ_CHAR);
7094   addptr(result, 2);
7095   subl(cnt1, 1);
7096   jccb(Assembler::zero, RET_NOT_FOUND);
7097   jmp(SCAN_TO_CHAR_LOOP);
7098 
7099   bind(RET_NOT_FOUND);
7100   movl(result, -1);
7101   jmpb(DONE_LABEL);
7102 
7103   bind(FOUND_CHAR);
7104   if (UseAVX >= 2) {
7105     vpmovmskb(tmp, vec3);
7106   } else {
7107     pmovmskb(tmp, vec3);
7108   }
7109   bsfl(ch, tmp);
7110   addl(result, ch);
7111 
7112   bind(FOUND_SEQ_CHAR);
7113   subptr(result, str1);
7114   shrl(result, 1);
7115 
7116   bind(DONE_LABEL);
7117 } // string_indexof_char
7118 
7119 // helper function for string_compare
7120 void MacroAssembler::load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
7121                                         Address::ScaleFactor scale, Address::ScaleFactor scale1,
7122                                         Address::ScaleFactor scale2, Register index, int ae) {
7123   if (ae == StrIntrinsicNode::LL) {
7124     load_unsigned_byte(elem1, Address(str1, index, scale, 0));
7125     load_unsigned_byte(elem2, Address(str2, index, scale, 0));
7126   } else if (ae == StrIntrinsicNode::UU) {
7127     load_unsigned_short(elem1, Address(str1, index, scale, 0));
7128     load_unsigned_short(elem2, Address(str2, index, scale, 0));
7129   } else {
7130     load_unsigned_byte(elem1, Address(str1, index, scale1, 0));
7131     load_unsigned_short(elem2, Address(str2, index, scale2, 0));
7132   }
7133 }
7134 
7135 // Compare strings, used for char[] and byte[].
7136 void MacroAssembler::string_compare(Register str1, Register str2,
7137                                     Register cnt1, Register cnt2, Register result,
7138                                     XMMRegister vec1, int ae) {
7139   ShortBranchVerifier sbv(this);
7140   Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
7141   Label COMPARE_WIDE_VECTORS_LOOP_FAILED;  // used only _LP64 && AVX3
7142   int stride, stride2, adr_stride, adr_stride1, adr_stride2;
7143   int stride2x2 = 0x40;
7144   Address::ScaleFactor scale = Address::no_scale;
7145   Address::ScaleFactor scale1 = Address::no_scale;
7146   Address::ScaleFactor scale2 = Address::no_scale;
7147 
7148   if (ae != StrIntrinsicNode::LL) {
7149     stride2x2 = 0x20;
7150   }
7151 
7152   if (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL) {
7153     shrl(cnt2, 1);
7154   }
7155   // Compute the minimum of the string lengths and the
7156   // difference of the string lengths (stack).
7157   // Do the conditional move stuff
7158   movl(result, cnt1);
7159   subl(cnt1, cnt2);
7160   push(cnt1);
7161   cmov32(Assembler::lessEqual, cnt2, result);    // cnt2 = min(cnt1, cnt2)
7162 
7163   // Is the minimum length zero?
7164   testl(cnt2, cnt2);
7165   jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7166   if (ae == StrIntrinsicNode::LL) {
7167     // Load first bytes
7168     load_unsigned_byte(result, Address(str1, 0));  // result = str1[0]
7169     load_unsigned_byte(cnt1, Address(str2, 0));    // cnt1   = str2[0]
7170   } else if (ae == StrIntrinsicNode::UU) {
7171     // Load first characters
7172     load_unsigned_short(result, Address(str1, 0));
7173     load_unsigned_short(cnt1, Address(str2, 0));
7174   } else {
7175     load_unsigned_byte(result, Address(str1, 0));
7176     load_unsigned_short(cnt1, Address(str2, 0));
7177   }
7178   subl(result, cnt1);
7179   jcc(Assembler::notZero,  POP_LABEL);
7180 
7181   if (ae == StrIntrinsicNode::UU) {
7182     // Divide length by 2 to get number of chars
7183     shrl(cnt2, 1);
7184   }
7185   cmpl(cnt2, 1);
7186   jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7187 
7188   // Check if the strings start at the same location and setup scale and stride
7189   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7190     cmpptr(str1, str2);
7191     jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7192     if (ae == StrIntrinsicNode::LL) {
7193       scale = Address::times_1;
7194       stride = 16;
7195     } else {
7196       scale = Address::times_2;
7197       stride = 8;
7198     }
7199   } else {
7200     scale1 = Address::times_1;
7201     scale2 = Address::times_2;
7202     // scale not used
7203     stride = 8;
7204   }
7205 
7206   if (UseAVX >= 2 && UseSSE42Intrinsics) {
7207     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_WIDE_TAIL, COMPARE_SMALL_STR;
7208     Label COMPARE_WIDE_VECTORS_LOOP, COMPARE_16_CHARS, COMPARE_INDEX_CHAR;
7209     Label COMPARE_WIDE_VECTORS_LOOP_AVX2;
7210     Label COMPARE_TAIL_LONG;
7211     Label COMPARE_WIDE_VECTORS_LOOP_AVX3;  // used only _LP64 && AVX3
7212 
7213     int pcmpmask = 0x19;
7214     if (ae == StrIntrinsicNode::LL) {
7215       pcmpmask &= ~0x01;
7216     }
7217 
7218     // Setup to compare 16-chars (32-bytes) vectors,
7219     // start from first character again because it has aligned address.
7220     if (ae == StrIntrinsicNode::LL) {
7221       stride2 = 32;
7222     } else {
7223       stride2 = 16;
7224     }
7225     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7226       adr_stride = stride << scale;
7227     } else {
7228       adr_stride1 = 8;  //stride << scale1;
7229       adr_stride2 = 16; //stride << scale2;
7230     }
7231 
7232     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
7233     // rax and rdx are used by pcmpestri as elements counters
7234     movl(result, cnt2);
7235     andl(cnt2, ~(stride2-1));   // cnt2 holds the vector count
7236     jcc(Assembler::zero, COMPARE_TAIL_LONG);
7237 
7238     // fast path : compare first 2 8-char vectors.
7239     bind(COMPARE_16_CHARS);
7240     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7241       movdqu(vec1, Address(str1, 0));
7242     } else {
7243       pmovzxbw(vec1, Address(str1, 0));
7244     }
7245     pcmpestri(vec1, Address(str2, 0), pcmpmask);
7246     jccb(Assembler::below, COMPARE_INDEX_CHAR);
7247 
7248     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7249       movdqu(vec1, Address(str1, adr_stride));
7250       pcmpestri(vec1, Address(str2, adr_stride), pcmpmask);
7251     } else {
7252       pmovzxbw(vec1, Address(str1, adr_stride1));
7253       pcmpestri(vec1, Address(str2, adr_stride2), pcmpmask);
7254     }
7255     jccb(Assembler::aboveEqual, COMPARE_WIDE_VECTORS);
7256     addl(cnt1, stride);
7257 
7258     // Compare the characters at index in cnt1
7259     bind(COMPARE_INDEX_CHAR); // cnt1 has the offset of the mismatching character
7260     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
7261     subl(result, cnt2);
7262     jmp(POP_LABEL);
7263 
7264     // Setup the registers to start vector comparison loop
7265     bind(COMPARE_WIDE_VECTORS);
7266     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7267       lea(str1, Address(str1, result, scale));
7268       lea(str2, Address(str2, result, scale));
7269     } else {
7270       lea(str1, Address(str1, result, scale1));
7271       lea(str2, Address(str2, result, scale2));
7272     }
7273     subl(result, stride2);
7274     subl(cnt2, stride2);
7275     jcc(Assembler::zero, COMPARE_WIDE_TAIL);
7276     negptr(result);
7277 
7278     //  In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest)
7279     bind(COMPARE_WIDE_VECTORS_LOOP);
7280 
7281 #ifdef _LP64
7282     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
7283       cmpl(cnt2, stride2x2);
7284       jccb(Assembler::below, COMPARE_WIDE_VECTORS_LOOP_AVX2);
7285       testl(cnt2, stride2x2-1);   // cnt2 holds the vector count
7286       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX2);   // means we cannot subtract by 0x40
7287 
7288       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
7289       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7290         evmovdquq(vec1, Address(str1, result, scale), Assembler::AVX_512bit);
7291         evpcmpeqb(k7, vec1, Address(str2, result, scale), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7292       } else {
7293         vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_512bit);
7294         evpcmpeqb(k7, vec1, Address(str2, result, scale2), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7295       }
7296       kortestql(k7, k7);
7297       jcc(Assembler::aboveEqual, COMPARE_WIDE_VECTORS_LOOP_FAILED);     // miscompare
7298       addptr(result, stride2x2);  // update since we already compared at this addr
7299       subl(cnt2, stride2x2);      // and sub the size too
7300       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX3);
7301 
7302       vpxor(vec1, vec1);
7303       jmpb(COMPARE_WIDE_TAIL);
7304     }//if (VM_Version::supports_avx512vlbw())
7305 #endif // _LP64
7306 
7307 
7308     bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7309     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7310       vmovdqu(vec1, Address(str1, result, scale));
7311       vpxor(vec1, Address(str2, result, scale));
7312     } else {
7313       vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_256bit);
7314       vpxor(vec1, Address(str2, result, scale2));
7315     }
7316     vptest(vec1, vec1);
7317     jcc(Assembler::notZero, VECTOR_NOT_EQUAL);
7318     addptr(result, stride2);
7319     subl(cnt2, stride2);
7320     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP);
7321     // clean upper bits of YMM registers
7322     vpxor(vec1, vec1);
7323 
7324     // compare wide vectors tail
7325     bind(COMPARE_WIDE_TAIL);
7326     testptr(result, result);
7327     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7328 
7329     movl(result, stride2);
7330     movl(cnt2, result);
7331     negptr(result);
7332     jmp(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7333 
7334     // Identifies the mismatching (higher or lower)16-bytes in the 32-byte vectors.
7335     bind(VECTOR_NOT_EQUAL);
7336     // clean upper bits of YMM registers
7337     vpxor(vec1, vec1);
7338     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7339       lea(str1, Address(str1, result, scale));
7340       lea(str2, Address(str2, result, scale));
7341     } else {
7342       lea(str1, Address(str1, result, scale1));
7343       lea(str2, Address(str2, result, scale2));
7344     }
7345     jmp(COMPARE_16_CHARS);
7346 
7347     // Compare tail chars, length between 1 to 15 chars
7348     bind(COMPARE_TAIL_LONG);
7349     movl(cnt2, result);
7350     cmpl(cnt2, stride);
7351     jcc(Assembler::less, COMPARE_SMALL_STR);
7352 
7353     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7354       movdqu(vec1, Address(str1, 0));
7355     } else {
7356       pmovzxbw(vec1, Address(str1, 0));
7357     }
7358     pcmpestri(vec1, Address(str2, 0), pcmpmask);
7359     jcc(Assembler::below, COMPARE_INDEX_CHAR);
7360     subptr(cnt2, stride);
7361     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7362     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7363       lea(str1, Address(str1, result, scale));
7364       lea(str2, Address(str2, result, scale));
7365     } else {
7366       lea(str1, Address(str1, result, scale1));
7367       lea(str2, Address(str2, result, scale2));
7368     }
7369     negptr(cnt2);
7370     jmpb(WHILE_HEAD_LABEL);
7371 
7372     bind(COMPARE_SMALL_STR);
7373   } else if (UseSSE42Intrinsics) {
7374     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
7375     int pcmpmask = 0x19;
7376     // Setup to compare 8-char (16-byte) vectors,
7377     // start from first character again because it has aligned address.
7378     movl(result, cnt2);
7379     andl(cnt2, ~(stride - 1));   // cnt2 holds the vector count
7380     if (ae == StrIntrinsicNode::LL) {
7381       pcmpmask &= ~0x01;
7382     }
7383     jcc(Assembler::zero, COMPARE_TAIL);
7384     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7385       lea(str1, Address(str1, result, scale));
7386       lea(str2, Address(str2, result, scale));
7387     } else {
7388       lea(str1, Address(str1, result, scale1));
7389       lea(str2, Address(str2, result, scale2));
7390     }
7391     negptr(result);
7392 
7393     // pcmpestri
7394     //   inputs:
7395     //     vec1- substring
7396     //     rax - negative string length (elements count)
7397     //     mem - scanned string
7398     //     rdx - string length (elements count)
7399     //     pcmpmask - cmp mode: 11000 (string compare with negated result)
7400     //               + 00 (unsigned bytes) or  + 01 (unsigned shorts)
7401     //   outputs:
7402     //     rcx - first mismatched element index
7403     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
7404 
7405     bind(COMPARE_WIDE_VECTORS);
7406     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7407       movdqu(vec1, Address(str1, result, scale));
7408       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
7409     } else {
7410       pmovzxbw(vec1, Address(str1, result, scale1));
7411       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
7412     }
7413     // After pcmpestri cnt1(rcx) contains mismatched element index
7414 
7415     jccb(Assembler::below, VECTOR_NOT_EQUAL);  // CF==1
7416     addptr(result, stride);
7417     subptr(cnt2, stride);
7418     jccb(Assembler::notZero, COMPARE_WIDE_VECTORS);
7419 
7420     // compare wide vectors tail
7421     testptr(result, result);
7422     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7423 
7424     movl(cnt2, stride);
7425     movl(result, stride);
7426     negptr(result);
7427     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7428       movdqu(vec1, Address(str1, result, scale));
7429       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
7430     } else {
7431       pmovzxbw(vec1, Address(str1, result, scale1));
7432       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
7433     }
7434     jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL);
7435 
7436     // Mismatched characters in the vectors
7437     bind(VECTOR_NOT_EQUAL);
7438     addptr(cnt1, result);
7439     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
7440     subl(result, cnt2);
7441     jmpb(POP_LABEL);
7442 
7443     bind(COMPARE_TAIL); // limit is zero
7444     movl(cnt2, result);
7445     // Fallthru to tail compare
7446   }
7447   // Shift str2 and str1 to the end of the arrays, negate min
7448   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7449     lea(str1, Address(str1, cnt2, scale));
7450     lea(str2, Address(str2, cnt2, scale));
7451   } else {
7452     lea(str1, Address(str1, cnt2, scale1));
7453     lea(str2, Address(str2, cnt2, scale2));
7454   }
7455   decrementl(cnt2);  // first character was compared already
7456   negptr(cnt2);
7457 
7458   // Compare the rest of the elements
7459   bind(WHILE_HEAD_LABEL);
7460   load_next_elements(result, cnt1, str1, str2, scale, scale1, scale2, cnt2, ae);
7461   subl(result, cnt1);
7462   jccb(Assembler::notZero, POP_LABEL);
7463   increment(cnt2);
7464   jccb(Assembler::notZero, WHILE_HEAD_LABEL);
7465 
7466   // Strings are equal up to min length.  Return the length difference.
7467   bind(LENGTH_DIFF_LABEL);
7468   pop(result);
7469   if (ae == StrIntrinsicNode::UU) {
7470     // Divide diff by 2 to get number of chars
7471     sarl(result, 1);
7472   }
7473   jmpb(DONE_LABEL);
7474 
7475 #ifdef _LP64
7476   if (VM_Version::supports_avx512vlbw()) {
7477 
7478     bind(COMPARE_WIDE_VECTORS_LOOP_FAILED);
7479 
7480     kmovql(cnt1, k7);
7481     notq(cnt1);
7482     bsfq(cnt2, cnt1);
7483     if (ae != StrIntrinsicNode::LL) {
7484       // Divide diff by 2 to get number of chars
7485       sarl(cnt2, 1);
7486     }
7487     addq(result, cnt2);
7488     if (ae == StrIntrinsicNode::LL) {
7489       load_unsigned_byte(cnt1, Address(str2, result));
7490       load_unsigned_byte(result, Address(str1, result));
7491     } else if (ae == StrIntrinsicNode::UU) {
7492       load_unsigned_short(cnt1, Address(str2, result, scale));
7493       load_unsigned_short(result, Address(str1, result, scale));
7494     } else {
7495       load_unsigned_short(cnt1, Address(str2, result, scale2));
7496       load_unsigned_byte(result, Address(str1, result, scale1));
7497     }
7498     subl(result, cnt1);
7499     jmpb(POP_LABEL);
7500   }//if (VM_Version::supports_avx512vlbw())
7501 #endif // _LP64
7502 
7503   // Discard the stored length difference
7504   bind(POP_LABEL);
7505   pop(cnt1);
7506 
7507   // That's it
7508   bind(DONE_LABEL);
7509   if(ae == StrIntrinsicNode::UL) {
7510     negl(result);
7511   }
7512 
7513 }
7514 
7515 // Search for Non-ASCII character (Negative byte value) in a byte array,
7516 // return true if it has any and false otherwise.
7517 //   ..\jdk\src\java.base\share\classes\java\lang\StringCoding.java
7518 //   @HotSpotIntrinsicCandidate
7519 //   private static boolean hasNegatives(byte[] ba, int off, int len) {
7520 //     for (int i = off; i < off + len; i++) {
7521 //       if (ba[i] < 0) {
7522 //         return true;
7523 //       }
7524 //     }
7525 //     return false;
7526 //   }
7527 void MacroAssembler::has_negatives(Register ary1, Register len,
7528   Register result, Register tmp1,
7529   XMMRegister vec1, XMMRegister vec2) {
7530   // rsi: byte array
7531   // rcx: len
7532   // rax: result
7533   ShortBranchVerifier sbv(this);
7534   assert_different_registers(ary1, len, result, tmp1);
7535   assert_different_registers(vec1, vec2);
7536   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_CHAR, COMPARE_VECTORS, COMPARE_BYTE;
7537 
7538   // len == 0
7539   testl(len, len);
7540   jcc(Assembler::zero, FALSE_LABEL);
7541 
7542   if ((UseAVX > 2) && // AVX512
7543     VM_Version::supports_avx512vlbw() &&
7544     VM_Version::supports_bmi2()) {
7545 
7546     set_vector_masking();  // opening of the stub context for programming mask registers
7547 
7548     Label test_64_loop, test_tail;
7549     Register tmp3_aliased = len;
7550 
7551     movl(tmp1, len);
7552     vpxor(vec2, vec2, vec2, Assembler::AVX_512bit);
7553 
7554     andl(tmp1, 64 - 1);   // tail count (in chars) 0x3F
7555     andl(len, ~(64 - 1));    // vector count (in chars)
7556     jccb(Assembler::zero, test_tail);
7557 
7558     lea(ary1, Address(ary1, len, Address::times_1));
7559     negptr(len);
7560 
7561     bind(test_64_loop);
7562     // Check whether our 64 elements of size byte contain negatives
7563     evpcmpgtb(k2, vec2, Address(ary1, len, Address::times_1), Assembler::AVX_512bit);
7564     kortestql(k2, k2);
7565     jcc(Assembler::notZero, TRUE_LABEL);
7566 
7567     addptr(len, 64);
7568     jccb(Assembler::notZero, test_64_loop);
7569 
7570 
7571     bind(test_tail);
7572     // bail out when there is nothing to be done
7573     testl(tmp1, -1);
7574     jcc(Assembler::zero, FALSE_LABEL);
7575 
7576     // Save k1
7577     kmovql(k3, k1);
7578 
7579     // ~(~0 << len) applied up to two times (for 32-bit scenario)
7580 #ifdef _LP64
7581     mov64(tmp3_aliased, 0xFFFFFFFFFFFFFFFF);
7582     shlxq(tmp3_aliased, tmp3_aliased, tmp1);
7583     notq(tmp3_aliased);
7584     kmovql(k1, tmp3_aliased);
7585 #else
7586     Label k_init;
7587     jmp(k_init);
7588 
7589     // We could not read 64-bits from a general purpose register thus we move
7590     // data required to compose 64 1's to the instruction stream
7591     // We emit 64 byte wide series of elements from 0..63 which later on would
7592     // be used as a compare targets with tail count contained in tmp1 register.
7593     // Result would be a k1 register having tmp1 consecutive number or 1
7594     // counting from least significant bit.
7595     address tmp = pc();
7596     emit_int64(0x0706050403020100);
7597     emit_int64(0x0F0E0D0C0B0A0908);
7598     emit_int64(0x1716151413121110);
7599     emit_int64(0x1F1E1D1C1B1A1918);
7600     emit_int64(0x2726252423222120);
7601     emit_int64(0x2F2E2D2C2B2A2928);
7602     emit_int64(0x3736353433323130);
7603     emit_int64(0x3F3E3D3C3B3A3938);
7604 
7605     bind(k_init);
7606     lea(len, InternalAddress(tmp));
7607     // create mask to test for negative byte inside a vector
7608     evpbroadcastb(vec1, tmp1, Assembler::AVX_512bit);
7609     evpcmpgtb(k1, vec1, Address(len, 0), Assembler::AVX_512bit);
7610 
7611 #endif
7612     evpcmpgtb(k2, k1, vec2, Address(ary1, 0), Assembler::AVX_512bit);
7613     ktestq(k2, k1);
7614     // Restore k1
7615     kmovql(k1, k3);
7616     jcc(Assembler::notZero, TRUE_LABEL);
7617 
7618     jmp(FALSE_LABEL);
7619 
7620     clear_vector_masking();   // closing of the stub context for programming mask registers
7621   } else {
7622     movl(result, len); // copy
7623 
7624     if (UseAVX == 2 && UseSSE >= 2) {
7625       // With AVX2, use 32-byte vector compare
7626       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7627 
7628       // Compare 32-byte vectors
7629       andl(result, 0x0000001f);  //   tail count (in bytes)
7630       andl(len, 0xffffffe0);   // vector count (in bytes)
7631       jccb(Assembler::zero, COMPARE_TAIL);
7632 
7633       lea(ary1, Address(ary1, len, Address::times_1));
7634       negptr(len);
7635 
7636       movl(tmp1, 0x80808080);   // create mask to test for Unicode chars in vector
7637       movdl(vec2, tmp1);
7638       vpbroadcastd(vec2, vec2);
7639 
7640       bind(COMPARE_WIDE_VECTORS);
7641       vmovdqu(vec1, Address(ary1, len, Address::times_1));
7642       vptest(vec1, vec2);
7643       jccb(Assembler::notZero, TRUE_LABEL);
7644       addptr(len, 32);
7645       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7646 
7647       testl(result, result);
7648       jccb(Assembler::zero, FALSE_LABEL);
7649 
7650       vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
7651       vptest(vec1, vec2);
7652       jccb(Assembler::notZero, TRUE_LABEL);
7653       jmpb(FALSE_LABEL);
7654 
7655       bind(COMPARE_TAIL); // len is zero
7656       movl(len, result);
7657       // Fallthru to tail compare
7658     } else if (UseSSE42Intrinsics) {
7659       // With SSE4.2, use double quad vector compare
7660       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7661 
7662       // Compare 16-byte vectors
7663       andl(result, 0x0000000f);  //   tail count (in bytes)
7664       andl(len, 0xfffffff0);   // vector count (in bytes)
7665       jcc(Assembler::zero, COMPARE_TAIL);
7666 
7667       lea(ary1, Address(ary1, len, Address::times_1));
7668       negptr(len);
7669 
7670       movl(tmp1, 0x80808080);
7671       movdl(vec2, tmp1);
7672       pshufd(vec2, vec2, 0);
7673 
7674       bind(COMPARE_WIDE_VECTORS);
7675       movdqu(vec1, Address(ary1, len, Address::times_1));
7676       ptest(vec1, vec2);
7677       jcc(Assembler::notZero, TRUE_LABEL);
7678       addptr(len, 16);
7679       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7680 
7681       testl(result, result);
7682       jcc(Assembler::zero, FALSE_LABEL);
7683 
7684       movdqu(vec1, Address(ary1, result, Address::times_1, -16));
7685       ptest(vec1, vec2);
7686       jccb(Assembler::notZero, TRUE_LABEL);
7687       jmpb(FALSE_LABEL);
7688 
7689       bind(COMPARE_TAIL); // len is zero
7690       movl(len, result);
7691       // Fallthru to tail compare
7692     }
7693   }
7694   // Compare 4-byte vectors
7695   andl(len, 0xfffffffc); // vector count (in bytes)
7696   jccb(Assembler::zero, COMPARE_CHAR);
7697 
7698   lea(ary1, Address(ary1, len, Address::times_1));
7699   negptr(len);
7700 
7701   bind(COMPARE_VECTORS);
7702   movl(tmp1, Address(ary1, len, Address::times_1));
7703   andl(tmp1, 0x80808080);
7704   jccb(Assembler::notZero, TRUE_LABEL);
7705   addptr(len, 4);
7706   jcc(Assembler::notZero, COMPARE_VECTORS);
7707 
7708   // Compare trailing char (final 2 bytes), if any
7709   bind(COMPARE_CHAR);
7710   testl(result, 0x2);   // tail  char
7711   jccb(Assembler::zero, COMPARE_BYTE);
7712   load_unsigned_short(tmp1, Address(ary1, 0));
7713   andl(tmp1, 0x00008080);
7714   jccb(Assembler::notZero, TRUE_LABEL);
7715   subptr(result, 2);
7716   lea(ary1, Address(ary1, 2));
7717 
7718   bind(COMPARE_BYTE);
7719   testl(result, 0x1);   // tail  byte
7720   jccb(Assembler::zero, FALSE_LABEL);
7721   load_unsigned_byte(tmp1, Address(ary1, 0));
7722   andl(tmp1, 0x00000080);
7723   jccb(Assembler::notEqual, TRUE_LABEL);
7724   jmpb(FALSE_LABEL);
7725 
7726   bind(TRUE_LABEL);
7727   movl(result, 1);   // return true
7728   jmpb(DONE);
7729 
7730   bind(FALSE_LABEL);
7731   xorl(result, result); // return false
7732 
7733   // That's it
7734   bind(DONE);
7735   if (UseAVX >= 2 && UseSSE >= 2) {
7736     // clean upper bits of YMM registers
7737     vpxor(vec1, vec1);
7738     vpxor(vec2, vec2);
7739   }
7740 }
7741 // Compare char[] or byte[] arrays aligned to 4 bytes or substrings.
7742 void MacroAssembler::arrays_equals(bool is_array_equ, Register ary1, Register ary2,
7743                                    Register limit, Register result, Register chr,
7744                                    XMMRegister vec1, XMMRegister vec2, bool is_char) {
7745   ShortBranchVerifier sbv(this);
7746   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR, COMPARE_BYTE;
7747 
7748   int length_offset  = arrayOopDesc::length_offset_in_bytes();
7749   int base_offset    = arrayOopDesc::base_offset_in_bytes(is_char ? T_CHAR : T_BYTE);
7750 
7751   if (is_array_equ) {
7752     // Check the input args
7753     cmpoop(ary1, ary2);
7754     jcc(Assembler::equal, TRUE_LABEL);
7755 
7756     // Need additional checks for arrays_equals.
7757     testptr(ary1, ary1);
7758     jcc(Assembler::zero, FALSE_LABEL);
7759     testptr(ary2, ary2);
7760     jcc(Assembler::zero, FALSE_LABEL);
7761 
7762     // Check the lengths
7763     movl(limit, Address(ary1, length_offset));
7764     cmpl(limit, Address(ary2, length_offset));
7765     jcc(Assembler::notEqual, FALSE_LABEL);
7766   }
7767 
7768   // count == 0
7769   testl(limit, limit);
7770   jcc(Assembler::zero, TRUE_LABEL);
7771 
7772   if (is_array_equ) {
7773     // Load array address
7774     lea(ary1, Address(ary1, base_offset));
7775     lea(ary2, Address(ary2, base_offset));
7776   }
7777 
7778   if (is_array_equ && is_char) {
7779     // arrays_equals when used for char[].
7780     shll(limit, 1);      // byte count != 0
7781   }
7782   movl(result, limit); // copy
7783 
7784   if (UseAVX >= 2) {
7785     // With AVX2, use 32-byte vector compare
7786     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7787 
7788     // Compare 32-byte vectors
7789     andl(result, 0x0000001f);  //   tail count (in bytes)
7790     andl(limit, 0xffffffe0);   // vector count (in bytes)
7791     jcc(Assembler::zero, COMPARE_TAIL);
7792 
7793     lea(ary1, Address(ary1, limit, Address::times_1));
7794     lea(ary2, Address(ary2, limit, Address::times_1));
7795     negptr(limit);
7796 
7797     bind(COMPARE_WIDE_VECTORS);
7798 
7799 #ifdef _LP64
7800     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
7801       Label COMPARE_WIDE_VECTORS_LOOP_AVX2, COMPARE_WIDE_VECTORS_LOOP_AVX3;
7802 
7803       cmpl(limit, -64);
7804       jccb(Assembler::greater, COMPARE_WIDE_VECTORS_LOOP_AVX2);
7805 
7806       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
7807 
7808       evmovdquq(vec1, Address(ary1, limit, Address::times_1), Assembler::AVX_512bit);
7809       evpcmpeqb(k7, vec1, Address(ary2, limit, Address::times_1), Assembler::AVX_512bit);
7810       kortestql(k7, k7);
7811       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
7812       addptr(limit, 64);  // update since we already compared at this addr
7813       cmpl(limit, -64);
7814       jccb(Assembler::lessEqual, COMPARE_WIDE_VECTORS_LOOP_AVX3);
7815 
7816       // At this point we may still need to compare -limit+result bytes.
7817       // We could execute the next two instruction and just continue via non-wide path:
7818       //  cmpl(limit, 0);
7819       //  jcc(Assembler::equal, COMPARE_TAIL);  // true
7820       // But since we stopped at the points ary{1,2}+limit which are
7821       // not farther than 64 bytes from the ends of arrays ary{1,2}+result
7822       // (|limit| <= 32 and result < 32),
7823       // we may just compare the last 64 bytes.
7824       //
7825       addptr(result, -64);   // it is safe, bc we just came from this area
7826       evmovdquq(vec1, Address(ary1, result, Address::times_1), Assembler::AVX_512bit);
7827       evpcmpeqb(k7, vec1, Address(ary2, result, Address::times_1), Assembler::AVX_512bit);
7828       kortestql(k7, k7);
7829       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
7830 
7831       jmp(TRUE_LABEL);
7832 
7833       bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7834 
7835     }//if (VM_Version::supports_avx512vlbw())
7836 #endif //_LP64
7837 
7838     vmovdqu(vec1, Address(ary1, limit, Address::times_1));
7839     vmovdqu(vec2, Address(ary2, limit, Address::times_1));
7840     vpxor(vec1, vec2);
7841 
7842     vptest(vec1, vec1);
7843     jcc(Assembler::notZero, FALSE_LABEL);
7844     addptr(limit, 32);
7845     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7846 
7847     testl(result, result);
7848     jcc(Assembler::zero, TRUE_LABEL);
7849 
7850     vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
7851     vmovdqu(vec2, Address(ary2, result, Address::times_1, -32));
7852     vpxor(vec1, vec2);
7853 
7854     vptest(vec1, vec1);
7855     jccb(Assembler::notZero, FALSE_LABEL);
7856     jmpb(TRUE_LABEL);
7857 
7858     bind(COMPARE_TAIL); // limit is zero
7859     movl(limit, result);
7860     // Fallthru to tail compare
7861   } else if (UseSSE42Intrinsics) {
7862     // With SSE4.2, use double quad vector compare
7863     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7864 
7865     // Compare 16-byte vectors
7866     andl(result, 0x0000000f);  //   tail count (in bytes)
7867     andl(limit, 0xfffffff0);   // vector count (in bytes)
7868     jcc(Assembler::zero, COMPARE_TAIL);
7869 
7870     lea(ary1, Address(ary1, limit, Address::times_1));
7871     lea(ary2, Address(ary2, limit, Address::times_1));
7872     negptr(limit);
7873 
7874     bind(COMPARE_WIDE_VECTORS);
7875     movdqu(vec1, Address(ary1, limit, Address::times_1));
7876     movdqu(vec2, Address(ary2, limit, Address::times_1));
7877     pxor(vec1, vec2);
7878 
7879     ptest(vec1, vec1);
7880     jcc(Assembler::notZero, FALSE_LABEL);
7881     addptr(limit, 16);
7882     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7883 
7884     testl(result, result);
7885     jcc(Assembler::zero, TRUE_LABEL);
7886 
7887     movdqu(vec1, Address(ary1, result, Address::times_1, -16));
7888     movdqu(vec2, Address(ary2, result, Address::times_1, -16));
7889     pxor(vec1, vec2);
7890 
7891     ptest(vec1, vec1);
7892     jccb(Assembler::notZero, FALSE_LABEL);
7893     jmpb(TRUE_LABEL);
7894 
7895     bind(COMPARE_TAIL); // limit is zero
7896     movl(limit, result);
7897     // Fallthru to tail compare
7898   }
7899 
7900   // Compare 4-byte vectors
7901   andl(limit, 0xfffffffc); // vector count (in bytes)
7902   jccb(Assembler::zero, COMPARE_CHAR);
7903 
7904   lea(ary1, Address(ary1, limit, Address::times_1));
7905   lea(ary2, Address(ary2, limit, Address::times_1));
7906   negptr(limit);
7907 
7908   bind(COMPARE_VECTORS);
7909   movl(chr, Address(ary1, limit, Address::times_1));
7910   cmpl(chr, Address(ary2, limit, Address::times_1));
7911   jccb(Assembler::notEqual, FALSE_LABEL);
7912   addptr(limit, 4);
7913   jcc(Assembler::notZero, COMPARE_VECTORS);
7914 
7915   // Compare trailing char (final 2 bytes), if any
7916   bind(COMPARE_CHAR);
7917   testl(result, 0x2);   // tail  char
7918   jccb(Assembler::zero, COMPARE_BYTE);
7919   load_unsigned_short(chr, Address(ary1, 0));
7920   load_unsigned_short(limit, Address(ary2, 0));
7921   cmpl(chr, limit);
7922   jccb(Assembler::notEqual, FALSE_LABEL);
7923 
7924   if (is_array_equ && is_char) {
7925     bind(COMPARE_BYTE);
7926   } else {
7927     lea(ary1, Address(ary1, 2));
7928     lea(ary2, Address(ary2, 2));
7929 
7930     bind(COMPARE_BYTE);
7931     testl(result, 0x1);   // tail  byte
7932     jccb(Assembler::zero, TRUE_LABEL);
7933     load_unsigned_byte(chr, Address(ary1, 0));
7934     load_unsigned_byte(limit, Address(ary2, 0));
7935     cmpl(chr, limit);
7936     jccb(Assembler::notEqual, FALSE_LABEL);
7937   }
7938   bind(TRUE_LABEL);
7939   movl(result, 1);   // return true
7940   jmpb(DONE);
7941 
7942   bind(FALSE_LABEL);
7943   xorl(result, result); // return false
7944 
7945   // That's it
7946   bind(DONE);
7947   if (UseAVX >= 2) {
7948     // clean upper bits of YMM registers
7949     vpxor(vec1, vec1);
7950     vpxor(vec2, vec2);
7951   }
7952 }
7953 
7954 #endif
7955 
7956 void MacroAssembler::generate_fill(BasicType t, bool aligned,
7957                                    Register to, Register value, Register count,
7958                                    Register rtmp, XMMRegister xtmp) {
7959   ShortBranchVerifier sbv(this);
7960   assert_different_registers(to, value, count, rtmp);
7961   Label L_exit;
7962   Label L_fill_2_bytes, L_fill_4_bytes;
7963 
7964   int shift = -1;
7965   switch (t) {
7966     case T_BYTE:
7967       shift = 2;
7968       break;
7969     case T_SHORT:
7970       shift = 1;
7971       break;
7972     case T_INT:
7973       shift = 0;
7974       break;
7975     default: ShouldNotReachHere();
7976   }
7977 
7978   if (t == T_BYTE) {
7979     andl(value, 0xff);
7980     movl(rtmp, value);
7981     shll(rtmp, 8);
7982     orl(value, rtmp);
7983   }
7984   if (t == T_SHORT) {
7985     andl(value, 0xffff);
7986   }
7987   if (t == T_BYTE || t == T_SHORT) {
7988     movl(rtmp, value);
7989     shll(rtmp, 16);
7990     orl(value, rtmp);
7991   }
7992 
7993   cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
7994   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
7995   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
7996     Label L_skip_align2;
7997     // align source address at 4 bytes address boundary
7998     if (t == T_BYTE) {
7999       Label L_skip_align1;
8000       // One byte misalignment happens only for byte arrays
8001       testptr(to, 1);
8002       jccb(Assembler::zero, L_skip_align1);
8003       movb(Address(to, 0), value);
8004       increment(to);
8005       decrement(count);
8006       BIND(L_skip_align1);
8007     }
8008     // Two bytes misalignment happens only for byte and short (char) arrays
8009     testptr(to, 2);
8010     jccb(Assembler::zero, L_skip_align2);
8011     movw(Address(to, 0), value);
8012     addptr(to, 2);
8013     subl(count, 1<<(shift-1));
8014     BIND(L_skip_align2);
8015   }
8016   if (UseSSE < 2) {
8017     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8018     // Fill 32-byte chunks
8019     subl(count, 8 << shift);
8020     jcc(Assembler::less, L_check_fill_8_bytes);
8021     align(16);
8022 
8023     BIND(L_fill_32_bytes_loop);
8024 
8025     for (int i = 0; i < 32; i += 4) {
8026       movl(Address(to, i), value);
8027     }
8028 
8029     addptr(to, 32);
8030     subl(count, 8 << shift);
8031     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8032     BIND(L_check_fill_8_bytes);
8033     addl(count, 8 << shift);
8034     jccb(Assembler::zero, L_exit);
8035     jmpb(L_fill_8_bytes);
8036 
8037     //
8038     // length is too short, just fill qwords
8039     //
8040     BIND(L_fill_8_bytes_loop);
8041     movl(Address(to, 0), value);
8042     movl(Address(to, 4), value);
8043     addptr(to, 8);
8044     BIND(L_fill_8_bytes);
8045     subl(count, 1 << (shift + 1));
8046     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8047     // fall through to fill 4 bytes
8048   } else {
8049     Label L_fill_32_bytes;
8050     if (!UseUnalignedLoadStores) {
8051       // align to 8 bytes, we know we are 4 byte aligned to start
8052       testptr(to, 4);
8053       jccb(Assembler::zero, L_fill_32_bytes);
8054       movl(Address(to, 0), value);
8055       addptr(to, 4);
8056       subl(count, 1<<shift);
8057     }
8058     BIND(L_fill_32_bytes);
8059     {
8060       assert( UseSSE >= 2, "supported cpu only" );
8061       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8062       if (UseAVX > 2) {
8063         movl(rtmp, 0xffff);
8064         kmovwl(k1, rtmp);
8065       }
8066       movdl(xtmp, value);
8067       if (UseAVX > 2 && UseUnalignedLoadStores) {
8068         // Fill 64-byte chunks
8069         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8070         evpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
8071 
8072         subl(count, 16 << shift);
8073         jcc(Assembler::less, L_check_fill_32_bytes);
8074         align(16);
8075 
8076         BIND(L_fill_64_bytes_loop);
8077         evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
8078         addptr(to, 64);
8079         subl(count, 16 << shift);
8080         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8081 
8082         BIND(L_check_fill_32_bytes);
8083         addl(count, 8 << shift);
8084         jccb(Assembler::less, L_check_fill_8_bytes);
8085         vmovdqu(Address(to, 0), xtmp);
8086         addptr(to, 32);
8087         subl(count, 8 << shift);
8088 
8089         BIND(L_check_fill_8_bytes);
8090       } else if (UseAVX == 2 && UseUnalignedLoadStores) {
8091         // Fill 64-byte chunks
8092         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8093         vpbroadcastd(xtmp, xtmp);
8094 
8095         subl(count, 16 << shift);
8096         jcc(Assembler::less, L_check_fill_32_bytes);
8097         align(16);
8098 
8099         BIND(L_fill_64_bytes_loop);
8100         vmovdqu(Address(to, 0), xtmp);
8101         vmovdqu(Address(to, 32), xtmp);
8102         addptr(to, 64);
8103         subl(count, 16 << shift);
8104         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8105 
8106         BIND(L_check_fill_32_bytes);
8107         addl(count, 8 << shift);
8108         jccb(Assembler::less, L_check_fill_8_bytes);
8109         vmovdqu(Address(to, 0), xtmp);
8110         addptr(to, 32);
8111         subl(count, 8 << shift);
8112 
8113         BIND(L_check_fill_8_bytes);
8114         // clean upper bits of YMM registers
8115         movdl(xtmp, value);
8116         pshufd(xtmp, xtmp, 0);
8117       } else {
8118         // Fill 32-byte chunks
8119         pshufd(xtmp, xtmp, 0);
8120 
8121         subl(count, 8 << shift);
8122         jcc(Assembler::less, L_check_fill_8_bytes);
8123         align(16);
8124 
8125         BIND(L_fill_32_bytes_loop);
8126 
8127         if (UseUnalignedLoadStores) {
8128           movdqu(Address(to, 0), xtmp);
8129           movdqu(Address(to, 16), xtmp);
8130         } else {
8131           movq(Address(to, 0), xtmp);
8132           movq(Address(to, 8), xtmp);
8133           movq(Address(to, 16), xtmp);
8134           movq(Address(to, 24), xtmp);
8135         }
8136 
8137         addptr(to, 32);
8138         subl(count, 8 << shift);
8139         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8140 
8141         BIND(L_check_fill_8_bytes);
8142       }
8143       addl(count, 8 << shift);
8144       jccb(Assembler::zero, L_exit);
8145       jmpb(L_fill_8_bytes);
8146 
8147       //
8148       // length is too short, just fill qwords
8149       //
8150       BIND(L_fill_8_bytes_loop);
8151       movq(Address(to, 0), xtmp);
8152       addptr(to, 8);
8153       BIND(L_fill_8_bytes);
8154       subl(count, 1 << (shift + 1));
8155       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8156     }
8157   }
8158   // fill trailing 4 bytes
8159   BIND(L_fill_4_bytes);
8160   testl(count, 1<<shift);
8161   jccb(Assembler::zero, L_fill_2_bytes);
8162   movl(Address(to, 0), value);
8163   if (t == T_BYTE || t == T_SHORT) {
8164     Label L_fill_byte;
8165     addptr(to, 4);
8166     BIND(L_fill_2_bytes);
8167     // fill trailing 2 bytes
8168     testl(count, 1<<(shift-1));
8169     jccb(Assembler::zero, L_fill_byte);
8170     movw(Address(to, 0), value);
8171     if (t == T_BYTE) {
8172       addptr(to, 2);
8173       BIND(L_fill_byte);
8174       // fill trailing byte
8175       testl(count, 1);
8176       jccb(Assembler::zero, L_exit);
8177       movb(Address(to, 0), value);
8178     } else {
8179       BIND(L_fill_byte);
8180     }
8181   } else {
8182     BIND(L_fill_2_bytes);
8183   }
8184   BIND(L_exit);
8185 }
8186 
8187 // encode char[] to byte[] in ISO_8859_1
8188    //@HotSpotIntrinsicCandidate
8189    //private static int implEncodeISOArray(byte[] sa, int sp,
8190    //byte[] da, int dp, int len) {
8191    //  int i = 0;
8192    //  for (; i < len; i++) {
8193    //    char c = StringUTF16.getChar(sa, sp++);
8194    //    if (c > '\u00FF')
8195    //      break;
8196    //    da[dp++] = (byte)c;
8197    //  }
8198    //  return i;
8199    //}
8200 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
8201   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
8202   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
8203   Register tmp5, Register result) {
8204 
8205   // rsi: src
8206   // rdi: dst
8207   // rdx: len
8208   // rcx: tmp5
8209   // rax: result
8210   ShortBranchVerifier sbv(this);
8211   assert_different_registers(src, dst, len, tmp5, result);
8212   Label L_done, L_copy_1_char, L_copy_1_char_exit;
8213 
8214   // set result
8215   xorl(result, result);
8216   // check for zero length
8217   testl(len, len);
8218   jcc(Assembler::zero, L_done);
8219 
8220   movl(result, len);
8221 
8222   // Setup pointers
8223   lea(src, Address(src, len, Address::times_2)); // char[]
8224   lea(dst, Address(dst, len, Address::times_1)); // byte[]
8225   negptr(len);
8226 
8227   if (UseSSE42Intrinsics || UseAVX >= 2) {
8228     Label L_copy_8_chars, L_copy_8_chars_exit;
8229     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
8230 
8231     if (UseAVX >= 2) {
8232       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
8233       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8234       movdl(tmp1Reg, tmp5);
8235       vpbroadcastd(tmp1Reg, tmp1Reg);
8236       jmp(L_chars_32_check);
8237 
8238       bind(L_copy_32_chars);
8239       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
8240       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
8241       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8242       vptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8243       jccb(Assembler::notZero, L_copy_32_chars_exit);
8244       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8245       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
8246       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
8247 
8248       bind(L_chars_32_check);
8249       addptr(len, 32);
8250       jcc(Assembler::lessEqual, L_copy_32_chars);
8251 
8252       bind(L_copy_32_chars_exit);
8253       subptr(len, 16);
8254       jccb(Assembler::greater, L_copy_16_chars_exit);
8255 
8256     } else if (UseSSE42Intrinsics) {
8257       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8258       movdl(tmp1Reg, tmp5);
8259       pshufd(tmp1Reg, tmp1Reg, 0);
8260       jmpb(L_chars_16_check);
8261     }
8262 
8263     bind(L_copy_16_chars);
8264     if (UseAVX >= 2) {
8265       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
8266       vptest(tmp2Reg, tmp1Reg);
8267       jcc(Assembler::notZero, L_copy_16_chars_exit);
8268       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
8269       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
8270     } else {
8271       if (UseAVX > 0) {
8272         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8273         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8274         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
8275       } else {
8276         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8277         por(tmp2Reg, tmp3Reg);
8278         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8279         por(tmp2Reg, tmp4Reg);
8280       }
8281       ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8282       jccb(Assembler::notZero, L_copy_16_chars_exit);
8283       packuswb(tmp3Reg, tmp4Reg);
8284     }
8285     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
8286 
8287     bind(L_chars_16_check);
8288     addptr(len, 16);
8289     jcc(Assembler::lessEqual, L_copy_16_chars);
8290 
8291     bind(L_copy_16_chars_exit);
8292     if (UseAVX >= 2) {
8293       // clean upper bits of YMM registers
8294       vpxor(tmp2Reg, tmp2Reg);
8295       vpxor(tmp3Reg, tmp3Reg);
8296       vpxor(tmp4Reg, tmp4Reg);
8297       movdl(tmp1Reg, tmp5);
8298       pshufd(tmp1Reg, tmp1Reg, 0);
8299     }
8300     subptr(len, 8);
8301     jccb(Assembler::greater, L_copy_8_chars_exit);
8302 
8303     bind(L_copy_8_chars);
8304     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
8305     ptest(tmp3Reg, tmp1Reg);
8306     jccb(Assembler::notZero, L_copy_8_chars_exit);
8307     packuswb(tmp3Reg, tmp1Reg);
8308     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
8309     addptr(len, 8);
8310     jccb(Assembler::lessEqual, L_copy_8_chars);
8311 
8312     bind(L_copy_8_chars_exit);
8313     subptr(len, 8);
8314     jccb(Assembler::zero, L_done);
8315   }
8316 
8317   bind(L_copy_1_char);
8318   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
8319   testl(tmp5, 0xff00);      // check if Unicode char
8320   jccb(Assembler::notZero, L_copy_1_char_exit);
8321   movb(Address(dst, len, Address::times_1, 0), tmp5);
8322   addptr(len, 1);
8323   jccb(Assembler::less, L_copy_1_char);
8324 
8325   bind(L_copy_1_char_exit);
8326   addptr(result, len); // len is negative count of not processed elements
8327 
8328   bind(L_done);
8329 }
8330 
8331 #ifdef _LP64
8332 /**
8333  * Helper for multiply_to_len().
8334  */
8335 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
8336   addq(dest_lo, src1);
8337   adcq(dest_hi, 0);
8338   addq(dest_lo, src2);
8339   adcq(dest_hi, 0);
8340 }
8341 
8342 /**
8343  * Multiply 64 bit by 64 bit first loop.
8344  */
8345 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
8346                                            Register y, Register y_idx, Register z,
8347                                            Register carry, Register product,
8348                                            Register idx, Register kdx) {
8349   //
8350   //  jlong carry, x[], y[], z[];
8351   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
8352   //    huge_128 product = y[idx] * x[xstart] + carry;
8353   //    z[kdx] = (jlong)product;
8354   //    carry  = (jlong)(product >>> 64);
8355   //  }
8356   //  z[xstart] = carry;
8357   //
8358 
8359   Label L_first_loop, L_first_loop_exit;
8360   Label L_one_x, L_one_y, L_multiply;
8361 
8362   decrementl(xstart);
8363   jcc(Assembler::negative, L_one_x);
8364 
8365   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
8366   rorq(x_xstart, 32); // convert big-endian to little-endian
8367 
8368   bind(L_first_loop);
8369   decrementl(idx);
8370   jcc(Assembler::negative, L_first_loop_exit);
8371   decrementl(idx);
8372   jcc(Assembler::negative, L_one_y);
8373   movq(y_idx, Address(y, idx, Address::times_4,  0));
8374   rorq(y_idx, 32); // convert big-endian to little-endian
8375   bind(L_multiply);
8376   movq(product, x_xstart);
8377   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
8378   addq(product, carry);
8379   adcq(rdx, 0);
8380   subl(kdx, 2);
8381   movl(Address(z, kdx, Address::times_4,  4), product);
8382   shrq(product, 32);
8383   movl(Address(z, kdx, Address::times_4,  0), product);
8384   movq(carry, rdx);
8385   jmp(L_first_loop);
8386 
8387   bind(L_one_y);
8388   movl(y_idx, Address(y,  0));
8389   jmp(L_multiply);
8390 
8391   bind(L_one_x);
8392   movl(x_xstart, Address(x,  0));
8393   jmp(L_first_loop);
8394 
8395   bind(L_first_loop_exit);
8396 }
8397 
8398 /**
8399  * Multiply 64 bit by 64 bit and add 128 bit.
8400  */
8401 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
8402                                             Register yz_idx, Register idx,
8403                                             Register carry, Register product, int offset) {
8404   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
8405   //     z[kdx] = (jlong)product;
8406 
8407   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
8408   rorq(yz_idx, 32); // convert big-endian to little-endian
8409   movq(product, x_xstart);
8410   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
8411   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
8412   rorq(yz_idx, 32); // convert big-endian to little-endian
8413 
8414   add2_with_carry(rdx, product, carry, yz_idx);
8415 
8416   movl(Address(z, idx, Address::times_4,  offset+4), product);
8417   shrq(product, 32);
8418   movl(Address(z, idx, Address::times_4,  offset), product);
8419 
8420 }
8421 
8422 /**
8423  * Multiply 128 bit by 128 bit. Unrolled inner loop.
8424  */
8425 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
8426                                              Register yz_idx, Register idx, Register jdx,
8427                                              Register carry, Register product,
8428                                              Register carry2) {
8429   //   jlong carry, x[], y[], z[];
8430   //   int kdx = ystart+1;
8431   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
8432   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
8433   //     z[kdx+idx+1] = (jlong)product;
8434   //     jlong carry2  = (jlong)(product >>> 64);
8435   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
8436   //     z[kdx+idx] = (jlong)product;
8437   //     carry  = (jlong)(product >>> 64);
8438   //   }
8439   //   idx += 2;
8440   //   if (idx > 0) {
8441   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
8442   //     z[kdx+idx] = (jlong)product;
8443   //     carry  = (jlong)(product >>> 64);
8444   //   }
8445   //
8446 
8447   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
8448 
8449   movl(jdx, idx);
8450   andl(jdx, 0xFFFFFFFC);
8451   shrl(jdx, 2);
8452 
8453   bind(L_third_loop);
8454   subl(jdx, 1);
8455   jcc(Assembler::negative, L_third_loop_exit);
8456   subl(idx, 4);
8457 
8458   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
8459   movq(carry2, rdx);
8460 
8461   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
8462   movq(carry, rdx);
8463   jmp(L_third_loop);
8464 
8465   bind (L_third_loop_exit);
8466 
8467   andl (idx, 0x3);
8468   jcc(Assembler::zero, L_post_third_loop_done);
8469 
8470   Label L_check_1;
8471   subl(idx, 2);
8472   jcc(Assembler::negative, L_check_1);
8473 
8474   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
8475   movq(carry, rdx);
8476 
8477   bind (L_check_1);
8478   addl (idx, 0x2);
8479   andl (idx, 0x1);
8480   subl(idx, 1);
8481   jcc(Assembler::negative, L_post_third_loop_done);
8482 
8483   movl(yz_idx, Address(y, idx, Address::times_4,  0));
8484   movq(product, x_xstart);
8485   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
8486   movl(yz_idx, Address(z, idx, Address::times_4,  0));
8487 
8488   add2_with_carry(rdx, product, yz_idx, carry);
8489 
8490   movl(Address(z, idx, Address::times_4,  0), product);
8491   shrq(product, 32);
8492 
8493   shlq(rdx, 32);
8494   orq(product, rdx);
8495   movq(carry, product);
8496 
8497   bind(L_post_third_loop_done);
8498 }
8499 
8500 /**
8501  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
8502  *
8503  */
8504 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
8505                                                   Register carry, Register carry2,
8506                                                   Register idx, Register jdx,
8507                                                   Register yz_idx1, Register yz_idx2,
8508                                                   Register tmp, Register tmp3, Register tmp4) {
8509   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
8510 
8511   //   jlong carry, x[], y[], z[];
8512   //   int kdx = ystart+1;
8513   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
8514   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
8515   //     jlong carry2  = (jlong)(tmp3 >>> 64);
8516   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
8517   //     carry  = (jlong)(tmp4 >>> 64);
8518   //     z[kdx+idx+1] = (jlong)tmp3;
8519   //     z[kdx+idx] = (jlong)tmp4;
8520   //   }
8521   //   idx += 2;
8522   //   if (idx > 0) {
8523   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
8524   //     z[kdx+idx] = (jlong)yz_idx1;
8525   //     carry  = (jlong)(yz_idx1 >>> 64);
8526   //   }
8527   //
8528 
8529   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
8530 
8531   movl(jdx, idx);
8532   andl(jdx, 0xFFFFFFFC);
8533   shrl(jdx, 2);
8534 
8535   bind(L_third_loop);
8536   subl(jdx, 1);
8537   jcc(Assembler::negative, L_third_loop_exit);
8538   subl(idx, 4);
8539 
8540   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
8541   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
8542   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
8543   rorxq(yz_idx2, yz_idx2, 32);
8544 
8545   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
8546   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
8547 
8548   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
8549   rorxq(yz_idx1, yz_idx1, 32);
8550   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
8551   rorxq(yz_idx2, yz_idx2, 32);
8552 
8553   if (VM_Version::supports_adx()) {
8554     adcxq(tmp3, carry);
8555     adoxq(tmp3, yz_idx1);
8556 
8557     adcxq(tmp4, tmp);
8558     adoxq(tmp4, yz_idx2);
8559 
8560     movl(carry, 0); // does not affect flags
8561     adcxq(carry2, carry);
8562     adoxq(carry2, carry);
8563   } else {
8564     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
8565     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
8566   }
8567   movq(carry, carry2);
8568 
8569   movl(Address(z, idx, Address::times_4, 12), tmp3);
8570   shrq(tmp3, 32);
8571   movl(Address(z, idx, Address::times_4,  8), tmp3);
8572 
8573   movl(Address(z, idx, Address::times_4,  4), tmp4);
8574   shrq(tmp4, 32);
8575   movl(Address(z, idx, Address::times_4,  0), tmp4);
8576 
8577   jmp(L_third_loop);
8578 
8579   bind (L_third_loop_exit);
8580 
8581   andl (idx, 0x3);
8582   jcc(Assembler::zero, L_post_third_loop_done);
8583 
8584   Label L_check_1;
8585   subl(idx, 2);
8586   jcc(Assembler::negative, L_check_1);
8587 
8588   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
8589   rorxq(yz_idx1, yz_idx1, 32);
8590   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
8591   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
8592   rorxq(yz_idx2, yz_idx2, 32);
8593 
8594   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
8595 
8596   movl(Address(z, idx, Address::times_4,  4), tmp3);
8597   shrq(tmp3, 32);
8598   movl(Address(z, idx, Address::times_4,  0), tmp3);
8599   movq(carry, tmp4);
8600 
8601   bind (L_check_1);
8602   addl (idx, 0x2);
8603   andl (idx, 0x1);
8604   subl(idx, 1);
8605   jcc(Assembler::negative, L_post_third_loop_done);
8606   movl(tmp4, Address(y, idx, Address::times_4,  0));
8607   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
8608   movl(tmp4, Address(z, idx, Address::times_4,  0));
8609 
8610   add2_with_carry(carry2, tmp3, tmp4, carry);
8611 
8612   movl(Address(z, idx, Address::times_4,  0), tmp3);
8613   shrq(tmp3, 32);
8614 
8615   shlq(carry2, 32);
8616   orq(tmp3, carry2);
8617   movq(carry, tmp3);
8618 
8619   bind(L_post_third_loop_done);
8620 }
8621 
8622 /**
8623  * Code for BigInteger::multiplyToLen() instrinsic.
8624  *
8625  * rdi: x
8626  * rax: xlen
8627  * rsi: y
8628  * rcx: ylen
8629  * r8:  z
8630  * r11: zlen
8631  * r12: tmp1
8632  * r13: tmp2
8633  * r14: tmp3
8634  * r15: tmp4
8635  * rbx: tmp5
8636  *
8637  */
8638 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
8639                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
8640   ShortBranchVerifier sbv(this);
8641   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
8642 
8643   push(tmp1);
8644   push(tmp2);
8645   push(tmp3);
8646   push(tmp4);
8647   push(tmp5);
8648 
8649   push(xlen);
8650   push(zlen);
8651 
8652   const Register idx = tmp1;
8653   const Register kdx = tmp2;
8654   const Register xstart = tmp3;
8655 
8656   const Register y_idx = tmp4;
8657   const Register carry = tmp5;
8658   const Register product  = xlen;
8659   const Register x_xstart = zlen;  // reuse register
8660 
8661   // First Loop.
8662   //
8663   //  final static long LONG_MASK = 0xffffffffL;
8664   //  int xstart = xlen - 1;
8665   //  int ystart = ylen - 1;
8666   //  long carry = 0;
8667   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
8668   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
8669   //    z[kdx] = (int)product;
8670   //    carry = product >>> 32;
8671   //  }
8672   //  z[xstart] = (int)carry;
8673   //
8674 
8675   movl(idx, ylen);      // idx = ylen;
8676   movl(kdx, zlen);      // kdx = xlen+ylen;
8677   xorq(carry, carry);   // carry = 0;
8678 
8679   Label L_done;
8680 
8681   movl(xstart, xlen);
8682   decrementl(xstart);
8683   jcc(Assembler::negative, L_done);
8684 
8685   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
8686 
8687   Label L_second_loop;
8688   testl(kdx, kdx);
8689   jcc(Assembler::zero, L_second_loop);
8690 
8691   Label L_carry;
8692   subl(kdx, 1);
8693   jcc(Assembler::zero, L_carry);
8694 
8695   movl(Address(z, kdx, Address::times_4,  0), carry);
8696   shrq(carry, 32);
8697   subl(kdx, 1);
8698 
8699   bind(L_carry);
8700   movl(Address(z, kdx, Address::times_4,  0), carry);
8701 
8702   // Second and third (nested) loops.
8703   //
8704   // for (int i = xstart-1; i >= 0; i--) { // Second loop
8705   //   carry = 0;
8706   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
8707   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
8708   //                    (z[k] & LONG_MASK) + carry;
8709   //     z[k] = (int)product;
8710   //     carry = product >>> 32;
8711   //   }
8712   //   z[i] = (int)carry;
8713   // }
8714   //
8715   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
8716 
8717   const Register jdx = tmp1;
8718 
8719   bind(L_second_loop);
8720   xorl(carry, carry);    // carry = 0;
8721   movl(jdx, ylen);       // j = ystart+1
8722 
8723   subl(xstart, 1);       // i = xstart-1;
8724   jcc(Assembler::negative, L_done);
8725 
8726   push (z);
8727 
8728   Label L_last_x;
8729   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
8730   subl(xstart, 1);       // i = xstart-1;
8731   jcc(Assembler::negative, L_last_x);
8732 
8733   if (UseBMI2Instructions) {
8734     movq(rdx,  Address(x, xstart, Address::times_4,  0));
8735     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
8736   } else {
8737     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
8738     rorq(x_xstart, 32);  // convert big-endian to little-endian
8739   }
8740 
8741   Label L_third_loop_prologue;
8742   bind(L_third_loop_prologue);
8743 
8744   push (x);
8745   push (xstart);
8746   push (ylen);
8747 
8748 
8749   if (UseBMI2Instructions) {
8750     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
8751   } else { // !UseBMI2Instructions
8752     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
8753   }
8754 
8755   pop(ylen);
8756   pop(xlen);
8757   pop(x);
8758   pop(z);
8759 
8760   movl(tmp3, xlen);
8761   addl(tmp3, 1);
8762   movl(Address(z, tmp3, Address::times_4,  0), carry);
8763   subl(tmp3, 1);
8764   jccb(Assembler::negative, L_done);
8765 
8766   shrq(carry, 32);
8767   movl(Address(z, tmp3, Address::times_4,  0), carry);
8768   jmp(L_second_loop);
8769 
8770   // Next infrequent code is moved outside loops.
8771   bind(L_last_x);
8772   if (UseBMI2Instructions) {
8773     movl(rdx, Address(x,  0));
8774   } else {
8775     movl(x_xstart, Address(x,  0));
8776   }
8777   jmp(L_third_loop_prologue);
8778 
8779   bind(L_done);
8780 
8781   pop(zlen);
8782   pop(xlen);
8783 
8784   pop(tmp5);
8785   pop(tmp4);
8786   pop(tmp3);
8787   pop(tmp2);
8788   pop(tmp1);
8789 }
8790 
8791 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
8792   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
8793   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
8794   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
8795   Label VECTOR8_TAIL, VECTOR4_TAIL;
8796   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
8797   Label SAME_TILL_END, DONE;
8798   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
8799 
8800   //scale is in rcx in both Win64 and Unix
8801   ShortBranchVerifier sbv(this);
8802 
8803   shlq(length);
8804   xorq(result, result);
8805 
8806   if ((UseAVX > 2) &&
8807       VM_Version::supports_avx512vlbw()) {
8808     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
8809 
8810     set_vector_masking();  // opening of the stub context for programming mask registers
8811     cmpq(length, 64);
8812     jcc(Assembler::less, VECTOR32_TAIL);
8813     movq(tmp1, length);
8814     andq(tmp1, 0x3F);      // tail count
8815     andq(length, ~(0x3F)); //vector count
8816 
8817     bind(VECTOR64_LOOP);
8818     // AVX512 code to compare 64 byte vectors.
8819     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
8820     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
8821     kortestql(k7, k7);
8822     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
8823     addq(result, 64);
8824     subq(length, 64);
8825     jccb(Assembler::notZero, VECTOR64_LOOP);
8826 
8827     //bind(VECTOR64_TAIL);
8828     testq(tmp1, tmp1);
8829     jcc(Assembler::zero, SAME_TILL_END);
8830 
8831     //bind(VECTOR64_TAIL);
8832     // AVX512 code to compare upto 63 byte vectors.
8833     // Save k1
8834     kmovql(k3, k1);
8835     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
8836     shlxq(tmp2, tmp2, tmp1);
8837     notq(tmp2);
8838     kmovql(k1, tmp2);
8839 
8840     evmovdqub(rymm0, k1, Address(obja, result), Assembler::AVX_512bit);
8841     evpcmpeqb(k7, k1, rymm0, Address(objb, result), Assembler::AVX_512bit);
8842 
8843     ktestql(k7, k1);
8844     // Restore k1
8845     kmovql(k1, k3);
8846     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
8847 
8848     bind(VECTOR64_NOT_EQUAL);
8849     kmovql(tmp1, k7);
8850     notq(tmp1);
8851     tzcntq(tmp1, tmp1);
8852     addq(result, tmp1);
8853     shrq(result);
8854     jmp(DONE);
8855     bind(VECTOR32_TAIL);
8856     clear_vector_masking();   // closing of the stub context for programming mask registers
8857   }
8858 
8859   cmpq(length, 8);
8860   jcc(Assembler::equal, VECTOR8_LOOP);
8861   jcc(Assembler::less, VECTOR4_TAIL);
8862 
8863   if (UseAVX >= 2) {
8864     Label VECTOR16_TAIL, VECTOR32_LOOP;
8865 
8866     cmpq(length, 16);
8867     jcc(Assembler::equal, VECTOR16_LOOP);
8868     jcc(Assembler::less, VECTOR8_LOOP);
8869 
8870     cmpq(length, 32);
8871     jccb(Assembler::less, VECTOR16_TAIL);
8872 
8873     subq(length, 32);
8874     bind(VECTOR32_LOOP);
8875     vmovdqu(rymm0, Address(obja, result));
8876     vmovdqu(rymm1, Address(objb, result));
8877     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
8878     vptest(rymm2, rymm2);
8879     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
8880     addq(result, 32);
8881     subq(length, 32);
8882     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
8883     addq(length, 32);
8884     jcc(Assembler::equal, SAME_TILL_END);
8885     //falling through if less than 32 bytes left //close the branch here.
8886 
8887     bind(VECTOR16_TAIL);
8888     cmpq(length, 16);
8889     jccb(Assembler::less, VECTOR8_TAIL);
8890     bind(VECTOR16_LOOP);
8891     movdqu(rymm0, Address(obja, result));
8892     movdqu(rymm1, Address(objb, result));
8893     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
8894     ptest(rymm2, rymm2);
8895     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
8896     addq(result, 16);
8897     subq(length, 16);
8898     jcc(Assembler::equal, SAME_TILL_END);
8899     //falling through if less than 16 bytes left
8900   } else {//regular intrinsics
8901 
8902     cmpq(length, 16);
8903     jccb(Assembler::less, VECTOR8_TAIL);
8904 
8905     subq(length, 16);
8906     bind(VECTOR16_LOOP);
8907     movdqu(rymm0, Address(obja, result));
8908     movdqu(rymm1, Address(objb, result));
8909     pxor(rymm0, rymm1);
8910     ptest(rymm0, rymm0);
8911     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
8912     addq(result, 16);
8913     subq(length, 16);
8914     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
8915     addq(length, 16);
8916     jcc(Assembler::equal, SAME_TILL_END);
8917     //falling through if less than 16 bytes left
8918   }
8919 
8920   bind(VECTOR8_TAIL);
8921   cmpq(length, 8);
8922   jccb(Assembler::less, VECTOR4_TAIL);
8923   bind(VECTOR8_LOOP);
8924   movq(tmp1, Address(obja, result));
8925   movq(tmp2, Address(objb, result));
8926   xorq(tmp1, tmp2);
8927   testq(tmp1, tmp1);
8928   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
8929   addq(result, 8);
8930   subq(length, 8);
8931   jcc(Assembler::equal, SAME_TILL_END);
8932   //falling through if less than 8 bytes left
8933 
8934   bind(VECTOR4_TAIL);
8935   cmpq(length, 4);
8936   jccb(Assembler::less, BYTES_TAIL);
8937   bind(VECTOR4_LOOP);
8938   movl(tmp1, Address(obja, result));
8939   xorl(tmp1, Address(objb, result));
8940   testl(tmp1, tmp1);
8941   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
8942   addq(result, 4);
8943   subq(length, 4);
8944   jcc(Assembler::equal, SAME_TILL_END);
8945   //falling through if less than 4 bytes left
8946 
8947   bind(BYTES_TAIL);
8948   bind(BYTES_LOOP);
8949   load_unsigned_byte(tmp1, Address(obja, result));
8950   load_unsigned_byte(tmp2, Address(objb, result));
8951   xorl(tmp1, tmp2);
8952   testl(tmp1, tmp1);
8953   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
8954   decq(length);
8955   jcc(Assembler::zero, SAME_TILL_END);
8956   incq(result);
8957   load_unsigned_byte(tmp1, Address(obja, result));
8958   load_unsigned_byte(tmp2, Address(objb, result));
8959   xorl(tmp1, tmp2);
8960   testl(tmp1, tmp1);
8961   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
8962   decq(length);
8963   jcc(Assembler::zero, SAME_TILL_END);
8964   incq(result);
8965   load_unsigned_byte(tmp1, Address(obja, result));
8966   load_unsigned_byte(tmp2, Address(objb, result));
8967   xorl(tmp1, tmp2);
8968   testl(tmp1, tmp1);
8969   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
8970   jmp(SAME_TILL_END);
8971 
8972   if (UseAVX >= 2) {
8973     bind(VECTOR32_NOT_EQUAL);
8974     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
8975     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
8976     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
8977     vpmovmskb(tmp1, rymm0);
8978     bsfq(tmp1, tmp1);
8979     addq(result, tmp1);
8980     shrq(result);
8981     jmp(DONE);
8982   }
8983 
8984   bind(VECTOR16_NOT_EQUAL);
8985   if (UseAVX >= 2) {
8986     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
8987     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
8988     pxor(rymm0, rymm2);
8989   } else {
8990     pcmpeqb(rymm2, rymm2);
8991     pxor(rymm0, rymm1);
8992     pcmpeqb(rymm0, rymm1);
8993     pxor(rymm0, rymm2);
8994   }
8995   pmovmskb(tmp1, rymm0);
8996   bsfq(tmp1, tmp1);
8997   addq(result, tmp1);
8998   shrq(result);
8999   jmpb(DONE);
9000 
9001   bind(VECTOR8_NOT_EQUAL);
9002   bind(VECTOR4_NOT_EQUAL);
9003   bsfq(tmp1, tmp1);
9004   shrq(tmp1, 3);
9005   addq(result, tmp1);
9006   bind(BYTES_NOT_EQUAL);
9007   shrq(result);
9008   jmpb(DONE);
9009 
9010   bind(SAME_TILL_END);
9011   mov64(result, -1);
9012 
9013   bind(DONE);
9014 }
9015 
9016 //Helper functions for square_to_len()
9017 
9018 /**
9019  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
9020  * Preserves x and z and modifies rest of the registers.
9021  */
9022 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9023   // Perform square and right shift by 1
9024   // Handle odd xlen case first, then for even xlen do the following
9025   // jlong carry = 0;
9026   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
9027   //     huge_128 product = x[j:j+1] * x[j:j+1];
9028   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
9029   //     z[i+2:i+3] = (jlong)(product >>> 1);
9030   //     carry = (jlong)product;
9031   // }
9032 
9033   xorq(tmp5, tmp5);     // carry
9034   xorq(rdxReg, rdxReg);
9035   xorl(tmp1, tmp1);     // index for x
9036   xorl(tmp4, tmp4);     // index for z
9037 
9038   Label L_first_loop, L_first_loop_exit;
9039 
9040   testl(xlen, 1);
9041   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
9042 
9043   // Square and right shift by 1 the odd element using 32 bit multiply
9044   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
9045   imulq(raxReg, raxReg);
9046   shrq(raxReg, 1);
9047   adcq(tmp5, 0);
9048   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
9049   incrementl(tmp1);
9050   addl(tmp4, 2);
9051 
9052   // Square and  right shift by 1 the rest using 64 bit multiply
9053   bind(L_first_loop);
9054   cmpptr(tmp1, xlen);
9055   jccb(Assembler::equal, L_first_loop_exit);
9056 
9057   // Square
9058   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
9059   rorq(raxReg, 32);    // convert big-endian to little-endian
9060   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
9061 
9062   // Right shift by 1 and save carry
9063   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
9064   rcrq(rdxReg, 1);
9065   rcrq(raxReg, 1);
9066   adcq(tmp5, 0);
9067 
9068   // Store result in z
9069   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
9070   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
9071 
9072   // Update indices for x and z
9073   addl(tmp1, 2);
9074   addl(tmp4, 4);
9075   jmp(L_first_loop);
9076 
9077   bind(L_first_loop_exit);
9078 }
9079 
9080 
9081 /**
9082  * Perform the following multiply add operation using BMI2 instructions
9083  * carry:sum = sum + op1*op2 + carry
9084  * op2 should be in rdx
9085  * op2 is preserved, all other registers are modified
9086  */
9087 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
9088   // assert op2 is rdx
9089   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
9090   addq(sum, carry);
9091   adcq(tmp2, 0);
9092   addq(sum, op1);
9093   adcq(tmp2, 0);
9094   movq(carry, tmp2);
9095 }
9096 
9097 /**
9098  * Perform the following multiply add operation:
9099  * carry:sum = sum + op1*op2 + carry
9100  * Preserves op1, op2 and modifies rest of registers
9101  */
9102 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
9103   // rdx:rax = op1 * op2
9104   movq(raxReg, op2);
9105   mulq(op1);
9106 
9107   //  rdx:rax = sum + carry + rdx:rax
9108   addq(sum, carry);
9109   adcq(rdxReg, 0);
9110   addq(sum, raxReg);
9111   adcq(rdxReg, 0);
9112 
9113   // carry:sum = rdx:sum
9114   movq(carry, rdxReg);
9115 }
9116 
9117 /**
9118  * Add 64 bit long carry into z[] with carry propogation.
9119  * Preserves z and carry register values and modifies rest of registers.
9120  *
9121  */
9122 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
9123   Label L_fourth_loop, L_fourth_loop_exit;
9124 
9125   movl(tmp1, 1);
9126   subl(zlen, 2);
9127   addq(Address(z, zlen, Address::times_4, 0), carry);
9128 
9129   bind(L_fourth_loop);
9130   jccb(Assembler::carryClear, L_fourth_loop_exit);
9131   subl(zlen, 2);
9132   jccb(Assembler::negative, L_fourth_loop_exit);
9133   addq(Address(z, zlen, Address::times_4, 0), tmp1);
9134   jmp(L_fourth_loop);
9135   bind(L_fourth_loop_exit);
9136 }
9137 
9138 /**
9139  * Shift z[] left by 1 bit.
9140  * Preserves x, len, z and zlen registers and modifies rest of the registers.
9141  *
9142  */
9143 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
9144 
9145   Label L_fifth_loop, L_fifth_loop_exit;
9146 
9147   // Fifth loop
9148   // Perform primitiveLeftShift(z, zlen, 1)
9149 
9150   const Register prev_carry = tmp1;
9151   const Register new_carry = tmp4;
9152   const Register value = tmp2;
9153   const Register zidx = tmp3;
9154 
9155   // int zidx, carry;
9156   // long value;
9157   // carry = 0;
9158   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
9159   //    (carry:value)  = (z[i] << 1) | carry ;
9160   //    z[i] = value;
9161   // }
9162 
9163   movl(zidx, zlen);
9164   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
9165 
9166   bind(L_fifth_loop);
9167   decl(zidx);  // Use decl to preserve carry flag
9168   decl(zidx);
9169   jccb(Assembler::negative, L_fifth_loop_exit);
9170 
9171   if (UseBMI2Instructions) {
9172      movq(value, Address(z, zidx, Address::times_4, 0));
9173      rclq(value, 1);
9174      rorxq(value, value, 32);
9175      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9176   }
9177   else {
9178     // clear new_carry
9179     xorl(new_carry, new_carry);
9180 
9181     // Shift z[i] by 1, or in previous carry and save new carry
9182     movq(value, Address(z, zidx, Address::times_4, 0));
9183     shlq(value, 1);
9184     adcl(new_carry, 0);
9185 
9186     orq(value, prev_carry);
9187     rorq(value, 0x20);
9188     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9189 
9190     // Set previous carry = new carry
9191     movl(prev_carry, new_carry);
9192   }
9193   jmp(L_fifth_loop);
9194 
9195   bind(L_fifth_loop_exit);
9196 }
9197 
9198 
9199 /**
9200  * Code for BigInteger::squareToLen() intrinsic
9201  *
9202  * rdi: x
9203  * rsi: len
9204  * r8:  z
9205  * rcx: zlen
9206  * r12: tmp1
9207  * r13: tmp2
9208  * r14: tmp3
9209  * r15: tmp4
9210  * rbx: tmp5
9211  *
9212  */
9213 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9214 
9215   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
9216   push(tmp1);
9217   push(tmp2);
9218   push(tmp3);
9219   push(tmp4);
9220   push(tmp5);
9221 
9222   // First loop
9223   // Store the squares, right shifted one bit (i.e., divided by 2).
9224   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
9225 
9226   // Add in off-diagonal sums.
9227   //
9228   // Second, third (nested) and fourth loops.
9229   // zlen +=2;
9230   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
9231   //    carry = 0;
9232   //    long op2 = x[xidx:xidx+1];
9233   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
9234   //       k -= 2;
9235   //       long op1 = x[j:j+1];
9236   //       long sum = z[k:k+1];
9237   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
9238   //       z[k:k+1] = sum;
9239   //    }
9240   //    add_one_64(z, k, carry, tmp_regs);
9241   // }
9242 
9243   const Register carry = tmp5;
9244   const Register sum = tmp3;
9245   const Register op1 = tmp4;
9246   Register op2 = tmp2;
9247 
9248   push(zlen);
9249   push(len);
9250   addl(zlen,2);
9251   bind(L_second_loop);
9252   xorq(carry, carry);
9253   subl(zlen, 4);
9254   subl(len, 2);
9255   push(zlen);
9256   push(len);
9257   cmpl(len, 0);
9258   jccb(Assembler::lessEqual, L_second_loop_exit);
9259 
9260   // Multiply an array by one 64 bit long.
9261   if (UseBMI2Instructions) {
9262     op2 = rdxReg;
9263     movq(op2, Address(x, len, Address::times_4,  0));
9264     rorxq(op2, op2, 32);
9265   }
9266   else {
9267     movq(op2, Address(x, len, Address::times_4,  0));
9268     rorq(op2, 32);
9269   }
9270 
9271   bind(L_third_loop);
9272   decrementl(len);
9273   jccb(Assembler::negative, L_third_loop_exit);
9274   decrementl(len);
9275   jccb(Assembler::negative, L_last_x);
9276 
9277   movq(op1, Address(x, len, Address::times_4,  0));
9278   rorq(op1, 32);
9279 
9280   bind(L_multiply);
9281   subl(zlen, 2);
9282   movq(sum, Address(z, zlen, Address::times_4,  0));
9283 
9284   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
9285   if (UseBMI2Instructions) {
9286     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
9287   }
9288   else {
9289     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9290   }
9291 
9292   movq(Address(z, zlen, Address::times_4, 0), sum);
9293 
9294   jmp(L_third_loop);
9295   bind(L_third_loop_exit);
9296 
9297   // Fourth loop
9298   // Add 64 bit long carry into z with carry propogation.
9299   // Uses offsetted zlen.
9300   add_one_64(z, zlen, carry, tmp1);
9301 
9302   pop(len);
9303   pop(zlen);
9304   jmp(L_second_loop);
9305 
9306   // Next infrequent code is moved outside loops.
9307   bind(L_last_x);
9308   movl(op1, Address(x, 0));
9309   jmp(L_multiply);
9310 
9311   bind(L_second_loop_exit);
9312   pop(len);
9313   pop(zlen);
9314   pop(len);
9315   pop(zlen);
9316 
9317   // Fifth loop
9318   // Shift z left 1 bit.
9319   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
9320 
9321   // z[zlen-1] |= x[len-1] & 1;
9322   movl(tmp3, Address(x, len, Address::times_4, -4));
9323   andl(tmp3, 1);
9324   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
9325 
9326   pop(tmp5);
9327   pop(tmp4);
9328   pop(tmp3);
9329   pop(tmp2);
9330   pop(tmp1);
9331 }
9332 
9333 /**
9334  * Helper function for mul_add()
9335  * Multiply the in[] by int k and add to out[] starting at offset offs using
9336  * 128 bit by 32 bit multiply and return the carry in tmp5.
9337  * Only quad int aligned length of in[] is operated on in this function.
9338  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
9339  * This function preserves out, in and k registers.
9340  * len and offset point to the appropriate index in "in" & "out" correspondingly
9341  * tmp5 has the carry.
9342  * other registers are temporary and are modified.
9343  *
9344  */
9345 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
9346   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
9347   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9348 
9349   Label L_first_loop, L_first_loop_exit;
9350 
9351   movl(tmp1, len);
9352   shrl(tmp1, 2);
9353 
9354   bind(L_first_loop);
9355   subl(tmp1, 1);
9356   jccb(Assembler::negative, L_first_loop_exit);
9357 
9358   subl(len, 4);
9359   subl(offset, 4);
9360 
9361   Register op2 = tmp2;
9362   const Register sum = tmp3;
9363   const Register op1 = tmp4;
9364   const Register carry = tmp5;
9365 
9366   if (UseBMI2Instructions) {
9367     op2 = rdxReg;
9368   }
9369 
9370   movq(op1, Address(in, len, Address::times_4,  8));
9371   rorq(op1, 32);
9372   movq(sum, Address(out, offset, Address::times_4,  8));
9373   rorq(sum, 32);
9374   if (UseBMI2Instructions) {
9375     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9376   }
9377   else {
9378     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9379   }
9380   // Store back in big endian from little endian
9381   rorq(sum, 0x20);
9382   movq(Address(out, offset, Address::times_4,  8), sum);
9383 
9384   movq(op1, Address(in, len, Address::times_4,  0));
9385   rorq(op1, 32);
9386   movq(sum, Address(out, offset, Address::times_4,  0));
9387   rorq(sum, 32);
9388   if (UseBMI2Instructions) {
9389     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9390   }
9391   else {
9392     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9393   }
9394   // Store back in big endian from little endian
9395   rorq(sum, 0x20);
9396   movq(Address(out, offset, Address::times_4,  0), sum);
9397 
9398   jmp(L_first_loop);
9399   bind(L_first_loop_exit);
9400 }
9401 
9402 /**
9403  * Code for BigInteger::mulAdd() intrinsic
9404  *
9405  * rdi: out
9406  * rsi: in
9407  * r11: offs (out.length - offset)
9408  * rcx: len
9409  * r8:  k
9410  * r12: tmp1
9411  * r13: tmp2
9412  * r14: tmp3
9413  * r15: tmp4
9414  * rbx: tmp5
9415  * Multiply the in[] by word k and add to out[], return the carry in rax
9416  */
9417 void MacroAssembler::mul_add(Register out, Register in, Register offs,
9418    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
9419    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9420 
9421   Label L_carry, L_last_in, L_done;
9422 
9423 // carry = 0;
9424 // for (int j=len-1; j >= 0; j--) {
9425 //    long product = (in[j] & LONG_MASK) * kLong +
9426 //                   (out[offs] & LONG_MASK) + carry;
9427 //    out[offs--] = (int)product;
9428 //    carry = product >>> 32;
9429 // }
9430 //
9431   push(tmp1);
9432   push(tmp2);
9433   push(tmp3);
9434   push(tmp4);
9435   push(tmp5);
9436 
9437   Register op2 = tmp2;
9438   const Register sum = tmp3;
9439   const Register op1 = tmp4;
9440   const Register carry =  tmp5;
9441 
9442   if (UseBMI2Instructions) {
9443     op2 = rdxReg;
9444     movl(op2, k);
9445   }
9446   else {
9447     movl(op2, k);
9448   }
9449 
9450   xorq(carry, carry);
9451 
9452   //First loop
9453 
9454   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
9455   //The carry is in tmp5
9456   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
9457 
9458   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
9459   decrementl(len);
9460   jccb(Assembler::negative, L_carry);
9461   decrementl(len);
9462   jccb(Assembler::negative, L_last_in);
9463 
9464   movq(op1, Address(in, len, Address::times_4,  0));
9465   rorq(op1, 32);
9466 
9467   subl(offs, 2);
9468   movq(sum, Address(out, offs, Address::times_4,  0));
9469   rorq(sum, 32);
9470 
9471   if (UseBMI2Instructions) {
9472     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9473   }
9474   else {
9475     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9476   }
9477 
9478   // Store back in big endian from little endian
9479   rorq(sum, 0x20);
9480   movq(Address(out, offs, Address::times_4,  0), sum);
9481 
9482   testl(len, len);
9483   jccb(Assembler::zero, L_carry);
9484 
9485   //Multiply the last in[] entry, if any
9486   bind(L_last_in);
9487   movl(op1, Address(in, 0));
9488   movl(sum, Address(out, offs, Address::times_4,  -4));
9489 
9490   movl(raxReg, k);
9491   mull(op1); //tmp4 * eax -> edx:eax
9492   addl(sum, carry);
9493   adcl(rdxReg, 0);
9494   addl(sum, raxReg);
9495   adcl(rdxReg, 0);
9496   movl(carry, rdxReg);
9497 
9498   movl(Address(out, offs, Address::times_4,  -4), sum);
9499 
9500   bind(L_carry);
9501   //return tmp5/carry as carry in rax
9502   movl(rax, carry);
9503 
9504   bind(L_done);
9505   pop(tmp5);
9506   pop(tmp4);
9507   pop(tmp3);
9508   pop(tmp2);
9509   pop(tmp1);
9510 }
9511 #endif
9512 
9513 /**
9514  * Emits code to update CRC-32 with a byte value according to constants in table
9515  *
9516  * @param [in,out]crc   Register containing the crc.
9517  * @param [in]val       Register containing the byte to fold into the CRC.
9518  * @param [in]table     Register containing the table of crc constants.
9519  *
9520  * uint32_t crc;
9521  * val = crc_table[(val ^ crc) & 0xFF];
9522  * crc = val ^ (crc >> 8);
9523  *
9524  */
9525 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
9526   xorl(val, crc);
9527   andl(val, 0xFF);
9528   shrl(crc, 8); // unsigned shift
9529   xorl(crc, Address(table, val, Address::times_4, 0));
9530 }
9531 
9532 /**
9533 * Fold four 128-bit data chunks
9534 */
9535 void MacroAssembler::fold_128bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
9536   evpclmulhdq(xtmp, xK, xcrc, Assembler::AVX_512bit); // [123:64]
9537   evpclmulldq(xcrc, xK, xcrc, Assembler::AVX_512bit); // [63:0]
9538   evpxorq(xcrc, xcrc, Address(buf, offset), Assembler::AVX_512bit /* vector_len */);
9539   evpxorq(xcrc, xcrc, xtmp, Assembler::AVX_512bit /* vector_len */);
9540 }
9541 
9542 /**
9543  * Fold 128-bit data chunk
9544  */
9545 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
9546   if (UseAVX > 0) {
9547     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
9548     vpclmulldq(xcrc, xK, xcrc); // [63:0]
9549     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
9550     pxor(xcrc, xtmp);
9551   } else {
9552     movdqa(xtmp, xcrc);
9553     pclmulhdq(xtmp, xK);   // [123:64]
9554     pclmulldq(xcrc, xK);   // [63:0]
9555     pxor(xcrc, xtmp);
9556     movdqu(xtmp, Address(buf, offset));
9557     pxor(xcrc, xtmp);
9558   }
9559 }
9560 
9561 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
9562   if (UseAVX > 0) {
9563     vpclmulhdq(xtmp, xK, xcrc);
9564     vpclmulldq(xcrc, xK, xcrc);
9565     pxor(xcrc, xbuf);
9566     pxor(xcrc, xtmp);
9567   } else {
9568     movdqa(xtmp, xcrc);
9569     pclmulhdq(xtmp, xK);
9570     pclmulldq(xcrc, xK);
9571     pxor(xcrc, xbuf);
9572     pxor(xcrc, xtmp);
9573   }
9574 }
9575 
9576 /**
9577  * 8-bit folds to compute 32-bit CRC
9578  *
9579  * uint64_t xcrc;
9580  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
9581  */
9582 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
9583   movdl(tmp, xcrc);
9584   andl(tmp, 0xFF);
9585   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
9586   psrldq(xcrc, 1); // unsigned shift one byte
9587   pxor(xcrc, xtmp);
9588 }
9589 
9590 /**
9591  * uint32_t crc;
9592  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
9593  */
9594 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
9595   movl(tmp, crc);
9596   andl(tmp, 0xFF);
9597   shrl(crc, 8);
9598   xorl(crc, Address(table, tmp, Address::times_4, 0));
9599 }
9600 
9601 /**
9602  * @param crc   register containing existing CRC (32-bit)
9603  * @param buf   register pointing to input byte buffer (byte*)
9604  * @param len   register containing number of bytes
9605  * @param table register that will contain address of CRC table
9606  * @param tmp   scratch register
9607  */
9608 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
9609   assert_different_registers(crc, buf, len, table, tmp, rax);
9610 
9611   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
9612   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
9613 
9614   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
9615   // context for the registers used, where all instructions below are using 128-bit mode
9616   // On EVEX without VL and BW, these instructions will all be AVX.
9617   if (VM_Version::supports_avx512vlbw()) {
9618     movl(tmp, 0xffff);
9619     kmovwl(k1, tmp);
9620   }
9621 
9622   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
9623   notl(crc); // ~crc
9624   cmpl(len, 16);
9625   jcc(Assembler::less, L_tail);
9626 
9627   // Align buffer to 16 bytes
9628   movl(tmp, buf);
9629   andl(tmp, 0xF);
9630   jccb(Assembler::zero, L_aligned);
9631   subl(tmp,  16);
9632   addl(len, tmp);
9633 
9634   align(4);
9635   BIND(L_align_loop);
9636   movsbl(rax, Address(buf, 0)); // load byte with sign extension
9637   update_byte_crc32(crc, rax, table);
9638   increment(buf);
9639   incrementl(tmp);
9640   jccb(Assembler::less, L_align_loop);
9641 
9642   BIND(L_aligned);
9643   movl(tmp, len); // save
9644   shrl(len, 4);
9645   jcc(Assembler::zero, L_tail_restore);
9646 
9647   // Fold total 512 bits of polynomial on each iteration
9648   if (VM_Version::supports_vpclmulqdq()) {
9649     Label Parallel_loop, L_No_Parallel;
9650 
9651     cmpl(len, 8);
9652     jccb(Assembler::less, L_No_Parallel);
9653 
9654     movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
9655     evmovdquq(xmm1, Address(buf, 0), Assembler::AVX_512bit);
9656     movdl(xmm5, crc);
9657     evpxorq(xmm1, xmm1, xmm5, Assembler::AVX_512bit);
9658     addptr(buf, 64);
9659     subl(len, 7);
9660     evshufi64x2(xmm0, xmm0, xmm0, 0x00, Assembler::AVX_512bit); //propagate the mask from 128 bits to 512 bits
9661 
9662     BIND(Parallel_loop);
9663     fold_128bit_crc32_avx512(xmm1, xmm0, xmm5, buf, 0);
9664     addptr(buf, 64);
9665     subl(len, 4);
9666     jcc(Assembler::greater, Parallel_loop);
9667 
9668     vextracti64x2(xmm2, xmm1, 0x01);
9669     vextracti64x2(xmm3, xmm1, 0x02);
9670     vextracti64x2(xmm4, xmm1, 0x03);
9671     jmp(L_fold_512b);
9672 
9673     BIND(L_No_Parallel);
9674   }
9675   // Fold crc into first bytes of vector
9676   movdqa(xmm1, Address(buf, 0));
9677   movdl(rax, xmm1);
9678   xorl(crc, rax);
9679   if (VM_Version::supports_sse4_1()) {
9680     pinsrd(xmm1, crc, 0);
9681   } else {
9682     pinsrw(xmm1, crc, 0);
9683     shrl(crc, 16);
9684     pinsrw(xmm1, crc, 1);
9685   }
9686   addptr(buf, 16);
9687   subl(len, 4); // len > 0
9688   jcc(Assembler::less, L_fold_tail);
9689 
9690   movdqa(xmm2, Address(buf,  0));
9691   movdqa(xmm3, Address(buf, 16));
9692   movdqa(xmm4, Address(buf, 32));
9693   addptr(buf, 48);
9694   subl(len, 3);
9695   jcc(Assembler::lessEqual, L_fold_512b);
9696 
9697   // Fold total 512 bits of polynomial on each iteration,
9698   // 128 bits per each of 4 parallel streams.
9699   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
9700 
9701   align(32);
9702   BIND(L_fold_512b_loop);
9703   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
9704   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
9705   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
9706   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
9707   addptr(buf, 64);
9708   subl(len, 4);
9709   jcc(Assembler::greater, L_fold_512b_loop);
9710 
9711   // Fold 512 bits to 128 bits.
9712   BIND(L_fold_512b);
9713   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
9714   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
9715   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
9716   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
9717 
9718   // Fold the rest of 128 bits data chunks
9719   BIND(L_fold_tail);
9720   addl(len, 3);
9721   jccb(Assembler::lessEqual, L_fold_128b);
9722   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
9723 
9724   BIND(L_fold_tail_loop);
9725   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
9726   addptr(buf, 16);
9727   decrementl(len);
9728   jccb(Assembler::greater, L_fold_tail_loop);
9729 
9730   // Fold 128 bits in xmm1 down into 32 bits in crc register.
9731   BIND(L_fold_128b);
9732   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()));
9733   if (UseAVX > 0) {
9734     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
9735     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
9736     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
9737   } else {
9738     movdqa(xmm2, xmm0);
9739     pclmulqdq(xmm2, xmm1, 0x1);
9740     movdqa(xmm3, xmm0);
9741     pand(xmm3, xmm2);
9742     pclmulqdq(xmm0, xmm3, 0x1);
9743   }
9744   psrldq(xmm1, 8);
9745   psrldq(xmm2, 4);
9746   pxor(xmm0, xmm1);
9747   pxor(xmm0, xmm2);
9748 
9749   // 8 8-bit folds to compute 32-bit CRC.
9750   for (int j = 0; j < 4; j++) {
9751     fold_8bit_crc32(xmm0, table, xmm1, rax);
9752   }
9753   movdl(crc, xmm0); // mov 32 bits to general register
9754   for (int j = 0; j < 4; j++) {
9755     fold_8bit_crc32(crc, table, rax);
9756   }
9757 
9758   BIND(L_tail_restore);
9759   movl(len, tmp); // restore
9760   BIND(L_tail);
9761   andl(len, 0xf);
9762   jccb(Assembler::zero, L_exit);
9763 
9764   // Fold the rest of bytes
9765   align(4);
9766   BIND(L_tail_loop);
9767   movsbl(rax, Address(buf, 0)); // load byte with sign extension
9768   update_byte_crc32(crc, rax, table);
9769   increment(buf);
9770   decrementl(len);
9771   jccb(Assembler::greater, L_tail_loop);
9772 
9773   BIND(L_exit);
9774   notl(crc); // ~c
9775 }
9776 
9777 #ifdef _LP64
9778 // S. Gueron / Information Processing Letters 112 (2012) 184
9779 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
9780 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
9781 // Output: the 64-bit carry-less product of B * CONST
9782 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
9783                                      Register tmp1, Register tmp2, Register tmp3) {
9784   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
9785   if (n > 0) {
9786     addq(tmp3, n * 256 * 8);
9787   }
9788   //    Q1 = TABLEExt[n][B & 0xFF];
9789   movl(tmp1, in);
9790   andl(tmp1, 0x000000FF);
9791   shll(tmp1, 3);
9792   addq(tmp1, tmp3);
9793   movq(tmp1, Address(tmp1, 0));
9794 
9795   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
9796   movl(tmp2, in);
9797   shrl(tmp2, 8);
9798   andl(tmp2, 0x000000FF);
9799   shll(tmp2, 3);
9800   addq(tmp2, tmp3);
9801   movq(tmp2, Address(tmp2, 0));
9802 
9803   shlq(tmp2, 8);
9804   xorq(tmp1, tmp2);
9805 
9806   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
9807   movl(tmp2, in);
9808   shrl(tmp2, 16);
9809   andl(tmp2, 0x000000FF);
9810   shll(tmp2, 3);
9811   addq(tmp2, tmp3);
9812   movq(tmp2, Address(tmp2, 0));
9813 
9814   shlq(tmp2, 16);
9815   xorq(tmp1, tmp2);
9816 
9817   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
9818   shrl(in, 24);
9819   andl(in, 0x000000FF);
9820   shll(in, 3);
9821   addq(in, tmp3);
9822   movq(in, Address(in, 0));
9823 
9824   shlq(in, 24);
9825   xorq(in, tmp1);
9826   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
9827 }
9828 
9829 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
9830                                       Register in_out,
9831                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
9832                                       XMMRegister w_xtmp2,
9833                                       Register tmp1,
9834                                       Register n_tmp2, Register n_tmp3) {
9835   if (is_pclmulqdq_supported) {
9836     movdl(w_xtmp1, in_out); // modified blindly
9837 
9838     movl(tmp1, const_or_pre_comp_const_index);
9839     movdl(w_xtmp2, tmp1);
9840     pclmulqdq(w_xtmp1, w_xtmp2, 0);
9841 
9842     movdq(in_out, w_xtmp1);
9843   } else {
9844     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
9845   }
9846 }
9847 
9848 // Recombination Alternative 2: No bit-reflections
9849 // T1 = (CRC_A * U1) << 1
9850 // T2 = (CRC_B * U2) << 1
9851 // C1 = T1 >> 32
9852 // C2 = T2 >> 32
9853 // T1 = T1 & 0xFFFFFFFF
9854 // T2 = T2 & 0xFFFFFFFF
9855 // T1 = CRC32(0, T1)
9856 // T2 = CRC32(0, T2)
9857 // C1 = C1 ^ T1
9858 // C2 = C2 ^ T2
9859 // CRC = C1 ^ C2 ^ CRC_C
9860 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
9861                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
9862                                      Register tmp1, Register tmp2,
9863                                      Register n_tmp3) {
9864   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
9865   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
9866   shlq(in_out, 1);
9867   movl(tmp1, in_out);
9868   shrq(in_out, 32);
9869   xorl(tmp2, tmp2);
9870   crc32(tmp2, tmp1, 4);
9871   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
9872   shlq(in1, 1);
9873   movl(tmp1, in1);
9874   shrq(in1, 32);
9875   xorl(tmp2, tmp2);
9876   crc32(tmp2, tmp1, 4);
9877   xorl(in1, tmp2);
9878   xorl(in_out, in1);
9879   xorl(in_out, in2);
9880 }
9881 
9882 // Set N to predefined value
9883 // Subtract from a lenght of a buffer
9884 // execute in a loop:
9885 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
9886 // for i = 1 to N do
9887 //  CRC_A = CRC32(CRC_A, A[i])
9888 //  CRC_B = CRC32(CRC_B, B[i])
9889 //  CRC_C = CRC32(CRC_C, C[i])
9890 // end for
9891 // Recombine
9892 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
9893                                        Register in_out1, Register in_out2, Register in_out3,
9894                                        Register tmp1, Register tmp2, Register tmp3,
9895                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
9896                                        Register tmp4, Register tmp5,
9897                                        Register n_tmp6) {
9898   Label L_processPartitions;
9899   Label L_processPartition;
9900   Label L_exit;
9901 
9902   bind(L_processPartitions);
9903   cmpl(in_out1, 3 * size);
9904   jcc(Assembler::less, L_exit);
9905     xorl(tmp1, tmp1);
9906     xorl(tmp2, tmp2);
9907     movq(tmp3, in_out2);
9908     addq(tmp3, size);
9909 
9910     bind(L_processPartition);
9911       crc32(in_out3, Address(in_out2, 0), 8);
9912       crc32(tmp1, Address(in_out2, size), 8);
9913       crc32(tmp2, Address(in_out2, size * 2), 8);
9914       addq(in_out2, 8);
9915       cmpq(in_out2, tmp3);
9916       jcc(Assembler::less, L_processPartition);
9917     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
9918             w_xtmp1, w_xtmp2, w_xtmp3,
9919             tmp4, tmp5,
9920             n_tmp6);
9921     addq(in_out2, 2 * size);
9922     subl(in_out1, 3 * size);
9923     jmp(L_processPartitions);
9924 
9925   bind(L_exit);
9926 }
9927 #else
9928 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
9929                                      Register tmp1, Register tmp2, Register tmp3,
9930                                      XMMRegister xtmp1, XMMRegister xtmp2) {
9931   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
9932   if (n > 0) {
9933     addl(tmp3, n * 256 * 8);
9934   }
9935   //    Q1 = TABLEExt[n][B & 0xFF];
9936   movl(tmp1, in_out);
9937   andl(tmp1, 0x000000FF);
9938   shll(tmp1, 3);
9939   addl(tmp1, tmp3);
9940   movq(xtmp1, Address(tmp1, 0));
9941 
9942   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
9943   movl(tmp2, in_out);
9944   shrl(tmp2, 8);
9945   andl(tmp2, 0x000000FF);
9946   shll(tmp2, 3);
9947   addl(tmp2, tmp3);
9948   movq(xtmp2, Address(tmp2, 0));
9949 
9950   psllq(xtmp2, 8);
9951   pxor(xtmp1, xtmp2);
9952 
9953   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
9954   movl(tmp2, in_out);
9955   shrl(tmp2, 16);
9956   andl(tmp2, 0x000000FF);
9957   shll(tmp2, 3);
9958   addl(tmp2, tmp3);
9959   movq(xtmp2, Address(tmp2, 0));
9960 
9961   psllq(xtmp2, 16);
9962   pxor(xtmp1, xtmp2);
9963 
9964   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
9965   shrl(in_out, 24);
9966   andl(in_out, 0x000000FF);
9967   shll(in_out, 3);
9968   addl(in_out, tmp3);
9969   movq(xtmp2, Address(in_out, 0));
9970 
9971   psllq(xtmp2, 24);
9972   pxor(xtmp1, xtmp2); // Result in CXMM
9973   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
9974 }
9975 
9976 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
9977                                       Register in_out,
9978                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
9979                                       XMMRegister w_xtmp2,
9980                                       Register tmp1,
9981                                       Register n_tmp2, Register n_tmp3) {
9982   if (is_pclmulqdq_supported) {
9983     movdl(w_xtmp1, in_out);
9984 
9985     movl(tmp1, const_or_pre_comp_const_index);
9986     movdl(w_xtmp2, tmp1);
9987     pclmulqdq(w_xtmp1, w_xtmp2, 0);
9988     // Keep result in XMM since GPR is 32 bit in length
9989   } else {
9990     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
9991   }
9992 }
9993 
9994 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
9995                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
9996                                      Register tmp1, Register tmp2,
9997                                      Register n_tmp3) {
9998   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
9999   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10000 
10001   psllq(w_xtmp1, 1);
10002   movdl(tmp1, w_xtmp1);
10003   psrlq(w_xtmp1, 32);
10004   movdl(in_out, w_xtmp1);
10005 
10006   xorl(tmp2, tmp2);
10007   crc32(tmp2, tmp1, 4);
10008   xorl(in_out, tmp2);
10009 
10010   psllq(w_xtmp2, 1);
10011   movdl(tmp1, w_xtmp2);
10012   psrlq(w_xtmp2, 32);
10013   movdl(in1, w_xtmp2);
10014 
10015   xorl(tmp2, tmp2);
10016   crc32(tmp2, tmp1, 4);
10017   xorl(in1, tmp2);
10018   xorl(in_out, in1);
10019   xorl(in_out, in2);
10020 }
10021 
10022 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
10023                                        Register in_out1, Register in_out2, Register in_out3,
10024                                        Register tmp1, Register tmp2, Register tmp3,
10025                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10026                                        Register tmp4, Register tmp5,
10027                                        Register n_tmp6) {
10028   Label L_processPartitions;
10029   Label L_processPartition;
10030   Label L_exit;
10031 
10032   bind(L_processPartitions);
10033   cmpl(in_out1, 3 * size);
10034   jcc(Assembler::less, L_exit);
10035     xorl(tmp1, tmp1);
10036     xorl(tmp2, tmp2);
10037     movl(tmp3, in_out2);
10038     addl(tmp3, size);
10039 
10040     bind(L_processPartition);
10041       crc32(in_out3, Address(in_out2, 0), 4);
10042       crc32(tmp1, Address(in_out2, size), 4);
10043       crc32(tmp2, Address(in_out2, size*2), 4);
10044       crc32(in_out3, Address(in_out2, 0+4), 4);
10045       crc32(tmp1, Address(in_out2, size+4), 4);
10046       crc32(tmp2, Address(in_out2, size*2+4), 4);
10047       addl(in_out2, 8);
10048       cmpl(in_out2, tmp3);
10049       jcc(Assembler::less, L_processPartition);
10050 
10051         push(tmp3);
10052         push(in_out1);
10053         push(in_out2);
10054         tmp4 = tmp3;
10055         tmp5 = in_out1;
10056         n_tmp6 = in_out2;
10057 
10058       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
10059             w_xtmp1, w_xtmp2, w_xtmp3,
10060             tmp4, tmp5,
10061             n_tmp6);
10062 
10063         pop(in_out2);
10064         pop(in_out1);
10065         pop(tmp3);
10066 
10067     addl(in_out2, 2 * size);
10068     subl(in_out1, 3 * size);
10069     jmp(L_processPartitions);
10070 
10071   bind(L_exit);
10072 }
10073 #endif //LP64
10074 
10075 #ifdef _LP64
10076 // Algorithm 2: Pipelined usage of the CRC32 instruction.
10077 // Input: A buffer I of L bytes.
10078 // Output: the CRC32C value of the buffer.
10079 // Notations:
10080 // Write L = 24N + r, with N = floor (L/24).
10081 // r = L mod 24 (0 <= r < 24).
10082 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
10083 // N quadwords, and R consists of r bytes.
10084 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
10085 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
10086 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
10087 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
10088 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10089                                           Register tmp1, Register tmp2, Register tmp3,
10090                                           Register tmp4, Register tmp5, Register tmp6,
10091                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10092                                           bool is_pclmulqdq_supported) {
10093   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10094   Label L_wordByWord;
10095   Label L_byteByByteProlog;
10096   Label L_byteByByte;
10097   Label L_exit;
10098 
10099   if (is_pclmulqdq_supported ) {
10100     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10101     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
10102 
10103     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10104     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10105 
10106     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10107     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10108     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
10109   } else {
10110     const_or_pre_comp_const_index[0] = 1;
10111     const_or_pre_comp_const_index[1] = 0;
10112 
10113     const_or_pre_comp_const_index[2] = 3;
10114     const_or_pre_comp_const_index[3] = 2;
10115 
10116     const_or_pre_comp_const_index[4] = 5;
10117     const_or_pre_comp_const_index[5] = 4;
10118    }
10119   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10120                     in2, in1, in_out,
10121                     tmp1, tmp2, tmp3,
10122                     w_xtmp1, w_xtmp2, w_xtmp3,
10123                     tmp4, tmp5,
10124                     tmp6);
10125   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10126                     in2, in1, in_out,
10127                     tmp1, tmp2, tmp3,
10128                     w_xtmp1, w_xtmp2, w_xtmp3,
10129                     tmp4, tmp5,
10130                     tmp6);
10131   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10132                     in2, in1, in_out,
10133                     tmp1, tmp2, tmp3,
10134                     w_xtmp1, w_xtmp2, w_xtmp3,
10135                     tmp4, tmp5,
10136                     tmp6);
10137   movl(tmp1, in2);
10138   andl(tmp1, 0x00000007);
10139   negl(tmp1);
10140   addl(tmp1, in2);
10141   addq(tmp1, in1);
10142 
10143   BIND(L_wordByWord);
10144   cmpq(in1, tmp1);
10145   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10146     crc32(in_out, Address(in1, 0), 4);
10147     addq(in1, 4);
10148     jmp(L_wordByWord);
10149 
10150   BIND(L_byteByByteProlog);
10151   andl(in2, 0x00000007);
10152   movl(tmp2, 1);
10153 
10154   BIND(L_byteByByte);
10155   cmpl(tmp2, in2);
10156   jccb(Assembler::greater, L_exit);
10157     crc32(in_out, Address(in1, 0), 1);
10158     incq(in1);
10159     incl(tmp2);
10160     jmp(L_byteByByte);
10161 
10162   BIND(L_exit);
10163 }
10164 #else
10165 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10166                                           Register tmp1, Register  tmp2, Register tmp3,
10167                                           Register tmp4, Register  tmp5, Register tmp6,
10168                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10169                                           bool is_pclmulqdq_supported) {
10170   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10171   Label L_wordByWord;
10172   Label L_byteByByteProlog;
10173   Label L_byteByByte;
10174   Label L_exit;
10175 
10176   if (is_pclmulqdq_supported) {
10177     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10178     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
10179 
10180     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10181     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10182 
10183     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10184     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10185   } else {
10186     const_or_pre_comp_const_index[0] = 1;
10187     const_or_pre_comp_const_index[1] = 0;
10188 
10189     const_or_pre_comp_const_index[2] = 3;
10190     const_or_pre_comp_const_index[3] = 2;
10191 
10192     const_or_pre_comp_const_index[4] = 5;
10193     const_or_pre_comp_const_index[5] = 4;
10194   }
10195   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10196                     in2, in1, in_out,
10197                     tmp1, tmp2, tmp3,
10198                     w_xtmp1, w_xtmp2, w_xtmp3,
10199                     tmp4, tmp5,
10200                     tmp6);
10201   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10202                     in2, in1, in_out,
10203                     tmp1, tmp2, tmp3,
10204                     w_xtmp1, w_xtmp2, w_xtmp3,
10205                     tmp4, tmp5,
10206                     tmp6);
10207   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10208                     in2, in1, in_out,
10209                     tmp1, tmp2, tmp3,
10210                     w_xtmp1, w_xtmp2, w_xtmp3,
10211                     tmp4, tmp5,
10212                     tmp6);
10213   movl(tmp1, in2);
10214   andl(tmp1, 0x00000007);
10215   negl(tmp1);
10216   addl(tmp1, in2);
10217   addl(tmp1, in1);
10218 
10219   BIND(L_wordByWord);
10220   cmpl(in1, tmp1);
10221   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10222     crc32(in_out, Address(in1,0), 4);
10223     addl(in1, 4);
10224     jmp(L_wordByWord);
10225 
10226   BIND(L_byteByByteProlog);
10227   andl(in2, 0x00000007);
10228   movl(tmp2, 1);
10229 
10230   BIND(L_byteByByte);
10231   cmpl(tmp2, in2);
10232   jccb(Assembler::greater, L_exit);
10233     movb(tmp1, Address(in1, 0));
10234     crc32(in_out, tmp1, 1);
10235     incl(in1);
10236     incl(tmp2);
10237     jmp(L_byteByByte);
10238 
10239   BIND(L_exit);
10240 }
10241 #endif // LP64
10242 #undef BIND
10243 #undef BLOCK_COMMENT
10244 
10245 // Compress char[] array to byte[].
10246 //   ..\jdk\src\java.base\share\classes\java\lang\StringUTF16.java
10247 //   @HotSpotIntrinsicCandidate
10248 //   private static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
10249 //     for (int i = 0; i < len; i++) {
10250 //       int c = src[srcOff++];
10251 //       if (c >>> 8 != 0) {
10252 //         return 0;
10253 //       }
10254 //       dst[dstOff++] = (byte)c;
10255 //     }
10256 //     return len;
10257 //   }
10258 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
10259   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
10260   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
10261   Register tmp5, Register result) {
10262   Label copy_chars_loop, return_length, return_zero, done;
10263 
10264   // rsi: src
10265   // rdi: dst
10266   // rdx: len
10267   // rcx: tmp5
10268   // rax: result
10269 
10270   // rsi holds start addr of source char[] to be compressed
10271   // rdi holds start addr of destination byte[]
10272   // rdx holds length
10273 
10274   assert(len != result, "");
10275 
10276   // save length for return
10277   push(len);
10278 
10279   if ((UseAVX > 2) && // AVX512
10280     VM_Version::supports_avx512vlbw() &&
10281     VM_Version::supports_bmi2()) {
10282 
10283     set_vector_masking();  // opening of the stub context for programming mask registers
10284 
10285     Label copy_32_loop, copy_loop_tail, restore_k1_return_zero, below_threshold;
10286 
10287     // alignment
10288     Label post_alignment;
10289 
10290     // if length of the string is less than 16, handle it in an old fashioned way
10291     testl(len, -32);
10292     jcc(Assembler::zero, below_threshold);
10293 
10294     // First check whether a character is compressable ( <= 0xFF).
10295     // Create mask to test for Unicode chars inside zmm vector
10296     movl(result, 0x00FF);
10297     evpbroadcastw(tmp2Reg, result, Assembler::AVX_512bit);
10298 
10299     // Save k1
10300     kmovql(k3, k1);
10301 
10302     testl(len, -64);
10303     jcc(Assembler::zero, post_alignment);
10304 
10305     movl(tmp5, dst);
10306     andl(tmp5, (32 - 1));
10307     negl(tmp5);
10308     andl(tmp5, (32 - 1));
10309 
10310     // bail out when there is nothing to be done
10311     testl(tmp5, 0xFFFFFFFF);
10312     jcc(Assembler::zero, post_alignment);
10313 
10314     // ~(~0 << len), where len is the # of remaining elements to process
10315     movl(result, 0xFFFFFFFF);
10316     shlxl(result, result, tmp5);
10317     notl(result);
10318     kmovdl(k1, result);
10319 
10320     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
10321     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10322     ktestd(k2, k1);
10323     jcc(Assembler::carryClear, restore_k1_return_zero);
10324 
10325     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
10326 
10327     addptr(src, tmp5);
10328     addptr(src, tmp5);
10329     addptr(dst, tmp5);
10330     subl(len, tmp5);
10331 
10332     bind(post_alignment);
10333     // end of alignment
10334 
10335     movl(tmp5, len);
10336     andl(tmp5, (32 - 1));    // tail count (in chars)
10337     andl(len, ~(32 - 1));    // vector count (in chars)
10338     jcc(Assembler::zero, copy_loop_tail);
10339 
10340     lea(src, Address(src, len, Address::times_2));
10341     lea(dst, Address(dst, len, Address::times_1));
10342     negptr(len);
10343 
10344     bind(copy_32_loop);
10345     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
10346     evpcmpuw(k2, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10347     kortestdl(k2, k2);
10348     jcc(Assembler::carryClear, restore_k1_return_zero);
10349 
10350     // All elements in current processed chunk are valid candidates for
10351     // compression. Write a truncated byte elements to the memory.
10352     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
10353     addptr(len, 32);
10354     jcc(Assembler::notZero, copy_32_loop);
10355 
10356     bind(copy_loop_tail);
10357     // bail out when there is nothing to be done
10358     testl(tmp5, 0xFFFFFFFF);
10359     // Restore k1
10360     kmovql(k1, k3);
10361     jcc(Assembler::zero, return_length);
10362 
10363     movl(len, tmp5);
10364 
10365     // ~(~0 << len), where len is the # of remaining elements to process
10366     movl(result, 0xFFFFFFFF);
10367     shlxl(result, result, len);
10368     notl(result);
10369 
10370     kmovdl(k1, result);
10371 
10372     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
10373     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10374     ktestd(k2, k1);
10375     jcc(Assembler::carryClear, restore_k1_return_zero);
10376 
10377     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
10378     // Restore k1
10379     kmovql(k1, k3);
10380     jmp(return_length);
10381 
10382     bind(restore_k1_return_zero);
10383     // Restore k1
10384     kmovql(k1, k3);
10385     jmp(return_zero);
10386 
10387     clear_vector_masking();   // closing of the stub context for programming mask registers
10388 
10389     bind(below_threshold);
10390   }
10391 
10392   if (UseSSE42Intrinsics) {
10393     Label copy_32_loop, copy_16, copy_tail;
10394 
10395     movl(result, len);
10396 
10397     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
10398 
10399     // vectored compression
10400     andl(len, 0xfffffff0);    // vector count (in chars)
10401     andl(result, 0x0000000f);    // tail count (in chars)
10402     testl(len, len);
10403     jcc(Assembler::zero, copy_16);
10404 
10405     // compress 16 chars per iter
10406     movdl(tmp1Reg, tmp5);
10407     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
10408     pxor(tmp4Reg, tmp4Reg);
10409 
10410     lea(src, Address(src, len, Address::times_2));
10411     lea(dst, Address(dst, len, Address::times_1));
10412     negptr(len);
10413 
10414     bind(copy_32_loop);
10415     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
10416     por(tmp4Reg, tmp2Reg);
10417     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
10418     por(tmp4Reg, tmp3Reg);
10419     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
10420     jcc(Assembler::notZero, return_zero);
10421     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
10422     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
10423     addptr(len, 16);
10424     jcc(Assembler::notZero, copy_32_loop);
10425 
10426     // compress next vector of 8 chars (if any)
10427     bind(copy_16);
10428     movl(len, result);
10429     andl(len, 0xfffffff8);    // vector count (in chars)
10430     andl(result, 0x00000007);    // tail count (in chars)
10431     testl(len, len);
10432     jccb(Assembler::zero, copy_tail);
10433 
10434     movdl(tmp1Reg, tmp5);
10435     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
10436     pxor(tmp3Reg, tmp3Reg);
10437 
10438     movdqu(tmp2Reg, Address(src, 0));
10439     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
10440     jccb(Assembler::notZero, return_zero);
10441     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
10442     movq(Address(dst, 0), tmp2Reg);
10443     addptr(src, 16);
10444     addptr(dst, 8);
10445 
10446     bind(copy_tail);
10447     movl(len, result);
10448   }
10449   // compress 1 char per iter
10450   testl(len, len);
10451   jccb(Assembler::zero, return_length);
10452   lea(src, Address(src, len, Address::times_2));
10453   lea(dst, Address(dst, len, Address::times_1));
10454   negptr(len);
10455 
10456   bind(copy_chars_loop);
10457   load_unsigned_short(result, Address(src, len, Address::times_2));
10458   testl(result, 0xff00);      // check if Unicode char
10459   jccb(Assembler::notZero, return_zero);
10460   movb(Address(dst, len, Address::times_1), result);  // ASCII char; compress to 1 byte
10461   increment(len);
10462   jcc(Assembler::notZero, copy_chars_loop);
10463 
10464   // if compression succeeded, return length
10465   bind(return_length);
10466   pop(result);
10467   jmpb(done);
10468 
10469   // if compression failed, return 0
10470   bind(return_zero);
10471   xorl(result, result);
10472   addptr(rsp, wordSize);
10473 
10474   bind(done);
10475 }
10476 
10477 // Inflate byte[] array to char[].
10478 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
10479 //   @HotSpotIntrinsicCandidate
10480 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
10481 //     for (int i = 0; i < len; i++) {
10482 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
10483 //     }
10484 //   }
10485 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
10486   XMMRegister tmp1, Register tmp2) {
10487   Label copy_chars_loop, done, below_threshold;
10488   // rsi: src
10489   // rdi: dst
10490   // rdx: len
10491   // rcx: tmp2
10492 
10493   // rsi holds start addr of source byte[] to be inflated
10494   // rdi holds start addr of destination char[]
10495   // rdx holds length
10496   assert_different_registers(src, dst, len, tmp2);
10497 
10498   if ((UseAVX > 2) && // AVX512
10499     VM_Version::supports_avx512vlbw() &&
10500     VM_Version::supports_bmi2()) {
10501 
10502     set_vector_masking();  // opening of the stub context for programming mask registers
10503 
10504     Label copy_32_loop, copy_tail;
10505     Register tmp3_aliased = len;
10506 
10507     // if length of the string is less than 16, handle it in an old fashioned way
10508     testl(len, -16);
10509     jcc(Assembler::zero, below_threshold);
10510 
10511     // In order to use only one arithmetic operation for the main loop we use
10512     // this pre-calculation
10513     movl(tmp2, len);
10514     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
10515     andl(len, -32);     // vector count
10516     jccb(Assembler::zero, copy_tail);
10517 
10518     lea(src, Address(src, len, Address::times_1));
10519     lea(dst, Address(dst, len, Address::times_2));
10520     negptr(len);
10521 
10522 
10523     // inflate 32 chars per iter
10524     bind(copy_32_loop);
10525     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
10526     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
10527     addptr(len, 32);
10528     jcc(Assembler::notZero, copy_32_loop);
10529 
10530     bind(copy_tail);
10531     // bail out when there is nothing to be done
10532     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
10533     jcc(Assembler::zero, done);
10534 
10535     // Save k1
10536     kmovql(k2, k1);
10537 
10538     // ~(~0 << length), where length is the # of remaining elements to process
10539     movl(tmp3_aliased, -1);
10540     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
10541     notl(tmp3_aliased);
10542     kmovdl(k1, tmp3_aliased);
10543     evpmovzxbw(tmp1, k1, Address(src, 0), Assembler::AVX_512bit);
10544     evmovdquw(Address(dst, 0), k1, tmp1, Assembler::AVX_512bit);
10545 
10546     // Restore k1
10547     kmovql(k1, k2);
10548     jmp(done);
10549 
10550     clear_vector_masking();   // closing of the stub context for programming mask registers
10551   }
10552   if (UseSSE42Intrinsics) {
10553     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
10554 
10555     movl(tmp2, len);
10556 
10557     if (UseAVX > 1) {
10558       andl(tmp2, (16 - 1));
10559       andl(len, -16);
10560       jccb(Assembler::zero, copy_new_tail);
10561     } else {
10562       andl(tmp2, 0x00000007);   // tail count (in chars)
10563       andl(len, 0xfffffff8);    // vector count (in chars)
10564       jccb(Assembler::zero, copy_tail);
10565     }
10566 
10567     // vectored inflation
10568     lea(src, Address(src, len, Address::times_1));
10569     lea(dst, Address(dst, len, Address::times_2));
10570     negptr(len);
10571 
10572     if (UseAVX > 1) {
10573       bind(copy_16_loop);
10574       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
10575       vmovdqu(Address(dst, len, Address::times_2), tmp1);
10576       addptr(len, 16);
10577       jcc(Assembler::notZero, copy_16_loop);
10578 
10579       bind(below_threshold);
10580       bind(copy_new_tail);
10581       if ((UseAVX > 2) &&
10582         VM_Version::supports_avx512vlbw() &&
10583         VM_Version::supports_bmi2()) {
10584         movl(tmp2, len);
10585       } else {
10586         movl(len, tmp2);
10587       }
10588       andl(tmp2, 0x00000007);
10589       andl(len, 0xFFFFFFF8);
10590       jccb(Assembler::zero, copy_tail);
10591 
10592       pmovzxbw(tmp1, Address(src, 0));
10593       movdqu(Address(dst, 0), tmp1);
10594       addptr(src, 8);
10595       addptr(dst, 2 * 8);
10596 
10597       jmp(copy_tail, true);
10598     }
10599 
10600     // inflate 8 chars per iter
10601     bind(copy_8_loop);
10602     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
10603     movdqu(Address(dst, len, Address::times_2), tmp1);
10604     addptr(len, 8);
10605     jcc(Assembler::notZero, copy_8_loop);
10606 
10607     bind(copy_tail);
10608     movl(len, tmp2);
10609 
10610     cmpl(len, 4);
10611     jccb(Assembler::less, copy_bytes);
10612 
10613     movdl(tmp1, Address(src, 0));  // load 4 byte chars
10614     pmovzxbw(tmp1, tmp1);
10615     movq(Address(dst, 0), tmp1);
10616     subptr(len, 4);
10617     addptr(src, 4);
10618     addptr(dst, 8);
10619 
10620     bind(copy_bytes);
10621   } else {
10622     bind(below_threshold);
10623   }
10624 
10625   testl(len, len);
10626   jccb(Assembler::zero, done);
10627   lea(src, Address(src, len, Address::times_1));
10628   lea(dst, Address(dst, len, Address::times_2));
10629   negptr(len);
10630 
10631   // inflate 1 char per iter
10632   bind(copy_chars_loop);
10633   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
10634   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
10635   increment(len);
10636   jcc(Assembler::notZero, copy_chars_loop);
10637 
10638   bind(done);
10639 }
10640 
10641 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
10642   switch (cond) {
10643     // Note some conditions are synonyms for others
10644     case Assembler::zero:         return Assembler::notZero;
10645     case Assembler::notZero:      return Assembler::zero;
10646     case Assembler::less:         return Assembler::greaterEqual;
10647     case Assembler::lessEqual:    return Assembler::greater;
10648     case Assembler::greater:      return Assembler::lessEqual;
10649     case Assembler::greaterEqual: return Assembler::less;
10650     case Assembler::below:        return Assembler::aboveEqual;
10651     case Assembler::belowEqual:   return Assembler::above;
10652     case Assembler::above:        return Assembler::belowEqual;
10653     case Assembler::aboveEqual:   return Assembler::below;
10654     case Assembler::overflow:     return Assembler::noOverflow;
10655     case Assembler::noOverflow:   return Assembler::overflow;
10656     case Assembler::negative:     return Assembler::positive;
10657     case Assembler::positive:     return Assembler::negative;
10658     case Assembler::parity:       return Assembler::noParity;
10659     case Assembler::noParity:     return Assembler::parity;
10660   }
10661   ShouldNotReachHere(); return Assembler::overflow;
10662 }
10663 
10664 SkipIfEqual::SkipIfEqual(
10665     MacroAssembler* masm, const bool* flag_addr, bool value) {
10666   _masm = masm;
10667   _masm->cmp8(ExternalAddress((address)flag_addr), value);
10668   _masm->jcc(Assembler::equal, _label);
10669 }
10670 
10671 SkipIfEqual::~SkipIfEqual() {
10672   _masm->bind(_label);
10673 }
10674 
10675 // 32-bit Windows has its own fast-path implementation
10676 // of get_thread
10677 #if !defined(WIN32) || defined(_LP64)
10678 
10679 // This is simply a call to Thread::current()
10680 void MacroAssembler::get_thread(Register thread) {
10681   if (thread != rax) {
10682     push(rax);
10683   }
10684   LP64_ONLY(push(rdi);)
10685   LP64_ONLY(push(rsi);)
10686   push(rdx);
10687   push(rcx);
10688 #ifdef _LP64
10689   push(r8);
10690   push(r9);
10691   push(r10);
10692   push(r11);
10693 #endif
10694 
10695   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
10696 
10697 #ifdef _LP64
10698   pop(r11);
10699   pop(r10);
10700   pop(r9);
10701   pop(r8);
10702 #endif
10703   pop(rcx);
10704   pop(rdx);
10705   LP64_ONLY(pop(rsi);)
10706   LP64_ONLY(pop(rdi);)
10707   if (thread != rax) {
10708     mov(thread, rax);
10709     pop(rax);
10710   }
10711 }
10712 
10713 #endif
--- EOF ---