1 /*
   2  * Copyright (c) 1997, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "asm/assembler.hpp"
  28 #include "asm/assembler.inline.hpp"
  29 #include "compiler/disassembler.hpp"
  30 #include "gc/shared/barrierSet.hpp"
  31 #include "gc/shared/barrierSetAssembler.hpp"
  32 #include "gc/shared/collectedHeap.inline.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "memory/universe.hpp"
  36 #include "oops/accessDecorators.hpp"
  37 #include "oops/klass.inline.hpp"
  38 #include "prims/methodHandles.hpp"
  39 #include "runtime/biasedLocking.hpp"
  40 #include "runtime/flags/flagSetting.hpp"
  41 #include "runtime/interfaceSupport.inline.hpp"
  42 #include "runtime/objectMonitor.hpp"
  43 #include "runtime/os.hpp"
  44 #include "runtime/safepoint.hpp"
  45 #include "runtime/safepointMechanism.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/stubRoutines.hpp"
  48 #include "runtime/thread.hpp"
  49 #include "utilities/macros.hpp"
  50 #include "crc32c.h"
  51 #ifdef COMPILER2
  52 #include "opto/intrinsicnode.hpp"
  53 #endif
  54 
  55 #ifdef PRODUCT
  56 #define BLOCK_COMMENT(str) /* nothing */
  57 #define STOP(error) stop(error)
  58 #else
  59 #define BLOCK_COMMENT(str) block_comment(str)
  60 #define STOP(error) block_comment(error); stop(error)
  61 #endif
  62 
  63 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  64 
  65 #ifdef ASSERT
  66 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  67 #endif
  68 
  69 static Assembler::Condition reverse[] = {
  70     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  71     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  72     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  73     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  74     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,
  75     Assembler::zero           /* notZero       = 0x5, notEqual      = 0x5 */ ,
  76     Assembler::above          /* belowEqual    = 0x6 */ ,
  77     Assembler::belowEqual     /* above         = 0x7 */ ,
  78     Assembler::positive       /* negative      = 0x8 */ ,
  79     Assembler::negative       /* positive      = 0x9 */ ,
  80     Assembler::noParity       /* parity        = 0xa */ ,
  81     Assembler::parity         /* noParity      = 0xb */ ,
  82     Assembler::greaterEqual   /* less          = 0xc */ ,
  83     Assembler::less           /* greaterEqual  = 0xd */ ,
  84     Assembler::greater        /* lessEqual     = 0xe */ ,
  85     Assembler::lessEqual      /* greater       = 0xf, */
  86 
  87 };
  88 
  89 
  90 // Implementation of MacroAssembler
  91 
  92 // First all the versions that have distinct versions depending on 32/64 bit
  93 // Unless the difference is trivial (1 line or so).
  94 
  95 #ifndef _LP64
  96 
  97 // 32bit versions
  98 
  99 Address MacroAssembler::as_Address(AddressLiteral adr) {
 100   return Address(adr.target(), adr.rspec());
 101 }
 102 
 103 Address MacroAssembler::as_Address(ArrayAddress adr) {
 104   return Address::make_array(adr);
 105 }
 106 
 107 void MacroAssembler::call_VM_leaf_base(address entry_point,
 108                                        int number_of_arguments) {
 109   call(RuntimeAddress(entry_point));
 110   increment(rsp, number_of_arguments * wordSize);
 111 }
 112 
 113 void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
 114   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 115 }
 116 
 117 void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
 118   cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 119 }
 120 
 121 void MacroAssembler::cmpoop_raw(Address src1, jobject obj) {
 122   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 123 }
 124 
 125 void MacroAssembler::cmpoop_raw(Register src1, jobject obj) {
 126   cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 127 }
 128 
 129 void MacroAssembler::cmpoop(Address src1, jobject obj) {
 130   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 131   bs->obj_equals(this, src1, obj);
 132 }
 133 
 134 void MacroAssembler::cmpoop(Register src1, jobject obj) {
 135   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 136   bs->obj_equals(this, src1, obj);
 137 }
 138 
 139 void MacroAssembler::extend_sign(Register hi, Register lo) {
 140   // According to Intel Doc. AP-526, "Integer Divide", p.18.
 141   if (VM_Version::is_P6() && hi == rdx && lo == rax) {
 142     cdql();
 143   } else {
 144     movl(hi, lo);
 145     sarl(hi, 31);
 146   }
 147 }
 148 
 149 void MacroAssembler::jC2(Register tmp, Label& L) {
 150   // set parity bit if FPU flag C2 is set (via rax)
 151   save_rax(tmp);
 152   fwait(); fnstsw_ax();
 153   sahf();
 154   restore_rax(tmp);
 155   // branch
 156   jcc(Assembler::parity, L);
 157 }
 158 
 159 void MacroAssembler::jnC2(Register tmp, Label& L) {
 160   // set parity bit if FPU flag C2 is set (via rax)
 161   save_rax(tmp);
 162   fwait(); fnstsw_ax();
 163   sahf();
 164   restore_rax(tmp);
 165   // branch
 166   jcc(Assembler::noParity, L);
 167 }
 168 
 169 // 32bit can do a case table jump in one instruction but we no longer allow the base
 170 // to be installed in the Address class
 171 void MacroAssembler::jump(ArrayAddress entry) {
 172   jmp(as_Address(entry));
 173 }
 174 
 175 // Note: y_lo will be destroyed
 176 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 177   // Long compare for Java (semantics as described in JVM spec.)
 178   Label high, low, done;
 179 
 180   cmpl(x_hi, y_hi);
 181   jcc(Assembler::less, low);
 182   jcc(Assembler::greater, high);
 183   // x_hi is the return register
 184   xorl(x_hi, x_hi);
 185   cmpl(x_lo, y_lo);
 186   jcc(Assembler::below, low);
 187   jcc(Assembler::equal, done);
 188 
 189   bind(high);
 190   xorl(x_hi, x_hi);
 191   increment(x_hi);
 192   jmp(done);
 193 
 194   bind(low);
 195   xorl(x_hi, x_hi);
 196   decrementl(x_hi);
 197 
 198   bind(done);
 199 }
 200 
 201 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 202     mov_literal32(dst, (int32_t)src.target(), src.rspec());
 203 }
 204 
 205 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 206   // leal(dst, as_Address(adr));
 207   // see note in movl as to why we must use a move
 208   mov_literal32(dst, (int32_t) adr.target(), adr.rspec());
 209 }
 210 
 211 void MacroAssembler::leave() {
 212   mov(rsp, rbp);
 213   pop(rbp);
 214 }
 215 
 216 void MacroAssembler::lmul(int x_rsp_offset, int y_rsp_offset) {
 217   // Multiplication of two Java long values stored on the stack
 218   // as illustrated below. Result is in rdx:rax.
 219   //
 220   // rsp ---> [  ??  ] \               \
 221   //            ....    | y_rsp_offset  |
 222   //          [ y_lo ] /  (in bytes)    | x_rsp_offset
 223   //          [ y_hi ]                  | (in bytes)
 224   //            ....                    |
 225   //          [ x_lo ]                 /
 226   //          [ x_hi ]
 227   //            ....
 228   //
 229   // Basic idea: lo(result) = lo(x_lo * y_lo)
 230   //             hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
 231   Address x_hi(rsp, x_rsp_offset + wordSize); Address x_lo(rsp, x_rsp_offset);
 232   Address y_hi(rsp, y_rsp_offset + wordSize); Address y_lo(rsp, y_rsp_offset);
 233   Label quick;
 234   // load x_hi, y_hi and check if quick
 235   // multiplication is possible
 236   movl(rbx, x_hi);
 237   movl(rcx, y_hi);
 238   movl(rax, rbx);
 239   orl(rbx, rcx);                                 // rbx, = 0 <=> x_hi = 0 and y_hi = 0
 240   jcc(Assembler::zero, quick);                   // if rbx, = 0 do quick multiply
 241   // do full multiplication
 242   // 1st step
 243   mull(y_lo);                                    // x_hi * y_lo
 244   movl(rbx, rax);                                // save lo(x_hi * y_lo) in rbx,
 245   // 2nd step
 246   movl(rax, x_lo);
 247   mull(rcx);                                     // x_lo * y_hi
 248   addl(rbx, rax);                                // add lo(x_lo * y_hi) to rbx,
 249   // 3rd step
 250   bind(quick);                                   // note: rbx, = 0 if quick multiply!
 251   movl(rax, x_lo);
 252   mull(y_lo);                                    // x_lo * y_lo
 253   addl(rdx, rbx);                                // correct hi(x_lo * y_lo)
 254 }
 255 
 256 void MacroAssembler::lneg(Register hi, Register lo) {
 257   negl(lo);
 258   adcl(hi, 0);
 259   negl(hi);
 260 }
 261 
 262 void MacroAssembler::lshl(Register hi, Register lo) {
 263   // Java shift left long support (semantics as described in JVM spec., p.305)
 264   // (basic idea for shift counts s >= n: x << s == (x << n) << (s - n))
 265   // shift value is in rcx !
 266   assert(hi != rcx, "must not use rcx");
 267   assert(lo != rcx, "must not use rcx");
 268   const Register s = rcx;                        // shift count
 269   const int      n = BitsPerWord;
 270   Label L;
 271   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 272   cmpl(s, n);                                    // if (s < n)
 273   jcc(Assembler::less, L);                       // else (s >= n)
 274   movl(hi, lo);                                  // x := x << n
 275   xorl(lo, lo);
 276   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 277   bind(L);                                       // s (mod n) < n
 278   shldl(hi, lo);                                 // x := x << s
 279   shll(lo);
 280 }
 281 
 282 
 283 void MacroAssembler::lshr(Register hi, Register lo, bool sign_extension) {
 284   // Java shift right long support (semantics as described in JVM spec., p.306 & p.310)
 285   // (basic idea for shift counts s >= n: x >> s == (x >> n) >> (s - n))
 286   assert(hi != rcx, "must not use rcx");
 287   assert(lo != rcx, "must not use rcx");
 288   const Register s = rcx;                        // shift count
 289   const int      n = BitsPerWord;
 290   Label L;
 291   andl(s, 0x3f);                                 // s := s & 0x3f (s < 0x40)
 292   cmpl(s, n);                                    // if (s < n)
 293   jcc(Assembler::less, L);                       // else (s >= n)
 294   movl(lo, hi);                                  // x := x >> n
 295   if (sign_extension) sarl(hi, 31);
 296   else                xorl(hi, hi);
 297   // Note: subl(s, n) is not needed since the Intel shift instructions work rcx mod n!
 298   bind(L);                                       // s (mod n) < n
 299   shrdl(lo, hi);                                 // x := x >> s
 300   if (sign_extension) sarl(hi);
 301   else                shrl(hi);
 302 }
 303 
 304 void MacroAssembler::movoop(Register dst, jobject obj) {
 305   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 306 }
 307 
 308 void MacroAssembler::movoop(Address dst, jobject obj) {
 309   mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 310 }
 311 
 312 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 313   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 314 }
 315 
 316 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 317   mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 318 }
 319 
 320 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 321   // scratch register is not used,
 322   // it is defined to match parameters of 64-bit version of this method.
 323   if (src.is_lval()) {
 324     mov_literal32(dst, (intptr_t)src.target(), src.rspec());
 325   } else {
 326     movl(dst, as_Address(src));
 327   }
 328 }
 329 
 330 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 331   movl(as_Address(dst), src);
 332 }
 333 
 334 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 335   movl(dst, as_Address(src));
 336 }
 337 
 338 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 339 void MacroAssembler::movptr(Address dst, intptr_t src) {
 340   movl(dst, src);
 341 }
 342 
 343 
 344 void MacroAssembler::pop_callee_saved_registers() {
 345   pop(rcx);
 346   pop(rdx);
 347   pop(rdi);
 348   pop(rsi);
 349 }
 350 
 351 void MacroAssembler::pop_fTOS() {
 352   fld_d(Address(rsp, 0));
 353   addl(rsp, 2 * wordSize);
 354 }
 355 
 356 void MacroAssembler::push_callee_saved_registers() {
 357   push(rsi);
 358   push(rdi);
 359   push(rdx);
 360   push(rcx);
 361 }
 362 
 363 void MacroAssembler::push_fTOS() {
 364   subl(rsp, 2 * wordSize);
 365   fstp_d(Address(rsp, 0));
 366 }
 367 
 368 
 369 void MacroAssembler::pushoop(jobject obj) {
 370   push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
 371 }
 372 
 373 void MacroAssembler::pushklass(Metadata* obj) {
 374   push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
 375 }
 376 
 377 void MacroAssembler::pushptr(AddressLiteral src) {
 378   if (src.is_lval()) {
 379     push_literal32((int32_t)src.target(), src.rspec());
 380   } else {
 381     pushl(as_Address(src));
 382   }
 383 }
 384 
 385 void MacroAssembler::set_word_if_not_zero(Register dst) {
 386   xorl(dst, dst);
 387   set_byte_if_not_zero(dst);
 388 }
 389 
 390 static void pass_arg0(MacroAssembler* masm, Register arg) {
 391   masm->push(arg);
 392 }
 393 
 394 static void pass_arg1(MacroAssembler* masm, Register arg) {
 395   masm->push(arg);
 396 }
 397 
 398 static void pass_arg2(MacroAssembler* masm, Register arg) {
 399   masm->push(arg);
 400 }
 401 
 402 static void pass_arg3(MacroAssembler* masm, Register arg) {
 403   masm->push(arg);
 404 }
 405 
 406 #ifndef PRODUCT
 407 extern "C" void findpc(intptr_t x);
 408 #endif
 409 
 410 void MacroAssembler::debug32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip, char* msg) {
 411   // In order to get locks to work, we need to fake a in_VM state
 412   JavaThread* thread = JavaThread::current();
 413   JavaThreadState saved_state = thread->thread_state();
 414   thread->set_thread_state(_thread_in_vm);
 415   if (ShowMessageBoxOnError) {
 416     JavaThread* thread = JavaThread::current();
 417     JavaThreadState saved_state = thread->thread_state();
 418     thread->set_thread_state(_thread_in_vm);
 419     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 420       ttyLocker ttyl;
 421       BytecodeCounter::print();
 422     }
 423     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 424     // This is the value of eip which points to where verify_oop will return.
 425     if (os::message_box(msg, "Execution stopped, print registers?")) {
 426       print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
 427       BREAKPOINT;
 428     }
 429   } else {
 430     ttyLocker ttyl;
 431     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
 432   }
 433   // Don't assert holding the ttyLock
 434     assert(false, "DEBUG MESSAGE: %s", msg);
 435   ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 436 }
 437 
 438 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
 439   ttyLocker ttyl;
 440   FlagSetting fs(Debugging, true);
 441   tty->print_cr("eip = 0x%08x", eip);
 442 #ifndef PRODUCT
 443   if ((WizardMode || Verbose) && PrintMiscellaneous) {
 444     tty->cr();
 445     findpc(eip);
 446     tty->cr();
 447   }
 448 #endif
 449 #define PRINT_REG(rax) \
 450   { tty->print("%s = ", #rax); os::print_location(tty, rax); }
 451   PRINT_REG(rax);
 452   PRINT_REG(rbx);
 453   PRINT_REG(rcx);
 454   PRINT_REG(rdx);
 455   PRINT_REG(rdi);
 456   PRINT_REG(rsi);
 457   PRINT_REG(rbp);
 458   PRINT_REG(rsp);
 459 #undef PRINT_REG
 460   // Print some words near top of staack.
 461   int* dump_sp = (int*) rsp;
 462   for (int col1 = 0; col1 < 8; col1++) {
 463     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 464     os::print_location(tty, *dump_sp++);
 465   }
 466   for (int row = 0; row < 16; row++) {
 467     tty->print("(rsp+0x%03x) 0x%08x: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 468     for (int col = 0; col < 8; col++) {
 469       tty->print(" 0x%08x", *dump_sp++);
 470     }
 471     tty->cr();
 472   }
 473   // Print some instructions around pc:
 474   Disassembler::decode((address)eip-64, (address)eip);
 475   tty->print_cr("--------");
 476   Disassembler::decode((address)eip, (address)eip+32);
 477 }
 478 
 479 void MacroAssembler::stop(const char* msg) {
 480   ExternalAddress message((address)msg);
 481   // push address of message
 482   pushptr(message.addr());
 483   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 484   pusha();                                            // push registers
 485   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 486   hlt();
 487 }
 488 
 489 void MacroAssembler::warn(const char* msg) {
 490   push_CPU_state();
 491 
 492   ExternalAddress message((address) msg);
 493   // push address of message
 494   pushptr(message.addr());
 495 
 496   call(RuntimeAddress(CAST_FROM_FN_PTR(address, warning)));
 497   addl(rsp, wordSize);       // discard argument
 498   pop_CPU_state();
 499 }
 500 
 501 void MacroAssembler::print_state() {
 502   { Label L; call(L, relocInfo::none); bind(L); }     // push eip
 503   pusha();                                            // push registers
 504 
 505   push_CPU_state();
 506   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::print_state32)));
 507   pop_CPU_state();
 508 
 509   popa();
 510   addl(rsp, wordSize);
 511 }
 512 
 513 #else // _LP64
 514 
 515 // 64 bit versions
 516 
 517 Address MacroAssembler::as_Address(AddressLiteral adr) {
 518   // amd64 always does this as a pc-rel
 519   // we can be absolute or disp based on the instruction type
 520   // jmp/call are displacements others are absolute
 521   assert(!adr.is_lval(), "must be rval");
 522   assert(reachable(adr), "must be");
 523   return Address((int32_t)(intptr_t)(adr.target() - pc()), adr.target(), adr.reloc());
 524 
 525 }
 526 
 527 Address MacroAssembler::as_Address(ArrayAddress adr) {
 528   AddressLiteral base = adr.base();
 529   lea(rscratch1, base);
 530   Address index = adr.index();
 531   assert(index._disp == 0, "must not have disp"); // maybe it can?
 532   Address array(rscratch1, index._index, index._scale, index._disp);
 533   return array;
 534 }
 535 
 536 void MacroAssembler::call_VM_leaf_base(address entry_point, int num_args) {
 537   Label L, E;
 538 
 539 #ifdef _WIN64
 540   // Windows always allocates space for it's register args
 541   assert(num_args <= 4, "only register arguments supported");
 542   subq(rsp,  frame::arg_reg_save_area_bytes);
 543 #endif
 544 
 545   // Align stack if necessary
 546   testl(rsp, 15);
 547   jcc(Assembler::zero, L);
 548 
 549   subq(rsp, 8);
 550   {
 551     call(RuntimeAddress(entry_point));
 552   }
 553   addq(rsp, 8);
 554   jmp(E);
 555 
 556   bind(L);
 557   {
 558     call(RuntimeAddress(entry_point));
 559   }
 560 
 561   bind(E);
 562 
 563 #ifdef _WIN64
 564   // restore stack pointer
 565   addq(rsp, frame::arg_reg_save_area_bytes);
 566 #endif
 567 
 568 }
 569 
 570 void MacroAssembler::cmp64(Register src1, AddressLiteral src2) {
 571   assert(!src2.is_lval(), "should use cmpptr");
 572 
 573   if (reachable(src2)) {
 574     cmpq(src1, as_Address(src2));
 575   } else {
 576     lea(rscratch1, src2);
 577     Assembler::cmpq(src1, Address(rscratch1, 0));
 578   }
 579 }
 580 
 581 int MacroAssembler::corrected_idivq(Register reg) {
 582   // Full implementation of Java ldiv and lrem; checks for special
 583   // case as described in JVM spec., p.243 & p.271.  The function
 584   // returns the (pc) offset of the idivl instruction - may be needed
 585   // for implicit exceptions.
 586   //
 587   //         normal case                           special case
 588   //
 589   // input : rax: dividend                         min_long
 590   //         reg: divisor   (may not be eax/edx)   -1
 591   //
 592   // output: rax: quotient  (= rax idiv reg)       min_long
 593   //         rdx: remainder (= rax irem reg)       0
 594   assert(reg != rax && reg != rdx, "reg cannot be rax or rdx register");
 595   static const int64_t min_long = 0x8000000000000000;
 596   Label normal_case, special_case;
 597 
 598   // check for special case
 599   cmp64(rax, ExternalAddress((address) &min_long));
 600   jcc(Assembler::notEqual, normal_case);
 601   xorl(rdx, rdx); // prepare rdx for possible special case (where
 602                   // remainder = 0)
 603   cmpq(reg, -1);
 604   jcc(Assembler::equal, special_case);
 605 
 606   // handle normal case
 607   bind(normal_case);
 608   cdqq();
 609   int idivq_offset = offset();
 610   idivq(reg);
 611 
 612   // normal and special case exit
 613   bind(special_case);
 614 
 615   return idivq_offset;
 616 }
 617 
 618 void MacroAssembler::decrementq(Register reg, int value) {
 619   if (value == min_jint) { subq(reg, value); return; }
 620   if (value <  0) { incrementq(reg, -value); return; }
 621   if (value == 0) {                        ; return; }
 622   if (value == 1 && UseIncDec) { decq(reg) ; return; }
 623   /* else */      { subq(reg, value)       ; return; }
 624 }
 625 
 626 void MacroAssembler::decrementq(Address dst, int value) {
 627   if (value == min_jint) { subq(dst, value); return; }
 628   if (value <  0) { incrementq(dst, -value); return; }
 629   if (value == 0) {                        ; return; }
 630   if (value == 1 && UseIncDec) { decq(dst) ; return; }
 631   /* else */      { subq(dst, value)       ; return; }
 632 }
 633 
 634 void MacroAssembler::incrementq(AddressLiteral dst) {
 635   if (reachable(dst)) {
 636     incrementq(as_Address(dst));
 637   } else {
 638     lea(rscratch1, dst);
 639     incrementq(Address(rscratch1, 0));
 640   }
 641 }
 642 
 643 void MacroAssembler::incrementq(Register reg, int value) {
 644   if (value == min_jint) { addq(reg, value); return; }
 645   if (value <  0) { decrementq(reg, -value); return; }
 646   if (value == 0) {                        ; return; }
 647   if (value == 1 && UseIncDec) { incq(reg) ; return; }
 648   /* else */      { addq(reg, value)       ; return; }
 649 }
 650 
 651 void MacroAssembler::incrementq(Address dst, int value) {
 652   if (value == min_jint) { addq(dst, value); return; }
 653   if (value <  0) { decrementq(dst, -value); return; }
 654   if (value == 0) {                        ; return; }
 655   if (value == 1 && UseIncDec) { incq(dst) ; return; }
 656   /* else */      { addq(dst, value)       ; return; }
 657 }
 658 
 659 // 32bit can do a case table jump in one instruction but we no longer allow the base
 660 // to be installed in the Address class
 661 void MacroAssembler::jump(ArrayAddress entry) {
 662   lea(rscratch1, entry.base());
 663   Address dispatch = entry.index();
 664   assert(dispatch._base == noreg, "must be");
 665   dispatch._base = rscratch1;
 666   jmp(dispatch);
 667 }
 668 
 669 void MacroAssembler::lcmp2int(Register x_hi, Register x_lo, Register y_hi, Register y_lo) {
 670   ShouldNotReachHere(); // 64bit doesn't use two regs
 671   cmpq(x_lo, y_lo);
 672 }
 673 
 674 void MacroAssembler::lea(Register dst, AddressLiteral src) {
 675     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 676 }
 677 
 678 void MacroAssembler::lea(Address dst, AddressLiteral adr) {
 679   mov_literal64(rscratch1, (intptr_t)adr.target(), adr.rspec());
 680   movptr(dst, rscratch1);
 681 }
 682 
 683 void MacroAssembler::leave() {
 684   // %%% is this really better? Why not on 32bit too?
 685   emit_int8((unsigned char)0xC9); // LEAVE
 686 }
 687 
 688 void MacroAssembler::lneg(Register hi, Register lo) {
 689   ShouldNotReachHere(); // 64bit doesn't use two regs
 690   negq(lo);
 691 }
 692 
 693 void MacroAssembler::movoop(Register dst, jobject obj) {
 694   mov_literal64(dst, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 695 }
 696 
 697 void MacroAssembler::movoop(Address dst, jobject obj) {
 698   mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 699   movq(dst, rscratch1);
 700 }
 701 
 702 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
 703   mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 704 }
 705 
 706 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 707   mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 708   movq(dst, rscratch1);
 709 }
 710 
 711 void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 712   if (src.is_lval()) {
 713     mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 714   } else {
 715     if (reachable(src)) {
 716       movq(dst, as_Address(src));
 717     } else {
 718       lea(scratch, src);
 719       movq(dst, Address(scratch, 0));
 720     }
 721   }
 722 }
 723 
 724 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 725   movq(as_Address(dst), src);
 726 }
 727 
 728 void MacroAssembler::movptr(Register dst, ArrayAddress src) {
 729   movq(dst, as_Address(src));
 730 }
 731 
 732 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
 733 void MacroAssembler::movptr(Address dst, intptr_t src) {
 734   mov64(rscratch1, src);
 735   movq(dst, rscratch1);
 736 }
 737 
 738 // These are mostly for initializing NULL
 739 void MacroAssembler::movptr(Address dst, int32_t src) {
 740   movslq(dst, src);
 741 }
 742 
 743 void MacroAssembler::movptr(Register dst, int32_t src) {
 744   mov64(dst, (intptr_t)src);
 745 }
 746 
 747 void MacroAssembler::pushoop(jobject obj) {
 748   movoop(rscratch1, obj);
 749   push(rscratch1);
 750 }
 751 
 752 void MacroAssembler::pushklass(Metadata* obj) {
 753   mov_metadata(rscratch1, obj);
 754   push(rscratch1);
 755 }
 756 
 757 void MacroAssembler::pushptr(AddressLiteral src) {
 758   lea(rscratch1, src);
 759   if (src.is_lval()) {
 760     push(rscratch1);
 761   } else {
 762     pushq(Address(rscratch1, 0));
 763   }
 764 }
 765 
 766 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
 767   // we must set sp to zero to clear frame
 768   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
 769   // must clear fp, so that compiled frames are not confused; it is
 770   // possible that we need it only for debugging
 771   if (clear_fp) {
 772     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
 773   }
 774 
 775   // Always clear the pc because it could have been set by make_walkable()
 776   movptr(Address(r15_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
 777   vzeroupper();
 778 }
 779 
 780 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 781                                          Register last_java_fp,
 782                                          address  last_java_pc) {
 783   vzeroupper();
 784   // determine last_java_sp register
 785   if (!last_java_sp->is_valid()) {
 786     last_java_sp = rsp;
 787   }
 788 
 789   // last_java_fp is optional
 790   if (last_java_fp->is_valid()) {
 791     movptr(Address(r15_thread, JavaThread::last_Java_fp_offset()),
 792            last_java_fp);
 793   }
 794 
 795   // last_java_pc is optional
 796   if (last_java_pc != NULL) {
 797     Address java_pc(r15_thread,
 798                     JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset());
 799     lea(rscratch1, InternalAddress(last_java_pc));
 800     movptr(java_pc, rscratch1);
 801   }
 802 
 803   movptr(Address(r15_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
 804 }
 805 
 806 static void pass_arg0(MacroAssembler* masm, Register arg) {
 807   if (c_rarg0 != arg ) {
 808     masm->mov(c_rarg0, arg);
 809   }
 810 }
 811 
 812 static void pass_arg1(MacroAssembler* masm, Register arg) {
 813   if (c_rarg1 != arg ) {
 814     masm->mov(c_rarg1, arg);
 815   }
 816 }
 817 
 818 static void pass_arg2(MacroAssembler* masm, Register arg) {
 819   if (c_rarg2 != arg ) {
 820     masm->mov(c_rarg2, arg);
 821   }
 822 }
 823 
 824 static void pass_arg3(MacroAssembler* masm, Register arg) {
 825   if (c_rarg3 != arg ) {
 826     masm->mov(c_rarg3, arg);
 827   }
 828 }
 829 
 830 void MacroAssembler::stop(const char* msg) {
 831   address rip = pc();
 832   pusha(); // get regs on stack
 833   lea(c_rarg0, ExternalAddress((address) msg));
 834   lea(c_rarg1, InternalAddress(rip));
 835   movq(c_rarg2, rsp); // pass pointer to regs array
 836   andq(rsp, -16); // align stack as required by ABI
 837   call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
 838   hlt();
 839 }
 840 
 841 void MacroAssembler::warn(const char* msg) {
 842   push(rbp);
 843   movq(rbp, rsp);
 844   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 845   push_CPU_state();   // keeps alignment at 16 bytes
 846   lea(c_rarg0, ExternalAddress((address) msg));
 847   lea(rax, ExternalAddress(CAST_FROM_FN_PTR(address, warning)));
 848   call(rax);
 849   pop_CPU_state();
 850   mov(rsp, rbp);
 851   pop(rbp);
 852 }
 853 
 854 void MacroAssembler::print_state() {
 855   address rip = pc();
 856   pusha();            // get regs on stack
 857   push(rbp);
 858   movq(rbp, rsp);
 859   andq(rsp, -16);     // align stack as required by push_CPU_state and call
 860   push_CPU_state();   // keeps alignment at 16 bytes
 861 
 862   lea(c_rarg0, InternalAddress(rip));
 863   lea(c_rarg1, Address(rbp, wordSize)); // pass pointer to regs array
 864   call_VM_leaf(CAST_FROM_FN_PTR(address, MacroAssembler::print_state64), c_rarg0, c_rarg1);
 865 
 866   pop_CPU_state();
 867   mov(rsp, rbp);
 868   pop(rbp);
 869   popa();
 870 }
 871 
 872 #ifndef PRODUCT
 873 extern "C" void findpc(intptr_t x);
 874 #endif
 875 
 876 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) {
 877   // In order to get locks to work, we need to fake a in_VM state
 878   if (ShowMessageBoxOnError) {
 879     JavaThread* thread = JavaThread::current();
 880     JavaThreadState saved_state = thread->thread_state();
 881     thread->set_thread_state(_thread_in_vm);
 882 #ifndef PRODUCT
 883     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 884       ttyLocker ttyl;
 885       BytecodeCounter::print();
 886     }
 887 #endif
 888     // To see where a verify_oop failed, get $ebx+40/X for this frame.
 889     // XXX correct this offset for amd64
 890     // This is the value of eip which points to where verify_oop will return.
 891     if (os::message_box(msg, "Execution stopped, print registers?")) {
 892       print_state64(pc, regs);
 893       BREAKPOINT;
 894       assert(false, "start up GDB");
 895     }
 896     ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 897   } else {
 898     ttyLocker ttyl;
 899     ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n",
 900                     msg);
 901     assert(false, "DEBUG MESSAGE: %s", msg);
 902   }
 903 }
 904 
 905 void MacroAssembler::print_state64(int64_t pc, int64_t regs[]) {
 906   ttyLocker ttyl;
 907   FlagSetting fs(Debugging, true);
 908   tty->print_cr("rip = 0x%016lx", (intptr_t)pc);
 909 #ifndef PRODUCT
 910   tty->cr();
 911   findpc(pc);
 912   tty->cr();
 913 #endif
 914 #define PRINT_REG(rax, value) \
 915   { tty->print("%s = ", #rax); os::print_location(tty, value); }
 916   PRINT_REG(rax, regs[15]);
 917   PRINT_REG(rbx, regs[12]);
 918   PRINT_REG(rcx, regs[14]);
 919   PRINT_REG(rdx, regs[13]);
 920   PRINT_REG(rdi, regs[8]);
 921   PRINT_REG(rsi, regs[9]);
 922   PRINT_REG(rbp, regs[10]);
 923   PRINT_REG(rsp, regs[11]);
 924   PRINT_REG(r8 , regs[7]);
 925   PRINT_REG(r9 , regs[6]);
 926   PRINT_REG(r10, regs[5]);
 927   PRINT_REG(r11, regs[4]);
 928   PRINT_REG(r12, regs[3]);
 929   PRINT_REG(r13, regs[2]);
 930   PRINT_REG(r14, regs[1]);
 931   PRINT_REG(r15, regs[0]);
 932 #undef PRINT_REG
 933   // Print some words near top of staack.
 934   int64_t* rsp = (int64_t*) regs[11];
 935   int64_t* dump_sp = rsp;
 936   for (int col1 = 0; col1 < 8; col1++) {
 937     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 938     os::print_location(tty, *dump_sp++);
 939   }
 940   for (int row = 0; row < 25; row++) {
 941     tty->print("(rsp+0x%03x) 0x%016lx: ", (int)((intptr_t)dump_sp - (intptr_t)rsp), (intptr_t)dump_sp);
 942     for (int col = 0; col < 4; col++) {
 943       tty->print(" 0x%016lx", (intptr_t)*dump_sp++);
 944     }
 945     tty->cr();
 946   }
 947   // Print some instructions around pc:
 948   Disassembler::decode((address)pc-64, (address)pc);
 949   tty->print_cr("--------");
 950   Disassembler::decode((address)pc, (address)pc+32);
 951 }
 952 
 953 #endif // _LP64
 954 
 955 // Now versions that are common to 32/64 bit
 956 
 957 void MacroAssembler::addptr(Register dst, int32_t imm32) {
 958   LP64_ONLY(addq(dst, imm32)) NOT_LP64(addl(dst, imm32));
 959 }
 960 
 961 void MacroAssembler::addptr(Register dst, Register src) {
 962   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 963 }
 964 
 965 void MacroAssembler::addptr(Address dst, Register src) {
 966   LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 967 }
 968 
 969 void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {
 970   if (reachable(src)) {
 971     Assembler::addsd(dst, as_Address(src));
 972   } else {
 973     lea(rscratch1, src);
 974     Assembler::addsd(dst, Address(rscratch1, 0));
 975   }
 976 }
 977 
 978 void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {
 979   if (reachable(src)) {
 980     addss(dst, as_Address(src));
 981   } else {
 982     lea(rscratch1, src);
 983     addss(dst, Address(rscratch1, 0));
 984   }
 985 }
 986 
 987 void MacroAssembler::addpd(XMMRegister dst, AddressLiteral src) {
 988   if (reachable(src)) {
 989     Assembler::addpd(dst, as_Address(src));
 990   } else {
 991     lea(rscratch1, src);
 992     Assembler::addpd(dst, Address(rscratch1, 0));
 993   }
 994 }
 995 
 996 void MacroAssembler::align(int modulus) {
 997   align(modulus, offset());
 998 }
 999 
1000 void MacroAssembler::align(int modulus, int target) {
1001   if (target % modulus != 0) {
1002     nop(modulus - (target % modulus));
1003   }
1004 }
1005 
1006 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {
1007   // Used in sign-masking with aligned address.
1008   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1009   if (reachable(src)) {
1010     Assembler::andpd(dst, as_Address(src));
1011   } else {
1012     lea(rscratch1, src);
1013     Assembler::andpd(dst, Address(rscratch1, 0));
1014   }
1015 }
1016 
1017 void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {
1018   // Used in sign-masking with aligned address.
1019   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
1020   if (reachable(src)) {
1021     Assembler::andps(dst, as_Address(src));
1022   } else {
1023     lea(rscratch1, src);
1024     Assembler::andps(dst, Address(rscratch1, 0));
1025   }
1026 }
1027 
1028 void MacroAssembler::andptr(Register dst, int32_t imm32) {
1029   LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
1030 }
1031 
1032 void MacroAssembler::atomic_incl(Address counter_addr) {
1033   if (os::is_MP())
1034     lock();
1035   incrementl(counter_addr);
1036 }
1037 
1038 void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
1039   if (reachable(counter_addr)) {
1040     atomic_incl(as_Address(counter_addr));
1041   } else {
1042     lea(scr, counter_addr);
1043     atomic_incl(Address(scr, 0));
1044   }
1045 }
1046 
1047 #ifdef _LP64
1048 void MacroAssembler::atomic_incq(Address counter_addr) {
1049   if (os::is_MP())
1050     lock();
1051   incrementq(counter_addr);
1052 }
1053 
1054 void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) {
1055   if (reachable(counter_addr)) {
1056     atomic_incq(as_Address(counter_addr));
1057   } else {
1058     lea(scr, counter_addr);
1059     atomic_incq(Address(scr, 0));
1060   }
1061 }
1062 #endif
1063 
1064 // Writes to stack successive pages until offset reached to check for
1065 // stack overflow + shadow pages.  This clobbers tmp.
1066 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
1067   movptr(tmp, rsp);
1068   // Bang stack for total size given plus shadow page size.
1069   // Bang one page at a time because large size can bang beyond yellow and
1070   // red zones.
1071   Label loop;
1072   bind(loop);
1073   movl(Address(tmp, (-os::vm_page_size())), size );
1074   subptr(tmp, os::vm_page_size());
1075   subl(size, os::vm_page_size());
1076   jcc(Assembler::greater, loop);
1077 
1078   // Bang down shadow pages too.
1079   // At this point, (tmp-0) is the last address touched, so don't
1080   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
1081   // was post-decremented.)  Skip this address by starting at i=1, and
1082   // touch a few more pages below.  N.B.  It is important to touch all
1083   // the way down including all pages in the shadow zone.
1084   for (int i = 1; i < ((int)JavaThread::stack_shadow_zone_size() / os::vm_page_size()); i++) {
1085     // this could be any sized move but this is can be a debugging crumb
1086     // so the bigger the better.
1087     movptr(Address(tmp, (-i*os::vm_page_size())), size );
1088   }
1089 }
1090 
1091 void MacroAssembler::reserved_stack_check() {
1092     // testing if reserved zone needs to be enabled
1093     Label no_reserved_zone_enabling;
1094     Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
1095     NOT_LP64(get_thread(rsi);)
1096 
1097     cmpptr(rsp, Address(thread, JavaThread::reserved_stack_activation_offset()));
1098     jcc(Assembler::below, no_reserved_zone_enabling);
1099 
1100     call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), thread);
1101     jump(RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()));
1102     should_not_reach_here();
1103 
1104     bind(no_reserved_zone_enabling);
1105 }
1106 
1107 int MacroAssembler::biased_locking_enter(Register lock_reg,
1108                                          Register obj_reg,
1109                                          Register swap_reg,
1110                                          Register tmp_reg,
1111                                          bool swap_reg_contains_mark,
1112                                          Label& done,
1113                                          Label* slow_case,
1114                                          BiasedLockingCounters* counters) {
1115   assert(UseBiasedLocking, "why call this otherwise?");
1116   assert(swap_reg == rax, "swap_reg must be rax for cmpxchgq");
1117   assert(tmp_reg != noreg, "tmp_reg must be supplied");
1118   assert_different_registers(lock_reg, obj_reg, swap_reg, tmp_reg);
1119   assert(markOopDesc::age_shift == markOopDesc::lock_bits + markOopDesc::biased_lock_bits, "biased locking makes assumptions about bit layout");
1120   Address mark_addr      (obj_reg, oopDesc::mark_offset_in_bytes());
1121   NOT_LP64( Address saved_mark_addr(lock_reg, 0); )
1122 
1123   if (PrintBiasedLockingStatistics && counters == NULL) {
1124     counters = BiasedLocking::counters();
1125   }
1126   // Biased locking
1127   // See whether the lock is currently biased toward our thread and
1128   // whether the epoch is still valid
1129   // Note that the runtime guarantees sufficient alignment of JavaThread
1130   // pointers to allow age to be placed into low bits
1131   // First check to see whether biasing is even enabled for this object
1132   Label cas_label;
1133   int null_check_offset = -1;
1134   if (!swap_reg_contains_mark) {
1135     null_check_offset = offset();
1136     movptr(swap_reg, mark_addr);
1137   }
1138   movptr(tmp_reg, swap_reg);
1139   andptr(tmp_reg, markOopDesc::biased_lock_mask_in_place);
1140   cmpptr(tmp_reg, markOopDesc::biased_lock_pattern);
1141   jcc(Assembler::notEqual, cas_label);
1142   // The bias pattern is present in the object's header. Need to check
1143   // whether the bias owner and the epoch are both still current.
1144 #ifndef _LP64
1145   // Note that because there is no current thread register on x86_32 we
1146   // need to store off the mark word we read out of the object to
1147   // avoid reloading it and needing to recheck invariants below. This
1148   // store is unfortunate but it makes the overall code shorter and
1149   // simpler.
1150   movptr(saved_mark_addr, swap_reg);
1151 #endif
1152   if (swap_reg_contains_mark) {
1153     null_check_offset = offset();
1154   }
1155   load_prototype_header(tmp_reg, obj_reg);
1156 #ifdef _LP64
1157   orptr(tmp_reg, r15_thread);
1158   xorptr(tmp_reg, swap_reg);
1159   Register header_reg = tmp_reg;
1160 #else
1161   xorptr(tmp_reg, swap_reg);
1162   get_thread(swap_reg);
1163   xorptr(swap_reg, tmp_reg);
1164   Register header_reg = swap_reg;
1165 #endif
1166   andptr(header_reg, ~((int) markOopDesc::age_mask_in_place));
1167   if (counters != NULL) {
1168     cond_inc32(Assembler::zero,
1169                ExternalAddress((address) counters->biased_lock_entry_count_addr()));
1170   }
1171   jcc(Assembler::equal, done);
1172 
1173   Label try_revoke_bias;
1174   Label try_rebias;
1175 
1176   // At this point we know that the header has the bias pattern and
1177   // that we are not the bias owner in the current epoch. We need to
1178   // figure out more details about the state of the header in order to
1179   // know what operations can be legally performed on the object's
1180   // header.
1181 
1182   // If the low three bits in the xor result aren't clear, that means
1183   // the prototype header is no longer biased and we have to revoke
1184   // the bias on this object.
1185   testptr(header_reg, markOopDesc::biased_lock_mask_in_place);
1186   jccb(Assembler::notZero, try_revoke_bias);
1187 
1188   // Biasing is still enabled for this data type. See whether the
1189   // epoch of the current bias is still valid, meaning that the epoch
1190   // bits of the mark word are equal to the epoch bits of the
1191   // prototype header. (Note that the prototype header's epoch bits
1192   // only change at a safepoint.) If not, attempt to rebias the object
1193   // toward the current thread. Note that we must be absolutely sure
1194   // that the current epoch is invalid in order to do this because
1195   // otherwise the manipulations it performs on the mark word are
1196   // illegal.
1197   testptr(header_reg, markOopDesc::epoch_mask_in_place);
1198   jccb(Assembler::notZero, try_rebias);
1199 
1200   // The epoch of the current bias is still valid but we know nothing
1201   // about the owner; it might be set or it might be clear. Try to
1202   // acquire the bias of the object using an atomic operation. If this
1203   // fails we will go in to the runtime to revoke the object's bias.
1204   // Note that we first construct the presumed unbiased header so we
1205   // don't accidentally blow away another thread's valid bias.
1206   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1207   andptr(swap_reg,
1208          markOopDesc::biased_lock_mask_in_place | markOopDesc::age_mask_in_place | markOopDesc::epoch_mask_in_place);
1209 #ifdef _LP64
1210   movptr(tmp_reg, swap_reg);
1211   orptr(tmp_reg, r15_thread);
1212 #else
1213   get_thread(tmp_reg);
1214   orptr(tmp_reg, swap_reg);
1215 #endif
1216   if (os::is_MP()) {
1217     lock();
1218   }
1219   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1220   // If the biasing toward our thread failed, this means that
1221   // another thread succeeded in biasing it toward itself and we
1222   // need to revoke that bias. The revocation will occur in the
1223   // interpreter runtime in the slow case.
1224   if (counters != NULL) {
1225     cond_inc32(Assembler::zero,
1226                ExternalAddress((address) counters->anonymously_biased_lock_entry_count_addr()));
1227   }
1228   if (slow_case != NULL) {
1229     jcc(Assembler::notZero, *slow_case);
1230   }
1231   jmp(done);
1232 
1233   bind(try_rebias);
1234   // At this point we know the epoch has expired, meaning that the
1235   // current "bias owner", if any, is actually invalid. Under these
1236   // circumstances _only_, we are allowed to use the current header's
1237   // value as the comparison value when doing the cas to acquire the
1238   // bias in the current epoch. In other words, we allow transfer of
1239   // the bias from one thread to another directly in this situation.
1240   //
1241   // FIXME: due to a lack of registers we currently blow away the age
1242   // bits in this situation. Should attempt to preserve them.
1243   load_prototype_header(tmp_reg, obj_reg);
1244 #ifdef _LP64
1245   orptr(tmp_reg, r15_thread);
1246 #else
1247   get_thread(swap_reg);
1248   orptr(tmp_reg, swap_reg);
1249   movptr(swap_reg, saved_mark_addr);
1250 #endif
1251   if (os::is_MP()) {
1252     lock();
1253   }
1254   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1255   // If the biasing toward our thread failed, then another thread
1256   // succeeded in biasing it toward itself and we need to revoke that
1257   // bias. The revocation will occur in the runtime in the slow case.
1258   if (counters != NULL) {
1259     cond_inc32(Assembler::zero,
1260                ExternalAddress((address) counters->rebiased_lock_entry_count_addr()));
1261   }
1262   if (slow_case != NULL) {
1263     jcc(Assembler::notZero, *slow_case);
1264   }
1265   jmp(done);
1266 
1267   bind(try_revoke_bias);
1268   // The prototype mark in the klass doesn't have the bias bit set any
1269   // more, indicating that objects of this data type are not supposed
1270   // to be biased any more. We are going to try to reset the mark of
1271   // this object to the prototype value and fall through to the
1272   // CAS-based locking scheme. Note that if our CAS fails, it means
1273   // that another thread raced us for the privilege of revoking the
1274   // bias of this particular object, so it's okay to continue in the
1275   // normal locking code.
1276   //
1277   // FIXME: due to a lack of registers we currently blow away the age
1278   // bits in this situation. Should attempt to preserve them.
1279   NOT_LP64( movptr(swap_reg, saved_mark_addr); )
1280   load_prototype_header(tmp_reg, obj_reg);
1281   if (os::is_MP()) {
1282     lock();
1283   }
1284   cmpxchgptr(tmp_reg, mark_addr); // compare tmp_reg and swap_reg
1285   // Fall through to the normal CAS-based lock, because no matter what
1286   // the result of the above CAS, some thread must have succeeded in
1287   // removing the bias bit from the object's header.
1288   if (counters != NULL) {
1289     cond_inc32(Assembler::zero,
1290                ExternalAddress((address) counters->revoked_lock_entry_count_addr()));
1291   }
1292 
1293   bind(cas_label);
1294 
1295   return null_check_offset;
1296 }
1297 
1298 void MacroAssembler::biased_locking_exit(Register obj_reg, Register temp_reg, Label& done) {
1299   assert(UseBiasedLocking, "why call this otherwise?");
1300 
1301   // Check for biased locking unlock case, which is a no-op
1302   // Note: we do not have to check the thread ID for two reasons.
1303   // First, the interpreter checks for IllegalMonitorStateException at
1304   // a higher level. Second, if the bias was revoked while we held the
1305   // lock, the object could not be rebiased toward another thread, so
1306   // the bias bit would be clear.
1307   movptr(temp_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1308   andptr(temp_reg, markOopDesc::biased_lock_mask_in_place);
1309   cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
1310   jcc(Assembler::equal, done);
1311 }
1312 
1313 #ifdef COMPILER2
1314 
1315 #if INCLUDE_RTM_OPT
1316 
1317 // Update rtm_counters based on abort status
1318 // input: abort_status
1319 //        rtm_counters (RTMLockingCounters*)
1320 // flags are killed
1321 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) {
1322 
1323   atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset()));
1324   if (PrintPreciseRTMLockingStatistics) {
1325     for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
1326       Label check_abort;
1327       testl(abort_status, (1<<i));
1328       jccb(Assembler::equal, check_abort);
1329       atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx))));
1330       bind(check_abort);
1331     }
1332   }
1333 }
1334 
1335 // Branch if (random & (count-1) != 0), count is 2^n
1336 // tmp, scr and flags are killed
1337 void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) {
1338   assert(tmp == rax, "");
1339   assert(scr == rdx, "");
1340   rdtsc(); // modifies EDX:EAX
1341   andptr(tmp, count-1);
1342   jccb(Assembler::notZero, brLabel);
1343 }
1344 
1345 // Perform abort ratio calculation, set no_rtm bit if high ratio
1346 // input:  rtm_counters_Reg (RTMLockingCounters* address)
1347 // tmpReg, rtm_counters_Reg and flags are killed
1348 void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg,
1349                                                  Register rtm_counters_Reg,
1350                                                  RTMLockingCounters* rtm_counters,
1351                                                  Metadata* method_data) {
1352   Label L_done, L_check_always_rtm1, L_check_always_rtm2;
1353 
1354   if (RTMLockingCalculationDelay > 0) {
1355     // Delay calculation
1356     movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg);
1357     testptr(tmpReg, tmpReg);
1358     jccb(Assembler::equal, L_done);
1359   }
1360   // Abort ratio calculation only if abort_count > RTMAbortThreshold
1361   //   Aborted transactions = abort_count * 100
1362   //   All transactions = total_count *  RTMTotalCountIncrRate
1363   //   Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio)
1364 
1365   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset()));
1366   cmpptr(tmpReg, RTMAbortThreshold);
1367   jccb(Assembler::below, L_check_always_rtm2);
1368   imulptr(tmpReg, tmpReg, 100);
1369 
1370   Register scrReg = rtm_counters_Reg;
1371   movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1372   imulptr(scrReg, scrReg, RTMTotalCountIncrRate);
1373   imulptr(scrReg, scrReg, RTMAbortRatio);
1374   cmpptr(tmpReg, scrReg);
1375   jccb(Assembler::below, L_check_always_rtm1);
1376   if (method_data != NULL) {
1377     // set rtm_state to "no rtm" in MDO
1378     mov_metadata(tmpReg, method_data);
1379     if (os::is_MP()) {
1380       lock();
1381     }
1382     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM);
1383   }
1384   jmpb(L_done);
1385   bind(L_check_always_rtm1);
1386   // Reload RTMLockingCounters* address
1387   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1388   bind(L_check_always_rtm2);
1389   movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
1390   cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate);
1391   jccb(Assembler::below, L_done);
1392   if (method_data != NULL) {
1393     // set rtm_state to "always rtm" in MDO
1394     mov_metadata(tmpReg, method_data);
1395     if (os::is_MP()) {
1396       lock();
1397     }
1398     orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM);
1399   }
1400   bind(L_done);
1401 }
1402 
1403 // Update counters and perform abort ratio calculation
1404 // input:  abort_status_Reg
1405 // rtm_counters_Reg, flags are killed
1406 void MacroAssembler::rtm_profiling(Register abort_status_Reg,
1407                                    Register rtm_counters_Reg,
1408                                    RTMLockingCounters* rtm_counters,
1409                                    Metadata* method_data,
1410                                    bool profile_rtm) {
1411 
1412   assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1413   // update rtm counters based on rax value at abort
1414   // reads abort_status_Reg, updates flags
1415   lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
1416   rtm_counters_update(abort_status_Reg, rtm_counters_Reg);
1417   if (profile_rtm) {
1418     // Save abort status because abort_status_Reg is used by following code.
1419     if (RTMRetryCount > 0) {
1420       push(abort_status_Reg);
1421     }
1422     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1423     rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data);
1424     // restore abort status
1425     if (RTMRetryCount > 0) {
1426       pop(abort_status_Reg);
1427     }
1428   }
1429 }
1430 
1431 // Retry on abort if abort's status is 0x6: can retry (0x2) | memory conflict (0x4)
1432 // inputs: retry_count_Reg
1433 //       : abort_status_Reg
1434 // output: retry_count_Reg decremented by 1
1435 // flags are killed
1436 void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) {
1437   Label doneRetry;
1438   assert(abort_status_Reg == rax, "");
1439   // The abort reason bits are in eax (see all states in rtmLocking.hpp)
1440   // 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4)
1441   // if reason is in 0x6 and retry count != 0 then retry
1442   andptr(abort_status_Reg, 0x6);
1443   jccb(Assembler::zero, doneRetry);
1444   testl(retry_count_Reg, retry_count_Reg);
1445   jccb(Assembler::zero, doneRetry);
1446   pause();
1447   decrementl(retry_count_Reg);
1448   jmp(retryLabel);
1449   bind(doneRetry);
1450 }
1451 
1452 // Spin and retry if lock is busy,
1453 // inputs: box_Reg (monitor address)
1454 //       : retry_count_Reg
1455 // output: retry_count_Reg decremented by 1
1456 //       : clear z flag if retry count exceeded
1457 // tmp_Reg, scr_Reg, flags are killed
1458 void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg,
1459                                             Register tmp_Reg, Register scr_Reg, Label& retryLabel) {
1460   Label SpinLoop, SpinExit, doneRetry;
1461   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1462 
1463   testl(retry_count_Reg, retry_count_Reg);
1464   jccb(Assembler::zero, doneRetry);
1465   decrementl(retry_count_Reg);
1466   movptr(scr_Reg, RTMSpinLoopCount);
1467 
1468   bind(SpinLoop);
1469   pause();
1470   decrementl(scr_Reg);
1471   jccb(Assembler::lessEqual, SpinExit);
1472   movptr(tmp_Reg, Address(box_Reg, owner_offset));
1473   testptr(tmp_Reg, tmp_Reg);
1474   jccb(Assembler::notZero, SpinLoop);
1475 
1476   bind(SpinExit);
1477   jmp(retryLabel);
1478   bind(doneRetry);
1479   incrementl(retry_count_Reg); // clear z flag
1480 }
1481 
1482 // Use RTM for normal stack locks
1483 // Input: objReg (object to lock)
1484 void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg,
1485                                        Register retry_on_abort_count_Reg,
1486                                        RTMLockingCounters* stack_rtm_counters,
1487                                        Metadata* method_data, bool profile_rtm,
1488                                        Label& DONE_LABEL, Label& IsInflated) {
1489   assert(UseRTMForStackLocks, "why call this otherwise?");
1490   assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1491   assert(tmpReg == rax, "");
1492   assert(scrReg == rdx, "");
1493   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1494 
1495   if (RTMRetryCount > 0) {
1496     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1497     bind(L_rtm_retry);
1498   }
1499   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1500   testptr(tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased
1501   jcc(Assembler::notZero, IsInflated);
1502 
1503   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1504     Label L_noincrement;
1505     if (RTMTotalCountIncrRate > 1) {
1506       // tmpReg, scrReg and flags are killed
1507       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1508     }
1509     assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM");
1510     atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg);
1511     bind(L_noincrement);
1512   }
1513   xbegin(L_on_abort);
1514   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));       // fetch markword
1515   andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
1516   cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
1517   jcc(Assembler::equal, DONE_LABEL);        // all done if unlocked
1518 
1519   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1520   if (UseRTMXendForLockBusy) {
1521     xend();
1522     movptr(abort_status_Reg, 0x2);   // Set the abort status to 2 (so we can retry)
1523     jmp(L_decrement_retry);
1524   }
1525   else {
1526     xabort(0);
1527   }
1528   bind(L_on_abort);
1529   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1530     rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm);
1531   }
1532   bind(L_decrement_retry);
1533   if (RTMRetryCount > 0) {
1534     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1535     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1536   }
1537 }
1538 
1539 // Use RTM for inflating locks
1540 // inputs: objReg (object to lock)
1541 //         boxReg (on-stack box address (displaced header location) - KILLED)
1542 //         tmpReg (ObjectMonitor address + markOopDesc::monitor_value)
1543 void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg,
1544                                           Register scrReg, Register retry_on_busy_count_Reg,
1545                                           Register retry_on_abort_count_Reg,
1546                                           RTMLockingCounters* rtm_counters,
1547                                           Metadata* method_data, bool profile_rtm,
1548                                           Label& DONE_LABEL) {
1549   assert(UseRTMLocking, "why call this otherwise?");
1550   assert(tmpReg == rax, "");
1551   assert(scrReg == rdx, "");
1552   Label L_rtm_retry, L_decrement_retry, L_on_abort;
1553   int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1554 
1555   // Without cast to int32_t a movptr will destroy r10 which is typically obj
1556   movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1557   movptr(boxReg, tmpReg); // Save ObjectMonitor address
1558 
1559   if (RTMRetryCount > 0) {
1560     movl(retry_on_busy_count_Reg, RTMRetryCount);  // Retry on lock busy
1561     movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
1562     bind(L_rtm_retry);
1563   }
1564   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1565     Label L_noincrement;
1566     if (RTMTotalCountIncrRate > 1) {
1567       // tmpReg, scrReg and flags are killed
1568       branch_on_random_using_rdtsc(tmpReg, scrReg, RTMTotalCountIncrRate, L_noincrement);
1569     }
1570     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
1571     atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);
1572     bind(L_noincrement);
1573   }
1574   xbegin(L_on_abort);
1575   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));
1576   movptr(tmpReg, Address(tmpReg, owner_offset));
1577   testptr(tmpReg, tmpReg);
1578   jcc(Assembler::zero, DONE_LABEL);
1579   if (UseRTMXendForLockBusy) {
1580     xend();
1581     jmp(L_decrement_retry);
1582   }
1583   else {
1584     xabort(0);
1585   }
1586   bind(L_on_abort);
1587   Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
1588   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
1589     rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm);
1590   }
1591   if (RTMRetryCount > 0) {
1592     // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
1593     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
1594   }
1595 
1596   movptr(tmpReg, Address(boxReg, owner_offset)) ;
1597   testptr(tmpReg, tmpReg) ;
1598   jccb(Assembler::notZero, L_decrement_retry) ;
1599 
1600   // Appears unlocked - try to swing _owner from null to non-null.
1601   // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1602 #ifdef _LP64
1603   Register threadReg = r15_thread;
1604 #else
1605   get_thread(scrReg);
1606   Register threadReg = scrReg;
1607 #endif
1608   if (os::is_MP()) {
1609     lock();
1610   }
1611   cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg
1612 
1613   if (RTMRetryCount > 0) {
1614     // success done else retry
1615     jccb(Assembler::equal, DONE_LABEL) ;
1616     bind(L_decrement_retry);
1617     // Spin and retry if lock is busy.
1618     rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
1619   }
1620   else {
1621     bind(L_decrement_retry);
1622   }
1623 }
1624 
1625 #endif //  INCLUDE_RTM_OPT
1626 
1627 // Fast_Lock and Fast_Unlock used by C2
1628 
1629 // Because the transitions from emitted code to the runtime
1630 // monitorenter/exit helper stubs are so slow it's critical that
1631 // we inline both the stack-locking fast-path and the inflated fast path.
1632 //
1633 // See also: cmpFastLock and cmpFastUnlock.
1634 //
1635 // What follows is a specialized inline transliteration of the code
1636 // in slow_enter() and slow_exit().  If we're concerned about I$ bloat
1637 // another option would be to emit TrySlowEnter and TrySlowExit methods
1638 // at startup-time.  These methods would accept arguments as
1639 // (rax,=Obj, rbx=Self, rcx=box, rdx=Scratch) and return success-failure
1640 // indications in the icc.ZFlag.  Fast_Lock and Fast_Unlock would simply
1641 // marshal the arguments and emit calls to TrySlowEnter and TrySlowExit.
1642 // In practice, however, the # of lock sites is bounded and is usually small.
1643 // Besides the call overhead, TrySlowEnter and TrySlowExit might suffer
1644 // if the processor uses simple bimodal branch predictors keyed by EIP
1645 // Since the helper routines would be called from multiple synchronization
1646 // sites.
1647 //
1648 // An even better approach would be write "MonitorEnter()" and "MonitorExit()"
1649 // in java - using j.u.c and unsafe - and just bind the lock and unlock sites
1650 // to those specialized methods.  That'd give us a mostly platform-independent
1651 // implementation that the JITs could optimize and inline at their pleasure.
1652 // Done correctly, the only time we'd need to cross to native could would be
1653 // to park() or unpark() threads.  We'd also need a few more unsafe operators
1654 // to (a) prevent compiler-JIT reordering of non-volatile accesses, and
1655 // (b) explicit barriers or fence operations.
1656 //
1657 // TODO:
1658 //
1659 // *  Arrange for C2 to pass "Self" into Fast_Lock and Fast_Unlock in one of the registers (scr).
1660 //    This avoids manifesting the Self pointer in the Fast_Lock and Fast_Unlock terminals.
1661 //    Given TLAB allocation, Self is usually manifested in a register, so passing it into
1662 //    the lock operators would typically be faster than reifying Self.
1663 //
1664 // *  Ideally I'd define the primitives as:
1665 //       fast_lock   (nax Obj, nax box, EAX tmp, nax scr) where box, tmp and scr are KILLED.
1666 //       fast_unlock (nax Obj, EAX box, nax tmp) where box and tmp are KILLED
1667 //    Unfortunately ADLC bugs prevent us from expressing the ideal form.
1668 //    Instead, we're stuck with a rather awkward and brittle register assignments below.
1669 //    Furthermore the register assignments are overconstrained, possibly resulting in
1670 //    sub-optimal code near the synchronization site.
1671 //
1672 // *  Eliminate the sp-proximity tests and just use "== Self" tests instead.
1673 //    Alternately, use a better sp-proximity test.
1674 //
1675 // *  Currently ObjectMonitor._Owner can hold either an sp value or a (THREAD *) value.
1676 //    Either one is sufficient to uniquely identify a thread.
1677 //    TODO: eliminate use of sp in _owner and use get_thread(tr) instead.
1678 //
1679 // *  Intrinsify notify() and notifyAll() for the common cases where the
1680 //    object is locked by the calling thread but the waitlist is empty.
1681 //    avoid the expensive JNI call to JVM_Notify() and JVM_NotifyAll().
1682 //
1683 // *  use jccb and jmpb instead of jcc and jmp to improve code density.
1684 //    But beware of excessive branch density on AMD Opterons.
1685 //
1686 // *  Both Fast_Lock and Fast_Unlock set the ICC.ZF to indicate success
1687 //    or failure of the fast-path.  If the fast-path fails then we pass
1688 //    control to the slow-path, typically in C.  In Fast_Lock and
1689 //    Fast_Unlock we often branch to DONE_LABEL, just to find that C2
1690 //    will emit a conditional branch immediately after the node.
1691 //    So we have branches to branches and lots of ICC.ZF games.
1692 //    Instead, it might be better to have C2 pass a "FailureLabel"
1693 //    into Fast_Lock and Fast_Unlock.  In the case of success, control
1694 //    will drop through the node.  ICC.ZF is undefined at exit.
1695 //    In the case of failure, the node will branch directly to the
1696 //    FailureLabel
1697 
1698 
1699 // obj: object to lock
1700 // box: on-stack box address (displaced header location) - KILLED
1701 // rax,: tmp -- KILLED
1702 // scr: tmp -- KILLED
1703 void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
1704                                Register scrReg, Register cx1Reg, Register cx2Reg,
1705                                BiasedLockingCounters* counters,
1706                                RTMLockingCounters* rtm_counters,
1707                                RTMLockingCounters* stack_rtm_counters,
1708                                Metadata* method_data,
1709                                bool use_rtm, bool profile_rtm) {
1710   // Ensure the register assignments are disjoint
1711   assert(tmpReg == rax, "");
1712 
1713   if (use_rtm) {
1714     assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg);
1715   } else {
1716     assert(cx1Reg == noreg, "");
1717     assert(cx2Reg == noreg, "");
1718     assert_different_registers(objReg, boxReg, tmpReg, scrReg);
1719   }
1720 
1721   if (counters != NULL) {
1722     atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);
1723   }
1724 
1725   // Possible cases that we'll encounter in fast_lock
1726   // ------------------------------------------------
1727   // * Inflated
1728   //    -- unlocked
1729   //    -- Locked
1730   //       = by self
1731   //       = by other
1732   // * biased
1733   //    -- by Self
1734   //    -- by other
1735   // * neutral
1736   // * stack-locked
1737   //    -- by self
1738   //       = sp-proximity test hits
1739   //       = sp-proximity test generates false-negative
1740   //    -- by other
1741   //
1742 
1743   Label IsInflated, DONE_LABEL;
1744 
1745   // it's stack-locked, biased or neutral
1746   // TODO: optimize away redundant LDs of obj->mark and improve the markword triage
1747   // order to reduce the number of conditional branches in the most common cases.
1748   // Beware -- there's a subtle invariant that fetch of the markword
1749   // at [FETCH], below, will never observe a biased encoding (*101b).
1750   // If this invariant is not held we risk exclusion (safety) failure.
1751   if (UseBiasedLocking && !UseOptoBiasInlining) {
1752     biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, counters);
1753   }
1754 
1755 #if INCLUDE_RTM_OPT
1756   if (UseRTMForStackLocks && use_rtm) {
1757     rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
1758                       stack_rtm_counters, method_data, profile_rtm,
1759                       DONE_LABEL, IsInflated);
1760   }
1761 #endif // INCLUDE_RTM_OPT
1762 
1763   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));          // [FETCH]
1764   testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
1765   jccb(Assembler::notZero, IsInflated);
1766 
1767   // Attempt stack-locking ...
1768   orptr (tmpReg, markOopDesc::unlocked_value);
1769   movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
1770   if (os::is_MP()) {
1771     lock();
1772   }
1773   cmpxchgptr(boxReg, Address(objReg, oopDesc::mark_offset_in_bytes()));      // Updates tmpReg
1774   if (counters != NULL) {
1775     cond_inc32(Assembler::equal,
1776                ExternalAddress((address)counters->fast_path_entry_count_addr()));
1777   }
1778   jcc(Assembler::equal, DONE_LABEL);           // Success
1779 
1780   // Recursive locking.
1781   // The object is stack-locked: markword contains stack pointer to BasicLock.
1782   // Locked by current thread if difference with current SP is less than one page.
1783   subptr(tmpReg, rsp);
1784   // Next instruction set ZFlag == 1 (Success) if difference is less then one page.
1785   andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
1786   movptr(Address(boxReg, 0), tmpReg);
1787   if (counters != NULL) {
1788     cond_inc32(Assembler::equal,
1789                ExternalAddress((address)counters->fast_path_entry_count_addr()));
1790   }
1791   jmp(DONE_LABEL);
1792 
1793   bind(IsInflated);
1794   // The object is inflated. tmpReg contains pointer to ObjectMonitor* + markOopDesc::monitor_value
1795 
1796 #if INCLUDE_RTM_OPT
1797   // Use the same RTM locking code in 32- and 64-bit VM.
1798   if (use_rtm) {
1799     rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg,
1800                          rtm_counters, method_data, profile_rtm, DONE_LABEL);
1801   } else {
1802 #endif // INCLUDE_RTM_OPT
1803 
1804 #ifndef _LP64
1805   // The object is inflated.
1806 
1807   // boxReg refers to the on-stack BasicLock in the current frame.
1808   // We'd like to write:
1809   //   set box->_displaced_header = markOopDesc::unused_mark().  Any non-0 value suffices.
1810   // This is convenient but results a ST-before-CAS penalty.  The following CAS suffers
1811   // additional latency as we have another ST in the store buffer that must drain.
1812 
1813   // avoid ST-before-CAS
1814   // register juggle because we need tmpReg for cmpxchgptr below
1815   movptr(scrReg, boxReg);
1816   movptr(boxReg, tmpReg);                   // consider: LEA box, [tmp-2]
1817 
1818   // Optimistic form: consider XORL tmpReg,tmpReg
1819   movptr(tmpReg, NULL_WORD);
1820 
1821   // Appears unlocked - try to swing _owner from null to non-null.
1822   // Ideally, I'd manifest "Self" with get_thread and then attempt
1823   // to CAS the register containing Self into m->Owner.
1824   // But we don't have enough registers, so instead we can either try to CAS
1825   // rsp or the address of the box (in scr) into &m->owner.  If the CAS succeeds
1826   // we later store "Self" into m->Owner.  Transiently storing a stack address
1827   // (rsp or the address of the box) into  m->owner is harmless.
1828   // Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
1829   if (os::is_MP()) {
1830     lock();
1831   }
1832   cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1833   movptr(Address(scrReg, 0), 3);          // box->_displaced_header = 3
1834   // If we weren't able to swing _owner from NULL to the BasicLock
1835   // then take the slow path.
1836   jccb  (Assembler::notZero, DONE_LABEL);
1837   // update _owner from BasicLock to thread
1838   get_thread (scrReg);                    // beware: clobbers ICCs
1839   movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
1840   xorptr(boxReg, boxReg);                 // set icc.ZFlag = 1 to indicate success
1841 
1842   // If the CAS fails we can either retry or pass control to the slow-path.
1843   // We use the latter tactic.
1844   // Pass the CAS result in the icc.ZFlag into DONE_LABEL
1845   // If the CAS was successful ...
1846   //   Self has acquired the lock
1847   //   Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
1848   // Intentional fall-through into DONE_LABEL ...
1849 #else // _LP64
1850   // It's inflated
1851   movq(scrReg, tmpReg);
1852   xorq(tmpReg, tmpReg);
1853 
1854   if (os::is_MP()) {
1855     lock();
1856   }
1857   cmpxchgptr(r15_thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
1858   // Unconditionally set box->_displaced_header = markOopDesc::unused_mark().
1859   // Without cast to int32_t movptr will destroy r10 which is typically obj.
1860   movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
1861   // Intentional fall-through into DONE_LABEL ...
1862   // Propagate ICC.ZF from CAS above into DONE_LABEL.
1863 #endif // _LP64
1864 #if INCLUDE_RTM_OPT
1865   } // use_rtm()
1866 #endif
1867   // DONE_LABEL is a hot target - we'd really like to place it at the
1868   // start of cache line by padding with NOPs.
1869   // See the AMD and Intel software optimization manuals for the
1870   // most efficient "long" NOP encodings.
1871   // Unfortunately none of our alignment mechanisms suffice.
1872   bind(DONE_LABEL);
1873 
1874   // At DONE_LABEL the icc ZFlag is set as follows ...
1875   // Fast_Unlock uses the same protocol.
1876   // ZFlag == 1 -> Success
1877   // ZFlag == 0 -> Failure - force control through the slow-path
1878 }
1879 
1880 // obj: object to unlock
1881 // box: box address (displaced header location), killed.  Must be EAX.
1882 // tmp: killed, cannot be obj nor box.
1883 //
1884 // Some commentary on balanced locking:
1885 //
1886 // Fast_Lock and Fast_Unlock are emitted only for provably balanced lock sites.
1887 // Methods that don't have provably balanced locking are forced to run in the
1888 // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
1889 // The interpreter provides two properties:
1890 // I1:  At return-time the interpreter automatically and quietly unlocks any
1891 //      objects acquired the current activation (frame).  Recall that the
1892 //      interpreter maintains an on-stack list of locks currently held by
1893 //      a frame.
1894 // I2:  If a method attempts to unlock an object that is not held by the
1895 //      the frame the interpreter throws IMSX.
1896 //
1897 // Lets say A(), which has provably balanced locking, acquires O and then calls B().
1898 // B() doesn't have provably balanced locking so it runs in the interpreter.
1899 // Control returns to A() and A() unlocks O.  By I1 and I2, above, we know that O
1900 // is still locked by A().
1901 //
1902 // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:
1903 // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
1904 // should not be unlocked by "normal" java-level locking and vice-versa.  The specification
1905 // doesn't specify what will occur if a program engages in such mixed-mode locking, however.
1906 // Arguably given that the spec legislates the JNI case as undefined our implementation
1907 // could reasonably *avoid* checking owner in Fast_Unlock().
1908 // In the interest of performance we elide m->Owner==Self check in unlock.
1909 // A perfectly viable alternative is to elide the owner check except when
1910 // Xcheck:jni is enabled.
1911 
1912 void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
1913   assert(boxReg == rax, "");
1914   assert_different_registers(objReg, boxReg, tmpReg);
1915 
1916   Label DONE_LABEL, Stacked, CheckSucc;
1917 
1918   // Critically, the biased locking test must have precedence over
1919   // and appear before the (box->dhw == 0) recursive stack-lock test.
1920   if (UseBiasedLocking && !UseOptoBiasInlining) {
1921     biased_locking_exit(objReg, tmpReg, DONE_LABEL);
1922   }
1923 
1924 #if INCLUDE_RTM_OPT
1925   if (UseRTMForStackLocks && use_rtm) {
1926     assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
1927     Label L_regular_unlock;
1928     movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));           // fetch markword
1929     andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
1930     cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
1931     jccb(Assembler::notEqual, L_regular_unlock);  // if !HLE RegularLock
1932     xend();                                       // otherwise end...
1933     jmp(DONE_LABEL);                              // ... and we're done
1934     bind(L_regular_unlock);
1935   }
1936 #endif
1937 
1938   cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
1939   jcc   (Assembler::zero, DONE_LABEL);            // 0 indicates recursive stack-lock
1940   movptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes()));             // Examine the object's markword
1941   testptr(tmpReg, markOopDesc::monitor_value);    // Inflated?
1942   jccb  (Assembler::zero, Stacked);
1943 
1944   // It's inflated.
1945 #if INCLUDE_RTM_OPT
1946   if (use_rtm) {
1947     Label L_regular_inflated_unlock;
1948     int owner_offset = OM_OFFSET_NO_MONITOR_VALUE_TAG(owner);
1949     movptr(boxReg, Address(tmpReg, owner_offset));
1950     testptr(boxReg, boxReg);
1951     jccb(Assembler::notZero, L_regular_inflated_unlock);
1952     xend();
1953     jmpb(DONE_LABEL);
1954     bind(L_regular_inflated_unlock);
1955   }
1956 #endif
1957 
1958   // Despite our balanced locking property we still check that m->_owner == Self
1959   // as java routines or native JNI code called by this thread might
1960   // have released the lock.
1961   // Refer to the comments in synchronizer.cpp for how we might encode extra
1962   // state in _succ so we can avoid fetching EntryList|cxq.
1963   //
1964   // I'd like to add more cases in fast_lock() and fast_unlock() --
1965   // such as recursive enter and exit -- but we have to be wary of
1966   // I$ bloat, T$ effects and BP$ effects.
1967   //
1968   // If there's no contention try a 1-0 exit.  That is, exit without
1969   // a costly MEMBAR or CAS.  See synchronizer.cpp for details on how
1970   // we detect and recover from the race that the 1-0 exit admits.
1971   //
1972   // Conceptually Fast_Unlock() must execute a STST|LDST "release" barrier
1973   // before it STs null into _owner, releasing the lock.  Updates
1974   // to data protected by the critical section must be visible before
1975   // we drop the lock (and thus before any other thread could acquire
1976   // the lock and observe the fields protected by the lock).
1977   // IA32's memory-model is SPO, so STs are ordered with respect to
1978   // each other and there's no need for an explicit barrier (fence).
1979   // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html.
1980 #ifndef _LP64
1981   get_thread (boxReg);
1982 
1983   // Note that we could employ various encoding schemes to reduce
1984   // the number of loads below (currently 4) to just 2 or 3.
1985   // Refer to the comments in synchronizer.cpp.
1986   // In practice the chain of fetches doesn't seem to impact performance, however.
1987   xorptr(boxReg, boxReg);
1988   orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
1989   jccb  (Assembler::notZero, DONE_LABEL);
1990   movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
1991   orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
1992   jccb  (Assembler::notZero, CheckSucc);
1993   movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), NULL_WORD);
1994   jmpb  (DONE_LABEL);
1995 
1996   bind (Stacked);
1997   // It's not inflated and it's not recursively stack-locked and it's not biased.
1998   // It must be stack-locked.
1999   // Try to reset the header to displaced header.
2000   // The "box" value on the stack is stable, so we can reload
2001   // and be assured we observe the same value as above.
2002   movptr(tmpReg, Address(boxReg, 0));
2003   if (os::is_MP()) {
2004     lock();
2005   }
2006   cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
2007   // Intention fall-thru into DONE_LABEL
2008 
2009   // DONE_LABEL is a hot target - we'd really like to place it at the
2010   // start of cache line by padding with NOPs.
2011   // See the AMD and Intel software optimization manuals for the
2012   // most efficient "long" NOP encodings.
2013   // Unfortunately none of our alignment mechanisms suffice.
2014   bind (CheckSucc);
2015 #else // _LP64
2016   // It's inflated
2017   xorptr(boxReg, boxReg);
2018   orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
2019   jccb  (Assembler::notZero, DONE_LABEL);
2020   movptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq)));
2021   orptr(boxReg, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList)));
2022   jccb  (Assembler::notZero, CheckSucc);
2023   movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2024   jmpb  (DONE_LABEL);
2025 
2026   // Try to avoid passing control into the slow_path ...
2027   Label LSuccess, LGoSlowPath ;
2028   bind  (CheckSucc);
2029 
2030   // The following optional optimization can be elided if necessary
2031   // Effectively: if (succ == null) goto SlowPath
2032   // The code reduces the window for a race, however,
2033   // and thus benefits performance.
2034   cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2035   jccb  (Assembler::zero, LGoSlowPath);
2036 
2037   xorptr(boxReg, boxReg);
2038   movptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), (int32_t)NULL_WORD);
2039   if (os::is_MP()) {
2040     // Memory barrier/fence
2041     // Dekker pivot point -- fulcrum : ST Owner; MEMBAR; LD Succ
2042     // Instead of MFENCE we use a dummy locked add of 0 to the top-of-stack.
2043     // This is faster on Nehalem and AMD Shanghai/Barcelona.
2044     // See https://blogs.oracle.com/dave/entry/instruction_selection_for_volatile_fences
2045     // We might also restructure (ST Owner=0;barrier;LD _Succ) to
2046     // (mov box,0; xchgq box, &m->Owner; LD _succ) .
2047     lock(); addl(Address(rsp, 0), 0);
2048   }
2049   cmpptr(Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(succ)), (int32_t)NULL_WORD);
2050   jccb  (Assembler::notZero, LSuccess);
2051 
2052   // Rare inopportune interleaving - race.
2053   // The successor vanished in the small window above.
2054   // The lock is contended -- (cxq|EntryList) != null -- and there's no apparent successor.
2055   // We need to ensure progress and succession.
2056   // Try to reacquire the lock.
2057   // If that fails then the new owner is responsible for succession and this
2058   // thread needs to take no further action and can exit via the fast path (success).
2059   // If the re-acquire succeeds then pass control into the slow path.
2060   // As implemented, this latter mode is horrible because we generated more
2061   // coherence traffic on the lock *and* artifically extended the critical section
2062   // length while by virtue of passing control into the slow path.
2063 
2064   // box is really RAX -- the following CMPXCHG depends on that binding
2065   // cmpxchg R,[M] is equivalent to rax = CAS(M,rax,R)
2066   if (os::is_MP()) { lock(); }
2067   cmpxchgptr(r15_thread, Address(tmpReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
2068   // There's no successor so we tried to regrab the lock.
2069   // If that didn't work, then another thread grabbed the
2070   // lock so we're done (and exit was a success).
2071   jccb  (Assembler::notEqual, LSuccess);
2072   // Intentional fall-through into slow-path
2073 
2074   bind  (LGoSlowPath);
2075   orl   (boxReg, 1);                      // set ICC.ZF=0 to indicate failure
2076   jmpb  (DONE_LABEL);
2077 
2078   bind  (LSuccess);
2079   testl (boxReg, 0);                      // set ICC.ZF=1 to indicate success
2080   jmpb  (DONE_LABEL);
2081 
2082   bind  (Stacked);
2083   movptr(tmpReg, Address (boxReg, 0));      // re-fetch
2084   if (os::is_MP()) { lock(); }
2085   cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
2086 
2087 #endif
2088   bind(DONE_LABEL);
2089 }
2090 #endif // COMPILER2
2091 
2092 void MacroAssembler::c2bool(Register x) {
2093   // implements x == 0 ? 0 : 1
2094   // note: must only look at least-significant byte of x
2095   //       since C-style booleans are stored in one byte
2096   //       only! (was bug)
2097   andl(x, 0xFF);
2098   setb(Assembler::notZero, x);
2099 }
2100 
2101 // Wouldn't need if AddressLiteral version had new name
2102 void MacroAssembler::call(Label& L, relocInfo::relocType rtype) {
2103   Assembler::call(L, rtype);
2104 }
2105 
2106 void MacroAssembler::call(Register entry) {
2107   Assembler::call(entry);
2108 }
2109 
2110 void MacroAssembler::call(AddressLiteral entry) {
2111   if (reachable(entry)) {
2112     Assembler::call_literal(entry.target(), entry.rspec());
2113   } else {
2114     lea(rscratch1, entry);
2115     Assembler::call(rscratch1);
2116   }
2117 }
2118 
2119 void MacroAssembler::ic_call(address entry, jint method_index) {
2120   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
2121   movptr(rax, (intptr_t)Universe::non_oop_word());
2122   call(AddressLiteral(entry, rh));
2123 }
2124 
2125 // Implementation of call_VM versions
2126 
2127 void MacroAssembler::call_VM(Register oop_result,
2128                              address entry_point,
2129                              bool check_exceptions) {
2130   Label C, E;
2131   call(C, relocInfo::none);
2132   jmp(E);
2133 
2134   bind(C);
2135   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
2136   ret(0);
2137 
2138   bind(E);
2139 }
2140 
2141 void MacroAssembler::call_VM(Register oop_result,
2142                              address entry_point,
2143                              Register arg_1,
2144                              bool check_exceptions) {
2145   Label C, E;
2146   call(C, relocInfo::none);
2147   jmp(E);
2148 
2149   bind(C);
2150   pass_arg1(this, arg_1);
2151   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
2152   ret(0);
2153 
2154   bind(E);
2155 }
2156 
2157 void MacroAssembler::call_VM(Register oop_result,
2158                              address entry_point,
2159                              Register arg_1,
2160                              Register arg_2,
2161                              bool check_exceptions) {
2162   Label C, E;
2163   call(C, relocInfo::none);
2164   jmp(E);
2165 
2166   bind(C);
2167 
2168   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2169 
2170   pass_arg2(this, arg_2);
2171   pass_arg1(this, arg_1);
2172   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
2173   ret(0);
2174 
2175   bind(E);
2176 }
2177 
2178 void MacroAssembler::call_VM(Register oop_result,
2179                              address entry_point,
2180                              Register arg_1,
2181                              Register arg_2,
2182                              Register arg_3,
2183                              bool check_exceptions) {
2184   Label C, E;
2185   call(C, relocInfo::none);
2186   jmp(E);
2187 
2188   bind(C);
2189 
2190   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2191   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2192   pass_arg3(this, arg_3);
2193 
2194   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2195   pass_arg2(this, arg_2);
2196 
2197   pass_arg1(this, arg_1);
2198   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
2199   ret(0);
2200 
2201   bind(E);
2202 }
2203 
2204 void MacroAssembler::call_VM(Register oop_result,
2205                              Register last_java_sp,
2206                              address entry_point,
2207                              int number_of_arguments,
2208                              bool check_exceptions) {
2209   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2210   call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2211 }
2212 
2213 void MacroAssembler::call_VM(Register oop_result,
2214                              Register last_java_sp,
2215                              address entry_point,
2216                              Register arg_1,
2217                              bool check_exceptions) {
2218   pass_arg1(this, arg_1);
2219   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2220 }
2221 
2222 void MacroAssembler::call_VM(Register oop_result,
2223                              Register last_java_sp,
2224                              address entry_point,
2225                              Register arg_1,
2226                              Register arg_2,
2227                              bool check_exceptions) {
2228 
2229   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2230   pass_arg2(this, arg_2);
2231   pass_arg1(this, arg_1);
2232   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2233 }
2234 
2235 void MacroAssembler::call_VM(Register oop_result,
2236                              Register last_java_sp,
2237                              address entry_point,
2238                              Register arg_1,
2239                              Register arg_2,
2240                              Register arg_3,
2241                              bool check_exceptions) {
2242   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2243   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2244   pass_arg3(this, arg_3);
2245   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2246   pass_arg2(this, arg_2);
2247   pass_arg1(this, arg_1);
2248   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2249 }
2250 
2251 void MacroAssembler::super_call_VM(Register oop_result,
2252                                    Register last_java_sp,
2253                                    address entry_point,
2254                                    int number_of_arguments,
2255                                    bool check_exceptions) {
2256   Register thread = LP64_ONLY(r15_thread) NOT_LP64(noreg);
2257   MacroAssembler::call_VM_base(oop_result, thread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
2258 }
2259 
2260 void MacroAssembler::super_call_VM(Register oop_result,
2261                                    Register last_java_sp,
2262                                    address entry_point,
2263                                    Register arg_1,
2264                                    bool check_exceptions) {
2265   pass_arg1(this, arg_1);
2266   super_call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
2267 }
2268 
2269 void MacroAssembler::super_call_VM(Register oop_result,
2270                                    Register last_java_sp,
2271                                    address entry_point,
2272                                    Register arg_1,
2273                                    Register arg_2,
2274                                    bool check_exceptions) {
2275 
2276   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2277   pass_arg2(this, arg_2);
2278   pass_arg1(this, arg_1);
2279   super_call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
2280 }
2281 
2282 void MacroAssembler::super_call_VM(Register oop_result,
2283                                    Register last_java_sp,
2284                                    address entry_point,
2285                                    Register arg_1,
2286                                    Register arg_2,
2287                                    Register arg_3,
2288                                    bool check_exceptions) {
2289   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2290   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2291   pass_arg3(this, arg_3);
2292   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2293   pass_arg2(this, arg_2);
2294   pass_arg1(this, arg_1);
2295   super_call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
2296 }
2297 
2298 void MacroAssembler::call_VM_base(Register oop_result,
2299                                   Register java_thread,
2300                                   Register last_java_sp,
2301                                   address  entry_point,
2302                                   int      number_of_arguments,
2303                                   bool     check_exceptions) {
2304   // determine java_thread register
2305   if (!java_thread->is_valid()) {
2306 #ifdef _LP64
2307     java_thread = r15_thread;
2308 #else
2309     java_thread = rdi;
2310     get_thread(java_thread);
2311 #endif // LP64
2312   }
2313   // determine last_java_sp register
2314   if (!last_java_sp->is_valid()) {
2315     last_java_sp = rsp;
2316   }
2317   // debugging support
2318   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
2319   LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
2320 #ifdef ASSERT
2321   // TraceBytecodes does not use r12 but saves it over the call, so don't verify
2322   // r12 is the heapbase.
2323   LP64_ONLY(if ((UseCompressedOops || UseCompressedClassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
2324 #endif // ASSERT
2325 
2326   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
2327   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
2328 
2329   // push java thread (becomes first argument of C function)
2330 
2331   NOT_LP64(push(java_thread); number_of_arguments++);
2332   LP64_ONLY(mov(c_rarg0, r15_thread));
2333 
2334   // set last Java frame before call
2335   assert(last_java_sp != rbp, "can't use ebp/rbp");
2336 
2337   // Only interpreter should have to set fp
2338   set_last_Java_frame(java_thread, last_java_sp, rbp, NULL);
2339 
2340   // do the call, remove parameters
2341   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments);
2342 
2343   // restore the thread (cannot use the pushed argument since arguments
2344   // may be overwritten by C code generated by an optimizing compiler);
2345   // however can use the register value directly if it is callee saved.
2346   if (LP64_ONLY(true ||) java_thread == rdi || java_thread == rsi) {
2347     // rdi & rsi (also r15) are callee saved -> nothing to do
2348 #ifdef ASSERT
2349     guarantee(java_thread != rax, "change this code");
2350     push(rax);
2351     { Label L;
2352       get_thread(rax);
2353       cmpptr(java_thread, rax);
2354       jcc(Assembler::equal, L);
2355       STOP("MacroAssembler::call_VM_base: rdi not callee saved?");
2356       bind(L);
2357     }
2358     pop(rax);
2359 #endif
2360   } else {
2361     get_thread(java_thread);
2362   }
2363   // reset last Java frame
2364   // Only interpreter should have to clear fp
2365   reset_last_Java_frame(java_thread, true);
2366 
2367    // C++ interp handles this in the interpreter
2368   check_and_handle_popframe(java_thread);
2369   check_and_handle_earlyret(java_thread);
2370 
2371   if (check_exceptions) {
2372     // check for pending exceptions (java_thread is set upon return)
2373     cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
2374 #ifndef _LP64
2375     jump_cc(Assembler::notEqual,
2376             RuntimeAddress(StubRoutines::forward_exception_entry()));
2377 #else
2378     // This used to conditionally jump to forward_exception however it is
2379     // possible if we relocate that the branch will not reach. So we must jump
2380     // around so we can always reach
2381 
2382     Label ok;
2383     jcc(Assembler::equal, ok);
2384     jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2385     bind(ok);
2386 #endif // LP64
2387   }
2388 
2389   // get oop result if there is one and reset the value in the thread
2390   if (oop_result->is_valid()) {
2391     get_vm_result(oop_result, java_thread);
2392   }
2393 }
2394 
2395 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
2396 
2397   // Calculate the value for last_Java_sp
2398   // somewhat subtle. call_VM does an intermediate call
2399   // which places a return address on the stack just under the
2400   // stack pointer as the user finsihed with it. This allows
2401   // use to retrieve last_Java_pc from last_Java_sp[-1].
2402   // On 32bit we then have to push additional args on the stack to accomplish
2403   // the actual requested call. On 64bit call_VM only can use register args
2404   // so the only extra space is the return address that call_VM created.
2405   // This hopefully explains the calculations here.
2406 
2407 #ifdef _LP64
2408   // We've pushed one address, correct last_Java_sp
2409   lea(rax, Address(rsp, wordSize));
2410 #else
2411   lea(rax, Address(rsp, (1 + number_of_arguments) * wordSize));
2412 #endif // LP64
2413 
2414   call_VM_base(oop_result, noreg, rax, entry_point, number_of_arguments, check_exceptions);
2415 
2416 }
2417 
2418 // Use this method when MacroAssembler version of call_VM_leaf_base() should be called from Interpreter.
2419 void MacroAssembler::call_VM_leaf0(address entry_point) {
2420   MacroAssembler::call_VM_leaf_base(entry_point, 0);
2421 }
2422 
2423 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
2424   call_VM_leaf_base(entry_point, number_of_arguments);
2425 }
2426 
2427 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
2428   pass_arg0(this, arg_0);
2429   call_VM_leaf(entry_point, 1);
2430 }
2431 
2432 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2433 
2434   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2435   pass_arg1(this, arg_1);
2436   pass_arg0(this, arg_0);
2437   call_VM_leaf(entry_point, 2);
2438 }
2439 
2440 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2441   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2442   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2443   pass_arg2(this, arg_2);
2444   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2445   pass_arg1(this, arg_1);
2446   pass_arg0(this, arg_0);
2447   call_VM_leaf(entry_point, 3);
2448 }
2449 
2450 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
2451   pass_arg0(this, arg_0);
2452   MacroAssembler::call_VM_leaf_base(entry_point, 1);
2453 }
2454 
2455 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
2456 
2457   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2458   pass_arg1(this, arg_1);
2459   pass_arg0(this, arg_0);
2460   MacroAssembler::call_VM_leaf_base(entry_point, 2);
2461 }
2462 
2463 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
2464   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2465   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2466   pass_arg2(this, arg_2);
2467   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2468   pass_arg1(this, arg_1);
2469   pass_arg0(this, arg_0);
2470   MacroAssembler::call_VM_leaf_base(entry_point, 3);
2471 }
2472 
2473 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
2474   LP64_ONLY(assert(arg_0 != c_rarg3, "smashed arg"));
2475   LP64_ONLY(assert(arg_1 != c_rarg3, "smashed arg"));
2476   LP64_ONLY(assert(arg_2 != c_rarg3, "smashed arg"));
2477   pass_arg3(this, arg_3);
2478   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
2479   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
2480   pass_arg2(this, arg_2);
2481   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
2482   pass_arg1(this, arg_1);
2483   pass_arg0(this, arg_0);
2484   MacroAssembler::call_VM_leaf_base(entry_point, 4);
2485 }
2486 
2487 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
2488   movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
2489   movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
2490   verify_oop(oop_result, "broken oop in call_VM_base");
2491 }
2492 
2493 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
2494   movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
2495   movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
2496 }
2497 
2498 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
2499 }
2500 
2501 void MacroAssembler::check_and_handle_popframe(Register java_thread) {
2502 }
2503 
2504 void MacroAssembler::cmp32(AddressLiteral src1, int32_t imm) {
2505   if (reachable(src1)) {
2506     cmpl(as_Address(src1), imm);
2507   } else {
2508     lea(rscratch1, src1);
2509     cmpl(Address(rscratch1, 0), imm);
2510   }
2511 }
2512 
2513 void MacroAssembler::cmp32(Register src1, AddressLiteral src2) {
2514   assert(!src2.is_lval(), "use cmpptr");
2515   if (reachable(src2)) {
2516     cmpl(src1, as_Address(src2));
2517   } else {
2518     lea(rscratch1, src2);
2519     cmpl(src1, Address(rscratch1, 0));
2520   }
2521 }
2522 
2523 void MacroAssembler::cmp32(Register src1, int32_t imm) {
2524   Assembler::cmpl(src1, imm);
2525 }
2526 
2527 void MacroAssembler::cmp32(Register src1, Address src2) {
2528   Assembler::cmpl(src1, src2);
2529 }
2530 
2531 void MacroAssembler::cmpsd2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2532   ucomisd(opr1, opr2);
2533 
2534   Label L;
2535   if (unordered_is_less) {
2536     movl(dst, -1);
2537     jcc(Assembler::parity, L);
2538     jcc(Assembler::below , L);
2539     movl(dst, 0);
2540     jcc(Assembler::equal , L);
2541     increment(dst);
2542   } else { // unordered is greater
2543     movl(dst, 1);
2544     jcc(Assembler::parity, L);
2545     jcc(Assembler::above , L);
2546     movl(dst, 0);
2547     jcc(Assembler::equal , L);
2548     decrementl(dst);
2549   }
2550   bind(L);
2551 }
2552 
2553 void MacroAssembler::cmpss2int(XMMRegister opr1, XMMRegister opr2, Register dst, bool unordered_is_less) {
2554   ucomiss(opr1, opr2);
2555 
2556   Label L;
2557   if (unordered_is_less) {
2558     movl(dst, -1);
2559     jcc(Assembler::parity, L);
2560     jcc(Assembler::below , L);
2561     movl(dst, 0);
2562     jcc(Assembler::equal , L);
2563     increment(dst);
2564   } else { // unordered is greater
2565     movl(dst, 1);
2566     jcc(Assembler::parity, L);
2567     jcc(Assembler::above , L);
2568     movl(dst, 0);
2569     jcc(Assembler::equal , L);
2570     decrementl(dst);
2571   }
2572   bind(L);
2573 }
2574 
2575 
2576 void MacroAssembler::cmp8(AddressLiteral src1, int imm) {
2577   if (reachable(src1)) {
2578     cmpb(as_Address(src1), imm);
2579   } else {
2580     lea(rscratch1, src1);
2581     cmpb(Address(rscratch1, 0), imm);
2582   }
2583 }
2584 
2585 void MacroAssembler::cmpptr(Register src1, AddressLiteral src2) {
2586 #ifdef _LP64
2587   if (src2.is_lval()) {
2588     movptr(rscratch1, src2);
2589     Assembler::cmpq(src1, rscratch1);
2590   } else if (reachable(src2)) {
2591     cmpq(src1, as_Address(src2));
2592   } else {
2593     lea(rscratch1, src2);
2594     Assembler::cmpq(src1, Address(rscratch1, 0));
2595   }
2596 #else
2597   if (src2.is_lval()) {
2598     cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2599   } else {
2600     cmpl(src1, as_Address(src2));
2601   }
2602 #endif // _LP64
2603 }
2604 
2605 void MacroAssembler::cmpptr(Address src1, AddressLiteral src2) {
2606   assert(src2.is_lval(), "not a mem-mem compare");
2607 #ifdef _LP64
2608   // moves src2's literal address
2609   movptr(rscratch1, src2);
2610   Assembler::cmpq(src1, rscratch1);
2611 #else
2612   cmp_literal32(src1, (int32_t) src2.target(), src2.rspec());
2613 #endif // _LP64
2614 }
2615 
2616 void MacroAssembler::cmpoop(Register src1, Register src2) {
2617   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2618   bs->obj_equals(this, src1, src2);
2619 }
2620 
2621 void MacroAssembler::cmpoop(Register src1, Address src2) {
2622   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2623   bs->obj_equals(this, src1, src2);
2624 }
2625 
2626 #ifdef _LP64
2627 void MacroAssembler::cmpoop(Register src1, jobject src2) {
2628   movoop(rscratch1, src2);
2629   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2630   bs->obj_equals(this, src1, rscratch1);
2631 }
2632 #endif
2633 
2634 void MacroAssembler::locked_cmpxchgptr(Register reg, AddressLiteral adr) {
2635   if (reachable(adr)) {
2636     if (os::is_MP())
2637       lock();
2638     cmpxchgptr(reg, as_Address(adr));
2639   } else {
2640     lea(rscratch1, adr);
2641     if (os::is_MP())
2642       lock();
2643     cmpxchgptr(reg, Address(rscratch1, 0));
2644   }
2645 }
2646 
2647 void MacroAssembler::cmpxchgptr(Register reg, Address adr) {
2648   LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
2649 }
2650 
2651 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {
2652   if (reachable(src)) {
2653     Assembler::comisd(dst, as_Address(src));
2654   } else {
2655     lea(rscratch1, src);
2656     Assembler::comisd(dst, Address(rscratch1, 0));
2657   }
2658 }
2659 
2660 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {
2661   if (reachable(src)) {
2662     Assembler::comiss(dst, as_Address(src));
2663   } else {
2664     lea(rscratch1, src);
2665     Assembler::comiss(dst, Address(rscratch1, 0));
2666   }
2667 }
2668 
2669 
2670 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
2671   Condition negated_cond = negate_condition(cond);
2672   Label L;
2673   jcc(negated_cond, L);
2674   pushf(); // Preserve flags
2675   atomic_incl(counter_addr);
2676   popf();
2677   bind(L);
2678 }
2679 
2680 int MacroAssembler::corrected_idivl(Register reg) {
2681   // Full implementation of Java idiv and irem; checks for
2682   // special case as described in JVM spec., p.243 & p.271.
2683   // The function returns the (pc) offset of the idivl
2684   // instruction - may be needed for implicit exceptions.
2685   //
2686   //         normal case                           special case
2687   //
2688   // input : rax,: dividend                         min_int
2689   //         reg: divisor   (may not be rax,/rdx)   -1
2690   //
2691   // output: rax,: quotient  (= rax, idiv reg)       min_int
2692   //         rdx: remainder (= rax, irem reg)       0
2693   assert(reg != rax && reg != rdx, "reg cannot be rax, or rdx register");
2694   const int min_int = 0x80000000;
2695   Label normal_case, special_case;
2696 
2697   // check for special case
2698   cmpl(rax, min_int);
2699   jcc(Assembler::notEqual, normal_case);
2700   xorl(rdx, rdx); // prepare rdx for possible special case (where remainder = 0)
2701   cmpl(reg, -1);
2702   jcc(Assembler::equal, special_case);
2703 
2704   // handle normal case
2705   bind(normal_case);
2706   cdql();
2707   int idivl_offset = offset();
2708   idivl(reg);
2709 
2710   // normal and special case exit
2711   bind(special_case);
2712 
2713   return idivl_offset;
2714 }
2715 
2716 
2717 
2718 void MacroAssembler::decrementl(Register reg, int value) {
2719   if (value == min_jint) {subl(reg, value) ; return; }
2720   if (value <  0) { incrementl(reg, -value); return; }
2721   if (value == 0) {                        ; return; }
2722   if (value == 1 && UseIncDec) { decl(reg) ; return; }
2723   /* else */      { subl(reg, value)       ; return; }
2724 }
2725 
2726 void MacroAssembler::decrementl(Address dst, int value) {
2727   if (value == min_jint) {subl(dst, value) ; return; }
2728   if (value <  0) { incrementl(dst, -value); return; }
2729   if (value == 0) {                        ; return; }
2730   if (value == 1 && UseIncDec) { decl(dst) ; return; }
2731   /* else */      { subl(dst, value)       ; return; }
2732 }
2733 
2734 void MacroAssembler::division_with_shift (Register reg, int shift_value) {
2735   assert (shift_value > 0, "illegal shift value");
2736   Label _is_positive;
2737   testl (reg, reg);
2738   jcc (Assembler::positive, _is_positive);
2739   int offset = (1 << shift_value) - 1 ;
2740 
2741   if (offset == 1) {
2742     incrementl(reg);
2743   } else {
2744     addl(reg, offset);
2745   }
2746 
2747   bind (_is_positive);
2748   sarl(reg, shift_value);
2749 }
2750 
2751 void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
2752   if (reachable(src)) {
2753     Assembler::divsd(dst, as_Address(src));
2754   } else {
2755     lea(rscratch1, src);
2756     Assembler::divsd(dst, Address(rscratch1, 0));
2757   }
2758 }
2759 
2760 void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
2761   if (reachable(src)) {
2762     Assembler::divss(dst, as_Address(src));
2763   } else {
2764     lea(rscratch1, src);
2765     Assembler::divss(dst, Address(rscratch1, 0));
2766   }
2767 }
2768 
2769 // !defined(COMPILER2) is because of stupid core builds
2770 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2) || INCLUDE_JVMCI
2771 void MacroAssembler::empty_FPU_stack() {
2772   if (VM_Version::supports_mmx()) {
2773     emms();
2774   } else {
2775     for (int i = 8; i-- > 0; ) ffree(i);
2776   }
2777 }
2778 #endif // !LP64 || C1 || !C2 || INCLUDE_JVMCI
2779 
2780 
2781 void MacroAssembler::enter() {
2782   push(rbp);
2783   mov(rbp, rsp);
2784 }
2785 
2786 // A 5 byte nop that is safe for patching (see patch_verified_entry)
2787 void MacroAssembler::fat_nop() {
2788   if (UseAddressNop) {
2789     addr_nop_5();
2790   } else {
2791     emit_int8(0x26); // es:
2792     emit_int8(0x2e); // cs:
2793     emit_int8(0x64); // fs:
2794     emit_int8(0x65); // gs:
2795     emit_int8((unsigned char)0x90);
2796   }
2797 }
2798 
2799 void MacroAssembler::fcmp(Register tmp) {
2800   fcmp(tmp, 1, true, true);
2801 }
2802 
2803 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
2804   assert(!pop_right || pop_left, "usage error");
2805   if (VM_Version::supports_cmov()) {
2806     assert(tmp == noreg, "unneeded temp");
2807     if (pop_left) {
2808       fucomip(index);
2809     } else {
2810       fucomi(index);
2811     }
2812     if (pop_right) {
2813       fpop();
2814     }
2815   } else {
2816     assert(tmp != noreg, "need temp");
2817     if (pop_left) {
2818       if (pop_right) {
2819         fcompp();
2820       } else {
2821         fcomp(index);
2822       }
2823     } else {
2824       fcom(index);
2825     }
2826     // convert FPU condition into eflags condition via rax,
2827     save_rax(tmp);
2828     fwait(); fnstsw_ax();
2829     sahf();
2830     restore_rax(tmp);
2831   }
2832   // condition codes set as follows:
2833   //
2834   // CF (corresponds to C0) if x < y
2835   // PF (corresponds to C2) if unordered
2836   // ZF (corresponds to C3) if x = y
2837 }
2838 
2839 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less) {
2840   fcmp2int(dst, unordered_is_less, 1, true, true);
2841 }
2842 
2843 void MacroAssembler::fcmp2int(Register dst, bool unordered_is_less, int index, bool pop_left, bool pop_right) {
2844   fcmp(VM_Version::supports_cmov() ? noreg : dst, index, pop_left, pop_right);
2845   Label L;
2846   if (unordered_is_less) {
2847     movl(dst, -1);
2848     jcc(Assembler::parity, L);
2849     jcc(Assembler::below , L);
2850     movl(dst, 0);
2851     jcc(Assembler::equal , L);
2852     increment(dst);
2853   } else { // unordered is greater
2854     movl(dst, 1);
2855     jcc(Assembler::parity, L);
2856     jcc(Assembler::above , L);
2857     movl(dst, 0);
2858     jcc(Assembler::equal , L);
2859     decrementl(dst);
2860   }
2861   bind(L);
2862 }
2863 
2864 void MacroAssembler::fld_d(AddressLiteral src) {
2865   fld_d(as_Address(src));
2866 }
2867 
2868 void MacroAssembler::fld_s(AddressLiteral src) {
2869   fld_s(as_Address(src));
2870 }
2871 
2872 void MacroAssembler::fld_x(AddressLiteral src) {
2873   Assembler::fld_x(as_Address(src));
2874 }
2875 
2876 void MacroAssembler::fldcw(AddressLiteral src) {
2877   Assembler::fldcw(as_Address(src));
2878 }
2879 
2880 void MacroAssembler::mulpd(XMMRegister dst, AddressLiteral src) {
2881   if (reachable(src)) {
2882     Assembler::mulpd(dst, as_Address(src));
2883   } else {
2884     lea(rscratch1, src);
2885     Assembler::mulpd(dst, Address(rscratch1, 0));
2886   }
2887 }
2888 
2889 void MacroAssembler::increase_precision() {
2890   subptr(rsp, BytesPerWord);
2891   fnstcw(Address(rsp, 0));
2892   movl(rax, Address(rsp, 0));
2893   orl(rax, 0x300);
2894   push(rax);
2895   fldcw(Address(rsp, 0));
2896   pop(rax);
2897 }
2898 
2899 void MacroAssembler::restore_precision() {
2900   fldcw(Address(rsp, 0));
2901   addptr(rsp, BytesPerWord);
2902 }
2903 
2904 void MacroAssembler::fpop() {
2905   ffree();
2906   fincstp();
2907 }
2908 
2909 void MacroAssembler::load_float(Address src) {
2910   if (UseSSE >= 1) {
2911     movflt(xmm0, src);
2912   } else {
2913     LP64_ONLY(ShouldNotReachHere());
2914     NOT_LP64(fld_s(src));
2915   }
2916 }
2917 
2918 void MacroAssembler::store_float(Address dst) {
2919   if (UseSSE >= 1) {
2920     movflt(dst, xmm0);
2921   } else {
2922     LP64_ONLY(ShouldNotReachHere());
2923     NOT_LP64(fstp_s(dst));
2924   }
2925 }
2926 
2927 void MacroAssembler::load_double(Address src) {
2928   if (UseSSE >= 2) {
2929     movdbl(xmm0, src);
2930   } else {
2931     LP64_ONLY(ShouldNotReachHere());
2932     NOT_LP64(fld_d(src));
2933   }
2934 }
2935 
2936 void MacroAssembler::store_double(Address dst) {
2937   if (UseSSE >= 2) {
2938     movdbl(dst, xmm0);
2939   } else {
2940     LP64_ONLY(ShouldNotReachHere());
2941     NOT_LP64(fstp_d(dst));
2942   }
2943 }
2944 
2945 void MacroAssembler::push_zmm(XMMRegister reg) {
2946   lea(rsp, Address(rsp, -64)); // Use lea to not affect flags
2947   evmovdqul(Address(rsp, 0), reg, Assembler::AVX_512bit);
2948 }
2949 
2950 void MacroAssembler::pop_zmm(XMMRegister reg) {
2951   evmovdqul(reg, Address(rsp, 0), Assembler::AVX_512bit);
2952   lea(rsp, Address(rsp, 64)); // Use lea to not affect flags
2953 }
2954 
2955 void MacroAssembler::fremr(Register tmp) {
2956   save_rax(tmp);
2957   { Label L;
2958     bind(L);
2959     fprem();
2960     fwait(); fnstsw_ax();
2961 #ifdef _LP64
2962     testl(rax, 0x400);
2963     jcc(Assembler::notEqual, L);
2964 #else
2965     sahf();
2966     jcc(Assembler::parity, L);
2967 #endif // _LP64
2968   }
2969   restore_rax(tmp);
2970   // Result is in ST0.
2971   // Note: fxch & fpop to get rid of ST1
2972   // (otherwise FPU stack could overflow eventually)
2973   fxch(1);
2974   fpop();
2975 }
2976 
2977 // dst = c = a * b + c
2978 void MacroAssembler::fmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
2979   Assembler::vfmadd231sd(c, a, b);
2980   if (dst != c) {
2981     movdbl(dst, c);
2982   }
2983 }
2984 
2985 // dst = c = a * b + c
2986 void MacroAssembler::fmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c) {
2987   Assembler::vfmadd231ss(c, a, b);
2988   if (dst != c) {
2989     movflt(dst, c);
2990   }
2991 }
2992 
2993 // dst = c = a * b + c
2994 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
2995   Assembler::vfmadd231pd(c, a, b, vector_len);
2996   if (dst != c) {
2997     vmovdqu(dst, c);
2998   }
2999 }
3000 
3001 // dst = c = a * b + c
3002 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, XMMRegister b, XMMRegister c, int vector_len) {
3003   Assembler::vfmadd231ps(c, a, b, vector_len);
3004   if (dst != c) {
3005     vmovdqu(dst, c);
3006   }
3007 }
3008 
3009 // dst = c = a * b + c
3010 void MacroAssembler::vfmad(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
3011   Assembler::vfmadd231pd(c, a, b, vector_len);
3012   if (dst != c) {
3013     vmovdqu(dst, c);
3014   }
3015 }
3016 
3017 // dst = c = a * b + c
3018 void MacroAssembler::vfmaf(XMMRegister dst, XMMRegister a, Address b, XMMRegister c, int vector_len) {
3019   Assembler::vfmadd231ps(c, a, b, vector_len);
3020   if (dst != c) {
3021     vmovdqu(dst, c);
3022   }
3023 }
3024 
3025 void MacroAssembler::incrementl(AddressLiteral dst) {
3026   if (reachable(dst)) {
3027     incrementl(as_Address(dst));
3028   } else {
3029     lea(rscratch1, dst);
3030     incrementl(Address(rscratch1, 0));
3031   }
3032 }
3033 
3034 void MacroAssembler::incrementl(ArrayAddress dst) {
3035   incrementl(as_Address(dst));
3036 }
3037 
3038 void MacroAssembler::incrementl(Register reg, int value) {
3039   if (value == min_jint) {addl(reg, value) ; return; }
3040   if (value <  0) { decrementl(reg, -value); return; }
3041   if (value == 0) {                        ; return; }
3042   if (value == 1 && UseIncDec) { incl(reg) ; return; }
3043   /* else */      { addl(reg, value)       ; return; }
3044 }
3045 
3046 void MacroAssembler::incrementl(Address dst, int value) {
3047   if (value == min_jint) {addl(dst, value) ; return; }
3048   if (value <  0) { decrementl(dst, -value); return; }
3049   if (value == 0) {                        ; return; }
3050   if (value == 1 && UseIncDec) { incl(dst) ; return; }
3051   /* else */      { addl(dst, value)       ; return; }
3052 }
3053 
3054 void MacroAssembler::jump(AddressLiteral dst) {
3055   if (reachable(dst)) {
3056     jmp_literal(dst.target(), dst.rspec());
3057   } else {
3058     lea(rscratch1, dst);
3059     jmp(rscratch1);
3060   }
3061 }
3062 
3063 void MacroAssembler::jump_cc(Condition cc, AddressLiteral dst) {
3064   if (reachable(dst)) {
3065     InstructionMark im(this);
3066     relocate(dst.reloc());
3067     const int short_size = 2;
3068     const int long_size = 6;
3069     int offs = (intptr_t)dst.target() - ((intptr_t)pc());
3070     if (dst.reloc() == relocInfo::none && is8bit(offs - short_size)) {
3071       // 0111 tttn #8-bit disp
3072       emit_int8(0x70 | cc);
3073       emit_int8((offs - short_size) & 0xFF);
3074     } else {
3075       // 0000 1111 1000 tttn #32-bit disp
3076       emit_int8(0x0F);
3077       emit_int8((unsigned char)(0x80 | cc));
3078       emit_int32(offs - long_size);
3079     }
3080   } else {
3081 #ifdef ASSERT
3082     warning("reversing conditional branch");
3083 #endif /* ASSERT */
3084     Label skip;
3085     jccb(reverse[cc], skip);
3086     lea(rscratch1, dst);
3087     Assembler::jmp(rscratch1);
3088     bind(skip);
3089   }
3090 }
3091 
3092 void MacroAssembler::ldmxcsr(AddressLiteral src) {
3093   if (reachable(src)) {
3094     Assembler::ldmxcsr(as_Address(src));
3095   } else {
3096     lea(rscratch1, src);
3097     Assembler::ldmxcsr(Address(rscratch1, 0));
3098   }
3099 }
3100 
3101 int MacroAssembler::load_signed_byte(Register dst, Address src) {
3102   int off;
3103   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3104     off = offset();
3105     movsbl(dst, src); // movsxb
3106   } else {
3107     off = load_unsigned_byte(dst, src);
3108     shll(dst, 24);
3109     sarl(dst, 24);
3110   }
3111   return off;
3112 }
3113 
3114 // Note: load_signed_short used to be called load_signed_word.
3115 // Although the 'w' in x86 opcodes refers to the term "word" in the assembler
3116 // manual, which means 16 bits, that usage is found nowhere in HotSpot code.
3117 // The term "word" in HotSpot means a 32- or 64-bit machine word.
3118 int MacroAssembler::load_signed_short(Register dst, Address src) {
3119   int off;
3120   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3121     // This is dubious to me since it seems safe to do a signed 16 => 64 bit
3122     // version but this is what 64bit has always done. This seems to imply
3123     // that users are only using 32bits worth.
3124     off = offset();
3125     movswl(dst, src); // movsxw
3126   } else {
3127     off = load_unsigned_short(dst, src);
3128     shll(dst, 16);
3129     sarl(dst, 16);
3130   }
3131   return off;
3132 }
3133 
3134 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
3135   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3136   // and "3.9 Partial Register Penalties", p. 22).
3137   int off;
3138   if (LP64_ONLY(true || ) VM_Version::is_P6() || src.uses(dst)) {
3139     off = offset();
3140     movzbl(dst, src); // movzxb
3141   } else {
3142     xorl(dst, dst);
3143     off = offset();
3144     movb(dst, src);
3145   }
3146   return off;
3147 }
3148 
3149 // Note: load_unsigned_short used to be called load_unsigned_word.
3150 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
3151   // According to Intel Doc. AP-526, "Zero-Extension of Short", p.16,
3152   // and "3.9 Partial Register Penalties", p. 22).
3153   int off;
3154   if (LP64_ONLY(true ||) VM_Version::is_P6() || src.uses(dst)) {
3155     off = offset();
3156     movzwl(dst, src); // movzxw
3157   } else {
3158     xorl(dst, dst);
3159     off = offset();
3160     movw(dst, src);
3161   }
3162   return off;
3163 }
3164 
3165 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed, Register dst2) {
3166   switch (size_in_bytes) {
3167 #ifndef _LP64
3168   case  8:
3169     assert(dst2 != noreg, "second dest register required");
3170     movl(dst,  src);
3171     movl(dst2, src.plus_disp(BytesPerInt));
3172     break;
3173 #else
3174   case  8:  movq(dst, src); break;
3175 #endif
3176   case  4:  movl(dst, src); break;
3177   case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
3178   case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
3179   default:  ShouldNotReachHere();
3180   }
3181 }
3182 
3183 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes, Register src2) {
3184   switch (size_in_bytes) {
3185 #ifndef _LP64
3186   case  8:
3187     assert(src2 != noreg, "second source register required");
3188     movl(dst,                        src);
3189     movl(dst.plus_disp(BytesPerInt), src2);
3190     break;
3191 #else
3192   case  8:  movq(dst, src); break;
3193 #endif
3194   case  4:  movl(dst, src); break;
3195   case  2:  movw(dst, src); break;
3196   case  1:  movb(dst, src); break;
3197   default:  ShouldNotReachHere();
3198   }
3199 }
3200 
3201 void MacroAssembler::mov32(AddressLiteral dst, Register src) {
3202   if (reachable(dst)) {
3203     movl(as_Address(dst), src);
3204   } else {
3205     lea(rscratch1, dst);
3206     movl(Address(rscratch1, 0), src);
3207   }
3208 }
3209 
3210 void MacroAssembler::mov32(Register dst, AddressLiteral src) {
3211   if (reachable(src)) {
3212     movl(dst, as_Address(src));
3213   } else {
3214     lea(rscratch1, src);
3215     movl(dst, Address(rscratch1, 0));
3216   }
3217 }
3218 
3219 // C++ bool manipulation
3220 
3221 void MacroAssembler::movbool(Register dst, Address src) {
3222   if(sizeof(bool) == 1)
3223     movb(dst, src);
3224   else if(sizeof(bool) == 2)
3225     movw(dst, src);
3226   else if(sizeof(bool) == 4)
3227     movl(dst, src);
3228   else
3229     // unsupported
3230     ShouldNotReachHere();
3231 }
3232 
3233 void MacroAssembler::movbool(Address dst, bool boolconst) {
3234   if(sizeof(bool) == 1)
3235     movb(dst, (int) boolconst);
3236   else if(sizeof(bool) == 2)
3237     movw(dst, (int) boolconst);
3238   else if(sizeof(bool) == 4)
3239     movl(dst, (int) boolconst);
3240   else
3241     // unsupported
3242     ShouldNotReachHere();
3243 }
3244 
3245 void MacroAssembler::movbool(Address dst, Register src) {
3246   if(sizeof(bool) == 1)
3247     movb(dst, src);
3248   else if(sizeof(bool) == 2)
3249     movw(dst, src);
3250   else if(sizeof(bool) == 4)
3251     movl(dst, src);
3252   else
3253     // unsupported
3254     ShouldNotReachHere();
3255 }
3256 
3257 void MacroAssembler::movbyte(ArrayAddress dst, int src) {
3258   movb(as_Address(dst), src);
3259 }
3260 
3261 void MacroAssembler::movdl(XMMRegister dst, AddressLiteral src) {
3262   if (reachable(src)) {
3263     movdl(dst, as_Address(src));
3264   } else {
3265     lea(rscratch1, src);
3266     movdl(dst, Address(rscratch1, 0));
3267   }
3268 }
3269 
3270 void MacroAssembler::movq(XMMRegister dst, AddressLiteral src) {
3271   if (reachable(src)) {
3272     movq(dst, as_Address(src));
3273   } else {
3274     lea(rscratch1, src);
3275     movq(dst, Address(rscratch1, 0));
3276   }
3277 }
3278 
3279 void MacroAssembler::setvectmask(Register dst, Register src) {
3280   Assembler::movl(dst, 1);
3281   Assembler::shlxl(dst, dst, src);
3282   Assembler::decl(dst);
3283   Assembler::kmovdl(k1, dst);
3284   Assembler::movl(dst, src);
3285 }
3286 
3287 void MacroAssembler::restorevectmask() {
3288   Assembler::knotwl(k1, k0);
3289 }
3290 
3291 void MacroAssembler::movdbl(XMMRegister dst, AddressLiteral src) {
3292   if (reachable(src)) {
3293     if (UseXmmLoadAndClearUpper) {
3294       movsd (dst, as_Address(src));
3295     } else {
3296       movlpd(dst, as_Address(src));
3297     }
3298   } else {
3299     lea(rscratch1, src);
3300     if (UseXmmLoadAndClearUpper) {
3301       movsd (dst, Address(rscratch1, 0));
3302     } else {
3303       movlpd(dst, Address(rscratch1, 0));
3304     }
3305   }
3306 }
3307 
3308 void MacroAssembler::movflt(XMMRegister dst, AddressLiteral src) {
3309   if (reachable(src)) {
3310     movss(dst, as_Address(src));
3311   } else {
3312     lea(rscratch1, src);
3313     movss(dst, Address(rscratch1, 0));
3314   }
3315 }
3316 
3317 void MacroAssembler::movptr(Register dst, Register src) {
3318   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3319 }
3320 
3321 void MacroAssembler::movptr(Register dst, Address src) {
3322   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3323 }
3324 
3325 // src should NEVER be a real pointer. Use AddressLiteral for true pointers
3326 void MacroAssembler::movptr(Register dst, intptr_t src) {
3327   LP64_ONLY(mov64(dst, src)) NOT_LP64(movl(dst, src));
3328 }
3329 
3330 void MacroAssembler::movptr(Address dst, Register src) {
3331   LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
3332 }
3333 
3334 void MacroAssembler::movdqu(Address dst, XMMRegister src) {
3335   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3336     Assembler::vextractf32x4(dst, src, 0);
3337   } else {
3338     Assembler::movdqu(dst, src);
3339   }
3340 }
3341 
3342 void MacroAssembler::movdqu(XMMRegister dst, Address src) {
3343   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3344     Assembler::vinsertf32x4(dst, dst, src, 0);
3345   } else {
3346     Assembler::movdqu(dst, src);
3347   }
3348 }
3349 
3350 void MacroAssembler::movdqu(XMMRegister dst, XMMRegister src) {
3351   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3352     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3353   } else {
3354     Assembler::movdqu(dst, src);
3355   }
3356 }
3357 
3358 void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src, Register scratchReg) {
3359   if (reachable(src)) {
3360     movdqu(dst, as_Address(src));
3361   } else {
3362     lea(scratchReg, src);
3363     movdqu(dst, Address(scratchReg, 0));
3364   }
3365 }
3366 
3367 void MacroAssembler::vmovdqu(Address dst, XMMRegister src) {
3368   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (src->encoding() > 15)) {
3369     vextractf64x4_low(dst, src);
3370   } else {
3371     Assembler::vmovdqu(dst, src);
3372   }
3373 }
3374 
3375 void MacroAssembler::vmovdqu(XMMRegister dst, Address src) {
3376   if (UseAVX > 2 && !VM_Version::supports_avx512vl() && (dst->encoding() > 15)) {
3377     vinsertf64x4_low(dst, src);
3378   } else {
3379     Assembler::vmovdqu(dst, src);
3380   }
3381 }
3382 
3383 void MacroAssembler::vmovdqu(XMMRegister dst, XMMRegister src) {
3384   if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3385     Assembler::evmovdqul(dst, src, Assembler::AVX_512bit);
3386   }
3387   else {
3388     Assembler::vmovdqu(dst, src);
3389   }
3390 }
3391 
3392 void MacroAssembler::vmovdqu(XMMRegister dst, AddressLiteral src) {
3393   if (reachable(src)) {
3394     vmovdqu(dst, as_Address(src));
3395   }
3396   else {
3397     lea(rscratch1, src);
3398     vmovdqu(dst, Address(rscratch1, 0));
3399   }
3400 }
3401 
3402 void MacroAssembler::evmovdquq(XMMRegister dst, AddressLiteral src, int vector_len, Register rscratch) {
3403   if (reachable(src)) {
3404     Assembler::evmovdquq(dst, as_Address(src), vector_len);
3405   } else {
3406     lea(rscratch, src);
3407     Assembler::evmovdquq(dst, Address(rscratch, 0), vector_len);
3408   }
3409 }
3410 
3411 void MacroAssembler::movdqa(XMMRegister dst, AddressLiteral src) {
3412   if (reachable(src)) {
3413     Assembler::movdqa(dst, as_Address(src));
3414   } else {
3415     lea(rscratch1, src);
3416     Assembler::movdqa(dst, Address(rscratch1, 0));
3417   }
3418 }
3419 
3420 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
3421   if (reachable(src)) {
3422     Assembler::movsd(dst, as_Address(src));
3423   } else {
3424     lea(rscratch1, src);
3425     Assembler::movsd(dst, Address(rscratch1, 0));
3426   }
3427 }
3428 
3429 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
3430   if (reachable(src)) {
3431     Assembler::movss(dst, as_Address(src));
3432   } else {
3433     lea(rscratch1, src);
3434     Assembler::movss(dst, Address(rscratch1, 0));
3435   }
3436 }
3437 
3438 void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) {
3439   if (reachable(src)) {
3440     Assembler::mulsd(dst, as_Address(src));
3441   } else {
3442     lea(rscratch1, src);
3443     Assembler::mulsd(dst, Address(rscratch1, 0));
3444   }
3445 }
3446 
3447 void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) {
3448   if (reachable(src)) {
3449     Assembler::mulss(dst, as_Address(src));
3450   } else {
3451     lea(rscratch1, src);
3452     Assembler::mulss(dst, Address(rscratch1, 0));
3453   }
3454 }
3455 
3456 void MacroAssembler::null_check(Register reg, int offset) {
3457   if (needs_explicit_null_check(offset)) {
3458     // provoke OS NULL exception if reg = NULL by
3459     // accessing M[reg] w/o changing any (non-CC) registers
3460     // NOTE: cmpl is plenty here to provoke a segv
3461     cmpptr(rax, Address(reg, 0));
3462     // Note: should probably use testl(rax, Address(reg, 0));
3463     //       may be shorter code (however, this version of
3464     //       testl needs to be implemented first)
3465   } else {
3466     // nothing to do, (later) access of M[reg + offset]
3467     // will provoke OS NULL exception if reg = NULL
3468   }
3469 }
3470 
3471 void MacroAssembler::os_breakpoint() {
3472   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
3473   // (e.g., MSVC can't call ps() otherwise)
3474   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
3475 }
3476 
3477 void MacroAssembler::unimplemented(const char* what) {
3478   const char* buf = NULL;
3479   {
3480     ResourceMark rm;
3481     stringStream ss;
3482     ss.print("unimplemented: %s", what);
3483     buf = code_string(ss.as_string());
3484   }
3485   stop(buf);
3486 }
3487 
3488 #ifdef _LP64
3489 #define XSTATE_BV 0x200
3490 #endif
3491 
3492 void MacroAssembler::pop_CPU_state() {
3493   pop_FPU_state();
3494   pop_IU_state();
3495 }
3496 
3497 void MacroAssembler::pop_FPU_state() {
3498 #ifndef _LP64
3499   frstor(Address(rsp, 0));
3500 #else
3501   fxrstor(Address(rsp, 0));
3502 #endif
3503   addptr(rsp, FPUStateSizeInWords * wordSize);
3504 }
3505 
3506 void MacroAssembler::pop_IU_state() {
3507   popa();
3508   LP64_ONLY(addq(rsp, 8));
3509   popf();
3510 }
3511 
3512 // Save Integer and Float state
3513 // Warning: Stack must be 16 byte aligned (64bit)
3514 void MacroAssembler::push_CPU_state() {
3515   push_IU_state();
3516   push_FPU_state();
3517 }
3518 
3519 void MacroAssembler::push_FPU_state() {
3520   subptr(rsp, FPUStateSizeInWords * wordSize);
3521 #ifndef _LP64
3522   fnsave(Address(rsp, 0));
3523   fwait();
3524 #else
3525   fxsave(Address(rsp, 0));
3526 #endif // LP64
3527 }
3528 
3529 void MacroAssembler::push_IU_state() {
3530   // Push flags first because pusha kills them
3531   pushf();
3532   // Make sure rsp stays 16-byte aligned
3533   LP64_ONLY(subq(rsp, 8));
3534   pusha();
3535 }
3536 
3537 void MacroAssembler::reset_last_Java_frame(Register java_thread, bool clear_fp) { // determine java_thread register
3538   if (!java_thread->is_valid()) {
3539     java_thread = rdi;
3540     get_thread(java_thread);
3541   }
3542   // we must set sp to zero to clear frame
3543   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), NULL_WORD);
3544   if (clear_fp) {
3545     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
3546   }
3547 
3548   // Always clear the pc because it could have been set by make_walkable()
3549   movptr(Address(java_thread, JavaThread::last_Java_pc_offset()), NULL_WORD);
3550 
3551   vzeroupper();
3552 }
3553 
3554 void MacroAssembler::restore_rax(Register tmp) {
3555   if (tmp == noreg) pop(rax);
3556   else if (tmp != rax) mov(rax, tmp);
3557 }
3558 
3559 void MacroAssembler::round_to(Register reg, int modulus) {
3560   addptr(reg, modulus - 1);
3561   andptr(reg, -modulus);
3562 }
3563 
3564 void MacroAssembler::save_rax(Register tmp) {
3565   if (tmp == noreg) push(rax);
3566   else if (tmp != rax) mov(tmp, rax);
3567 }
3568 
3569 // Write serialization page so VM thread can do a pseudo remote membar.
3570 // We use the current thread pointer to calculate a thread specific
3571 // offset to write to within the page. This minimizes bus traffic
3572 // due to cache line collision.
3573 void MacroAssembler::serialize_memory(Register thread, Register tmp) {
3574   movl(tmp, thread);
3575   shrl(tmp, os::get_serialize_page_shift_count());
3576   andl(tmp, (os::vm_page_size() - sizeof(int)));
3577 
3578   Address index(noreg, tmp, Address::times_1);
3579   ExternalAddress page(os::get_memory_serialize_page());
3580 
3581   // Size of store must match masking code above
3582   movl(as_Address(ArrayAddress(page, index)), tmp);
3583 }
3584 
3585 void MacroAssembler::safepoint_poll(Label& slow_path, Register thread_reg, Register temp_reg) {
3586   if (SafepointMechanism::uses_thread_local_poll()) {
3587 #ifdef _LP64
3588     assert(thread_reg == r15_thread, "should be");
3589 #else
3590     if (thread_reg == noreg) {
3591       thread_reg = temp_reg;
3592       get_thread(thread_reg);
3593     }
3594 #endif
3595     testb(Address(thread_reg, Thread::polling_page_offset()), SafepointMechanism::poll_bit());
3596     jcc(Assembler::notZero, slow_path); // handshake bit set implies poll
3597   } else {
3598     cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
3599         SafepointSynchronize::_not_synchronized);
3600     jcc(Assembler::notEqual, slow_path);
3601   }
3602 }
3603 
3604 // Calls to C land
3605 //
3606 // When entering C land, the rbp, & rsp of the last Java frame have to be recorded
3607 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
3608 // has to be reset to 0. This is required to allow proper stack traversal.
3609 void MacroAssembler::set_last_Java_frame(Register java_thread,
3610                                          Register last_java_sp,
3611                                          Register last_java_fp,
3612                                          address  last_java_pc) {
3613   vzeroupper();
3614   // determine java_thread register
3615   if (!java_thread->is_valid()) {
3616     java_thread = rdi;
3617     get_thread(java_thread);
3618   }
3619   // determine last_java_sp register
3620   if (!last_java_sp->is_valid()) {
3621     last_java_sp = rsp;
3622   }
3623 
3624   // last_java_fp is optional
3625 
3626   if (last_java_fp->is_valid()) {
3627     movptr(Address(java_thread, JavaThread::last_Java_fp_offset()), last_java_fp);
3628   }
3629 
3630   // last_java_pc is optional
3631 
3632   if (last_java_pc != NULL) {
3633     lea(Address(java_thread,
3634                  JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()),
3635         InternalAddress(last_java_pc));
3636 
3637   }
3638   movptr(Address(java_thread, JavaThread::last_Java_sp_offset()), last_java_sp);
3639 }
3640 
3641 void MacroAssembler::shlptr(Register dst, int imm8) {
3642   LP64_ONLY(shlq(dst, imm8)) NOT_LP64(shll(dst, imm8));
3643 }
3644 
3645 void MacroAssembler::shrptr(Register dst, int imm8) {
3646   LP64_ONLY(shrq(dst, imm8)) NOT_LP64(shrl(dst, imm8));
3647 }
3648 
3649 void MacroAssembler::sign_extend_byte(Register reg) {
3650   if (LP64_ONLY(true ||) (VM_Version::is_P6() && reg->has_byte_register())) {
3651     movsbl(reg, reg); // movsxb
3652   } else {
3653     shll(reg, 24);
3654     sarl(reg, 24);
3655   }
3656 }
3657 
3658 void MacroAssembler::sign_extend_short(Register reg) {
3659   if (LP64_ONLY(true ||) VM_Version::is_P6()) {
3660     movswl(reg, reg); // movsxw
3661   } else {
3662     shll(reg, 16);
3663     sarl(reg, 16);
3664   }
3665 }
3666 
3667 void MacroAssembler::testl(Register dst, AddressLiteral src) {
3668   assert(reachable(src), "Address should be reachable");
3669   testl(dst, as_Address(src));
3670 }
3671 
3672 void MacroAssembler::pcmpeqb(XMMRegister dst, XMMRegister src) {
3673   int dst_enc = dst->encoding();
3674   int src_enc = src->encoding();
3675   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3676     Assembler::pcmpeqb(dst, src);
3677   } else if ((dst_enc < 16) && (src_enc < 16)) {
3678     Assembler::pcmpeqb(dst, src);
3679   } else if (src_enc < 16) {
3680     push_zmm(xmm0);
3681     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3682     Assembler::pcmpeqb(xmm0, src);
3683     movdqu(dst, xmm0);
3684     pop_zmm(xmm0);
3685   } else if (dst_enc < 16) {
3686     push_zmm(xmm0);
3687     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3688     Assembler::pcmpeqb(dst, xmm0);
3689     pop_zmm(xmm0);
3690   } else {
3691     push_zmm(xmm0);
3692     push_zmm(xmm1);
3693     movdqu(xmm0, src);
3694     movdqu(xmm1, dst);
3695     Assembler::pcmpeqb(xmm1, xmm0);
3696     movdqu(dst, xmm1);
3697     pop_zmm(xmm1);
3698     pop_zmm(xmm0);
3699   }
3700 }
3701 
3702 void MacroAssembler::pcmpeqw(XMMRegister dst, XMMRegister src) {
3703   int dst_enc = dst->encoding();
3704   int src_enc = src->encoding();
3705   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3706     Assembler::pcmpeqw(dst, src);
3707   } else if ((dst_enc < 16) && (src_enc < 16)) {
3708     Assembler::pcmpeqw(dst, src);
3709   } else if (src_enc < 16) {
3710     push_zmm(xmm0);
3711     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3712     Assembler::pcmpeqw(xmm0, src);
3713     movdqu(dst, xmm0);
3714     pop_zmm(xmm0);
3715   } else if (dst_enc < 16) {
3716     push_zmm(xmm0);
3717     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3718     Assembler::pcmpeqw(dst, xmm0);
3719     pop_zmm(xmm0);
3720   } else {
3721     push_zmm(xmm0);
3722     push_zmm(xmm1);
3723     movdqu(xmm0, src);
3724     movdqu(xmm1, dst);
3725     Assembler::pcmpeqw(xmm1, xmm0);
3726     movdqu(dst, xmm1);
3727     pop_zmm(xmm1);
3728     pop_zmm(xmm0);
3729   }
3730 }
3731 
3732 void MacroAssembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
3733   int dst_enc = dst->encoding();
3734   if (dst_enc < 16) {
3735     Assembler::pcmpestri(dst, src, imm8);
3736   } else {
3737     push_zmm(xmm0);
3738     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3739     Assembler::pcmpestri(xmm0, src, imm8);
3740     movdqu(dst, xmm0);
3741     pop_zmm(xmm0);
3742   }
3743 }
3744 
3745 void MacroAssembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
3746   int dst_enc = dst->encoding();
3747   int src_enc = src->encoding();
3748   if ((dst_enc < 16) && (src_enc < 16)) {
3749     Assembler::pcmpestri(dst, src, imm8);
3750   } else if (src_enc < 16) {
3751     push_zmm(xmm0);
3752     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3753     Assembler::pcmpestri(xmm0, src, imm8);
3754     movdqu(dst, xmm0);
3755     pop_zmm(xmm0);
3756   } else if (dst_enc < 16) {
3757     push_zmm(xmm0);
3758     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3759     Assembler::pcmpestri(dst, xmm0, imm8);
3760     pop_zmm(xmm0);
3761   } else {
3762     push_zmm(xmm0);
3763     push_zmm(xmm1);
3764     movdqu(xmm0, src);
3765     movdqu(xmm1, dst);
3766     Assembler::pcmpestri(xmm1, xmm0, imm8);
3767     movdqu(dst, xmm1);
3768     pop_zmm(xmm1);
3769     pop_zmm(xmm0);
3770   }
3771 }
3772 
3773 void MacroAssembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
3774   int dst_enc = dst->encoding();
3775   int src_enc = src->encoding();
3776   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3777     Assembler::pmovzxbw(dst, src);
3778   } else if ((dst_enc < 16) && (src_enc < 16)) {
3779     Assembler::pmovzxbw(dst, src);
3780   } else if (src_enc < 16) {
3781     push_zmm(xmm0);
3782     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3783     Assembler::pmovzxbw(xmm0, src);
3784     movdqu(dst, xmm0);
3785     pop_zmm(xmm0);
3786   } else if (dst_enc < 16) {
3787     push_zmm(xmm0);
3788     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3789     Assembler::pmovzxbw(dst, xmm0);
3790     pop_zmm(xmm0);
3791   } else {
3792     push_zmm(xmm0);
3793     push_zmm(xmm1);
3794     movdqu(xmm0, src);
3795     movdqu(xmm1, dst);
3796     Assembler::pmovzxbw(xmm1, xmm0);
3797     movdqu(dst, xmm1);
3798     pop_zmm(xmm1);
3799     pop_zmm(xmm0);
3800   }
3801 }
3802 
3803 void MacroAssembler::pmovzxbw(XMMRegister dst, Address src) {
3804   int dst_enc = dst->encoding();
3805   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
3806     Assembler::pmovzxbw(dst, src);
3807   } else if (dst_enc < 16) {
3808     Assembler::pmovzxbw(dst, src);
3809   } else {
3810     push_zmm(xmm0);
3811     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3812     Assembler::pmovzxbw(xmm0, src);
3813     movdqu(dst, xmm0);
3814     pop_zmm(xmm0);
3815   }
3816 }
3817 
3818 void MacroAssembler::pmovmskb(Register dst, XMMRegister src) {
3819   int src_enc = src->encoding();
3820   if (src_enc < 16) {
3821     Assembler::pmovmskb(dst, src);
3822   } else {
3823     push_zmm(xmm0);
3824     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3825     Assembler::pmovmskb(dst, xmm0);
3826     pop_zmm(xmm0);
3827   }
3828 }
3829 
3830 void MacroAssembler::ptest(XMMRegister dst, XMMRegister src) {
3831   int dst_enc = dst->encoding();
3832   int src_enc = src->encoding();
3833   if ((dst_enc < 16) && (src_enc < 16)) {
3834     Assembler::ptest(dst, src);
3835   } else if (src_enc < 16) {
3836     push_zmm(xmm0);
3837     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
3838     Assembler::ptest(xmm0, src);
3839     pop_zmm(xmm0);
3840   } else if (dst_enc < 16) {
3841     push_zmm(xmm0);
3842     evmovdqul(xmm0, src, Assembler::AVX_512bit);
3843     Assembler::ptest(dst, xmm0);
3844     pop_zmm(xmm0);
3845   } else {
3846     push_zmm(xmm0);
3847     push_zmm(xmm1);
3848     movdqu(xmm0, src);
3849     movdqu(xmm1, dst);
3850     Assembler::ptest(xmm1, xmm0);
3851     pop_zmm(xmm1);
3852     pop_zmm(xmm0);
3853   }
3854 }
3855 
3856 void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) {
3857   if (reachable(src)) {
3858     Assembler::sqrtsd(dst, as_Address(src));
3859   } else {
3860     lea(rscratch1, src);
3861     Assembler::sqrtsd(dst, Address(rscratch1, 0));
3862   }
3863 }
3864 
3865 void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) {
3866   if (reachable(src)) {
3867     Assembler::sqrtss(dst, as_Address(src));
3868   } else {
3869     lea(rscratch1, src);
3870     Assembler::sqrtss(dst, Address(rscratch1, 0));
3871   }
3872 }
3873 
3874 void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) {
3875   if (reachable(src)) {
3876     Assembler::subsd(dst, as_Address(src));
3877   } else {
3878     lea(rscratch1, src);
3879     Assembler::subsd(dst, Address(rscratch1, 0));
3880   }
3881 }
3882 
3883 void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) {
3884   if (reachable(src)) {
3885     Assembler::subss(dst, as_Address(src));
3886   } else {
3887     lea(rscratch1, src);
3888     Assembler::subss(dst, Address(rscratch1, 0));
3889   }
3890 }
3891 
3892 void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
3893   if (reachable(src)) {
3894     Assembler::ucomisd(dst, as_Address(src));
3895   } else {
3896     lea(rscratch1, src);
3897     Assembler::ucomisd(dst, Address(rscratch1, 0));
3898   }
3899 }
3900 
3901 void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
3902   if (reachable(src)) {
3903     Assembler::ucomiss(dst, as_Address(src));
3904   } else {
3905     lea(rscratch1, src);
3906     Assembler::ucomiss(dst, Address(rscratch1, 0));
3907   }
3908 }
3909 
3910 void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
3911   // Used in sign-bit flipping with aligned address.
3912   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3913   if (reachable(src)) {
3914     Assembler::xorpd(dst, as_Address(src));
3915   } else {
3916     lea(rscratch1, src);
3917     Assembler::xorpd(dst, Address(rscratch1, 0));
3918   }
3919 }
3920 
3921 void MacroAssembler::xorpd(XMMRegister dst, XMMRegister src) {
3922   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
3923     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
3924   }
3925   else {
3926     Assembler::xorpd(dst, src);
3927   }
3928 }
3929 
3930 void MacroAssembler::xorps(XMMRegister dst, XMMRegister src) {
3931   if (UseAVX > 2 && !VM_Version::supports_avx512dq() && (dst->encoding() == src->encoding())) {
3932     Assembler::vpxor(dst, dst, src, Assembler::AVX_512bit);
3933   } else {
3934     Assembler::xorps(dst, src);
3935   }
3936 }
3937 
3938 void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
3939   // Used in sign-bit flipping with aligned address.
3940   assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
3941   if (reachable(src)) {
3942     Assembler::xorps(dst, as_Address(src));
3943   } else {
3944     lea(rscratch1, src);
3945     Assembler::xorps(dst, Address(rscratch1, 0));
3946   }
3947 }
3948 
3949 void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) {
3950   // Used in sign-bit flipping with aligned address.
3951   bool aligned_adr = (((intptr_t)src.target() & 15) == 0);
3952   assert((UseAVX > 0) || aligned_adr, "SSE mode requires address alignment 16 bytes");
3953   if (reachable(src)) {
3954     Assembler::pshufb(dst, as_Address(src));
3955   } else {
3956     lea(rscratch1, src);
3957     Assembler::pshufb(dst, Address(rscratch1, 0));
3958   }
3959 }
3960 
3961 // AVX 3-operands instructions
3962 
3963 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3964   if (reachable(src)) {
3965     vaddsd(dst, nds, as_Address(src));
3966   } else {
3967     lea(rscratch1, src);
3968     vaddsd(dst, nds, Address(rscratch1, 0));
3969   }
3970 }
3971 
3972 void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
3973   if (reachable(src)) {
3974     vaddss(dst, nds, as_Address(src));
3975   } else {
3976     lea(rscratch1, src);
3977     vaddss(dst, nds, Address(rscratch1, 0));
3978   }
3979 }
3980 
3981 void MacroAssembler::vabsss(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
3982   int dst_enc = dst->encoding();
3983   int nds_enc = nds->encoding();
3984   int src_enc = src->encoding();
3985   if ((dst_enc < 16) && (nds_enc < 16)) {
3986     vandps(dst, nds, negate_field, vector_len);
3987   } else if ((src_enc < 16) && (dst_enc < 16)) {
3988     // Use src scratch register
3989     evmovdqul(src, nds, Assembler::AVX_512bit);
3990     vandps(dst, src, negate_field, vector_len);
3991   } else if (dst_enc < 16) {
3992     evmovdqul(dst, nds, Assembler::AVX_512bit);
3993     vandps(dst, dst, negate_field, vector_len);
3994   } else if (nds_enc < 16) {
3995     vandps(nds, nds, negate_field, vector_len);
3996     evmovdqul(dst, nds, Assembler::AVX_512bit);
3997   } else if (src_enc < 16) {
3998     evmovdqul(src, nds, Assembler::AVX_512bit);
3999     vandps(src, src, negate_field, vector_len);
4000     evmovdqul(dst, src, Assembler::AVX_512bit);
4001   } else {
4002     if (src_enc != dst_enc) {
4003       // Use src scratch register
4004       evmovdqul(src, xmm0, Assembler::AVX_512bit);
4005       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4006       vandps(xmm0, xmm0, negate_field, vector_len);
4007       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4008       evmovdqul(xmm0, src, Assembler::AVX_512bit);
4009     } else {
4010       push_zmm(xmm0);
4011       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4012       vandps(xmm0, xmm0, negate_field, vector_len);
4013       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4014       pop_zmm(xmm0);
4015     }
4016   }
4017 }
4018 
4019 void MacroAssembler::vabssd(XMMRegister dst, XMMRegister nds, XMMRegister src, AddressLiteral negate_field, int vector_len) {
4020   int dst_enc = dst->encoding();
4021   int nds_enc = nds->encoding();
4022   int src_enc = src->encoding();
4023   if ((dst_enc < 16) && (nds_enc < 16)) {
4024     vandpd(dst, nds, negate_field, vector_len);
4025   } else if ((src_enc < 16) && (dst_enc < 16)) {
4026     // Use src scratch register
4027     evmovdqul(src, nds, Assembler::AVX_512bit);
4028     vandpd(dst, src, negate_field, vector_len);
4029   } else if (dst_enc < 16) {
4030     evmovdqul(dst, nds, Assembler::AVX_512bit);
4031     vandpd(dst, dst, negate_field, vector_len);
4032   } else if (nds_enc < 16) {
4033     vandpd(nds, nds, negate_field, vector_len);
4034     evmovdqul(dst, nds, Assembler::AVX_512bit);
4035   } else if (src_enc < 16) {
4036     evmovdqul(src, nds, Assembler::AVX_512bit);
4037     vandpd(src, src, negate_field, vector_len);
4038     evmovdqul(dst, src, Assembler::AVX_512bit);
4039   } else {
4040     if (src_enc != dst_enc) {
4041       evmovdqul(src, xmm0, Assembler::AVX_512bit);
4042       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4043       vandpd(xmm0, xmm0, negate_field, vector_len);
4044       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4045       evmovdqul(xmm0, src, Assembler::AVX_512bit);
4046     } else {
4047       push_zmm(xmm0);
4048       evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4049       vandpd(xmm0, xmm0, negate_field, vector_len);
4050       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4051       pop_zmm(xmm0);
4052     }
4053   }
4054 }
4055 
4056 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4057   int dst_enc = dst->encoding();
4058   int nds_enc = nds->encoding();
4059   int src_enc = src->encoding();
4060   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4061     Assembler::vpaddb(dst, nds, src, vector_len);
4062   } else if ((dst_enc < 16) && (src_enc < 16)) {
4063     Assembler::vpaddb(dst, dst, src, vector_len);
4064   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4065     // use nds as scratch for src
4066     evmovdqul(nds, src, Assembler::AVX_512bit);
4067     Assembler::vpaddb(dst, dst, nds, vector_len);
4068   } else if ((src_enc < 16) && (nds_enc < 16)) {
4069     // use nds as scratch for dst
4070     evmovdqul(nds, dst, Assembler::AVX_512bit);
4071     Assembler::vpaddb(nds, nds, src, vector_len);
4072     evmovdqul(dst, nds, Assembler::AVX_512bit);
4073   } else if (dst_enc < 16) {
4074     // use nds as scatch for xmm0 to hold src
4075     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4076     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4077     Assembler::vpaddb(dst, dst, xmm0, vector_len);
4078     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4079   } else {
4080     // worse case scenario, all regs are in the upper bank
4081     push_zmm(xmm1);
4082     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4083     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4084     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4085     Assembler::vpaddb(xmm0, xmm0, xmm1, vector_len);
4086     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4087     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4088     pop_zmm(xmm1);
4089   }
4090 }
4091 
4092 void MacroAssembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4093   int dst_enc = dst->encoding();
4094   int nds_enc = nds->encoding();
4095   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4096     Assembler::vpaddb(dst, nds, src, vector_len);
4097   } else if (dst_enc < 16) {
4098     Assembler::vpaddb(dst, dst, src, vector_len);
4099   } else if (nds_enc < 16) {
4100     // implies dst_enc in upper bank with src as scratch
4101     evmovdqul(nds, dst, Assembler::AVX_512bit);
4102     Assembler::vpaddb(nds, nds, src, vector_len);
4103     evmovdqul(dst, nds, Assembler::AVX_512bit);
4104   } else {
4105     // worse case scenario, all regs in upper bank
4106     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4107     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4108     Assembler::vpaddb(xmm0, xmm0, src, vector_len);
4109     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4110     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4111   }
4112 }
4113 
4114 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4115   int dst_enc = dst->encoding();
4116   int nds_enc = nds->encoding();
4117   int src_enc = src->encoding();
4118   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4119     Assembler::vpaddw(dst, nds, src, vector_len);
4120   } else if ((dst_enc < 16) && (src_enc < 16)) {
4121     Assembler::vpaddw(dst, dst, src, vector_len);
4122   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4123     // use nds as scratch for src
4124     evmovdqul(nds, src, Assembler::AVX_512bit);
4125     Assembler::vpaddw(dst, dst, nds, vector_len);
4126   } else if ((src_enc < 16) && (nds_enc < 16)) {
4127     // use nds as scratch for dst
4128     evmovdqul(nds, dst, Assembler::AVX_512bit);
4129     Assembler::vpaddw(nds, nds, src, vector_len);
4130     evmovdqul(dst, nds, Assembler::AVX_512bit);
4131   } else if (dst_enc < 16) {
4132     // use nds as scatch for xmm0 to hold src
4133     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4134     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4135     Assembler::vpaddw(dst, dst, xmm0, vector_len);
4136     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4137   } else {
4138     // worse case scenario, all regs are in the upper bank
4139     push_zmm(xmm1);
4140     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4141     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4142     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4143     Assembler::vpaddw(xmm0, xmm0, xmm1, vector_len);
4144     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4145     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4146     pop_zmm(xmm1);
4147   }
4148 }
4149 
4150 void MacroAssembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4151   int dst_enc = dst->encoding();
4152   int nds_enc = nds->encoding();
4153   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4154     Assembler::vpaddw(dst, nds, src, vector_len);
4155   } else if (dst_enc < 16) {
4156     Assembler::vpaddw(dst, dst, src, vector_len);
4157   } else if (nds_enc < 16) {
4158     // implies dst_enc in upper bank with nds as scratch
4159     evmovdqul(nds, dst, Assembler::AVX_512bit);
4160     Assembler::vpaddw(nds, nds, src, vector_len);
4161     evmovdqul(dst, nds, Assembler::AVX_512bit);
4162   } else {
4163     // worse case scenario, all regs in upper bank
4164     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4165     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4166     Assembler::vpaddw(xmm0, xmm0, src, vector_len);
4167     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4168     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4169   }
4170 }
4171 
4172 void MacroAssembler::vpand(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4173   if (reachable(src)) {
4174     Assembler::vpand(dst, nds, as_Address(src), vector_len);
4175   } else {
4176     lea(rscratch1, src);
4177     Assembler::vpand(dst, nds, Address(rscratch1, 0), vector_len);
4178   }
4179 }
4180 
4181 void MacroAssembler::vpbroadcastw(XMMRegister dst, XMMRegister src) {
4182   int dst_enc = dst->encoding();
4183   int src_enc = src->encoding();
4184   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4185     Assembler::vpbroadcastw(dst, src);
4186   } else if ((dst_enc < 16) && (src_enc < 16)) {
4187     Assembler::vpbroadcastw(dst, src);
4188   } else if (src_enc < 16) {
4189     push_zmm(xmm0);
4190     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4191     Assembler::vpbroadcastw(xmm0, src);
4192     movdqu(dst, xmm0);
4193     pop_zmm(xmm0);
4194   } else if (dst_enc < 16) {
4195     push_zmm(xmm0);
4196     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4197     Assembler::vpbroadcastw(dst, xmm0);
4198     pop_zmm(xmm0);
4199   } else {
4200     push_zmm(xmm0);
4201     push_zmm(xmm1);
4202     movdqu(xmm0, src);
4203     movdqu(xmm1, dst);
4204     Assembler::vpbroadcastw(xmm1, xmm0);
4205     movdqu(dst, xmm1);
4206     pop_zmm(xmm1);
4207     pop_zmm(xmm0);
4208   }
4209 }
4210 
4211 void MacroAssembler::vpcmpeqb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4212   int dst_enc = dst->encoding();
4213   int nds_enc = nds->encoding();
4214   int src_enc = src->encoding();
4215   assert(dst_enc == nds_enc, "");
4216   if ((dst_enc < 16) && (src_enc < 16)) {
4217     Assembler::vpcmpeqb(dst, nds, src, vector_len);
4218   } else if (src_enc < 16) {
4219     push_zmm(xmm0);
4220     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4221     Assembler::vpcmpeqb(xmm0, xmm0, src, vector_len);
4222     movdqu(dst, xmm0);
4223     pop_zmm(xmm0);
4224   } else if (dst_enc < 16) {
4225     push_zmm(xmm0);
4226     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4227     Assembler::vpcmpeqb(dst, dst, xmm0, vector_len);
4228     pop_zmm(xmm0);
4229   } else {
4230     push_zmm(xmm0);
4231     push_zmm(xmm1);
4232     movdqu(xmm0, src);
4233     movdqu(xmm1, dst);
4234     Assembler::vpcmpeqb(xmm1, xmm1, xmm0, vector_len);
4235     movdqu(dst, xmm1);
4236     pop_zmm(xmm1);
4237     pop_zmm(xmm0);
4238   }
4239 }
4240 
4241 void MacroAssembler::vpcmpeqw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4242   int dst_enc = dst->encoding();
4243   int nds_enc = nds->encoding();
4244   int src_enc = src->encoding();
4245   assert(dst_enc == nds_enc, "");
4246   if ((dst_enc < 16) && (src_enc < 16)) {
4247     Assembler::vpcmpeqw(dst, nds, src, vector_len);
4248   } else if (src_enc < 16) {
4249     push_zmm(xmm0);
4250     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4251     Assembler::vpcmpeqw(xmm0, xmm0, src, vector_len);
4252     movdqu(dst, xmm0);
4253     pop_zmm(xmm0);
4254   } else if (dst_enc < 16) {
4255     push_zmm(xmm0);
4256     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4257     Assembler::vpcmpeqw(dst, dst, xmm0, vector_len);
4258     pop_zmm(xmm0);
4259   } else {
4260     push_zmm(xmm0);
4261     push_zmm(xmm1);
4262     movdqu(xmm0, src);
4263     movdqu(xmm1, dst);
4264     Assembler::vpcmpeqw(xmm1, xmm1, xmm0, vector_len);
4265     movdqu(dst, xmm1);
4266     pop_zmm(xmm1);
4267     pop_zmm(xmm0);
4268   }
4269 }
4270 
4271 void MacroAssembler::vpmovzxbw(XMMRegister dst, Address src, int vector_len) {
4272   int dst_enc = dst->encoding();
4273   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4274     Assembler::vpmovzxbw(dst, src, vector_len);
4275   } else if (dst_enc < 16) {
4276     Assembler::vpmovzxbw(dst, src, vector_len);
4277   } else {
4278     push_zmm(xmm0);
4279     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4280     Assembler::vpmovzxbw(xmm0, src, vector_len);
4281     movdqu(dst, xmm0);
4282     pop_zmm(xmm0);
4283   }
4284 }
4285 
4286 void MacroAssembler::vpmovmskb(Register dst, XMMRegister src) {
4287   int src_enc = src->encoding();
4288   if (src_enc < 16) {
4289     Assembler::vpmovmskb(dst, src);
4290   } else {
4291     push_zmm(xmm0);
4292     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4293     Assembler::vpmovmskb(dst, xmm0);
4294     pop_zmm(xmm0);
4295   }
4296 }
4297 
4298 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4299   int dst_enc = dst->encoding();
4300   int nds_enc = nds->encoding();
4301   int src_enc = src->encoding();
4302   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4303     Assembler::vpmullw(dst, nds, src, vector_len);
4304   } else if ((dst_enc < 16) && (src_enc < 16)) {
4305     Assembler::vpmullw(dst, dst, src, vector_len);
4306   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4307     // use nds as scratch for src
4308     evmovdqul(nds, src, Assembler::AVX_512bit);
4309     Assembler::vpmullw(dst, dst, nds, vector_len);
4310   } else if ((src_enc < 16) && (nds_enc < 16)) {
4311     // use nds as scratch for dst
4312     evmovdqul(nds, dst, Assembler::AVX_512bit);
4313     Assembler::vpmullw(nds, nds, src, vector_len);
4314     evmovdqul(dst, nds, Assembler::AVX_512bit);
4315   } else if (dst_enc < 16) {
4316     // use nds as scatch for xmm0 to hold src
4317     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4318     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4319     Assembler::vpmullw(dst, dst, xmm0, vector_len);
4320     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4321   } else {
4322     // worse case scenario, all regs are in the upper bank
4323     push_zmm(xmm1);
4324     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4325     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4326     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4327     Assembler::vpmullw(xmm0, xmm0, xmm1, vector_len);
4328     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4329     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4330     pop_zmm(xmm1);
4331   }
4332 }
4333 
4334 void MacroAssembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4335   int dst_enc = dst->encoding();
4336   int nds_enc = nds->encoding();
4337   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4338     Assembler::vpmullw(dst, nds, src, vector_len);
4339   } else if (dst_enc < 16) {
4340     Assembler::vpmullw(dst, dst, src, vector_len);
4341   } else if (nds_enc < 16) {
4342     // implies dst_enc in upper bank with src as scratch
4343     evmovdqul(nds, dst, Assembler::AVX_512bit);
4344     Assembler::vpmullw(nds, nds, src, vector_len);
4345     evmovdqul(dst, nds, Assembler::AVX_512bit);
4346   } else {
4347     // worse case scenario, all regs in upper bank
4348     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4349     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4350     Assembler::vpmullw(xmm0, xmm0, src, vector_len);
4351     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4352     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4353   }
4354 }
4355 
4356 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4357   int dst_enc = dst->encoding();
4358   int nds_enc = nds->encoding();
4359   int src_enc = src->encoding();
4360   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4361     Assembler::vpsubb(dst, nds, src, vector_len);
4362   } else if ((dst_enc < 16) && (src_enc < 16)) {
4363     Assembler::vpsubb(dst, dst, src, vector_len);
4364   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4365     // use nds as scratch for src
4366     evmovdqul(nds, src, Assembler::AVX_512bit);
4367     Assembler::vpsubb(dst, dst, nds, vector_len);
4368   } else if ((src_enc < 16) && (nds_enc < 16)) {
4369     // use nds as scratch for dst
4370     evmovdqul(nds, dst, Assembler::AVX_512bit);
4371     Assembler::vpsubb(nds, nds, src, vector_len);
4372     evmovdqul(dst, nds, Assembler::AVX_512bit);
4373   } else if (dst_enc < 16) {
4374     // use nds as scatch for xmm0 to hold src
4375     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4376     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4377     Assembler::vpsubb(dst, dst, xmm0, vector_len);
4378     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4379   } else {
4380     // worse case scenario, all regs are in the upper bank
4381     push_zmm(xmm1);
4382     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4383     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4384     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4385     Assembler::vpsubb(xmm0, xmm0, xmm1, vector_len);
4386     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4387     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4388     pop_zmm(xmm1);
4389   }
4390 }
4391 
4392 void MacroAssembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4393   int dst_enc = dst->encoding();
4394   int nds_enc = nds->encoding();
4395   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4396     Assembler::vpsubb(dst, nds, src, vector_len);
4397   } else if (dst_enc < 16) {
4398     Assembler::vpsubb(dst, dst, src, vector_len);
4399   } else if (nds_enc < 16) {
4400     // implies dst_enc in upper bank with src as scratch
4401     evmovdqul(nds, dst, Assembler::AVX_512bit);
4402     Assembler::vpsubb(nds, nds, src, vector_len);
4403     evmovdqul(dst, nds, Assembler::AVX_512bit);
4404   } else {
4405     // worse case scenario, all regs in upper bank
4406     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4407     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4408     Assembler::vpsubb(xmm0, xmm0, src, vector_len);
4409     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4410     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4411   }
4412 }
4413 
4414 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, int vector_len) {
4415   int dst_enc = dst->encoding();
4416   int nds_enc = nds->encoding();
4417   int src_enc = src->encoding();
4418   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4419     Assembler::vpsubw(dst, nds, src, vector_len);
4420   } else if ((dst_enc < 16) && (src_enc < 16)) {
4421     Assembler::vpsubw(dst, dst, src, vector_len);
4422   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4423     // use nds as scratch for src
4424     evmovdqul(nds, src, Assembler::AVX_512bit);
4425     Assembler::vpsubw(dst, dst, nds, vector_len);
4426   } else if ((src_enc < 16) && (nds_enc < 16)) {
4427     // use nds as scratch for dst
4428     evmovdqul(nds, dst, Assembler::AVX_512bit);
4429     Assembler::vpsubw(nds, nds, src, vector_len);
4430     evmovdqul(dst, nds, Assembler::AVX_512bit);
4431   } else if (dst_enc < 16) {
4432     // use nds as scatch for xmm0 to hold src
4433     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4434     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4435     Assembler::vpsubw(dst, dst, xmm0, vector_len);
4436     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4437   } else {
4438     // worse case scenario, all regs are in the upper bank
4439     push_zmm(xmm1);
4440     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4441     evmovdqul(xmm1, src, Assembler::AVX_512bit);
4442     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4443     Assembler::vpsubw(xmm0, xmm0, xmm1, vector_len);
4444     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4445     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4446     pop_zmm(xmm1);
4447   }
4448 }
4449 
4450 void MacroAssembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, int vector_len) {
4451   int dst_enc = dst->encoding();
4452   int nds_enc = nds->encoding();
4453   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4454     Assembler::vpsubw(dst, nds, src, vector_len);
4455   } else if (dst_enc < 16) {
4456     Assembler::vpsubw(dst, dst, src, vector_len);
4457   } else if (nds_enc < 16) {
4458     // implies dst_enc in upper bank with src as scratch
4459     evmovdqul(nds, dst, Assembler::AVX_512bit);
4460     Assembler::vpsubw(nds, nds, src, vector_len);
4461     evmovdqul(dst, nds, Assembler::AVX_512bit);
4462   } else {
4463     // worse case scenario, all regs in upper bank
4464     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4465     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4466     Assembler::vpsubw(xmm0, xmm0, src, vector_len);
4467     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4468     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4469   }
4470 }
4471 
4472 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4473   int dst_enc = dst->encoding();
4474   int nds_enc = nds->encoding();
4475   int shift_enc = shift->encoding();
4476   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4477     Assembler::vpsraw(dst, nds, shift, vector_len);
4478   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4479     Assembler::vpsraw(dst, dst, shift, vector_len);
4480   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4481     // use nds as scratch with shift
4482     evmovdqul(nds, shift, Assembler::AVX_512bit);
4483     Assembler::vpsraw(dst, dst, nds, vector_len);
4484   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4485     // use nds as scratch with dst
4486     evmovdqul(nds, dst, Assembler::AVX_512bit);
4487     Assembler::vpsraw(nds, nds, shift, vector_len);
4488     evmovdqul(dst, nds, Assembler::AVX_512bit);
4489   } else if (dst_enc < 16) {
4490     // use nds to save a copy of xmm0 and hold shift
4491     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4492     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4493     Assembler::vpsraw(dst, dst, xmm0, vector_len);
4494     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4495   } else if (nds_enc < 16) {
4496     // use nds and dst as temps
4497     evmovdqul(nds, dst, Assembler::AVX_512bit);
4498     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4499     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4500     Assembler::vpsraw(nds, nds, xmm0, vector_len);
4501     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4502     evmovdqul(dst, nds, Assembler::AVX_512bit);
4503   } else {
4504     // worse case scenario, all regs are in the upper bank
4505     push_zmm(xmm1);
4506     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4507     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4508     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4509     Assembler::vpsraw(xmm0, xmm0, xmm1, vector_len);
4510     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4511     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4512     pop_zmm(xmm1);
4513   }
4514 }
4515 
4516 void MacroAssembler::vpsraw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4517   int dst_enc = dst->encoding();
4518   int nds_enc = nds->encoding();
4519   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4520     Assembler::vpsraw(dst, nds, shift, vector_len);
4521   } else if (dst_enc < 16) {
4522     Assembler::vpsraw(dst, dst, shift, vector_len);
4523   } else if (nds_enc < 16) {
4524     // use nds as scratch
4525     evmovdqul(nds, dst, Assembler::AVX_512bit);
4526     Assembler::vpsraw(nds, nds, shift, vector_len);
4527     evmovdqul(dst, nds, Assembler::AVX_512bit);
4528   } else {
4529     // use nds as scratch for xmm0
4530     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4531     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4532     Assembler::vpsraw(xmm0, xmm0, shift, vector_len);
4533     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4534     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4535   }
4536 }
4537 
4538 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4539   int dst_enc = dst->encoding();
4540   int nds_enc = nds->encoding();
4541   int shift_enc = shift->encoding();
4542   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4543     Assembler::vpsrlw(dst, nds, shift, vector_len);
4544   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4545     Assembler::vpsrlw(dst, dst, shift, vector_len);
4546   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4547     // use nds as scratch with shift
4548     evmovdqul(nds, shift, Assembler::AVX_512bit);
4549     Assembler::vpsrlw(dst, dst, nds, vector_len);
4550   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4551     // use nds as scratch with dst
4552     evmovdqul(nds, dst, Assembler::AVX_512bit);
4553     Assembler::vpsrlw(nds, nds, shift, vector_len);
4554     evmovdqul(dst, nds, Assembler::AVX_512bit);
4555   } else if (dst_enc < 16) {
4556     // use nds to save a copy of xmm0 and hold shift
4557     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4558     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4559     Assembler::vpsrlw(dst, dst, xmm0, vector_len);
4560     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4561   } else if (nds_enc < 16) {
4562     // use nds and dst as temps
4563     evmovdqul(nds, dst, Assembler::AVX_512bit);
4564     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4565     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4566     Assembler::vpsrlw(nds, nds, xmm0, vector_len);
4567     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4568     evmovdqul(dst, nds, Assembler::AVX_512bit);
4569   } else {
4570     // worse case scenario, all regs are in the upper bank
4571     push_zmm(xmm1);
4572     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4573     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4574     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4575     Assembler::vpsrlw(xmm0, xmm0, xmm1, vector_len);
4576     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4577     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4578     pop_zmm(xmm1);
4579   }
4580 }
4581 
4582 void MacroAssembler::vpsrlw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4583   int dst_enc = dst->encoding();
4584   int nds_enc = nds->encoding();
4585   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4586     Assembler::vpsrlw(dst, nds, shift, vector_len);
4587   } else if (dst_enc < 16) {
4588     Assembler::vpsrlw(dst, dst, shift, vector_len);
4589   } else if (nds_enc < 16) {
4590     // use nds as scratch
4591     evmovdqul(nds, dst, Assembler::AVX_512bit);
4592     Assembler::vpsrlw(nds, nds, shift, vector_len);
4593     evmovdqul(dst, nds, Assembler::AVX_512bit);
4594   } else {
4595     // use nds as scratch for xmm0
4596     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4597     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4598     Assembler::vpsrlw(xmm0, xmm0, shift, vector_len);
4599     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4600     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4601   }
4602 }
4603 
4604 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, XMMRegister shift, int vector_len) {
4605   int dst_enc = dst->encoding();
4606   int nds_enc = nds->encoding();
4607   int shift_enc = shift->encoding();
4608   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4609     Assembler::vpsllw(dst, nds, shift, vector_len);
4610   } else if ((dst_enc < 16) && (shift_enc < 16)) {
4611     Assembler::vpsllw(dst, dst, shift, vector_len);
4612   } else if ((dst_enc < 16) && (nds_enc < 16)) {
4613     // use nds as scratch with shift
4614     evmovdqul(nds, shift, Assembler::AVX_512bit);
4615     Assembler::vpsllw(dst, dst, nds, vector_len);
4616   } else if ((shift_enc < 16) && (nds_enc < 16)) {
4617     // use nds as scratch with dst
4618     evmovdqul(nds, dst, Assembler::AVX_512bit);
4619     Assembler::vpsllw(nds, nds, shift, vector_len);
4620     evmovdqul(dst, nds, Assembler::AVX_512bit);
4621   } else if (dst_enc < 16) {
4622     // use nds to save a copy of xmm0 and hold shift
4623     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4624     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4625     Assembler::vpsllw(dst, dst, xmm0, vector_len);
4626     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4627   } else if (nds_enc < 16) {
4628     // use nds and dst as temps
4629     evmovdqul(nds, dst, Assembler::AVX_512bit);
4630     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4631     evmovdqul(xmm0, shift, Assembler::AVX_512bit);
4632     Assembler::vpsllw(nds, nds, xmm0, vector_len);
4633     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4634     evmovdqul(dst, nds, Assembler::AVX_512bit);
4635   } else {
4636     // worse case scenario, all regs are in the upper bank
4637     push_zmm(xmm1);
4638     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4639     evmovdqul(xmm1, shift, Assembler::AVX_512bit);
4640     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4641     Assembler::vpsllw(xmm0, xmm0, xmm1, vector_len);
4642     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4643     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4644     pop_zmm(xmm1);
4645   }
4646 }
4647 
4648 void MacroAssembler::vpsllw(XMMRegister dst, XMMRegister nds, int shift, int vector_len) {
4649   int dst_enc = dst->encoding();
4650   int nds_enc = nds->encoding();
4651   if (VM_Version::supports_avxonly() || VM_Version::supports_avx512bw()) {
4652     Assembler::vpsllw(dst, nds, shift, vector_len);
4653   } else if (dst_enc < 16) {
4654     Assembler::vpsllw(dst, dst, shift, vector_len);
4655   } else if (nds_enc < 16) {
4656     // use nds as scratch
4657     evmovdqul(nds, dst, Assembler::AVX_512bit);
4658     Assembler::vpsllw(nds, nds, shift, vector_len);
4659     evmovdqul(dst, nds, Assembler::AVX_512bit);
4660   } else {
4661     // use nds as scratch for xmm0
4662     evmovdqul(nds, xmm0, Assembler::AVX_512bit);
4663     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4664     Assembler::vpsllw(xmm0, xmm0, shift, vector_len);
4665     evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4666     evmovdqul(xmm0, nds, Assembler::AVX_512bit);
4667   }
4668 }
4669 
4670 void MacroAssembler::vptest(XMMRegister dst, XMMRegister src) {
4671   int dst_enc = dst->encoding();
4672   int src_enc = src->encoding();
4673   if ((dst_enc < 16) && (src_enc < 16)) {
4674     Assembler::vptest(dst, src);
4675   } else if (src_enc < 16) {
4676     push_zmm(xmm0);
4677     evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4678     Assembler::vptest(xmm0, src);
4679     pop_zmm(xmm0);
4680   } else if (dst_enc < 16) {
4681     push_zmm(xmm0);
4682     evmovdqul(xmm0, src, Assembler::AVX_512bit);
4683     Assembler::vptest(dst, xmm0);
4684     pop_zmm(xmm0);
4685   } else {
4686     push_zmm(xmm0);
4687     push_zmm(xmm1);
4688     movdqu(xmm0, src);
4689     movdqu(xmm1, dst);
4690     Assembler::vptest(xmm1, xmm0);
4691     pop_zmm(xmm1);
4692     pop_zmm(xmm0);
4693   }
4694 }
4695 
4696 // This instruction exists within macros, ergo we cannot control its input
4697 // when emitted through those patterns.
4698 void MacroAssembler::punpcklbw(XMMRegister dst, XMMRegister src) {
4699   if (VM_Version::supports_avx512nobw()) {
4700     int dst_enc = dst->encoding();
4701     int src_enc = src->encoding();
4702     if (dst_enc == src_enc) {
4703       if (dst_enc < 16) {
4704         Assembler::punpcklbw(dst, src);
4705       } else {
4706         push_zmm(xmm0);
4707         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4708         Assembler::punpcklbw(xmm0, xmm0);
4709         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4710         pop_zmm(xmm0);
4711       }
4712     } else {
4713       if ((src_enc < 16) && (dst_enc < 16)) {
4714         Assembler::punpcklbw(dst, src);
4715       } else if (src_enc < 16) {
4716         push_zmm(xmm0);
4717         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4718         Assembler::punpcklbw(xmm0, src);
4719         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4720         pop_zmm(xmm0);
4721       } else if (dst_enc < 16) {
4722         push_zmm(xmm0);
4723         evmovdqul(xmm0, src, Assembler::AVX_512bit);
4724         Assembler::punpcklbw(dst, xmm0);
4725         pop_zmm(xmm0);
4726       } else {
4727         push_zmm(xmm0);
4728         push_zmm(xmm1);
4729         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4730         evmovdqul(xmm1, src, Assembler::AVX_512bit);
4731         Assembler::punpcklbw(xmm0, xmm1);
4732         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4733         pop_zmm(xmm1);
4734         pop_zmm(xmm0);
4735       }
4736     }
4737   } else {
4738     Assembler::punpcklbw(dst, src);
4739   }
4740 }
4741 
4742 void MacroAssembler::pshufd(XMMRegister dst, Address src, int mode) {
4743   if (VM_Version::supports_avx512vl()) {
4744     Assembler::pshufd(dst, src, mode);
4745   } else {
4746     int dst_enc = dst->encoding();
4747     if (dst_enc < 16) {
4748       Assembler::pshufd(dst, src, mode);
4749     } else {
4750       push_zmm(xmm0);
4751       Assembler::pshufd(xmm0, src, mode);
4752       evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4753       pop_zmm(xmm0);
4754     }
4755   }
4756 }
4757 
4758 // This instruction exists within macros, ergo we cannot control its input
4759 // when emitted through those patterns.
4760 void MacroAssembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
4761   if (VM_Version::supports_avx512nobw()) {
4762     int dst_enc = dst->encoding();
4763     int src_enc = src->encoding();
4764     if (dst_enc == src_enc) {
4765       if (dst_enc < 16) {
4766         Assembler::pshuflw(dst, src, mode);
4767       } else {
4768         push_zmm(xmm0);
4769         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4770         Assembler::pshuflw(xmm0, xmm0, mode);
4771         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4772         pop_zmm(xmm0);
4773       }
4774     } else {
4775       if ((src_enc < 16) && (dst_enc < 16)) {
4776         Assembler::pshuflw(dst, src, mode);
4777       } else if (src_enc < 16) {
4778         push_zmm(xmm0);
4779         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4780         Assembler::pshuflw(xmm0, src, mode);
4781         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4782         pop_zmm(xmm0);
4783       } else if (dst_enc < 16) {
4784         push_zmm(xmm0);
4785         evmovdqul(xmm0, src, Assembler::AVX_512bit);
4786         Assembler::pshuflw(dst, xmm0, mode);
4787         pop_zmm(xmm0);
4788       } else {
4789         push_zmm(xmm0);
4790         push_zmm(xmm1);
4791         evmovdqul(xmm0, dst, Assembler::AVX_512bit);
4792         evmovdqul(xmm1, src, Assembler::AVX_512bit);
4793         Assembler::pshuflw(xmm0, xmm1, mode);
4794         evmovdqul(dst, xmm0, Assembler::AVX_512bit);
4795         pop_zmm(xmm1);
4796         pop_zmm(xmm0);
4797       }
4798     }
4799   } else {
4800     Assembler::pshuflw(dst, src, mode);
4801   }
4802 }
4803 
4804 void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4805   if (reachable(src)) {
4806     vandpd(dst, nds, as_Address(src), vector_len);
4807   } else {
4808     lea(rscratch1, src);
4809     vandpd(dst, nds, Address(rscratch1, 0), vector_len);
4810   }
4811 }
4812 
4813 void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4814   if (reachable(src)) {
4815     vandps(dst, nds, as_Address(src), vector_len);
4816   } else {
4817     lea(rscratch1, src);
4818     vandps(dst, nds, Address(rscratch1, 0), vector_len);
4819   }
4820 }
4821 
4822 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4823   if (reachable(src)) {
4824     vdivsd(dst, nds, as_Address(src));
4825   } else {
4826     lea(rscratch1, src);
4827     vdivsd(dst, nds, Address(rscratch1, 0));
4828   }
4829 }
4830 
4831 void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4832   if (reachable(src)) {
4833     vdivss(dst, nds, as_Address(src));
4834   } else {
4835     lea(rscratch1, src);
4836     vdivss(dst, nds, Address(rscratch1, 0));
4837   }
4838 }
4839 
4840 void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4841   if (reachable(src)) {
4842     vmulsd(dst, nds, as_Address(src));
4843   } else {
4844     lea(rscratch1, src);
4845     vmulsd(dst, nds, Address(rscratch1, 0));
4846   }
4847 }
4848 
4849 void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4850   if (reachable(src)) {
4851     vmulss(dst, nds, as_Address(src));
4852   } else {
4853     lea(rscratch1, src);
4854     vmulss(dst, nds, Address(rscratch1, 0));
4855   }
4856 }
4857 
4858 void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4859   if (reachable(src)) {
4860     vsubsd(dst, nds, as_Address(src));
4861   } else {
4862     lea(rscratch1, src);
4863     vsubsd(dst, nds, Address(rscratch1, 0));
4864   }
4865 }
4866 
4867 void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4868   if (reachable(src)) {
4869     vsubss(dst, nds, as_Address(src));
4870   } else {
4871     lea(rscratch1, src);
4872     vsubss(dst, nds, Address(rscratch1, 0));
4873   }
4874 }
4875 
4876 void MacroAssembler::vnegatess(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4877   int nds_enc = nds->encoding();
4878   int dst_enc = dst->encoding();
4879   bool dst_upper_bank = (dst_enc > 15);
4880   bool nds_upper_bank = (nds_enc > 15);
4881   if (VM_Version::supports_avx512novl() &&
4882       (nds_upper_bank || dst_upper_bank)) {
4883     if (dst_upper_bank) {
4884       push_zmm(xmm0);
4885       movflt(xmm0, nds);
4886       vxorps(xmm0, xmm0, src, Assembler::AVX_128bit);
4887       movflt(dst, xmm0);
4888       pop_zmm(xmm0);
4889     } else {
4890       movflt(dst, nds);
4891       vxorps(dst, dst, src, Assembler::AVX_128bit);
4892     }
4893   } else {
4894     vxorps(dst, nds, src, Assembler::AVX_128bit);
4895   }
4896 }
4897 
4898 void MacroAssembler::vnegatesd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
4899   int nds_enc = nds->encoding();
4900   int dst_enc = dst->encoding();
4901   bool dst_upper_bank = (dst_enc > 15);
4902   bool nds_upper_bank = (nds_enc > 15);
4903   if (VM_Version::supports_avx512novl() &&
4904       (nds_upper_bank || dst_upper_bank)) {
4905     if (dst_upper_bank) {
4906       push_zmm(xmm0);
4907       movdbl(xmm0, nds);
4908       vxorpd(xmm0, xmm0, src, Assembler::AVX_128bit);
4909       movdbl(dst, xmm0);
4910       pop_zmm(xmm0);
4911     } else {
4912       movdbl(dst, nds);
4913       vxorpd(dst, dst, src, Assembler::AVX_128bit);
4914     }
4915   } else {
4916     vxorpd(dst, nds, src, Assembler::AVX_128bit);
4917   }
4918 }
4919 
4920 void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4921   if (reachable(src)) {
4922     vxorpd(dst, nds, as_Address(src), vector_len);
4923   } else {
4924     lea(rscratch1, src);
4925     vxorpd(dst, nds, Address(rscratch1, 0), vector_len);
4926   }
4927 }
4928 
4929 void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, int vector_len) {
4930   if (reachable(src)) {
4931     vxorps(dst, nds, as_Address(src), vector_len);
4932   } else {
4933     lea(rscratch1, src);
4934     vxorps(dst, nds, Address(rscratch1, 0), vector_len);
4935   }
4936 }
4937 
4938 void MacroAssembler::clear_jweak_tag(Register possibly_jweak) {
4939   const int32_t inverted_jweak_mask = ~static_cast<int32_t>(JNIHandles::weak_tag_mask);
4940   STATIC_ASSERT(inverted_jweak_mask == -2); // otherwise check this code
4941   // The inverted mask is sign-extended
4942   andptr(possibly_jweak, inverted_jweak_mask);
4943 }
4944 
4945 void MacroAssembler::resolve_jobject(Register value,
4946                                      Register thread,
4947                                      Register tmp) {
4948   assert_different_registers(value, thread, tmp);
4949   Label done, not_weak;
4950   testptr(value, value);
4951   jcc(Assembler::zero, done);                // Use NULL as-is.
4952   testptr(value, JNIHandles::weak_tag_mask); // Test for jweak tag.
4953   jcc(Assembler::zero, not_weak);
4954   // Resolve jweak.
4955   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
4956                  value, Address(value, -JNIHandles::weak_tag_value), tmp, thread);
4957   verify_oop(value);
4958   jmp(done);
4959   bind(not_weak);
4960   // Resolve (untagged) jobject.
4961   access_load_at(T_OBJECT, IN_NATIVE, value, Address(value, 0), tmp, thread);
4962   verify_oop(value);
4963   bind(done);
4964 }
4965 
4966 void MacroAssembler::subptr(Register dst, int32_t imm32) {
4967   LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
4968 }
4969 
4970 // Force generation of a 4 byte immediate value even if it fits into 8bit
4971 void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
4972   LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
4973 }
4974 
4975 void MacroAssembler::subptr(Register dst, Register src) {
4976   LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
4977 }
4978 
4979 // C++ bool manipulation
4980 void MacroAssembler::testbool(Register dst) {
4981   if(sizeof(bool) == 1)
4982     testb(dst, 0xff);
4983   else if(sizeof(bool) == 2) {
4984     // testw implementation needed for two byte bools
4985     ShouldNotReachHere();
4986   } else if(sizeof(bool) == 4)
4987     testl(dst, dst);
4988   else
4989     // unsupported
4990     ShouldNotReachHere();
4991 }
4992 
4993 void MacroAssembler::testptr(Register dst, Register src) {
4994   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
4995 }
4996 
4997 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4998 void MacroAssembler::tlab_allocate(Register thread, Register obj,
4999                                    Register var_size_in_bytes,
5000                                    int con_size_in_bytes,
5001                                    Register t1,
5002                                    Register t2,
5003                                    Label& slow_case) {
5004   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5005   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
5006 }
5007 
5008 // Defines obj, preserves var_size_in_bytes
5009 void MacroAssembler::eden_allocate(Register thread, Register obj,
5010                                    Register var_size_in_bytes,
5011                                    int con_size_in_bytes,
5012                                    Register t1,
5013                                    Label& slow_case) {
5014   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5015   bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
5016 }
5017 
5018 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
5019 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
5020   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
5021   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
5022   Label done;
5023 
5024   testptr(length_in_bytes, length_in_bytes);
5025   jcc(Assembler::zero, done);
5026 
5027   // initialize topmost word, divide index by 2, check if odd and test if zero
5028   // note: for the remaining code to work, index must be a multiple of BytesPerWord
5029 #ifdef ASSERT
5030   {
5031     Label L;
5032     testptr(length_in_bytes, BytesPerWord - 1);
5033     jcc(Assembler::zero, L);
5034     stop("length must be a multiple of BytesPerWord");
5035     bind(L);
5036   }
5037 #endif
5038   Register index = length_in_bytes;
5039   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
5040   if (UseIncDec) {
5041     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
5042   } else {
5043     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
5044     shrptr(index, 1);
5045   }
5046 #ifndef _LP64
5047   // index could have not been a multiple of 8 (i.e., bit 2 was set)
5048   {
5049     Label even;
5050     // note: if index was a multiple of 8, then it cannot
5051     //       be 0 now otherwise it must have been 0 before
5052     //       => if it is even, we don't need to check for 0 again
5053     jcc(Assembler::carryClear, even);
5054     // clear topmost word (no jump would be needed if conditional assignment worked here)
5055     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
5056     // index could be 0 now, must check again
5057     jcc(Assembler::zero, done);
5058     bind(even);
5059   }
5060 #endif // !_LP64
5061   // initialize remaining object fields: index is a multiple of 2 now
5062   {
5063     Label loop;
5064     bind(loop);
5065     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
5066     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
5067     decrement(index);
5068     jcc(Assembler::notZero, loop);
5069   }
5070 
5071   bind(done);
5072 }
5073 
5074 // Look up the method for a megamorphic invokeinterface call.
5075 // The target method is determined by <intf_klass, itable_index>.
5076 // The receiver klass is in recv_klass.
5077 // On success, the result will be in method_result, and execution falls through.
5078 // On failure, execution transfers to the given label.
5079 void MacroAssembler::lookup_interface_method(Register recv_klass,
5080                                              Register intf_klass,
5081                                              RegisterOrConstant itable_index,
5082                                              Register method_result,
5083                                              Register scan_temp,
5084                                              Label& L_no_such_interface,
5085                                              bool return_method) {
5086   assert_different_registers(recv_klass, intf_klass, scan_temp);
5087   assert_different_registers(method_result, intf_klass, scan_temp);
5088   assert(recv_klass != method_result || !return_method,
5089          "recv_klass can be destroyed when method isn't needed");
5090 
5091   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
5092          "caller must use same register for non-constant itable index as for method");
5093 
5094   // Compute start of first itableOffsetEntry (which is at the end of the vtable)
5095   int vtable_base = in_bytes(Klass::vtable_start_offset());
5096   int itentry_off = itableMethodEntry::method_offset_in_bytes();
5097   int scan_step   = itableOffsetEntry::size() * wordSize;
5098   int vte_size    = vtableEntry::size_in_bytes();
5099   Address::ScaleFactor times_vte_scale = Address::times_ptr;
5100   assert(vte_size == wordSize, "else adjust times_vte_scale");
5101 
5102   movl(scan_temp, Address(recv_klass, Klass::vtable_length_offset()));
5103 
5104   // %%% Could store the aligned, prescaled offset in the klassoop.
5105   lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
5106 
5107   if (return_method) {
5108     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
5109     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
5110     lea(recv_klass, Address(recv_klass, itable_index, Address::times_ptr, itentry_off));
5111   }
5112 
5113   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
5114   //   if (scan->interface() == intf) {
5115   //     result = (klass + scan->offset() + itable_index);
5116   //   }
5117   // }
5118   Label search, found_method;
5119 
5120   for (int peel = 1; peel >= 0; peel--) {
5121     movptr(method_result, Address(scan_temp, itableOffsetEntry::interface_offset_in_bytes()));
5122     cmpptr(intf_klass, method_result);
5123 
5124     if (peel) {
5125       jccb(Assembler::equal, found_method);
5126     } else {
5127       jccb(Assembler::notEqual, search);
5128       // (invert the test to fall through to found_method...)
5129     }
5130 
5131     if (!peel)  break;
5132 
5133     bind(search);
5134 
5135     // Check that the previous entry is non-null.  A null entry means that
5136     // the receiver class doesn't implement the interface, and wasn't the
5137     // same as when the caller was compiled.
5138     testptr(method_result, method_result);
5139     jcc(Assembler::zero, L_no_such_interface);
5140     addptr(scan_temp, scan_step);
5141   }
5142 
5143   bind(found_method);
5144 
5145   if (return_method) {
5146     // Got a hit.
5147     movl(scan_temp, Address(scan_temp, itableOffsetEntry::offset_offset_in_bytes()));
5148     movptr(method_result, Address(recv_klass, scan_temp, Address::times_1));
5149   }
5150 }
5151 
5152 
5153 // virtual method calling
5154 void MacroAssembler::lookup_virtual_method(Register recv_klass,
5155                                            RegisterOrConstant vtable_index,
5156                                            Register method_result) {
5157   const int base = in_bytes(Klass::vtable_start_offset());
5158   assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
5159   Address vtable_entry_addr(recv_klass,
5160                             vtable_index, Address::times_ptr,
5161                             base + vtableEntry::method_offset_in_bytes());
5162   movptr(method_result, vtable_entry_addr);
5163 }
5164 
5165 
5166 void MacroAssembler::check_klass_subtype(Register sub_klass,
5167                            Register super_klass,
5168                            Register temp_reg,
5169                            Label& L_success) {
5170   Label L_failure;
5171   check_klass_subtype_fast_path(sub_klass, super_klass, temp_reg,        &L_success, &L_failure, NULL);
5172   check_klass_subtype_slow_path(sub_klass, super_klass, temp_reg, noreg, &L_success, NULL);
5173   bind(L_failure);
5174 }
5175 
5176 
5177 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
5178                                                    Register super_klass,
5179                                                    Register temp_reg,
5180                                                    Label* L_success,
5181                                                    Label* L_failure,
5182                                                    Label* L_slow_path,
5183                                         RegisterOrConstant super_check_offset) {
5184   assert_different_registers(sub_klass, super_klass, temp_reg);
5185   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
5186   if (super_check_offset.is_register()) {
5187     assert_different_registers(sub_klass, super_klass,
5188                                super_check_offset.as_register());
5189   } else if (must_load_sco) {
5190     assert(temp_reg != noreg, "supply either a temp or a register offset");
5191   }
5192 
5193   Label L_fallthrough;
5194   int label_nulls = 0;
5195   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
5196   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
5197   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
5198   assert(label_nulls <= 1, "at most one NULL in the batch");
5199 
5200   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
5201   int sco_offset = in_bytes(Klass::super_check_offset_offset());
5202   Address super_check_offset_addr(super_klass, sco_offset);
5203 
5204   // Hacked jcc, which "knows" that L_fallthrough, at least, is in
5205   // range of a jccb.  If this routine grows larger, reconsider at
5206   // least some of these.
5207 #define local_jcc(assembler_cond, label)                                \
5208   if (&(label) == &L_fallthrough)  jccb(assembler_cond, label);         \
5209   else                             jcc( assembler_cond, label) /*omit semi*/
5210 
5211   // Hacked jmp, which may only be used just before L_fallthrough.
5212 #define final_jmp(label)                                                \
5213   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
5214   else                            jmp(label)                /*omit semi*/
5215 
5216   // If the pointers are equal, we are done (e.g., String[] elements).
5217   // This self-check enables sharing of secondary supertype arrays among
5218   // non-primary types such as array-of-interface.  Otherwise, each such
5219   // type would need its own customized SSA.
5220   // We move this check to the front of the fast path because many
5221   // type checks are in fact trivially successful in this manner,
5222   // so we get a nicely predicted branch right at the start of the check.
5223   cmpptr(sub_klass, super_klass);
5224   local_jcc(Assembler::equal, *L_success);
5225 
5226   // Check the supertype display:
5227   if (must_load_sco) {
5228     // Positive movl does right thing on LP64.
5229     movl(temp_reg, super_check_offset_addr);
5230     super_check_offset = RegisterOrConstant(temp_reg);
5231   }
5232   Address super_check_addr(sub_klass, super_check_offset, Address::times_1, 0);
5233   cmpptr(super_klass, super_check_addr); // load displayed supertype
5234 
5235   // This check has worked decisively for primary supers.
5236   // Secondary supers are sought in the super_cache ('super_cache_addr').
5237   // (Secondary supers are interfaces and very deeply nested subtypes.)
5238   // This works in the same check above because of a tricky aliasing
5239   // between the super_cache and the primary super display elements.
5240   // (The 'super_check_addr' can address either, as the case requires.)
5241   // Note that the cache is updated below if it does not help us find
5242   // what we need immediately.
5243   // So if it was a primary super, we can just fail immediately.
5244   // Otherwise, it's the slow path for us (no success at this point).
5245 
5246   if (super_check_offset.is_register()) {
5247     local_jcc(Assembler::equal, *L_success);
5248     cmpl(super_check_offset.as_register(), sc_offset);
5249     if (L_failure == &L_fallthrough) {
5250       local_jcc(Assembler::equal, *L_slow_path);
5251     } else {
5252       local_jcc(Assembler::notEqual, *L_failure);
5253       final_jmp(*L_slow_path);
5254     }
5255   } else if (super_check_offset.as_constant() == sc_offset) {
5256     // Need a slow path; fast failure is impossible.
5257     if (L_slow_path == &L_fallthrough) {
5258       local_jcc(Assembler::equal, *L_success);
5259     } else {
5260       local_jcc(Assembler::notEqual, *L_slow_path);
5261       final_jmp(*L_success);
5262     }
5263   } else {
5264     // No slow path; it's a fast decision.
5265     if (L_failure == &L_fallthrough) {
5266       local_jcc(Assembler::equal, *L_success);
5267     } else {
5268       local_jcc(Assembler::notEqual, *L_failure);
5269       final_jmp(*L_success);
5270     }
5271   }
5272 
5273   bind(L_fallthrough);
5274 
5275 #undef local_jcc
5276 #undef final_jmp
5277 }
5278 
5279 
5280 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
5281                                                    Register super_klass,
5282                                                    Register temp_reg,
5283                                                    Register temp2_reg,
5284                                                    Label* L_success,
5285                                                    Label* L_failure,
5286                                                    bool set_cond_codes) {
5287   assert_different_registers(sub_klass, super_klass, temp_reg);
5288   if (temp2_reg != noreg)
5289     assert_different_registers(sub_klass, super_klass, temp_reg, temp2_reg);
5290 #define IS_A_TEMP(reg) ((reg) == temp_reg || (reg) == temp2_reg)
5291 
5292   Label L_fallthrough;
5293   int label_nulls = 0;
5294   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
5295   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
5296   assert(label_nulls <= 1, "at most one NULL in the batch");
5297 
5298   // a couple of useful fields in sub_klass:
5299   int ss_offset = in_bytes(Klass::secondary_supers_offset());
5300   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
5301   Address secondary_supers_addr(sub_klass, ss_offset);
5302   Address super_cache_addr(     sub_klass, sc_offset);
5303 
5304   // Do a linear scan of the secondary super-klass chain.
5305   // This code is rarely used, so simplicity is a virtue here.
5306   // The repne_scan instruction uses fixed registers, which we must spill.
5307   // Don't worry too much about pre-existing connections with the input regs.
5308 
5309   assert(sub_klass != rax, "killed reg"); // killed by mov(rax, super)
5310   assert(sub_klass != rcx, "killed reg"); // killed by lea(rcx, &pst_counter)
5311 
5312   // Get super_klass value into rax (even if it was in rdi or rcx).
5313   bool pushed_rax = false, pushed_rcx = false, pushed_rdi = false;
5314   if (super_klass != rax || UseCompressedOops) {
5315     if (!IS_A_TEMP(rax)) { push(rax); pushed_rax = true; }
5316     mov(rax, super_klass);
5317   }
5318   if (!IS_A_TEMP(rcx)) { push(rcx); pushed_rcx = true; }
5319   if (!IS_A_TEMP(rdi)) { push(rdi); pushed_rdi = true; }
5320 
5321 #ifndef PRODUCT
5322   int* pst_counter = &SharedRuntime::_partial_subtype_ctr;
5323   ExternalAddress pst_counter_addr((address) pst_counter);
5324   NOT_LP64(  incrementl(pst_counter_addr) );
5325   LP64_ONLY( lea(rcx, pst_counter_addr) );
5326   LP64_ONLY( incrementl(Address(rcx, 0)) );
5327 #endif //PRODUCT
5328 
5329   // We will consult the secondary-super array.
5330   movptr(rdi, secondary_supers_addr);
5331   // Load the array length.  (Positive movl does right thing on LP64.)
5332   movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
5333   // Skip to start of data.
5334   addptr(rdi, Array<Klass*>::base_offset_in_bytes());
5335 
5336   // Scan RCX words at [RDI] for an occurrence of RAX.
5337   // Set NZ/Z based on last compare.
5338   // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
5339   // not change flags (only scas instruction which is repeated sets flags).
5340   // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
5341 
5342     testptr(rax,rax); // Set Z = 0
5343     repne_scan();
5344 
5345   // Unspill the temp. registers:
5346   if (pushed_rdi)  pop(rdi);
5347   if (pushed_rcx)  pop(rcx);
5348   if (pushed_rax)  pop(rax);
5349 
5350   if (set_cond_codes) {
5351     // Special hack for the AD files:  rdi is guaranteed non-zero.
5352     assert(!pushed_rdi, "rdi must be left non-NULL");
5353     // Also, the condition codes are properly set Z/NZ on succeed/failure.
5354   }
5355 
5356   if (L_failure == &L_fallthrough)
5357         jccb(Assembler::notEqual, *L_failure);
5358   else  jcc(Assembler::notEqual, *L_failure);
5359 
5360   // Success.  Cache the super we found and proceed in triumph.
5361   movptr(super_cache_addr, super_klass);
5362 
5363   if (L_success != &L_fallthrough) {
5364     jmp(*L_success);
5365   }
5366 
5367 #undef IS_A_TEMP
5368 
5369   bind(L_fallthrough);
5370 }
5371 
5372 
5373 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
5374   if (VM_Version::supports_cmov()) {
5375     cmovl(cc, dst, src);
5376   } else {
5377     Label L;
5378     jccb(negate_condition(cc), L);
5379     movl(dst, src);
5380     bind(L);
5381   }
5382 }
5383 
5384 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
5385   if (VM_Version::supports_cmov()) {
5386     cmovl(cc, dst, src);
5387   } else {
5388     Label L;
5389     jccb(negate_condition(cc), L);
5390     movl(dst, src);
5391     bind(L);
5392   }
5393 }
5394 
5395 void MacroAssembler::verify_oop(Register reg, const char* s) {
5396   if (!VerifyOops) return;
5397 
5398   // Pass register number to verify_oop_subroutine
5399   const char* b = NULL;
5400   {
5401     ResourceMark rm;
5402     stringStream ss;
5403     ss.print("verify_oop: %s: %s", reg->name(), s);
5404     b = code_string(ss.as_string());
5405   }
5406   BLOCK_COMMENT("verify_oop {");
5407 #ifdef _LP64
5408   push(rscratch1);                    // save r10, trashed by movptr()
5409 #endif
5410   push(rax);                          // save rax,
5411   push(reg);                          // pass register argument
5412   ExternalAddress buffer((address) b);
5413   // avoid using pushptr, as it modifies scratch registers
5414   // and our contract is not to modify anything
5415   movptr(rax, buffer.addr());
5416   push(rax);
5417   // call indirectly to solve generation ordering problem
5418   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
5419   call(rax);
5420   // Caller pops the arguments (oop, message) and restores rax, r10
5421   BLOCK_COMMENT("} verify_oop");
5422 }
5423 
5424 
5425 RegisterOrConstant MacroAssembler::delayed_value_impl(intptr_t* delayed_value_addr,
5426                                                       Register tmp,
5427                                                       int offset) {
5428   intptr_t value = *delayed_value_addr;
5429   if (value != 0)
5430     return RegisterOrConstant(value + offset);
5431 
5432   // load indirectly to solve generation ordering problem
5433   movptr(tmp, ExternalAddress((address) delayed_value_addr));
5434 
5435 #ifdef ASSERT
5436   { Label L;
5437     testptr(tmp, tmp);
5438     if (WizardMode) {
5439       const char* buf = NULL;
5440       {
5441         ResourceMark rm;
5442         stringStream ss;
5443         ss.print("DelayedValue=" INTPTR_FORMAT, delayed_value_addr[1]);
5444         buf = code_string(ss.as_string());
5445       }
5446       jcc(Assembler::notZero, L);
5447       STOP(buf);
5448     } else {
5449       jccb(Assembler::notZero, L);
5450       hlt();
5451     }
5452     bind(L);
5453   }
5454 #endif
5455 
5456   if (offset != 0)
5457     addptr(tmp, offset);
5458 
5459   return RegisterOrConstant(tmp);
5460 }
5461 
5462 
5463 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
5464                                          int extra_slot_offset) {
5465   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
5466   int stackElementSize = Interpreter::stackElementSize;
5467   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
5468 #ifdef ASSERT
5469   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
5470   assert(offset1 - offset == stackElementSize, "correct arithmetic");
5471 #endif
5472   Register             scale_reg    = noreg;
5473   Address::ScaleFactor scale_factor = Address::no_scale;
5474   if (arg_slot.is_constant()) {
5475     offset += arg_slot.as_constant() * stackElementSize;
5476   } else {
5477     scale_reg    = arg_slot.as_register();
5478     scale_factor = Address::times(stackElementSize);
5479   }
5480   offset += wordSize;           // return PC is on stack
5481   return Address(rsp, scale_reg, scale_factor, offset);
5482 }
5483 
5484 
5485 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
5486   if (!VerifyOops) return;
5487 
5488   // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
5489   // Pass register number to verify_oop_subroutine
5490   const char* b = NULL;
5491   {
5492     ResourceMark rm;
5493     stringStream ss;
5494     ss.print("verify_oop_addr: %s", s);
5495     b = code_string(ss.as_string());
5496   }
5497 #ifdef _LP64
5498   push(rscratch1);                    // save r10, trashed by movptr()
5499 #endif
5500   push(rax);                          // save rax,
5501   // addr may contain rsp so we will have to adjust it based on the push
5502   // we just did (and on 64 bit we do two pushes)
5503   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
5504   // stores rax into addr which is backwards of what was intended.
5505   if (addr.uses(rsp)) {
5506     lea(rax, addr);
5507     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
5508   } else {
5509     pushptr(addr);
5510   }
5511 
5512   ExternalAddress buffer((address) b);
5513   // pass msg argument
5514   // avoid using pushptr, as it modifies scratch registers
5515   // and our contract is not to modify anything
5516   movptr(rax, buffer.addr());
5517   push(rax);
5518 
5519   // call indirectly to solve generation ordering problem
5520   movptr(rax, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
5521   call(rax);
5522   // Caller pops the arguments (addr, message) and restores rax, r10.
5523 }
5524 
5525 void MacroAssembler::verify_tlab() {
5526 #ifdef ASSERT
5527   if (UseTLAB && VerifyOops) {
5528     Label next, ok;
5529     Register t1 = rsi;
5530     Register thread_reg = NOT_LP64(rbx) LP64_ONLY(r15_thread);
5531 
5532     push(t1);
5533     NOT_LP64(push(thread_reg));
5534     NOT_LP64(get_thread(thread_reg));
5535 
5536     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5537     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_start_offset())));
5538     jcc(Assembler::aboveEqual, next);
5539     STOP("assert(top >= start)");
5540     should_not_reach_here();
5541 
5542     bind(next);
5543     movptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_end_offset())));
5544     cmpptr(t1, Address(thread_reg, in_bytes(JavaThread::tlab_top_offset())));
5545     jcc(Assembler::aboveEqual, ok);
5546     STOP("assert(top <= end)");
5547     should_not_reach_here();
5548 
5549     bind(ok);
5550     NOT_LP64(pop(thread_reg));
5551     pop(t1);
5552   }
5553 #endif
5554 }
5555 
5556 class ControlWord {
5557  public:
5558   int32_t _value;
5559 
5560   int  rounding_control() const        { return  (_value >> 10) & 3      ; }
5561   int  precision_control() const       { return  (_value >>  8) & 3      ; }
5562   bool precision() const               { return ((_value >>  5) & 1) != 0; }
5563   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
5564   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
5565   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
5566   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
5567   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
5568 
5569   void print() const {
5570     // rounding control
5571     const char* rc;
5572     switch (rounding_control()) {
5573       case 0: rc = "round near"; break;
5574       case 1: rc = "round down"; break;
5575       case 2: rc = "round up  "; break;
5576       case 3: rc = "chop      "; break;
5577     };
5578     // precision control
5579     const char* pc;
5580     switch (precision_control()) {
5581       case 0: pc = "24 bits "; break;
5582       case 1: pc = "reserved"; break;
5583       case 2: pc = "53 bits "; break;
5584       case 3: pc = "64 bits "; break;
5585     };
5586     // flags
5587     char f[9];
5588     f[0] = ' ';
5589     f[1] = ' ';
5590     f[2] = (precision   ()) ? 'P' : 'p';
5591     f[3] = (underflow   ()) ? 'U' : 'u';
5592     f[4] = (overflow    ()) ? 'O' : 'o';
5593     f[5] = (zero_divide ()) ? 'Z' : 'z';
5594     f[6] = (denormalized()) ? 'D' : 'd';
5595     f[7] = (invalid     ()) ? 'I' : 'i';
5596     f[8] = '\x0';
5597     // output
5598     printf("%04x  masks = %s, %s, %s", _value & 0xFFFF, f, rc, pc);
5599   }
5600 
5601 };
5602 
5603 class StatusWord {
5604  public:
5605   int32_t _value;
5606 
5607   bool busy() const                    { return ((_value >> 15) & 1) != 0; }
5608   bool C3() const                      { return ((_value >> 14) & 1) != 0; }
5609   bool C2() const                      { return ((_value >> 10) & 1) != 0; }
5610   bool C1() const                      { return ((_value >>  9) & 1) != 0; }
5611   bool C0() const                      { return ((_value >>  8) & 1) != 0; }
5612   int  top() const                     { return  (_value >> 11) & 7      ; }
5613   bool error_status() const            { return ((_value >>  7) & 1) != 0; }
5614   bool stack_fault() const             { return ((_value >>  6) & 1) != 0; }
5615   bool precision() const               { return ((_value >>  5) & 1) != 0; }
5616   bool underflow() const               { return ((_value >>  4) & 1) != 0; }
5617   bool overflow() const                { return ((_value >>  3) & 1) != 0; }
5618   bool zero_divide() const             { return ((_value >>  2) & 1) != 0; }
5619   bool denormalized() const            { return ((_value >>  1) & 1) != 0; }
5620   bool invalid() const                 { return ((_value >>  0) & 1) != 0; }
5621 
5622   void print() const {
5623     // condition codes
5624     char c[5];
5625     c[0] = (C3()) ? '3' : '-';
5626     c[1] = (C2()) ? '2' : '-';
5627     c[2] = (C1()) ? '1' : '-';
5628     c[3] = (C0()) ? '0' : '-';
5629     c[4] = '\x0';
5630     // flags
5631     char f[9];
5632     f[0] = (error_status()) ? 'E' : '-';
5633     f[1] = (stack_fault ()) ? 'S' : '-';
5634     f[2] = (precision   ()) ? 'P' : '-';
5635     f[3] = (underflow   ()) ? 'U' : '-';
5636     f[4] = (overflow    ()) ? 'O' : '-';
5637     f[5] = (zero_divide ()) ? 'Z' : '-';
5638     f[6] = (denormalized()) ? 'D' : '-';
5639     f[7] = (invalid     ()) ? 'I' : '-';
5640     f[8] = '\x0';
5641     // output
5642     printf("%04x  flags = %s, cc =  %s, top = %d", _value & 0xFFFF, f, c, top());
5643   }
5644 
5645 };
5646 
5647 class TagWord {
5648  public:
5649   int32_t _value;
5650 
5651   int tag_at(int i) const              { return (_value >> (i*2)) & 3; }
5652 
5653   void print() const {
5654     printf("%04x", _value & 0xFFFF);
5655   }
5656 
5657 };
5658 
5659 class FPU_Register {
5660  public:
5661   int32_t _m0;
5662   int32_t _m1;
5663   int16_t _ex;
5664 
5665   bool is_indefinite() const           {
5666     return _ex == -1 && _m1 == (int32_t)0xC0000000 && _m0 == 0;
5667   }
5668 
5669   void print() const {
5670     char  sign = (_ex < 0) ? '-' : '+';
5671     const char* kind = (_ex == 0x7FFF || _ex == (int16_t)-1) ? "NaN" : "   ";
5672     printf("%c%04hx.%08x%08x  %s", sign, _ex, _m1, _m0, kind);
5673   };
5674 
5675 };
5676 
5677 class FPU_State {
5678  public:
5679   enum {
5680     register_size       = 10,
5681     number_of_registers =  8,
5682     register_mask       =  7
5683   };
5684 
5685   ControlWord  _control_word;
5686   StatusWord   _status_word;
5687   TagWord      _tag_word;
5688   int32_t      _error_offset;
5689   int32_t      _error_selector;
5690   int32_t      _data_offset;
5691   int32_t      _data_selector;
5692   int8_t       _register[register_size * number_of_registers];
5693 
5694   int tag_for_st(int i) const          { return _tag_word.tag_at((_status_word.top() + i) & register_mask); }
5695   FPU_Register* st(int i) const        { return (FPU_Register*)&_register[register_size * i]; }
5696 
5697   const char* tag_as_string(int tag) const {
5698     switch (tag) {
5699       case 0: return "valid";
5700       case 1: return "zero";
5701       case 2: return "special";
5702       case 3: return "empty";
5703     }
5704     ShouldNotReachHere();
5705     return NULL;
5706   }
5707 
5708   void print() const {
5709     // print computation registers
5710     { int t = _status_word.top();
5711       for (int i = 0; i < number_of_registers; i++) {
5712         int j = (i - t) & register_mask;
5713         printf("%c r%d = ST%d = ", (j == 0 ? '*' : ' '), i, j);
5714         st(j)->print();
5715         printf(" %s\n", tag_as_string(_tag_word.tag_at(i)));
5716       }
5717     }
5718     printf("\n");
5719     // print control registers
5720     printf("ctrl = "); _control_word.print(); printf("\n");
5721     printf("stat = "); _status_word .print(); printf("\n");
5722     printf("tags = "); _tag_word    .print(); printf("\n");
5723   }
5724 
5725 };
5726 
5727 class Flag_Register {
5728  public:
5729   int32_t _value;
5730 
5731   bool overflow() const                { return ((_value >> 11) & 1) != 0; }
5732   bool direction() const               { return ((_value >> 10) & 1) != 0; }
5733   bool sign() const                    { return ((_value >>  7) & 1) != 0; }
5734   bool zero() const                    { return ((_value >>  6) & 1) != 0; }
5735   bool auxiliary_carry() const         { return ((_value >>  4) & 1) != 0; }
5736   bool parity() const                  { return ((_value >>  2) & 1) != 0; }
5737   bool carry() const                   { return ((_value >>  0) & 1) != 0; }
5738 
5739   void print() const {
5740     // flags
5741     char f[8];
5742     f[0] = (overflow       ()) ? 'O' : '-';
5743     f[1] = (direction      ()) ? 'D' : '-';
5744     f[2] = (sign           ()) ? 'S' : '-';
5745     f[3] = (zero           ()) ? 'Z' : '-';
5746     f[4] = (auxiliary_carry()) ? 'A' : '-';
5747     f[5] = (parity         ()) ? 'P' : '-';
5748     f[6] = (carry          ()) ? 'C' : '-';
5749     f[7] = '\x0';
5750     // output
5751     printf("%08x  flags = %s", _value, f);
5752   }
5753 
5754 };
5755 
5756 class IU_Register {
5757  public:
5758   int32_t _value;
5759 
5760   void print() const {
5761     printf("%08x  %11d", _value, _value);
5762   }
5763 
5764 };
5765 
5766 class IU_State {
5767  public:
5768   Flag_Register _eflags;
5769   IU_Register   _rdi;
5770   IU_Register   _rsi;
5771   IU_Register   _rbp;
5772   IU_Register   _rsp;
5773   IU_Register   _rbx;
5774   IU_Register   _rdx;
5775   IU_Register   _rcx;
5776   IU_Register   _rax;
5777 
5778   void print() const {
5779     // computation registers
5780     printf("rax,  = "); _rax.print(); printf("\n");
5781     printf("rbx,  = "); _rbx.print(); printf("\n");
5782     printf("rcx  = "); _rcx.print(); printf("\n");
5783     printf("rdx  = "); _rdx.print(); printf("\n");
5784     printf("rdi  = "); _rdi.print(); printf("\n");
5785     printf("rsi  = "); _rsi.print(); printf("\n");
5786     printf("rbp,  = "); _rbp.print(); printf("\n");
5787     printf("rsp  = "); _rsp.print(); printf("\n");
5788     printf("\n");
5789     // control registers
5790     printf("flgs = "); _eflags.print(); printf("\n");
5791   }
5792 };
5793 
5794 
5795 class CPU_State {
5796  public:
5797   FPU_State _fpu_state;
5798   IU_State  _iu_state;
5799 
5800   void print() const {
5801     printf("--------------------------------------------------\n");
5802     _iu_state .print();
5803     printf("\n");
5804     _fpu_state.print();
5805     printf("--------------------------------------------------\n");
5806   }
5807 
5808 };
5809 
5810 
5811 static void _print_CPU_state(CPU_State* state) {
5812   state->print();
5813 };
5814 
5815 
5816 void MacroAssembler::print_CPU_state() {
5817   push_CPU_state();
5818   push(rsp);                // pass CPU state
5819   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _print_CPU_state)));
5820   addptr(rsp, wordSize);       // discard argument
5821   pop_CPU_state();
5822 }
5823 
5824 
5825 static bool _verify_FPU(int stack_depth, char* s, CPU_State* state) {
5826   static int counter = 0;
5827   FPU_State* fs = &state->_fpu_state;
5828   counter++;
5829   // For leaf calls, only verify that the top few elements remain empty.
5830   // We only need 1 empty at the top for C2 code.
5831   if( stack_depth < 0 ) {
5832     if( fs->tag_for_st(7) != 3 ) {
5833       printf("FPR7 not empty\n");
5834       state->print();
5835       assert(false, "error");
5836       return false;
5837     }
5838     return true;                // All other stack states do not matter
5839   }
5840 
5841   assert((fs->_control_word._value & 0xffff) == StubRoutines::_fpu_cntrl_wrd_std,
5842          "bad FPU control word");
5843 
5844   // compute stack depth
5845   int i = 0;
5846   while (i < FPU_State::number_of_registers && fs->tag_for_st(i)  < 3) i++;
5847   int d = i;
5848   while (i < FPU_State::number_of_registers && fs->tag_for_st(i) == 3) i++;
5849   // verify findings
5850   if (i != FPU_State::number_of_registers) {
5851     // stack not contiguous
5852     printf("%s: stack not contiguous at ST%d\n", s, i);
5853     state->print();
5854     assert(false, "error");
5855     return false;
5856   }
5857   // check if computed stack depth corresponds to expected stack depth
5858   if (stack_depth < 0) {
5859     // expected stack depth is -stack_depth or less
5860     if (d > -stack_depth) {
5861       // too many elements on the stack
5862       printf("%s: <= %d stack elements expected but found %d\n", s, -stack_depth, d);
5863       state->print();
5864       assert(false, "error");
5865       return false;
5866     }
5867   } else {
5868     // expected stack depth is stack_depth
5869     if (d != stack_depth) {
5870       // wrong stack depth
5871       printf("%s: %d stack elements expected but found %d\n", s, stack_depth, d);
5872       state->print();
5873       assert(false, "error");
5874       return false;
5875     }
5876   }
5877   // everything is cool
5878   return true;
5879 }
5880 
5881 
5882 void MacroAssembler::verify_FPU(int stack_depth, const char* s) {
5883   if (!VerifyFPU) return;
5884   push_CPU_state();
5885   push(rsp);                // pass CPU state
5886   ExternalAddress msg((address) s);
5887   // pass message string s
5888   pushptr(msg.addr());
5889   push(stack_depth);        // pass stack depth
5890   call(RuntimeAddress(CAST_FROM_FN_PTR(address, _verify_FPU)));
5891   addptr(rsp, 3 * wordSize);   // discard arguments
5892   // check for error
5893   { Label L;
5894     testl(rax, rax);
5895     jcc(Assembler::notZero, L);
5896     int3();                  // break if error condition
5897     bind(L);
5898   }
5899   pop_CPU_state();
5900 }
5901 
5902 void MacroAssembler::restore_cpu_control_state_after_jni() {
5903   // Either restore the MXCSR register after returning from the JNI Call
5904   // or verify that it wasn't changed (with -Xcheck:jni flag).
5905   if (VM_Version::supports_sse()) {
5906     if (RestoreMXCSROnJNICalls) {
5907       ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std()));
5908     } else if (CheckJNICalls) {
5909       call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
5910     }
5911   }
5912   // Clear upper bits of YMM registers to avoid SSE <-> AVX transition penalty.
5913   vzeroupper();
5914   // Reset k1 to 0xffff.
5915   if (VM_Version::supports_evex()) {
5916     push(rcx);
5917     movl(rcx, 0xffff);
5918     kmovwl(k1, rcx);
5919     pop(rcx);
5920   }
5921 
5922 #ifndef _LP64
5923   // Either restore the x87 floating pointer control word after returning
5924   // from the JNI call or verify that it wasn't changed.
5925   if (CheckJNICalls) {
5926     call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
5927   }
5928 #endif // _LP64
5929 }
5930 
5931 // ((OopHandle)result).resolve();
5932 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
5933   assert_different_registers(result, tmp);
5934 
5935   // Only 64 bit platforms support GCs that require a tmp register
5936   // Only IN_HEAP loads require a thread_tmp register
5937   // OopHandle::resolve is an indirection like jobject.
5938   access_load_at(T_OBJECT, IN_NATIVE,
5939                  result, Address(result, 0), tmp, /*tmp_thread*/noreg);
5940 }
5941 
5942 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
5943   // get mirror
5944   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
5945   movptr(mirror, Address(method, Method::const_offset()));
5946   movptr(mirror, Address(mirror, ConstMethod::constants_offset()));
5947   movptr(mirror, Address(mirror, ConstantPool::pool_holder_offset_in_bytes()));
5948   movptr(mirror, Address(mirror, mirror_offset));
5949   resolve_oop_handle(mirror, tmp);
5950 }
5951 
5952 void MacroAssembler::load_klass(Register dst, Register src) {
5953 #ifdef _LP64
5954   if (UseCompressedClassPointers) {
5955     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5956     decode_klass_not_null(dst);
5957   } else
5958 #endif
5959     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5960 }
5961 
5962 void MacroAssembler::load_prototype_header(Register dst, Register src) {
5963   load_klass(dst, src);
5964   movptr(dst, Address(dst, Klass::prototype_header_offset()));
5965 }
5966 
5967 void MacroAssembler::store_klass(Register dst, Register src) {
5968 #ifdef _LP64
5969   if (UseCompressedClassPointers) {
5970     encode_klass_not_null(src);
5971     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5972   } else
5973 #endif
5974     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5975 }
5976 
5977 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
5978                                     Register tmp1, Register thread_tmp) {
5979   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5980   decorators = AccessInternal::decorator_fixup(decorators);
5981   bool as_raw = (decorators & AS_RAW) != 0;
5982   if (as_raw) {
5983     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
5984   } else {
5985     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
5986   }
5987 }
5988 
5989 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
5990                                      Register tmp1, Register tmp2) {
5991   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5992   decorators = AccessInternal::decorator_fixup(decorators);
5993   bool as_raw = (decorators & AS_RAW) != 0;
5994   if (as_raw) {
5995     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2);
5996   } else {
5997     bs->store_at(this, decorators, type, dst, src, tmp1, tmp2);
5998   }
5999 }
6000 
6001 void MacroAssembler::resolve(DecoratorSet decorators, Register obj) {
6002   // Use stronger ACCESS_WRITE|ACCESS_READ by default.
6003   if ((decorators & (ACCESS_READ | ACCESS_WRITE)) == 0) {
6004     decorators |= ACCESS_READ | ACCESS_WRITE;
6005   }
6006   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
6007   return bs->resolve(this, decorators, obj);
6008 }
6009 
6010 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
6011                                    Register thread_tmp, DecoratorSet decorators) {
6012   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
6013 }
6014 
6015 // Doesn't do verfication, generates fixed size code
6016 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
6017                                             Register thread_tmp, DecoratorSet decorators) {
6018   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
6019 }
6020 
6021 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
6022                                     Register tmp2, DecoratorSet decorators) {
6023   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
6024 }
6025 
6026 // Used for storing NULLs.
6027 void MacroAssembler::store_heap_oop_null(Address dst) {
6028   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg);
6029 }
6030 
6031 #ifdef _LP64
6032 void MacroAssembler::store_klass_gap(Register dst, Register src) {
6033   if (UseCompressedClassPointers) {
6034     // Store to klass gap in destination
6035     movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
6036   }
6037 }
6038 
6039 #ifdef ASSERT
6040 void MacroAssembler::verify_heapbase(const char* msg) {
6041   assert (UseCompressedOops, "should be compressed");
6042   assert (Universe::heap() != NULL, "java heap should be initialized");
6043   if (CheckCompressedOops) {
6044     Label ok;
6045     push(rscratch1); // cmpptr trashes rscratch1
6046     cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6047     jcc(Assembler::equal, ok);
6048     STOP(msg);
6049     bind(ok);
6050     pop(rscratch1);
6051   }
6052 }
6053 #endif
6054 
6055 // Algorithm must match oop.inline.hpp encode_heap_oop.
6056 void MacroAssembler::encode_heap_oop(Register r) {
6057 #ifdef ASSERT
6058   verify_heapbase("MacroAssembler::encode_heap_oop: heap base corrupted?");
6059 #endif
6060   verify_oop(r, "broken oop in encode_heap_oop");
6061   if (Universe::narrow_oop_base() == NULL) {
6062     if (Universe::narrow_oop_shift() != 0) {
6063       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6064       shrq(r, LogMinObjAlignmentInBytes);
6065     }
6066     return;
6067   }
6068   testq(r, r);
6069   cmovq(Assembler::equal, r, r12_heapbase);
6070   subq(r, r12_heapbase);
6071   shrq(r, LogMinObjAlignmentInBytes);
6072 }
6073 
6074 void MacroAssembler::encode_heap_oop_not_null(Register r) {
6075 #ifdef ASSERT
6076   verify_heapbase("MacroAssembler::encode_heap_oop_not_null: heap base corrupted?");
6077   if (CheckCompressedOops) {
6078     Label ok;
6079     testq(r, r);
6080     jcc(Assembler::notEqual, ok);
6081     STOP("null oop passed to encode_heap_oop_not_null");
6082     bind(ok);
6083   }
6084 #endif
6085   verify_oop(r, "broken oop in encode_heap_oop_not_null");
6086   if (Universe::narrow_oop_base() != NULL) {
6087     subq(r, r12_heapbase);
6088   }
6089   if (Universe::narrow_oop_shift() != 0) {
6090     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6091     shrq(r, LogMinObjAlignmentInBytes);
6092   }
6093 }
6094 
6095 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) {
6096 #ifdef ASSERT
6097   verify_heapbase("MacroAssembler::encode_heap_oop_not_null2: heap base corrupted?");
6098   if (CheckCompressedOops) {
6099     Label ok;
6100     testq(src, src);
6101     jcc(Assembler::notEqual, ok);
6102     STOP("null oop passed to encode_heap_oop_not_null2");
6103     bind(ok);
6104   }
6105 #endif
6106   verify_oop(src, "broken oop in encode_heap_oop_not_null2");
6107   if (dst != src) {
6108     movq(dst, src);
6109   }
6110   if (Universe::narrow_oop_base() != NULL) {
6111     subq(dst, r12_heapbase);
6112   }
6113   if (Universe::narrow_oop_shift() != 0) {
6114     assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6115     shrq(dst, LogMinObjAlignmentInBytes);
6116   }
6117 }
6118 
6119 void  MacroAssembler::decode_heap_oop(Register r) {
6120 #ifdef ASSERT
6121   verify_heapbase("MacroAssembler::decode_heap_oop: heap base corrupted?");
6122 #endif
6123   if (Universe::narrow_oop_base() == NULL) {
6124     if (Universe::narrow_oop_shift() != 0) {
6125       assert (LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6126       shlq(r, LogMinObjAlignmentInBytes);
6127     }
6128   } else {
6129     Label done;
6130     shlq(r, LogMinObjAlignmentInBytes);
6131     jccb(Assembler::equal, done);
6132     addq(r, r12_heapbase);
6133     bind(done);
6134   }
6135   verify_oop(r, "broken oop in decode_heap_oop");
6136 }
6137 
6138 void  MacroAssembler::decode_heap_oop_not_null(Register r) {
6139   // Note: it will change flags
6140   assert (UseCompressedOops, "should only be used for compressed headers");
6141   assert (Universe::heap() != NULL, "java heap should be initialized");
6142   // Cannot assert, unverified entry point counts instructions (see .ad file)
6143   // vtableStubs also counts instructions in pd_code_size_limit.
6144   // Also do not verify_oop as this is called by verify_oop.
6145   if (Universe::narrow_oop_shift() != 0) {
6146     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6147     shlq(r, LogMinObjAlignmentInBytes);
6148     if (Universe::narrow_oop_base() != NULL) {
6149       addq(r, r12_heapbase);
6150     }
6151   } else {
6152     assert (Universe::narrow_oop_base() == NULL, "sanity");
6153   }
6154 }
6155 
6156 void  MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
6157   // Note: it will change flags
6158   assert (UseCompressedOops, "should only be used for compressed headers");
6159   assert (Universe::heap() != NULL, "java heap should be initialized");
6160   // Cannot assert, unverified entry point counts instructions (see .ad file)
6161   // vtableStubs also counts instructions in pd_code_size_limit.
6162   // Also do not verify_oop as this is called by verify_oop.
6163   if (Universe::narrow_oop_shift() != 0) {
6164     assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
6165     if (LogMinObjAlignmentInBytes == Address::times_8) {
6166       leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
6167     } else {
6168       if (dst != src) {
6169         movq(dst, src);
6170       }
6171       shlq(dst, LogMinObjAlignmentInBytes);
6172       if (Universe::narrow_oop_base() != NULL) {
6173         addq(dst, r12_heapbase);
6174       }
6175     }
6176   } else {
6177     assert (Universe::narrow_oop_base() == NULL, "sanity");
6178     if (dst != src) {
6179       movq(dst, src);
6180     }
6181   }
6182 }
6183 
6184 void MacroAssembler::encode_klass_not_null(Register r) {
6185   if (Universe::narrow_klass_base() != NULL) {
6186     // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6187     assert(r != r12_heapbase, "Encoding a klass in r12");
6188     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6189     subq(r, r12_heapbase);
6190   }
6191   if (Universe::narrow_klass_shift() != 0) {
6192     assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6193     shrq(r, LogKlassAlignmentInBytes);
6194   }
6195   if (Universe::narrow_klass_base() != NULL) {
6196     reinit_heapbase();
6197   }
6198 }
6199 
6200 void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
6201   if (dst == src) {
6202     encode_klass_not_null(src);
6203   } else {
6204     if (Universe::narrow_klass_base() != NULL) {
6205       mov64(dst, (int64_t)Universe::narrow_klass_base());
6206       negq(dst);
6207       addq(dst, src);
6208     } else {
6209       movptr(dst, src);
6210     }
6211     if (Universe::narrow_klass_shift() != 0) {
6212       assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6213       shrq(dst, LogKlassAlignmentInBytes);
6214     }
6215   }
6216 }
6217 
6218 // Function instr_size_for_decode_klass_not_null() counts the instructions
6219 // generated by decode_klass_not_null(register r) and reinit_heapbase(),
6220 // when (Universe::heap() != NULL).  Hence, if the instructions they
6221 // generate change, then this method needs to be updated.
6222 int MacroAssembler::instr_size_for_decode_klass_not_null() {
6223   assert (UseCompressedClassPointers, "only for compressed klass ptrs");
6224   if (Universe::narrow_klass_base() != NULL) {
6225     // mov64 + addq + shlq? + mov64  (for reinit_heapbase()).
6226     return (Universe::narrow_klass_shift() == 0 ? 20 : 24);
6227   } else {
6228     // longest load decode klass function, mov64, leaq
6229     return 16;
6230   }
6231 }
6232 
6233 // !!! If the instructions that get generated here change then function
6234 // instr_size_for_decode_klass_not_null() needs to get updated.
6235 void  MacroAssembler::decode_klass_not_null(Register r) {
6236   // Note: it will change flags
6237   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6238   assert(r != r12_heapbase, "Decoding a klass in r12");
6239   // Cannot assert, unverified entry point counts instructions (see .ad file)
6240   // vtableStubs also counts instructions in pd_code_size_limit.
6241   // Also do not verify_oop as this is called by verify_oop.
6242   if (Universe::narrow_klass_shift() != 0) {
6243     assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6244     shlq(r, LogKlassAlignmentInBytes);
6245   }
6246   // Use r12 as a scratch register in which to temporarily load the narrow_klass_base.
6247   if (Universe::narrow_klass_base() != NULL) {
6248     mov64(r12_heapbase, (int64_t)Universe::narrow_klass_base());
6249     addq(r, r12_heapbase);
6250     reinit_heapbase();
6251   }
6252 }
6253 
6254 void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
6255   // Note: it will change flags
6256   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6257   if (dst == src) {
6258     decode_klass_not_null(dst);
6259   } else {
6260     // Cannot assert, unverified entry point counts instructions (see .ad file)
6261     // vtableStubs also counts instructions in pd_code_size_limit.
6262     // Also do not verify_oop as this is called by verify_oop.
6263     mov64(dst, (int64_t)Universe::narrow_klass_base());
6264     if (Universe::narrow_klass_shift() != 0) {
6265       assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
6266       assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
6267       leaq(dst, Address(dst, src, Address::times_8, 0));
6268     } else {
6269       addq(dst, src);
6270     }
6271   }
6272 }
6273 
6274 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
6275   assert (UseCompressedOops, "should only be used for compressed headers");
6276   assert (Universe::heap() != NULL, "java heap should be initialized");
6277   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6278   int oop_index = oop_recorder()->find_index(obj);
6279   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6280   mov_narrow_oop(dst, oop_index, rspec);
6281 }
6282 
6283 void  MacroAssembler::set_narrow_oop(Address dst, jobject obj) {
6284   assert (UseCompressedOops, "should only be used for compressed headers");
6285   assert (Universe::heap() != NULL, "java heap should be initialized");
6286   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6287   int oop_index = oop_recorder()->find_index(obj);
6288   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6289   mov_narrow_oop(dst, oop_index, rspec);
6290 }
6291 
6292 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
6293   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6294   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6295   int klass_index = oop_recorder()->find_index(k);
6296   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6297   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6298 }
6299 
6300 void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
6301   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6302   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6303   int klass_index = oop_recorder()->find_index(k);
6304   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6305   mov_narrow_oop(dst, Klass::encode_klass(k), rspec);
6306 }
6307 
6308 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
6309   assert (UseCompressedOops, "should only be used for compressed headers");
6310   assert (Universe::heap() != NULL, "java heap should be initialized");
6311   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6312   int oop_index = oop_recorder()->find_index(obj);
6313   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6314   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6315 }
6316 
6317 void  MacroAssembler::cmp_narrow_oop(Address dst, jobject obj) {
6318   assert (UseCompressedOops, "should only be used for compressed headers");
6319   assert (Universe::heap() != NULL, "java heap should be initialized");
6320   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6321   int oop_index = oop_recorder()->find_index(obj);
6322   RelocationHolder rspec = oop_Relocation::spec(oop_index);
6323   Assembler::cmp_narrow_oop(dst, oop_index, rspec);
6324 }
6325 
6326 void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
6327   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6328   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6329   int klass_index = oop_recorder()->find_index(k);
6330   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6331   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6332 }
6333 
6334 void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
6335   assert (UseCompressedClassPointers, "should only be used for compressed headers");
6336   assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
6337   int klass_index = oop_recorder()->find_index(k);
6338   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
6339   Assembler::cmp_narrow_oop(dst, Klass::encode_klass(k), rspec);
6340 }
6341 
6342 void MacroAssembler::reinit_heapbase() {
6343   if (UseCompressedOops || UseCompressedClassPointers) {
6344     if (Universe::heap() != NULL) {
6345       if (Universe::narrow_oop_base() == NULL) {
6346         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
6347       } else {
6348         mov64(r12_heapbase, (int64_t)Universe::narrow_ptrs_base());
6349       }
6350     } else {
6351       movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
6352     }
6353   }
6354 }
6355 
6356 #endif // _LP64
6357 
6358 // C2 compiled method's prolog code.
6359 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b) {
6360 
6361   // WARNING: Initial instruction MUST be 5 bytes or longer so that
6362   // NativeJump::patch_verified_entry will be able to patch out the entry
6363   // code safely. The push to verify stack depth is ok at 5 bytes,
6364   // the frame allocation can be either 3 or 6 bytes. So if we don't do
6365   // stack bang then we must use the 6 byte frame allocation even if
6366   // we have no frame. :-(
6367   assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
6368 
6369   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
6370   // Remove word for return addr
6371   framesize -= wordSize;
6372   stack_bang_size -= wordSize;
6373 
6374   // Calls to C2R adapters often do not accept exceptional returns.
6375   // We require that their callers must bang for them.  But be careful, because
6376   // some VM calls (such as call site linkage) can use several kilobytes of
6377   // stack.  But the stack safety zone should account for that.
6378   // See bugs 4446381, 4468289, 4497237.
6379   if (stack_bang_size > 0) {
6380     generate_stack_overflow_check(stack_bang_size);
6381 
6382     // We always push rbp, so that on return to interpreter rbp, will be
6383     // restored correctly and we can correct the stack.
6384     push(rbp);
6385     // Save caller's stack pointer into RBP if the frame pointer is preserved.
6386     if (PreserveFramePointer) {
6387       mov(rbp, rsp);
6388     }
6389     // Remove word for ebp
6390     framesize -= wordSize;
6391 
6392     // Create frame
6393     if (framesize) {
6394       subptr(rsp, framesize);
6395     }
6396   } else {
6397     // Create frame (force generation of a 4 byte immediate value)
6398     subptr_imm32(rsp, framesize);
6399 
6400     // Save RBP register now.
6401     framesize -= wordSize;
6402     movptr(Address(rsp, framesize), rbp);
6403     // Save caller's stack pointer into RBP if the frame pointer is preserved.
6404     if (PreserveFramePointer) {
6405       movptr(rbp, rsp);
6406       if (framesize > 0) {
6407         addptr(rbp, framesize);
6408       }
6409     }
6410   }
6411 
6412   if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
6413     framesize -= wordSize;
6414     movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
6415   }
6416 
6417 #ifndef _LP64
6418   // If method sets FPU control word do it now
6419   if (fp_mode_24b) {
6420     fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
6421   }
6422   if (UseSSE >= 2 && VerifyFPU) {
6423     verify_FPU(0, "FPU stack must be clean on entry");
6424   }
6425 #endif
6426 
6427 #ifdef ASSERT
6428   if (VerifyStackAtCalls) {
6429     Label L;
6430     push(rax);
6431     mov(rax, rsp);
6432     andptr(rax, StackAlignmentInBytes-1);
6433     cmpptr(rax, StackAlignmentInBytes-wordSize);
6434     pop(rax);
6435     jcc(Assembler::equal, L);
6436     STOP("Stack is not properly aligned!");
6437     bind(L);
6438   }
6439 #endif
6440 
6441 }
6442 
6443 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM registers
6444 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, XMMRegister xtmp) {
6445   // cnt - number of qwords (8-byte words).
6446   // base - start address, qword aligned.
6447   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
6448   if (UseAVX >= 2) {
6449     vpxor(xtmp, xtmp, xtmp, AVX_256bit);
6450   } else {
6451     pxor(xtmp, xtmp);
6452   }
6453   jmp(L_zero_64_bytes);
6454 
6455   BIND(L_loop);
6456   if (UseAVX >= 2) {
6457     vmovdqu(Address(base,  0), xtmp);
6458     vmovdqu(Address(base, 32), xtmp);
6459   } else {
6460     movdqu(Address(base,  0), xtmp);
6461     movdqu(Address(base, 16), xtmp);
6462     movdqu(Address(base, 32), xtmp);
6463     movdqu(Address(base, 48), xtmp);
6464   }
6465   addptr(base, 64);
6466 
6467   BIND(L_zero_64_bytes);
6468   subptr(cnt, 8);
6469   jccb(Assembler::greaterEqual, L_loop);
6470   addptr(cnt, 4);
6471   jccb(Assembler::less, L_tail);
6472   // Copy trailing 32 bytes
6473   if (UseAVX >= 2) {
6474     vmovdqu(Address(base, 0), xtmp);
6475   } else {
6476     movdqu(Address(base,  0), xtmp);
6477     movdqu(Address(base, 16), xtmp);
6478   }
6479   addptr(base, 32);
6480   subptr(cnt, 4);
6481 
6482   BIND(L_tail);
6483   addptr(cnt, 4);
6484   jccb(Assembler::lessEqual, L_end);
6485   decrement(cnt);
6486 
6487   BIND(L_sloop);
6488   movq(Address(base, 0), xtmp);
6489   addptr(base, 8);
6490   decrement(cnt);
6491   jccb(Assembler::greaterEqual, L_sloop);
6492   BIND(L_end);
6493 }
6494 
6495 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp, bool is_large) {
6496   // cnt - number of qwords (8-byte words).
6497   // base - start address, qword aligned.
6498   // is_large - if optimizers know cnt is larger than InitArrayShortSize
6499   assert(base==rdi, "base register must be edi for rep stos");
6500   assert(tmp==rax,   "tmp register must be eax for rep stos");
6501   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
6502   assert(InitArrayShortSize % BytesPerLong == 0,
6503     "InitArrayShortSize should be the multiple of BytesPerLong");
6504 
6505   Label DONE;
6506 
6507   if (!is_large || !UseXMMForObjInit) {
6508     xorptr(tmp, tmp);
6509   }
6510 
6511   if (!is_large) {
6512     Label LOOP, LONG;
6513     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
6514     jccb(Assembler::greater, LONG);
6515 
6516     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
6517 
6518     decrement(cnt);
6519     jccb(Assembler::negative, DONE); // Zero length
6520 
6521     // Use individual pointer-sized stores for small counts:
6522     BIND(LOOP);
6523     movptr(Address(base, cnt, Address::times_ptr), tmp);
6524     decrement(cnt);
6525     jccb(Assembler::greaterEqual, LOOP);
6526     jmpb(DONE);
6527 
6528     BIND(LONG);
6529   }
6530 
6531   // Use longer rep-prefixed ops for non-small counts:
6532   if (UseFastStosb) {
6533     shlptr(cnt, 3); // convert to number of bytes
6534     rep_stosb();
6535   } else if (UseXMMForObjInit) {
6536     movptr(tmp, base);
6537     xmm_clear_mem(tmp, cnt, xtmp);
6538   } else {
6539     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
6540     rep_stos();
6541   }
6542 
6543   BIND(DONE);
6544 }
6545 
6546 #ifdef COMPILER2
6547 
6548 // IndexOf for constant substrings with size >= 8 chars
6549 // which don't need to be loaded through stack.
6550 void MacroAssembler::string_indexofC8(Register str1, Register str2,
6551                                       Register cnt1, Register cnt2,
6552                                       int int_cnt2,  Register result,
6553                                       XMMRegister vec, Register tmp,
6554                                       int ae) {
6555   ShortBranchVerifier sbv(this);
6556   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
6557   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
6558 
6559   // This method uses the pcmpestri instruction with bound registers
6560   //   inputs:
6561   //     xmm - substring
6562   //     rax - substring length (elements count)
6563   //     mem - scanned string
6564   //     rdx - string length (elements count)
6565   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
6566   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
6567   //   outputs:
6568   //     rcx - matched index in string
6569   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
6570   int mode   = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
6571   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
6572   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
6573   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
6574 
6575   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR,
6576         RET_FOUND, RET_NOT_FOUND, EXIT, FOUND_SUBSTR,
6577         MATCH_SUBSTR_HEAD, RELOAD_STR, FOUND_CANDIDATE;
6578 
6579   // Note, inline_string_indexOf() generates checks:
6580   // if (substr.count > string.count) return -1;
6581   // if (substr.count == 0) return 0;
6582   assert(int_cnt2 >= stride, "this code is used only for cnt2 >= 8 chars");
6583 
6584   // Load substring.
6585   if (ae == StrIntrinsicNode::UL) {
6586     pmovzxbw(vec, Address(str2, 0));
6587   } else {
6588     movdqu(vec, Address(str2, 0));
6589   }
6590   movl(cnt2, int_cnt2);
6591   movptr(result, str1); // string addr
6592 
6593   if (int_cnt2 > stride) {
6594     jmpb(SCAN_TO_SUBSTR);
6595 
6596     // Reload substr for rescan, this code
6597     // is executed only for large substrings (> 8 chars)
6598     bind(RELOAD_SUBSTR);
6599     if (ae == StrIntrinsicNode::UL) {
6600       pmovzxbw(vec, Address(str2, 0));
6601     } else {
6602       movdqu(vec, Address(str2, 0));
6603     }
6604     negptr(cnt2); // Jumped here with negative cnt2, convert to positive
6605 
6606     bind(RELOAD_STR);
6607     // We came here after the beginning of the substring was
6608     // matched but the rest of it was not so we need to search
6609     // again. Start from the next element after the previous match.
6610 
6611     // cnt2 is number of substring reminding elements and
6612     // cnt1 is number of string reminding elements when cmp failed.
6613     // Restored cnt1 = cnt1 - cnt2 + int_cnt2
6614     subl(cnt1, cnt2);
6615     addl(cnt1, int_cnt2);
6616     movl(cnt2, int_cnt2); // Now restore cnt2
6617 
6618     decrementl(cnt1);     // Shift to next element
6619     cmpl(cnt1, cnt2);
6620     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6621 
6622     addptr(result, (1<<scale1));
6623 
6624   } // (int_cnt2 > 8)
6625 
6626   // Scan string for start of substr in 16-byte vectors
6627   bind(SCAN_TO_SUBSTR);
6628   pcmpestri(vec, Address(result, 0), mode);
6629   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
6630   subl(cnt1, stride);
6631   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
6632   cmpl(cnt1, cnt2);
6633   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6634   addptr(result, 16);
6635   jmpb(SCAN_TO_SUBSTR);
6636 
6637   // Found a potential substr
6638   bind(FOUND_CANDIDATE);
6639   // Matched whole vector if first element matched (tmp(rcx) == 0).
6640   if (int_cnt2 == stride) {
6641     jccb(Assembler::overflow, RET_FOUND);    // OF == 1
6642   } else { // int_cnt2 > 8
6643     jccb(Assembler::overflow, FOUND_SUBSTR);
6644   }
6645   // After pcmpestri tmp(rcx) contains matched element index
6646   // Compute start addr of substr
6647   lea(result, Address(result, tmp, scale1));
6648 
6649   // Make sure string is still long enough
6650   subl(cnt1, tmp);
6651   cmpl(cnt1, cnt2);
6652   if (int_cnt2 == stride) {
6653     jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
6654   } else { // int_cnt2 > 8
6655     jccb(Assembler::greaterEqual, MATCH_SUBSTR_HEAD);
6656   }
6657   // Left less then substring.
6658 
6659   bind(RET_NOT_FOUND);
6660   movl(result, -1);
6661   jmp(EXIT);
6662 
6663   if (int_cnt2 > stride) {
6664     // This code is optimized for the case when whole substring
6665     // is matched if its head is matched.
6666     bind(MATCH_SUBSTR_HEAD);
6667     pcmpestri(vec, Address(result, 0), mode);
6668     // Reload only string if does not match
6669     jcc(Assembler::noOverflow, RELOAD_STR); // OF == 0
6670 
6671     Label CONT_SCAN_SUBSTR;
6672     // Compare the rest of substring (> 8 chars).
6673     bind(FOUND_SUBSTR);
6674     // First 8 chars are already matched.
6675     negptr(cnt2);
6676     addptr(cnt2, stride);
6677 
6678     bind(SCAN_SUBSTR);
6679     subl(cnt1, stride);
6680     cmpl(cnt2, -stride); // Do not read beyond substring
6681     jccb(Assembler::lessEqual, CONT_SCAN_SUBSTR);
6682     // Back-up strings to avoid reading beyond substring:
6683     // cnt1 = cnt1 - cnt2 + 8
6684     addl(cnt1, cnt2); // cnt2 is negative
6685     addl(cnt1, stride);
6686     movl(cnt2, stride); negptr(cnt2);
6687     bind(CONT_SCAN_SUBSTR);
6688     if (int_cnt2 < (int)G) {
6689       int tail_off1 = int_cnt2<<scale1;
6690       int tail_off2 = int_cnt2<<scale2;
6691       if (ae == StrIntrinsicNode::UL) {
6692         pmovzxbw(vec, Address(str2, cnt2, scale2, tail_off2));
6693       } else {
6694         movdqu(vec, Address(str2, cnt2, scale2, tail_off2));
6695       }
6696       pcmpestri(vec, Address(result, cnt2, scale1, tail_off1), mode);
6697     } else {
6698       // calculate index in register to avoid integer overflow (int_cnt2*2)
6699       movl(tmp, int_cnt2);
6700       addptr(tmp, cnt2);
6701       if (ae == StrIntrinsicNode::UL) {
6702         pmovzxbw(vec, Address(str2, tmp, scale2, 0));
6703       } else {
6704         movdqu(vec, Address(str2, tmp, scale2, 0));
6705       }
6706       pcmpestri(vec, Address(result, tmp, scale1, 0), mode);
6707     }
6708     // Need to reload strings pointers if not matched whole vector
6709     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
6710     addptr(cnt2, stride);
6711     jcc(Assembler::negative, SCAN_SUBSTR);
6712     // Fall through if found full substring
6713 
6714   } // (int_cnt2 > 8)
6715 
6716   bind(RET_FOUND);
6717   // Found result if we matched full small substring.
6718   // Compute substr offset
6719   subptr(result, str1);
6720   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
6721     shrl(result, 1); // index
6722   }
6723   bind(EXIT);
6724 
6725 } // string_indexofC8
6726 
6727 // Small strings are loaded through stack if they cross page boundary.
6728 void MacroAssembler::string_indexof(Register str1, Register str2,
6729                                     Register cnt1, Register cnt2,
6730                                     int int_cnt2,  Register result,
6731                                     XMMRegister vec, Register tmp,
6732                                     int ae) {
6733   ShortBranchVerifier sbv(this);
6734   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
6735   assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
6736 
6737   //
6738   // int_cnt2 is length of small (< 8 chars) constant substring
6739   // or (-1) for non constant substring in which case its length
6740   // is in cnt2 register.
6741   //
6742   // Note, inline_string_indexOf() generates checks:
6743   // if (substr.count > string.count) return -1;
6744   // if (substr.count == 0) return 0;
6745   //
6746   int stride = (ae == StrIntrinsicNode::LL) ? 16 : 8; //UU, UL -> 8
6747   assert(int_cnt2 == -1 || (0 < int_cnt2 && int_cnt2 < stride), "should be != 0");
6748   // This method uses the pcmpestri instruction with bound registers
6749   //   inputs:
6750   //     xmm - substring
6751   //     rax - substring length (elements count)
6752   //     mem - scanned string
6753   //     rdx - string length (elements count)
6754   //     0xd - mode: 1100 (substring search) + 01 (unsigned shorts)
6755   //     0xc - mode: 1100 (substring search) + 00 (unsigned bytes)
6756   //   outputs:
6757   //     rcx - matched index in string
6758   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
6759   int mode = (ae == StrIntrinsicNode::LL) ? 0x0c : 0x0d; // bytes or shorts
6760   Address::ScaleFactor scale1 = (ae == StrIntrinsicNode::LL) ? Address::times_1 : Address::times_2;
6761   Address::ScaleFactor scale2 = (ae == StrIntrinsicNode::UL) ? Address::times_1 : scale1;
6762 
6763   Label RELOAD_SUBSTR, SCAN_TO_SUBSTR, SCAN_SUBSTR, ADJUST_STR,
6764         RET_FOUND, RET_NOT_FOUND, CLEANUP, FOUND_SUBSTR,
6765         FOUND_CANDIDATE;
6766 
6767   { //========================================================
6768     // We don't know where these strings are located
6769     // and we can't read beyond them. Load them through stack.
6770     Label BIG_STRINGS, CHECK_STR, COPY_SUBSTR, COPY_STR;
6771 
6772     movptr(tmp, rsp); // save old SP
6773 
6774     if (int_cnt2 > 0) {     // small (< 8 chars) constant substring
6775       if (int_cnt2 == (1>>scale2)) { // One byte
6776         assert((ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL), "Only possible for latin1 encoding");
6777         load_unsigned_byte(result, Address(str2, 0));
6778         movdl(vec, result); // move 32 bits
6779       } else if (ae == StrIntrinsicNode::LL && int_cnt2 == 3) {  // Three bytes
6780         // Not enough header space in 32-bit VM: 12+3 = 15.
6781         movl(result, Address(str2, -1));
6782         shrl(result, 8);
6783         movdl(vec, result); // move 32 bits
6784       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (2>>scale2)) {  // One char
6785         load_unsigned_short(result, Address(str2, 0));
6786         movdl(vec, result); // move 32 bits
6787       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (4>>scale2)) { // Two chars
6788         movdl(vec, Address(str2, 0)); // move 32 bits
6789       } else if (ae != StrIntrinsicNode::UL && int_cnt2 == (8>>scale2)) { // Four chars
6790         movq(vec, Address(str2, 0));  // move 64 bits
6791       } else { // cnt2 = { 3, 5, 6, 7 } || (ae == StrIntrinsicNode::UL && cnt2 ={2, ..., 7})
6792         // Array header size is 12 bytes in 32-bit VM
6793         // + 6 bytes for 3 chars == 18 bytes,
6794         // enough space to load vec and shift.
6795         assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity");
6796         if (ae == StrIntrinsicNode::UL) {
6797           int tail_off = int_cnt2-8;
6798           pmovzxbw(vec, Address(str2, tail_off));
6799           psrldq(vec, -2*tail_off);
6800         }
6801         else {
6802           int tail_off = int_cnt2*(1<<scale2);
6803           movdqu(vec, Address(str2, tail_off-16));
6804           psrldq(vec, 16-tail_off);
6805         }
6806       }
6807     } else { // not constant substring
6808       cmpl(cnt2, stride);
6809       jccb(Assembler::aboveEqual, BIG_STRINGS); // Both strings are big enough
6810 
6811       // We can read beyond string if srt+16 does not cross page boundary
6812       // since heaps are aligned and mapped by pages.
6813       assert(os::vm_page_size() < (int)G, "default page should be small");
6814       movl(result, str2); // We need only low 32 bits
6815       andl(result, (os::vm_page_size()-1));
6816       cmpl(result, (os::vm_page_size()-16));
6817       jccb(Assembler::belowEqual, CHECK_STR);
6818 
6819       // Move small strings to stack to allow load 16 bytes into vec.
6820       subptr(rsp, 16);
6821       int stk_offset = wordSize-(1<<scale2);
6822       push(cnt2);
6823 
6824       bind(COPY_SUBSTR);
6825       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UL) {
6826         load_unsigned_byte(result, Address(str2, cnt2, scale2, -1));
6827         movb(Address(rsp, cnt2, scale2, stk_offset), result);
6828       } else if (ae == StrIntrinsicNode::UU) {
6829         load_unsigned_short(result, Address(str2, cnt2, scale2, -2));
6830         movw(Address(rsp, cnt2, scale2, stk_offset), result);
6831       }
6832       decrement(cnt2);
6833       jccb(Assembler::notZero, COPY_SUBSTR);
6834 
6835       pop(cnt2);
6836       movptr(str2, rsp);  // New substring address
6837     } // non constant
6838 
6839     bind(CHECK_STR);
6840     cmpl(cnt1, stride);
6841     jccb(Assembler::aboveEqual, BIG_STRINGS);
6842 
6843     // Check cross page boundary.
6844     movl(result, str1); // We need only low 32 bits
6845     andl(result, (os::vm_page_size()-1));
6846     cmpl(result, (os::vm_page_size()-16));
6847     jccb(Assembler::belowEqual, BIG_STRINGS);
6848 
6849     subptr(rsp, 16);
6850     int stk_offset = -(1<<scale1);
6851     if (int_cnt2 < 0) { // not constant
6852       push(cnt2);
6853       stk_offset += wordSize;
6854     }
6855     movl(cnt2, cnt1);
6856 
6857     bind(COPY_STR);
6858     if (ae == StrIntrinsicNode::LL) {
6859       load_unsigned_byte(result, Address(str1, cnt2, scale1, -1));
6860       movb(Address(rsp, cnt2, scale1, stk_offset), result);
6861     } else {
6862       load_unsigned_short(result, Address(str1, cnt2, scale1, -2));
6863       movw(Address(rsp, cnt2, scale1, stk_offset), result);
6864     }
6865     decrement(cnt2);
6866     jccb(Assembler::notZero, COPY_STR);
6867 
6868     if (int_cnt2 < 0) { // not constant
6869       pop(cnt2);
6870     }
6871     movptr(str1, rsp);  // New string address
6872 
6873     bind(BIG_STRINGS);
6874     // Load substring.
6875     if (int_cnt2 < 0) { // -1
6876       if (ae == StrIntrinsicNode::UL) {
6877         pmovzxbw(vec, Address(str2, 0));
6878       } else {
6879         movdqu(vec, Address(str2, 0));
6880       }
6881       push(cnt2);       // substr count
6882       push(str2);       // substr addr
6883       push(str1);       // string addr
6884     } else {
6885       // Small (< 8 chars) constant substrings are loaded already.
6886       movl(cnt2, int_cnt2);
6887     }
6888     push(tmp);  // original SP
6889 
6890   } // Finished loading
6891 
6892   //========================================================
6893   // Start search
6894   //
6895 
6896   movptr(result, str1); // string addr
6897 
6898   if (int_cnt2  < 0) {  // Only for non constant substring
6899     jmpb(SCAN_TO_SUBSTR);
6900 
6901     // SP saved at sp+0
6902     // String saved at sp+1*wordSize
6903     // Substr saved at sp+2*wordSize
6904     // Substr count saved at sp+3*wordSize
6905 
6906     // Reload substr for rescan, this code
6907     // is executed only for large substrings (> 8 chars)
6908     bind(RELOAD_SUBSTR);
6909     movptr(str2, Address(rsp, 2*wordSize));
6910     movl(cnt2, Address(rsp, 3*wordSize));
6911     if (ae == StrIntrinsicNode::UL) {
6912       pmovzxbw(vec, Address(str2, 0));
6913     } else {
6914       movdqu(vec, Address(str2, 0));
6915     }
6916     // We came here after the beginning of the substring was
6917     // matched but the rest of it was not so we need to search
6918     // again. Start from the next element after the previous match.
6919     subptr(str1, result); // Restore counter
6920     if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
6921       shrl(str1, 1);
6922     }
6923     addl(cnt1, str1);
6924     decrementl(cnt1);   // Shift to next element
6925     cmpl(cnt1, cnt2);
6926     jcc(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6927 
6928     addptr(result, (1<<scale1));
6929   } // non constant
6930 
6931   // Scan string for start of substr in 16-byte vectors
6932   bind(SCAN_TO_SUBSTR);
6933   assert(cnt1 == rdx && cnt2 == rax && tmp == rcx, "pcmpestri");
6934   pcmpestri(vec, Address(result, 0), mode);
6935   jccb(Assembler::below, FOUND_CANDIDATE);   // CF == 1
6936   subl(cnt1, stride);
6937   jccb(Assembler::lessEqual, RET_NOT_FOUND); // Scanned full string
6938   cmpl(cnt1, cnt2);
6939   jccb(Assembler::negative, RET_NOT_FOUND);  // Left less then substring
6940   addptr(result, 16);
6941 
6942   bind(ADJUST_STR);
6943   cmpl(cnt1, stride); // Do not read beyond string
6944   jccb(Assembler::greaterEqual, SCAN_TO_SUBSTR);
6945   // Back-up string to avoid reading beyond string.
6946   lea(result, Address(result, cnt1, scale1, -16));
6947   movl(cnt1, stride);
6948   jmpb(SCAN_TO_SUBSTR);
6949 
6950   // Found a potential substr
6951   bind(FOUND_CANDIDATE);
6952   // After pcmpestri tmp(rcx) contains matched element index
6953 
6954   // Make sure string is still long enough
6955   subl(cnt1, tmp);
6956   cmpl(cnt1, cnt2);
6957   jccb(Assembler::greaterEqual, FOUND_SUBSTR);
6958   // Left less then substring.
6959 
6960   bind(RET_NOT_FOUND);
6961   movl(result, -1);
6962   jmp(CLEANUP);
6963 
6964   bind(FOUND_SUBSTR);
6965   // Compute start addr of substr
6966   lea(result, Address(result, tmp, scale1));
6967   if (int_cnt2 > 0) { // Constant substring
6968     // Repeat search for small substring (< 8 chars)
6969     // from new point without reloading substring.
6970     // Have to check that we don't read beyond string.
6971     cmpl(tmp, stride-int_cnt2);
6972     jccb(Assembler::greater, ADJUST_STR);
6973     // Fall through if matched whole substring.
6974   } else { // non constant
6975     assert(int_cnt2 == -1, "should be != 0");
6976 
6977     addl(tmp, cnt2);
6978     // Found result if we matched whole substring.
6979     cmpl(tmp, stride);
6980     jcc(Assembler::lessEqual, RET_FOUND);
6981 
6982     // Repeat search for small substring (<= 8 chars)
6983     // from new point 'str1' without reloading substring.
6984     cmpl(cnt2, stride);
6985     // Have to check that we don't read beyond string.
6986     jccb(Assembler::lessEqual, ADJUST_STR);
6987 
6988     Label CHECK_NEXT, CONT_SCAN_SUBSTR, RET_FOUND_LONG;
6989     // Compare the rest of substring (> 8 chars).
6990     movptr(str1, result);
6991 
6992     cmpl(tmp, cnt2);
6993     // First 8 chars are already matched.
6994     jccb(Assembler::equal, CHECK_NEXT);
6995 
6996     bind(SCAN_SUBSTR);
6997     pcmpestri(vec, Address(str1, 0), mode);
6998     // Need to reload strings pointers if not matched whole vector
6999     jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
7000 
7001     bind(CHECK_NEXT);
7002     subl(cnt2, stride);
7003     jccb(Assembler::lessEqual, RET_FOUND_LONG); // Found full substring
7004     addptr(str1, 16);
7005     if (ae == StrIntrinsicNode::UL) {
7006       addptr(str2, 8);
7007     } else {
7008       addptr(str2, 16);
7009     }
7010     subl(cnt1, stride);
7011     cmpl(cnt2, stride); // Do not read beyond substring
7012     jccb(Assembler::greaterEqual, CONT_SCAN_SUBSTR);
7013     // Back-up strings to avoid reading beyond substring.
7014 
7015     if (ae == StrIntrinsicNode::UL) {
7016       lea(str2, Address(str2, cnt2, scale2, -8));
7017       lea(str1, Address(str1, cnt2, scale1, -16));
7018     } else {
7019       lea(str2, Address(str2, cnt2, scale2, -16));
7020       lea(str1, Address(str1, cnt2, scale1, -16));
7021     }
7022     subl(cnt1, cnt2);
7023     movl(cnt2, stride);
7024     addl(cnt1, stride);
7025     bind(CONT_SCAN_SUBSTR);
7026     if (ae == StrIntrinsicNode::UL) {
7027       pmovzxbw(vec, Address(str2, 0));
7028     } else {
7029       movdqu(vec, Address(str2, 0));
7030     }
7031     jmp(SCAN_SUBSTR);
7032 
7033     bind(RET_FOUND_LONG);
7034     movptr(str1, Address(rsp, wordSize));
7035   } // non constant
7036 
7037   bind(RET_FOUND);
7038   // Compute substr offset
7039   subptr(result, str1);
7040   if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) {
7041     shrl(result, 1); // index
7042   }
7043   bind(CLEANUP);
7044   pop(rsp); // restore SP
7045 
7046 } // string_indexof
7047 
7048 void MacroAssembler::string_indexof_char(Register str1, Register cnt1, Register ch, Register result,
7049                                          XMMRegister vec1, XMMRegister vec2, XMMRegister vec3, Register tmp) {
7050   ShortBranchVerifier sbv(this);
7051   assert(UseSSE42Intrinsics, "SSE4.2 intrinsics are required");
7052 
7053   int stride = 8;
7054 
7055   Label FOUND_CHAR, SCAN_TO_CHAR, SCAN_TO_CHAR_LOOP,
7056         SCAN_TO_8_CHAR, SCAN_TO_8_CHAR_LOOP, SCAN_TO_16_CHAR_LOOP,
7057         RET_NOT_FOUND, SCAN_TO_8_CHAR_INIT,
7058         FOUND_SEQ_CHAR, DONE_LABEL;
7059 
7060   movptr(result, str1);
7061   if (UseAVX >= 2) {
7062     cmpl(cnt1, stride);
7063     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7064     cmpl(cnt1, 2*stride);
7065     jcc(Assembler::less, SCAN_TO_8_CHAR_INIT);
7066     movdl(vec1, ch);
7067     vpbroadcastw(vec1, vec1);
7068     vpxor(vec2, vec2);
7069     movl(tmp, cnt1);
7070     andl(tmp, 0xFFFFFFF0);  //vector count (in chars)
7071     andl(cnt1,0x0000000F);  //tail count (in chars)
7072 
7073     bind(SCAN_TO_16_CHAR_LOOP);
7074     vmovdqu(vec3, Address(result, 0));
7075     vpcmpeqw(vec3, vec3, vec1, 1);
7076     vptest(vec2, vec3);
7077     jcc(Assembler::carryClear, FOUND_CHAR);
7078     addptr(result, 32);
7079     subl(tmp, 2*stride);
7080     jcc(Assembler::notZero, SCAN_TO_16_CHAR_LOOP);
7081     jmp(SCAN_TO_8_CHAR);
7082     bind(SCAN_TO_8_CHAR_INIT);
7083     movdl(vec1, ch);
7084     pshuflw(vec1, vec1, 0x00);
7085     pshufd(vec1, vec1, 0);
7086     pxor(vec2, vec2);
7087   }
7088   bind(SCAN_TO_8_CHAR);
7089   cmpl(cnt1, stride);
7090   if (UseAVX >= 2) {
7091     jcc(Assembler::less, SCAN_TO_CHAR);
7092   } else {
7093     jcc(Assembler::less, SCAN_TO_CHAR_LOOP);
7094     movdl(vec1, ch);
7095     pshuflw(vec1, vec1, 0x00);
7096     pshufd(vec1, vec1, 0);
7097     pxor(vec2, vec2);
7098   }
7099   movl(tmp, cnt1);
7100   andl(tmp, 0xFFFFFFF8);  //vector count (in chars)
7101   andl(cnt1,0x00000007);  //tail count (in chars)
7102 
7103   bind(SCAN_TO_8_CHAR_LOOP);
7104   movdqu(vec3, Address(result, 0));
7105   pcmpeqw(vec3, vec1);
7106   ptest(vec2, vec3);
7107   jcc(Assembler::carryClear, FOUND_CHAR);
7108   addptr(result, 16);
7109   subl(tmp, stride);
7110   jcc(Assembler::notZero, SCAN_TO_8_CHAR_LOOP);
7111   bind(SCAN_TO_CHAR);
7112   testl(cnt1, cnt1);
7113   jcc(Assembler::zero, RET_NOT_FOUND);
7114   bind(SCAN_TO_CHAR_LOOP);
7115   load_unsigned_short(tmp, Address(result, 0));
7116   cmpl(ch, tmp);
7117   jccb(Assembler::equal, FOUND_SEQ_CHAR);
7118   addptr(result, 2);
7119   subl(cnt1, 1);
7120   jccb(Assembler::zero, RET_NOT_FOUND);
7121   jmp(SCAN_TO_CHAR_LOOP);
7122 
7123   bind(RET_NOT_FOUND);
7124   movl(result, -1);
7125   jmpb(DONE_LABEL);
7126 
7127   bind(FOUND_CHAR);
7128   if (UseAVX >= 2) {
7129     vpmovmskb(tmp, vec3);
7130   } else {
7131     pmovmskb(tmp, vec3);
7132   }
7133   bsfl(ch, tmp);
7134   addl(result, ch);
7135 
7136   bind(FOUND_SEQ_CHAR);
7137   subptr(result, str1);
7138   shrl(result, 1);
7139 
7140   bind(DONE_LABEL);
7141 } // string_indexof_char
7142 
7143 // helper function for string_compare
7144 void MacroAssembler::load_next_elements(Register elem1, Register elem2, Register str1, Register str2,
7145                                         Address::ScaleFactor scale, Address::ScaleFactor scale1,
7146                                         Address::ScaleFactor scale2, Register index, int ae) {
7147   if (ae == StrIntrinsicNode::LL) {
7148     load_unsigned_byte(elem1, Address(str1, index, scale, 0));
7149     load_unsigned_byte(elem2, Address(str2, index, scale, 0));
7150   } else if (ae == StrIntrinsicNode::UU) {
7151     load_unsigned_short(elem1, Address(str1, index, scale, 0));
7152     load_unsigned_short(elem2, Address(str2, index, scale, 0));
7153   } else {
7154     load_unsigned_byte(elem1, Address(str1, index, scale1, 0));
7155     load_unsigned_short(elem2, Address(str2, index, scale2, 0));
7156   }
7157 }
7158 
7159 // Compare strings, used for char[] and byte[].
7160 void MacroAssembler::string_compare(Register str1, Register str2,
7161                                     Register cnt1, Register cnt2, Register result,
7162                                     XMMRegister vec1, int ae) {
7163   ShortBranchVerifier sbv(this);
7164   Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
7165   Label COMPARE_WIDE_VECTORS_LOOP_FAILED;  // used only _LP64 && AVX3
7166   int stride, stride2, adr_stride, adr_stride1, adr_stride2;
7167   int stride2x2 = 0x40;
7168   Address::ScaleFactor scale = Address::no_scale;
7169   Address::ScaleFactor scale1 = Address::no_scale;
7170   Address::ScaleFactor scale2 = Address::no_scale;
7171 
7172   if (ae != StrIntrinsicNode::LL) {
7173     stride2x2 = 0x20;
7174   }
7175 
7176   if (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL) {
7177     shrl(cnt2, 1);
7178   }
7179   // Compute the minimum of the string lengths and the
7180   // difference of the string lengths (stack).
7181   // Do the conditional move stuff
7182   movl(result, cnt1);
7183   subl(cnt1, cnt2);
7184   push(cnt1);
7185   cmov32(Assembler::lessEqual, cnt2, result);    // cnt2 = min(cnt1, cnt2)
7186 
7187   // Is the minimum length zero?
7188   testl(cnt2, cnt2);
7189   jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7190   if (ae == StrIntrinsicNode::LL) {
7191     // Load first bytes
7192     load_unsigned_byte(result, Address(str1, 0));  // result = str1[0]
7193     load_unsigned_byte(cnt1, Address(str2, 0));    // cnt1   = str2[0]
7194   } else if (ae == StrIntrinsicNode::UU) {
7195     // Load first characters
7196     load_unsigned_short(result, Address(str1, 0));
7197     load_unsigned_short(cnt1, Address(str2, 0));
7198   } else {
7199     load_unsigned_byte(result, Address(str1, 0));
7200     load_unsigned_short(cnt1, Address(str2, 0));
7201   }
7202   subl(result, cnt1);
7203   jcc(Assembler::notZero,  POP_LABEL);
7204 
7205   if (ae == StrIntrinsicNode::UU) {
7206     // Divide length by 2 to get number of chars
7207     shrl(cnt2, 1);
7208   }
7209   cmpl(cnt2, 1);
7210   jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7211 
7212   // Check if the strings start at the same location and setup scale and stride
7213   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7214     cmpptr(str1, str2);
7215     jcc(Assembler::equal, LENGTH_DIFF_LABEL);
7216     if (ae == StrIntrinsicNode::LL) {
7217       scale = Address::times_1;
7218       stride = 16;
7219     } else {
7220       scale = Address::times_2;
7221       stride = 8;
7222     }
7223   } else {
7224     scale1 = Address::times_1;
7225     scale2 = Address::times_2;
7226     // scale not used
7227     stride = 8;
7228   }
7229 
7230   if (UseAVX >= 2 && UseSSE42Intrinsics) {
7231     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_WIDE_TAIL, COMPARE_SMALL_STR;
7232     Label COMPARE_WIDE_VECTORS_LOOP, COMPARE_16_CHARS, COMPARE_INDEX_CHAR;
7233     Label COMPARE_WIDE_VECTORS_LOOP_AVX2;
7234     Label COMPARE_TAIL_LONG;
7235     Label COMPARE_WIDE_VECTORS_LOOP_AVX3;  // used only _LP64 && AVX3
7236 
7237     int pcmpmask = 0x19;
7238     if (ae == StrIntrinsicNode::LL) {
7239       pcmpmask &= ~0x01;
7240     }
7241 
7242     // Setup to compare 16-chars (32-bytes) vectors,
7243     // start from first character again because it has aligned address.
7244     if (ae == StrIntrinsicNode::LL) {
7245       stride2 = 32;
7246     } else {
7247       stride2 = 16;
7248     }
7249     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7250       adr_stride = stride << scale;
7251     } else {
7252       adr_stride1 = 8;  //stride << scale1;
7253       adr_stride2 = 16; //stride << scale2;
7254     }
7255 
7256     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
7257     // rax and rdx are used by pcmpestri as elements counters
7258     movl(result, cnt2);
7259     andl(cnt2, ~(stride2-1));   // cnt2 holds the vector count
7260     jcc(Assembler::zero, COMPARE_TAIL_LONG);
7261 
7262     // fast path : compare first 2 8-char vectors.
7263     bind(COMPARE_16_CHARS);
7264     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7265       movdqu(vec1, Address(str1, 0));
7266     } else {
7267       pmovzxbw(vec1, Address(str1, 0));
7268     }
7269     pcmpestri(vec1, Address(str2, 0), pcmpmask);
7270     jccb(Assembler::below, COMPARE_INDEX_CHAR);
7271 
7272     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7273       movdqu(vec1, Address(str1, adr_stride));
7274       pcmpestri(vec1, Address(str2, adr_stride), pcmpmask);
7275     } else {
7276       pmovzxbw(vec1, Address(str1, adr_stride1));
7277       pcmpestri(vec1, Address(str2, adr_stride2), pcmpmask);
7278     }
7279     jccb(Assembler::aboveEqual, COMPARE_WIDE_VECTORS);
7280     addl(cnt1, stride);
7281 
7282     // Compare the characters at index in cnt1
7283     bind(COMPARE_INDEX_CHAR); // cnt1 has the offset of the mismatching character
7284     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
7285     subl(result, cnt2);
7286     jmp(POP_LABEL);
7287 
7288     // Setup the registers to start vector comparison loop
7289     bind(COMPARE_WIDE_VECTORS);
7290     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7291       lea(str1, Address(str1, result, scale));
7292       lea(str2, Address(str2, result, scale));
7293     } else {
7294       lea(str1, Address(str1, result, scale1));
7295       lea(str2, Address(str2, result, scale2));
7296     }
7297     subl(result, stride2);
7298     subl(cnt2, stride2);
7299     jcc(Assembler::zero, COMPARE_WIDE_TAIL);
7300     negptr(result);
7301 
7302     //  In a loop, compare 16-chars (32-bytes) at once using (vpxor+vptest)
7303     bind(COMPARE_WIDE_VECTORS_LOOP);
7304 
7305 #ifdef _LP64
7306     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
7307       cmpl(cnt2, stride2x2);
7308       jccb(Assembler::below, COMPARE_WIDE_VECTORS_LOOP_AVX2);
7309       testl(cnt2, stride2x2-1);   // cnt2 holds the vector count
7310       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX2);   // means we cannot subtract by 0x40
7311 
7312       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
7313       if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7314         evmovdquq(vec1, Address(str1, result, scale), Assembler::AVX_512bit);
7315         evpcmpeqb(k7, vec1, Address(str2, result, scale), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7316       } else {
7317         vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_512bit);
7318         evpcmpeqb(k7, vec1, Address(str2, result, scale2), Assembler::AVX_512bit); // k7 == 11..11, if operands equal, otherwise k7 has some 0
7319       }
7320       kortestql(k7, k7);
7321       jcc(Assembler::aboveEqual, COMPARE_WIDE_VECTORS_LOOP_FAILED);     // miscompare
7322       addptr(result, stride2x2);  // update since we already compared at this addr
7323       subl(cnt2, stride2x2);      // and sub the size too
7324       jccb(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP_AVX3);
7325 
7326       vpxor(vec1, vec1);
7327       jmpb(COMPARE_WIDE_TAIL);
7328     }//if (VM_Version::supports_avx512vlbw())
7329 #endif // _LP64
7330 
7331 
7332     bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7333     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7334       vmovdqu(vec1, Address(str1, result, scale));
7335       vpxor(vec1, Address(str2, result, scale));
7336     } else {
7337       vpmovzxbw(vec1, Address(str1, result, scale1), Assembler::AVX_256bit);
7338       vpxor(vec1, Address(str2, result, scale2));
7339     }
7340     vptest(vec1, vec1);
7341     jcc(Assembler::notZero, VECTOR_NOT_EQUAL);
7342     addptr(result, stride2);
7343     subl(cnt2, stride2);
7344     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS_LOOP);
7345     // clean upper bits of YMM registers
7346     vpxor(vec1, vec1);
7347 
7348     // compare wide vectors tail
7349     bind(COMPARE_WIDE_TAIL);
7350     testptr(result, result);
7351     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7352 
7353     movl(result, stride2);
7354     movl(cnt2, result);
7355     negptr(result);
7356     jmp(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7357 
7358     // Identifies the mismatching (higher or lower)16-bytes in the 32-byte vectors.
7359     bind(VECTOR_NOT_EQUAL);
7360     // clean upper bits of YMM registers
7361     vpxor(vec1, vec1);
7362     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7363       lea(str1, Address(str1, result, scale));
7364       lea(str2, Address(str2, result, scale));
7365     } else {
7366       lea(str1, Address(str1, result, scale1));
7367       lea(str2, Address(str2, result, scale2));
7368     }
7369     jmp(COMPARE_16_CHARS);
7370 
7371     // Compare tail chars, length between 1 to 15 chars
7372     bind(COMPARE_TAIL_LONG);
7373     movl(cnt2, result);
7374     cmpl(cnt2, stride);
7375     jcc(Assembler::less, COMPARE_SMALL_STR);
7376 
7377     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7378       movdqu(vec1, Address(str1, 0));
7379     } else {
7380       pmovzxbw(vec1, Address(str1, 0));
7381     }
7382     pcmpestri(vec1, Address(str2, 0), pcmpmask);
7383     jcc(Assembler::below, COMPARE_INDEX_CHAR);
7384     subptr(cnt2, stride);
7385     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7386     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7387       lea(str1, Address(str1, result, scale));
7388       lea(str2, Address(str2, result, scale));
7389     } else {
7390       lea(str1, Address(str1, result, scale1));
7391       lea(str2, Address(str2, result, scale2));
7392     }
7393     negptr(cnt2);
7394     jmpb(WHILE_HEAD_LABEL);
7395 
7396     bind(COMPARE_SMALL_STR);
7397   } else if (UseSSE42Intrinsics) {
7398     Label COMPARE_WIDE_VECTORS, VECTOR_NOT_EQUAL, COMPARE_TAIL;
7399     int pcmpmask = 0x19;
7400     // Setup to compare 8-char (16-byte) vectors,
7401     // start from first character again because it has aligned address.
7402     movl(result, cnt2);
7403     andl(cnt2, ~(stride - 1));   // cnt2 holds the vector count
7404     if (ae == StrIntrinsicNode::LL) {
7405       pcmpmask &= ~0x01;
7406     }
7407     jcc(Assembler::zero, COMPARE_TAIL);
7408     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7409       lea(str1, Address(str1, result, scale));
7410       lea(str2, Address(str2, result, scale));
7411     } else {
7412       lea(str1, Address(str1, result, scale1));
7413       lea(str2, Address(str2, result, scale2));
7414     }
7415     negptr(result);
7416 
7417     // pcmpestri
7418     //   inputs:
7419     //     vec1- substring
7420     //     rax - negative string length (elements count)
7421     //     mem - scanned string
7422     //     rdx - string length (elements count)
7423     //     pcmpmask - cmp mode: 11000 (string compare with negated result)
7424     //               + 00 (unsigned bytes) or  + 01 (unsigned shorts)
7425     //   outputs:
7426     //     rcx - first mismatched element index
7427     assert(result == rax && cnt2 == rdx && cnt1 == rcx, "pcmpestri");
7428 
7429     bind(COMPARE_WIDE_VECTORS);
7430     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7431       movdqu(vec1, Address(str1, result, scale));
7432       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
7433     } else {
7434       pmovzxbw(vec1, Address(str1, result, scale1));
7435       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
7436     }
7437     // After pcmpestri cnt1(rcx) contains mismatched element index
7438 
7439     jccb(Assembler::below, VECTOR_NOT_EQUAL);  // CF==1
7440     addptr(result, stride);
7441     subptr(cnt2, stride);
7442     jccb(Assembler::notZero, COMPARE_WIDE_VECTORS);
7443 
7444     // compare wide vectors tail
7445     testptr(result, result);
7446     jcc(Assembler::zero, LENGTH_DIFF_LABEL);
7447 
7448     movl(cnt2, stride);
7449     movl(result, stride);
7450     negptr(result);
7451     if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7452       movdqu(vec1, Address(str1, result, scale));
7453       pcmpestri(vec1, Address(str2, result, scale), pcmpmask);
7454     } else {
7455       pmovzxbw(vec1, Address(str1, result, scale1));
7456       pcmpestri(vec1, Address(str2, result, scale2), pcmpmask);
7457     }
7458     jccb(Assembler::aboveEqual, LENGTH_DIFF_LABEL);
7459 
7460     // Mismatched characters in the vectors
7461     bind(VECTOR_NOT_EQUAL);
7462     addptr(cnt1, result);
7463     load_next_elements(result, cnt2, str1, str2, scale, scale1, scale2, cnt1, ae);
7464     subl(result, cnt2);
7465     jmpb(POP_LABEL);
7466 
7467     bind(COMPARE_TAIL); // limit is zero
7468     movl(cnt2, result);
7469     // Fallthru to tail compare
7470   }
7471   // Shift str2 and str1 to the end of the arrays, negate min
7472   if (ae == StrIntrinsicNode::LL || ae == StrIntrinsicNode::UU) {
7473     lea(str1, Address(str1, cnt2, scale));
7474     lea(str2, Address(str2, cnt2, scale));
7475   } else {
7476     lea(str1, Address(str1, cnt2, scale1));
7477     lea(str2, Address(str2, cnt2, scale2));
7478   }
7479   decrementl(cnt2);  // first character was compared already
7480   negptr(cnt2);
7481 
7482   // Compare the rest of the elements
7483   bind(WHILE_HEAD_LABEL);
7484   load_next_elements(result, cnt1, str1, str2, scale, scale1, scale2, cnt2, ae);
7485   subl(result, cnt1);
7486   jccb(Assembler::notZero, POP_LABEL);
7487   increment(cnt2);
7488   jccb(Assembler::notZero, WHILE_HEAD_LABEL);
7489 
7490   // Strings are equal up to min length.  Return the length difference.
7491   bind(LENGTH_DIFF_LABEL);
7492   pop(result);
7493   if (ae == StrIntrinsicNode::UU) {
7494     // Divide diff by 2 to get number of chars
7495     sarl(result, 1);
7496   }
7497   jmpb(DONE_LABEL);
7498 
7499 #ifdef _LP64
7500   if (VM_Version::supports_avx512vlbw()) {
7501 
7502     bind(COMPARE_WIDE_VECTORS_LOOP_FAILED);
7503 
7504     kmovql(cnt1, k7);
7505     notq(cnt1);
7506     bsfq(cnt2, cnt1);
7507     if (ae != StrIntrinsicNode::LL) {
7508       // Divide diff by 2 to get number of chars
7509       sarl(cnt2, 1);
7510     }
7511     addq(result, cnt2);
7512     if (ae == StrIntrinsicNode::LL) {
7513       load_unsigned_byte(cnt1, Address(str2, result));
7514       load_unsigned_byte(result, Address(str1, result));
7515     } else if (ae == StrIntrinsicNode::UU) {
7516       load_unsigned_short(cnt1, Address(str2, result, scale));
7517       load_unsigned_short(result, Address(str1, result, scale));
7518     } else {
7519       load_unsigned_short(cnt1, Address(str2, result, scale2));
7520       load_unsigned_byte(result, Address(str1, result, scale1));
7521     }
7522     subl(result, cnt1);
7523     jmpb(POP_LABEL);
7524   }//if (VM_Version::supports_avx512vlbw())
7525 #endif // _LP64
7526 
7527   // Discard the stored length difference
7528   bind(POP_LABEL);
7529   pop(cnt1);
7530 
7531   // That's it
7532   bind(DONE_LABEL);
7533   if(ae == StrIntrinsicNode::UL) {
7534     negl(result);
7535   }
7536 
7537 }
7538 
7539 // Search for Non-ASCII character (Negative byte value) in a byte array,
7540 // return true if it has any and false otherwise.
7541 //   ..\jdk\src\java.base\share\classes\java\lang\StringCoding.java
7542 //   @HotSpotIntrinsicCandidate
7543 //   private static boolean hasNegatives(byte[] ba, int off, int len) {
7544 //     for (int i = off; i < off + len; i++) {
7545 //       if (ba[i] < 0) {
7546 //         return true;
7547 //       }
7548 //     }
7549 //     return false;
7550 //   }
7551 void MacroAssembler::has_negatives(Register ary1, Register len,
7552   Register result, Register tmp1,
7553   XMMRegister vec1, XMMRegister vec2) {
7554   // rsi: byte array
7555   // rcx: len
7556   // rax: result
7557   ShortBranchVerifier sbv(this);
7558   assert_different_registers(ary1, len, result, tmp1);
7559   assert_different_registers(vec1, vec2);
7560   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_CHAR, COMPARE_VECTORS, COMPARE_BYTE;
7561 
7562   // len == 0
7563   testl(len, len);
7564   jcc(Assembler::zero, FALSE_LABEL);
7565 
7566   if ((UseAVX > 2) && // AVX512
7567     VM_Version::supports_avx512vlbw() &&
7568     VM_Version::supports_bmi2()) {
7569 
7570     set_vector_masking();  // opening of the stub context for programming mask registers
7571 
7572     Label test_64_loop, test_tail;
7573     Register tmp3_aliased = len;
7574 
7575     movl(tmp1, len);
7576     vpxor(vec2, vec2, vec2, Assembler::AVX_512bit);
7577 
7578     andl(tmp1, 64 - 1);   // tail count (in chars) 0x3F
7579     andl(len, ~(64 - 1));    // vector count (in chars)
7580     jccb(Assembler::zero, test_tail);
7581 
7582     lea(ary1, Address(ary1, len, Address::times_1));
7583     negptr(len);
7584 
7585     bind(test_64_loop);
7586     // Check whether our 64 elements of size byte contain negatives
7587     evpcmpgtb(k2, vec2, Address(ary1, len, Address::times_1), Assembler::AVX_512bit);
7588     kortestql(k2, k2);
7589     jcc(Assembler::notZero, TRUE_LABEL);
7590 
7591     addptr(len, 64);
7592     jccb(Assembler::notZero, test_64_loop);
7593 
7594 
7595     bind(test_tail);
7596     // bail out when there is nothing to be done
7597     testl(tmp1, -1);
7598     jcc(Assembler::zero, FALSE_LABEL);
7599 
7600     // Save k1
7601     kmovql(k3, k1);
7602 
7603     // ~(~0 << len) applied up to two times (for 32-bit scenario)
7604 #ifdef _LP64
7605     mov64(tmp3_aliased, 0xFFFFFFFFFFFFFFFF);
7606     shlxq(tmp3_aliased, tmp3_aliased, tmp1);
7607     notq(tmp3_aliased);
7608     kmovql(k1, tmp3_aliased);
7609 #else
7610     Label k_init;
7611     jmp(k_init);
7612 
7613     // We could not read 64-bits from a general purpose register thus we move
7614     // data required to compose 64 1's to the instruction stream
7615     // We emit 64 byte wide series of elements from 0..63 which later on would
7616     // be used as a compare targets with tail count contained in tmp1 register.
7617     // Result would be a k1 register having tmp1 consecutive number or 1
7618     // counting from least significant bit.
7619     address tmp = pc();
7620     emit_int64(0x0706050403020100);
7621     emit_int64(0x0F0E0D0C0B0A0908);
7622     emit_int64(0x1716151413121110);
7623     emit_int64(0x1F1E1D1C1B1A1918);
7624     emit_int64(0x2726252423222120);
7625     emit_int64(0x2F2E2D2C2B2A2928);
7626     emit_int64(0x3736353433323130);
7627     emit_int64(0x3F3E3D3C3B3A3938);
7628 
7629     bind(k_init);
7630     lea(len, InternalAddress(tmp));
7631     // create mask to test for negative byte inside a vector
7632     evpbroadcastb(vec1, tmp1, Assembler::AVX_512bit);
7633     evpcmpgtb(k1, vec1, Address(len, 0), Assembler::AVX_512bit);
7634 
7635 #endif
7636     evpcmpgtb(k2, k1, vec2, Address(ary1, 0), Assembler::AVX_512bit);
7637     ktestq(k2, k1);
7638     // Restore k1
7639     kmovql(k1, k3);
7640     jcc(Assembler::notZero, TRUE_LABEL);
7641 
7642     jmp(FALSE_LABEL);
7643 
7644     clear_vector_masking();   // closing of the stub context for programming mask registers
7645   } else {
7646     movl(result, len); // copy
7647 
7648     if (UseAVX == 2 && UseSSE >= 2) {
7649       // With AVX2, use 32-byte vector compare
7650       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7651 
7652       // Compare 32-byte vectors
7653       andl(result, 0x0000001f);  //   tail count (in bytes)
7654       andl(len, 0xffffffe0);   // vector count (in bytes)
7655       jccb(Assembler::zero, COMPARE_TAIL);
7656 
7657       lea(ary1, Address(ary1, len, Address::times_1));
7658       negptr(len);
7659 
7660       movl(tmp1, 0x80808080);   // create mask to test for Unicode chars in vector
7661       movdl(vec2, tmp1);
7662       vpbroadcastd(vec2, vec2);
7663 
7664       bind(COMPARE_WIDE_VECTORS);
7665       vmovdqu(vec1, Address(ary1, len, Address::times_1));
7666       vptest(vec1, vec2);
7667       jccb(Assembler::notZero, TRUE_LABEL);
7668       addptr(len, 32);
7669       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7670 
7671       testl(result, result);
7672       jccb(Assembler::zero, FALSE_LABEL);
7673 
7674       vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
7675       vptest(vec1, vec2);
7676       jccb(Assembler::notZero, TRUE_LABEL);
7677       jmpb(FALSE_LABEL);
7678 
7679       bind(COMPARE_TAIL); // len is zero
7680       movl(len, result);
7681       // Fallthru to tail compare
7682     } else if (UseSSE42Intrinsics) {
7683       // With SSE4.2, use double quad vector compare
7684       Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7685 
7686       // Compare 16-byte vectors
7687       andl(result, 0x0000000f);  //   tail count (in bytes)
7688       andl(len, 0xfffffff0);   // vector count (in bytes)
7689       jcc(Assembler::zero, COMPARE_TAIL);
7690 
7691       lea(ary1, Address(ary1, len, Address::times_1));
7692       negptr(len);
7693 
7694       movl(tmp1, 0x80808080);
7695       movdl(vec2, tmp1);
7696       pshufd(vec2, vec2, 0);
7697 
7698       bind(COMPARE_WIDE_VECTORS);
7699       movdqu(vec1, Address(ary1, len, Address::times_1));
7700       ptest(vec1, vec2);
7701       jcc(Assembler::notZero, TRUE_LABEL);
7702       addptr(len, 16);
7703       jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7704 
7705       testl(result, result);
7706       jcc(Assembler::zero, FALSE_LABEL);
7707 
7708       movdqu(vec1, Address(ary1, result, Address::times_1, -16));
7709       ptest(vec1, vec2);
7710       jccb(Assembler::notZero, TRUE_LABEL);
7711       jmpb(FALSE_LABEL);
7712 
7713       bind(COMPARE_TAIL); // len is zero
7714       movl(len, result);
7715       // Fallthru to tail compare
7716     }
7717   }
7718   // Compare 4-byte vectors
7719   andl(len, 0xfffffffc); // vector count (in bytes)
7720   jccb(Assembler::zero, COMPARE_CHAR);
7721 
7722   lea(ary1, Address(ary1, len, Address::times_1));
7723   negptr(len);
7724 
7725   bind(COMPARE_VECTORS);
7726   movl(tmp1, Address(ary1, len, Address::times_1));
7727   andl(tmp1, 0x80808080);
7728   jccb(Assembler::notZero, TRUE_LABEL);
7729   addptr(len, 4);
7730   jcc(Assembler::notZero, COMPARE_VECTORS);
7731 
7732   // Compare trailing char (final 2 bytes), if any
7733   bind(COMPARE_CHAR);
7734   testl(result, 0x2);   // tail  char
7735   jccb(Assembler::zero, COMPARE_BYTE);
7736   load_unsigned_short(tmp1, Address(ary1, 0));
7737   andl(tmp1, 0x00008080);
7738   jccb(Assembler::notZero, TRUE_LABEL);
7739   subptr(result, 2);
7740   lea(ary1, Address(ary1, 2));
7741 
7742   bind(COMPARE_BYTE);
7743   testl(result, 0x1);   // tail  byte
7744   jccb(Assembler::zero, FALSE_LABEL);
7745   load_unsigned_byte(tmp1, Address(ary1, 0));
7746   andl(tmp1, 0x00000080);
7747   jccb(Assembler::notEqual, TRUE_LABEL);
7748   jmpb(FALSE_LABEL);
7749 
7750   bind(TRUE_LABEL);
7751   movl(result, 1);   // return true
7752   jmpb(DONE);
7753 
7754   bind(FALSE_LABEL);
7755   xorl(result, result); // return false
7756 
7757   // That's it
7758   bind(DONE);
7759   if (UseAVX >= 2 && UseSSE >= 2) {
7760     // clean upper bits of YMM registers
7761     vpxor(vec1, vec1);
7762     vpxor(vec2, vec2);
7763   }
7764 }
7765 // Compare char[] or byte[] arrays aligned to 4 bytes or substrings.
7766 void MacroAssembler::arrays_equals(bool is_array_equ, Register ary1, Register ary2,
7767                                    Register limit, Register result, Register chr,
7768                                    XMMRegister vec1, XMMRegister vec2, bool is_char) {
7769   ShortBranchVerifier sbv(this);
7770   Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR, COMPARE_BYTE;
7771 
7772   int length_offset  = arrayOopDesc::length_offset_in_bytes();
7773   int base_offset    = arrayOopDesc::base_offset_in_bytes(is_char ? T_CHAR : T_BYTE);
7774 
7775   if (is_array_equ) {
7776     // Check the input args
7777     cmpoop(ary1, ary2);
7778     jcc(Assembler::equal, TRUE_LABEL);
7779 
7780     // Need additional checks for arrays_equals.
7781     testptr(ary1, ary1);
7782     jcc(Assembler::zero, FALSE_LABEL);
7783     testptr(ary2, ary2);
7784     jcc(Assembler::zero, FALSE_LABEL);
7785 
7786     // Check the lengths
7787     movl(limit, Address(ary1, length_offset));
7788     cmpl(limit, Address(ary2, length_offset));
7789     jcc(Assembler::notEqual, FALSE_LABEL);
7790   }
7791 
7792   // count == 0
7793   testl(limit, limit);
7794   jcc(Assembler::zero, TRUE_LABEL);
7795 
7796   if (is_array_equ) {
7797     // Load array address
7798     lea(ary1, Address(ary1, base_offset));
7799     lea(ary2, Address(ary2, base_offset));
7800   }
7801 
7802   if (is_array_equ && is_char) {
7803     // arrays_equals when used for char[].
7804     shll(limit, 1);      // byte count != 0
7805   }
7806   movl(result, limit); // copy
7807 
7808   if (UseAVX >= 2) {
7809     // With AVX2, use 32-byte vector compare
7810     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7811 
7812     // Compare 32-byte vectors
7813     andl(result, 0x0000001f);  //   tail count (in bytes)
7814     andl(limit, 0xffffffe0);   // vector count (in bytes)
7815     jcc(Assembler::zero, COMPARE_TAIL);
7816 
7817     lea(ary1, Address(ary1, limit, Address::times_1));
7818     lea(ary2, Address(ary2, limit, Address::times_1));
7819     negptr(limit);
7820 
7821     bind(COMPARE_WIDE_VECTORS);
7822 
7823 #ifdef _LP64
7824     if (VM_Version::supports_avx512vlbw()) { // trying 64 bytes fast loop
7825       Label COMPARE_WIDE_VECTORS_LOOP_AVX2, COMPARE_WIDE_VECTORS_LOOP_AVX3;
7826 
7827       cmpl(limit, -64);
7828       jccb(Assembler::greater, COMPARE_WIDE_VECTORS_LOOP_AVX2);
7829 
7830       bind(COMPARE_WIDE_VECTORS_LOOP_AVX3); // the hottest loop
7831 
7832       evmovdquq(vec1, Address(ary1, limit, Address::times_1), Assembler::AVX_512bit);
7833       evpcmpeqb(k7, vec1, Address(ary2, limit, Address::times_1), Assembler::AVX_512bit);
7834       kortestql(k7, k7);
7835       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
7836       addptr(limit, 64);  // update since we already compared at this addr
7837       cmpl(limit, -64);
7838       jccb(Assembler::lessEqual, COMPARE_WIDE_VECTORS_LOOP_AVX3);
7839 
7840       // At this point we may still need to compare -limit+result bytes.
7841       // We could execute the next two instruction and just continue via non-wide path:
7842       //  cmpl(limit, 0);
7843       //  jcc(Assembler::equal, COMPARE_TAIL);  // true
7844       // But since we stopped at the points ary{1,2}+limit which are
7845       // not farther than 64 bytes from the ends of arrays ary{1,2}+result
7846       // (|limit| <= 32 and result < 32),
7847       // we may just compare the last 64 bytes.
7848       //
7849       addptr(result, -64);   // it is safe, bc we just came from this area
7850       evmovdquq(vec1, Address(ary1, result, Address::times_1), Assembler::AVX_512bit);
7851       evpcmpeqb(k7, vec1, Address(ary2, result, Address::times_1), Assembler::AVX_512bit);
7852       kortestql(k7, k7);
7853       jcc(Assembler::aboveEqual, FALSE_LABEL);     // miscompare
7854 
7855       jmp(TRUE_LABEL);
7856 
7857       bind(COMPARE_WIDE_VECTORS_LOOP_AVX2);
7858 
7859     }//if (VM_Version::supports_avx512vlbw())
7860 #endif //_LP64
7861 
7862     vmovdqu(vec1, Address(ary1, limit, Address::times_1));
7863     vmovdqu(vec2, Address(ary2, limit, Address::times_1));
7864     vpxor(vec1, vec2);
7865 
7866     vptest(vec1, vec1);
7867     jcc(Assembler::notZero, FALSE_LABEL);
7868     addptr(limit, 32);
7869     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7870 
7871     testl(result, result);
7872     jcc(Assembler::zero, TRUE_LABEL);
7873 
7874     vmovdqu(vec1, Address(ary1, result, Address::times_1, -32));
7875     vmovdqu(vec2, Address(ary2, result, Address::times_1, -32));
7876     vpxor(vec1, vec2);
7877 
7878     vptest(vec1, vec1);
7879     jccb(Assembler::notZero, FALSE_LABEL);
7880     jmpb(TRUE_LABEL);
7881 
7882     bind(COMPARE_TAIL); // limit is zero
7883     movl(limit, result);
7884     // Fallthru to tail compare
7885   } else if (UseSSE42Intrinsics) {
7886     // With SSE4.2, use double quad vector compare
7887     Label COMPARE_WIDE_VECTORS, COMPARE_TAIL;
7888 
7889     // Compare 16-byte vectors
7890     andl(result, 0x0000000f);  //   tail count (in bytes)
7891     andl(limit, 0xfffffff0);   // vector count (in bytes)
7892     jcc(Assembler::zero, COMPARE_TAIL);
7893 
7894     lea(ary1, Address(ary1, limit, Address::times_1));
7895     lea(ary2, Address(ary2, limit, Address::times_1));
7896     negptr(limit);
7897 
7898     bind(COMPARE_WIDE_VECTORS);
7899     movdqu(vec1, Address(ary1, limit, Address::times_1));
7900     movdqu(vec2, Address(ary2, limit, Address::times_1));
7901     pxor(vec1, vec2);
7902 
7903     ptest(vec1, vec1);
7904     jcc(Assembler::notZero, FALSE_LABEL);
7905     addptr(limit, 16);
7906     jcc(Assembler::notZero, COMPARE_WIDE_VECTORS);
7907 
7908     testl(result, result);
7909     jcc(Assembler::zero, TRUE_LABEL);
7910 
7911     movdqu(vec1, Address(ary1, result, Address::times_1, -16));
7912     movdqu(vec2, Address(ary2, result, Address::times_1, -16));
7913     pxor(vec1, vec2);
7914 
7915     ptest(vec1, vec1);
7916     jccb(Assembler::notZero, FALSE_LABEL);
7917     jmpb(TRUE_LABEL);
7918 
7919     bind(COMPARE_TAIL); // limit is zero
7920     movl(limit, result);
7921     // Fallthru to tail compare
7922   }
7923 
7924   // Compare 4-byte vectors
7925   andl(limit, 0xfffffffc); // vector count (in bytes)
7926   jccb(Assembler::zero, COMPARE_CHAR);
7927 
7928   lea(ary1, Address(ary1, limit, Address::times_1));
7929   lea(ary2, Address(ary2, limit, Address::times_1));
7930   negptr(limit);
7931 
7932   bind(COMPARE_VECTORS);
7933   movl(chr, Address(ary1, limit, Address::times_1));
7934   cmpl(chr, Address(ary2, limit, Address::times_1));
7935   jccb(Assembler::notEqual, FALSE_LABEL);
7936   addptr(limit, 4);
7937   jcc(Assembler::notZero, COMPARE_VECTORS);
7938 
7939   // Compare trailing char (final 2 bytes), if any
7940   bind(COMPARE_CHAR);
7941   testl(result, 0x2);   // tail  char
7942   jccb(Assembler::zero, COMPARE_BYTE);
7943   load_unsigned_short(chr, Address(ary1, 0));
7944   load_unsigned_short(limit, Address(ary2, 0));
7945   cmpl(chr, limit);
7946   jccb(Assembler::notEqual, FALSE_LABEL);
7947 
7948   if (is_array_equ && is_char) {
7949     bind(COMPARE_BYTE);
7950   } else {
7951     lea(ary1, Address(ary1, 2));
7952     lea(ary2, Address(ary2, 2));
7953 
7954     bind(COMPARE_BYTE);
7955     testl(result, 0x1);   // tail  byte
7956     jccb(Assembler::zero, TRUE_LABEL);
7957     load_unsigned_byte(chr, Address(ary1, 0));
7958     load_unsigned_byte(limit, Address(ary2, 0));
7959     cmpl(chr, limit);
7960     jccb(Assembler::notEqual, FALSE_LABEL);
7961   }
7962   bind(TRUE_LABEL);
7963   movl(result, 1);   // return true
7964   jmpb(DONE);
7965 
7966   bind(FALSE_LABEL);
7967   xorl(result, result); // return false
7968 
7969   // That's it
7970   bind(DONE);
7971   if (UseAVX >= 2) {
7972     // clean upper bits of YMM registers
7973     vpxor(vec1, vec1);
7974     vpxor(vec2, vec2);
7975   }
7976 }
7977 
7978 #endif
7979 
7980 void MacroAssembler::generate_fill(BasicType t, bool aligned,
7981                                    Register to, Register value, Register count,
7982                                    Register rtmp, XMMRegister xtmp) {
7983   ShortBranchVerifier sbv(this);
7984   assert_different_registers(to, value, count, rtmp);
7985   Label L_exit;
7986   Label L_fill_2_bytes, L_fill_4_bytes;
7987 
7988   int shift = -1;
7989   switch (t) {
7990     case T_BYTE:
7991       shift = 2;
7992       break;
7993     case T_SHORT:
7994       shift = 1;
7995       break;
7996     case T_INT:
7997       shift = 0;
7998       break;
7999     default: ShouldNotReachHere();
8000   }
8001 
8002   if (t == T_BYTE) {
8003     andl(value, 0xff);
8004     movl(rtmp, value);
8005     shll(rtmp, 8);
8006     orl(value, rtmp);
8007   }
8008   if (t == T_SHORT) {
8009     andl(value, 0xffff);
8010   }
8011   if (t == T_BYTE || t == T_SHORT) {
8012     movl(rtmp, value);
8013     shll(rtmp, 16);
8014     orl(value, rtmp);
8015   }
8016 
8017   cmpl(count, 2<<shift); // Short arrays (< 8 bytes) fill by element
8018   jcc(Assembler::below, L_fill_4_bytes); // use unsigned cmp
8019   if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
8020     Label L_skip_align2;
8021     // align source address at 4 bytes address boundary
8022     if (t == T_BYTE) {
8023       Label L_skip_align1;
8024       // One byte misalignment happens only for byte arrays
8025       testptr(to, 1);
8026       jccb(Assembler::zero, L_skip_align1);
8027       movb(Address(to, 0), value);
8028       increment(to);
8029       decrement(count);
8030       BIND(L_skip_align1);
8031     }
8032     // Two bytes misalignment happens only for byte and short (char) arrays
8033     testptr(to, 2);
8034     jccb(Assembler::zero, L_skip_align2);
8035     movw(Address(to, 0), value);
8036     addptr(to, 2);
8037     subl(count, 1<<(shift-1));
8038     BIND(L_skip_align2);
8039   }
8040   if (UseSSE < 2) {
8041     Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8042     // Fill 32-byte chunks
8043     subl(count, 8 << shift);
8044     jcc(Assembler::less, L_check_fill_8_bytes);
8045     align(16);
8046 
8047     BIND(L_fill_32_bytes_loop);
8048 
8049     for (int i = 0; i < 32; i += 4) {
8050       movl(Address(to, i), value);
8051     }
8052 
8053     addptr(to, 32);
8054     subl(count, 8 << shift);
8055     jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8056     BIND(L_check_fill_8_bytes);
8057     addl(count, 8 << shift);
8058     jccb(Assembler::zero, L_exit);
8059     jmpb(L_fill_8_bytes);
8060 
8061     //
8062     // length is too short, just fill qwords
8063     //
8064     BIND(L_fill_8_bytes_loop);
8065     movl(Address(to, 0), value);
8066     movl(Address(to, 4), value);
8067     addptr(to, 8);
8068     BIND(L_fill_8_bytes);
8069     subl(count, 1 << (shift + 1));
8070     jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8071     // fall through to fill 4 bytes
8072   } else {
8073     Label L_fill_32_bytes;
8074     if (!UseUnalignedLoadStores) {
8075       // align to 8 bytes, we know we are 4 byte aligned to start
8076       testptr(to, 4);
8077       jccb(Assembler::zero, L_fill_32_bytes);
8078       movl(Address(to, 0), value);
8079       addptr(to, 4);
8080       subl(count, 1<<shift);
8081     }
8082     BIND(L_fill_32_bytes);
8083     {
8084       assert( UseSSE >= 2, "supported cpu only" );
8085       Label L_fill_32_bytes_loop, L_check_fill_8_bytes, L_fill_8_bytes_loop, L_fill_8_bytes;
8086       if (UseAVX > 2) {
8087         movl(rtmp, 0xffff);
8088         kmovwl(k1, rtmp);
8089       }
8090       movdl(xtmp, value);
8091       if (UseAVX > 2 && UseUnalignedLoadStores) {
8092         // Fill 64-byte chunks
8093         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8094         evpbroadcastd(xtmp, xtmp, Assembler::AVX_512bit);
8095 
8096         subl(count, 16 << shift);
8097         jcc(Assembler::less, L_check_fill_32_bytes);
8098         align(16);
8099 
8100         BIND(L_fill_64_bytes_loop);
8101         evmovdqul(Address(to, 0), xtmp, Assembler::AVX_512bit);
8102         addptr(to, 64);
8103         subl(count, 16 << shift);
8104         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8105 
8106         BIND(L_check_fill_32_bytes);
8107         addl(count, 8 << shift);
8108         jccb(Assembler::less, L_check_fill_8_bytes);
8109         vmovdqu(Address(to, 0), xtmp);
8110         addptr(to, 32);
8111         subl(count, 8 << shift);
8112 
8113         BIND(L_check_fill_8_bytes);
8114       } else if (UseAVX == 2 && UseUnalignedLoadStores) {
8115         // Fill 64-byte chunks
8116         Label L_fill_64_bytes_loop, L_check_fill_32_bytes;
8117         vpbroadcastd(xtmp, xtmp);
8118 
8119         subl(count, 16 << shift);
8120         jcc(Assembler::less, L_check_fill_32_bytes);
8121         align(16);
8122 
8123         BIND(L_fill_64_bytes_loop);
8124         vmovdqu(Address(to, 0), xtmp);
8125         vmovdqu(Address(to, 32), xtmp);
8126         addptr(to, 64);
8127         subl(count, 16 << shift);
8128         jcc(Assembler::greaterEqual, L_fill_64_bytes_loop);
8129 
8130         BIND(L_check_fill_32_bytes);
8131         addl(count, 8 << shift);
8132         jccb(Assembler::less, L_check_fill_8_bytes);
8133         vmovdqu(Address(to, 0), xtmp);
8134         addptr(to, 32);
8135         subl(count, 8 << shift);
8136 
8137         BIND(L_check_fill_8_bytes);
8138         // clean upper bits of YMM registers
8139         movdl(xtmp, value);
8140         pshufd(xtmp, xtmp, 0);
8141       } else {
8142         // Fill 32-byte chunks
8143         pshufd(xtmp, xtmp, 0);
8144 
8145         subl(count, 8 << shift);
8146         jcc(Assembler::less, L_check_fill_8_bytes);
8147         align(16);
8148 
8149         BIND(L_fill_32_bytes_loop);
8150 
8151         if (UseUnalignedLoadStores) {
8152           movdqu(Address(to, 0), xtmp);
8153           movdqu(Address(to, 16), xtmp);
8154         } else {
8155           movq(Address(to, 0), xtmp);
8156           movq(Address(to, 8), xtmp);
8157           movq(Address(to, 16), xtmp);
8158           movq(Address(to, 24), xtmp);
8159         }
8160 
8161         addptr(to, 32);
8162         subl(count, 8 << shift);
8163         jcc(Assembler::greaterEqual, L_fill_32_bytes_loop);
8164 
8165         BIND(L_check_fill_8_bytes);
8166       }
8167       addl(count, 8 << shift);
8168       jccb(Assembler::zero, L_exit);
8169       jmpb(L_fill_8_bytes);
8170 
8171       //
8172       // length is too short, just fill qwords
8173       //
8174       BIND(L_fill_8_bytes_loop);
8175       movq(Address(to, 0), xtmp);
8176       addptr(to, 8);
8177       BIND(L_fill_8_bytes);
8178       subl(count, 1 << (shift + 1));
8179       jcc(Assembler::greaterEqual, L_fill_8_bytes_loop);
8180     }
8181   }
8182   // fill trailing 4 bytes
8183   BIND(L_fill_4_bytes);
8184   testl(count, 1<<shift);
8185   jccb(Assembler::zero, L_fill_2_bytes);
8186   movl(Address(to, 0), value);
8187   if (t == T_BYTE || t == T_SHORT) {
8188     Label L_fill_byte;
8189     addptr(to, 4);
8190     BIND(L_fill_2_bytes);
8191     // fill trailing 2 bytes
8192     testl(count, 1<<(shift-1));
8193     jccb(Assembler::zero, L_fill_byte);
8194     movw(Address(to, 0), value);
8195     if (t == T_BYTE) {
8196       addptr(to, 2);
8197       BIND(L_fill_byte);
8198       // fill trailing byte
8199       testl(count, 1);
8200       jccb(Assembler::zero, L_exit);
8201       movb(Address(to, 0), value);
8202     } else {
8203       BIND(L_fill_byte);
8204     }
8205   } else {
8206     BIND(L_fill_2_bytes);
8207   }
8208   BIND(L_exit);
8209 }
8210 
8211 // encode char[] to byte[] in ISO_8859_1
8212    //@HotSpotIntrinsicCandidate
8213    //private static int implEncodeISOArray(byte[] sa, int sp,
8214    //byte[] da, int dp, int len) {
8215    //  int i = 0;
8216    //  for (; i < len; i++) {
8217    //    char c = StringUTF16.getChar(sa, sp++);
8218    //    if (c > '\u00FF')
8219    //      break;
8220    //    da[dp++] = (byte)c;
8221    //  }
8222    //  return i;
8223    //}
8224 void MacroAssembler::encode_iso_array(Register src, Register dst, Register len,
8225   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
8226   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
8227   Register tmp5, Register result) {
8228 
8229   // rsi: src
8230   // rdi: dst
8231   // rdx: len
8232   // rcx: tmp5
8233   // rax: result
8234   ShortBranchVerifier sbv(this);
8235   assert_different_registers(src, dst, len, tmp5, result);
8236   Label L_done, L_copy_1_char, L_copy_1_char_exit;
8237 
8238   // set result
8239   xorl(result, result);
8240   // check for zero length
8241   testl(len, len);
8242   jcc(Assembler::zero, L_done);
8243 
8244   movl(result, len);
8245 
8246   // Setup pointers
8247   lea(src, Address(src, len, Address::times_2)); // char[]
8248   lea(dst, Address(dst, len, Address::times_1)); // byte[]
8249   negptr(len);
8250 
8251   if (UseSSE42Intrinsics || UseAVX >= 2) {
8252     Label L_copy_8_chars, L_copy_8_chars_exit;
8253     Label L_chars_16_check, L_copy_16_chars, L_copy_16_chars_exit;
8254 
8255     if (UseAVX >= 2) {
8256       Label L_chars_32_check, L_copy_32_chars, L_copy_32_chars_exit;
8257       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8258       movdl(tmp1Reg, tmp5);
8259       vpbroadcastd(tmp1Reg, tmp1Reg);
8260       jmp(L_chars_32_check);
8261 
8262       bind(L_copy_32_chars);
8263       vmovdqu(tmp3Reg, Address(src, len, Address::times_2, -64));
8264       vmovdqu(tmp4Reg, Address(src, len, Address::times_2, -32));
8265       vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8266       vptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8267       jccb(Assembler::notZero, L_copy_32_chars_exit);
8268       vpackuswb(tmp3Reg, tmp3Reg, tmp4Reg, /* vector_len */ 1);
8269       vpermq(tmp4Reg, tmp3Reg, 0xD8, /* vector_len */ 1);
8270       vmovdqu(Address(dst, len, Address::times_1, -32), tmp4Reg);
8271 
8272       bind(L_chars_32_check);
8273       addptr(len, 32);
8274       jcc(Assembler::lessEqual, L_copy_32_chars);
8275 
8276       bind(L_copy_32_chars_exit);
8277       subptr(len, 16);
8278       jccb(Assembler::greater, L_copy_16_chars_exit);
8279 
8280     } else if (UseSSE42Intrinsics) {
8281       movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vector
8282       movdl(tmp1Reg, tmp5);
8283       pshufd(tmp1Reg, tmp1Reg, 0);
8284       jmpb(L_chars_16_check);
8285     }
8286 
8287     bind(L_copy_16_chars);
8288     if (UseAVX >= 2) {
8289       vmovdqu(tmp2Reg, Address(src, len, Address::times_2, -32));
8290       vptest(tmp2Reg, tmp1Reg);
8291       jcc(Assembler::notZero, L_copy_16_chars_exit);
8292       vpackuswb(tmp2Reg, tmp2Reg, tmp1Reg, /* vector_len */ 1);
8293       vpermq(tmp3Reg, tmp2Reg, 0xD8, /* vector_len */ 1);
8294     } else {
8295       if (UseAVX > 0) {
8296         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8297         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8298         vpor(tmp2Reg, tmp3Reg, tmp4Reg, /* vector_len */ 0);
8299       } else {
8300         movdqu(tmp3Reg, Address(src, len, Address::times_2, -32));
8301         por(tmp2Reg, tmp3Reg);
8302         movdqu(tmp4Reg, Address(src, len, Address::times_2, -16));
8303         por(tmp2Reg, tmp4Reg);
8304       }
8305       ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in  vector
8306       jccb(Assembler::notZero, L_copy_16_chars_exit);
8307       packuswb(tmp3Reg, tmp4Reg);
8308     }
8309     movdqu(Address(dst, len, Address::times_1, -16), tmp3Reg);
8310 
8311     bind(L_chars_16_check);
8312     addptr(len, 16);
8313     jcc(Assembler::lessEqual, L_copy_16_chars);
8314 
8315     bind(L_copy_16_chars_exit);
8316     if (UseAVX >= 2) {
8317       // clean upper bits of YMM registers
8318       vpxor(tmp2Reg, tmp2Reg);
8319       vpxor(tmp3Reg, tmp3Reg);
8320       vpxor(tmp4Reg, tmp4Reg);
8321       movdl(tmp1Reg, tmp5);
8322       pshufd(tmp1Reg, tmp1Reg, 0);
8323     }
8324     subptr(len, 8);
8325     jccb(Assembler::greater, L_copy_8_chars_exit);
8326 
8327     bind(L_copy_8_chars);
8328     movdqu(tmp3Reg, Address(src, len, Address::times_2, -16));
8329     ptest(tmp3Reg, tmp1Reg);
8330     jccb(Assembler::notZero, L_copy_8_chars_exit);
8331     packuswb(tmp3Reg, tmp1Reg);
8332     movq(Address(dst, len, Address::times_1, -8), tmp3Reg);
8333     addptr(len, 8);
8334     jccb(Assembler::lessEqual, L_copy_8_chars);
8335 
8336     bind(L_copy_8_chars_exit);
8337     subptr(len, 8);
8338     jccb(Assembler::zero, L_done);
8339   }
8340 
8341   bind(L_copy_1_char);
8342   load_unsigned_short(tmp5, Address(src, len, Address::times_2, 0));
8343   testl(tmp5, 0xff00);      // check if Unicode char
8344   jccb(Assembler::notZero, L_copy_1_char_exit);
8345   movb(Address(dst, len, Address::times_1, 0), tmp5);
8346   addptr(len, 1);
8347   jccb(Assembler::less, L_copy_1_char);
8348 
8349   bind(L_copy_1_char_exit);
8350   addptr(result, len); // len is negative count of not processed elements
8351 
8352   bind(L_done);
8353 }
8354 
8355 #ifdef _LP64
8356 /**
8357  * Helper for multiply_to_len().
8358  */
8359 void MacroAssembler::add2_with_carry(Register dest_hi, Register dest_lo, Register src1, Register src2) {
8360   addq(dest_lo, src1);
8361   adcq(dest_hi, 0);
8362   addq(dest_lo, src2);
8363   adcq(dest_hi, 0);
8364 }
8365 
8366 /**
8367  * Multiply 64 bit by 64 bit first loop.
8368  */
8369 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
8370                                            Register y, Register y_idx, Register z,
8371                                            Register carry, Register product,
8372                                            Register idx, Register kdx) {
8373   //
8374   //  jlong carry, x[], y[], z[];
8375   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
8376   //    huge_128 product = y[idx] * x[xstart] + carry;
8377   //    z[kdx] = (jlong)product;
8378   //    carry  = (jlong)(product >>> 64);
8379   //  }
8380   //  z[xstart] = carry;
8381   //
8382 
8383   Label L_first_loop, L_first_loop_exit;
8384   Label L_one_x, L_one_y, L_multiply;
8385 
8386   decrementl(xstart);
8387   jcc(Assembler::negative, L_one_x);
8388 
8389   movq(x_xstart, Address(x, xstart, Address::times_4,  0));
8390   rorq(x_xstart, 32); // convert big-endian to little-endian
8391 
8392   bind(L_first_loop);
8393   decrementl(idx);
8394   jcc(Assembler::negative, L_first_loop_exit);
8395   decrementl(idx);
8396   jcc(Assembler::negative, L_one_y);
8397   movq(y_idx, Address(y, idx, Address::times_4,  0));
8398   rorq(y_idx, 32); // convert big-endian to little-endian
8399   bind(L_multiply);
8400   movq(product, x_xstart);
8401   mulq(y_idx); // product(rax) * y_idx -> rdx:rax
8402   addq(product, carry);
8403   adcq(rdx, 0);
8404   subl(kdx, 2);
8405   movl(Address(z, kdx, Address::times_4,  4), product);
8406   shrq(product, 32);
8407   movl(Address(z, kdx, Address::times_4,  0), product);
8408   movq(carry, rdx);
8409   jmp(L_first_loop);
8410 
8411   bind(L_one_y);
8412   movl(y_idx, Address(y,  0));
8413   jmp(L_multiply);
8414 
8415   bind(L_one_x);
8416   movl(x_xstart, Address(x,  0));
8417   jmp(L_first_loop);
8418 
8419   bind(L_first_loop_exit);
8420 }
8421 
8422 /**
8423  * Multiply 64 bit by 64 bit and add 128 bit.
8424  */
8425 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y, Register z,
8426                                             Register yz_idx, Register idx,
8427                                             Register carry, Register product, int offset) {
8428   //     huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
8429   //     z[kdx] = (jlong)product;
8430 
8431   movq(yz_idx, Address(y, idx, Address::times_4,  offset));
8432   rorq(yz_idx, 32); // convert big-endian to little-endian
8433   movq(product, x_xstart);
8434   mulq(yz_idx);     // product(rax) * yz_idx -> rdx:product(rax)
8435   movq(yz_idx, Address(z, idx, Address::times_4,  offset));
8436   rorq(yz_idx, 32); // convert big-endian to little-endian
8437 
8438   add2_with_carry(rdx, product, carry, yz_idx);
8439 
8440   movl(Address(z, idx, Address::times_4,  offset+4), product);
8441   shrq(product, 32);
8442   movl(Address(z, idx, Address::times_4,  offset), product);
8443 
8444 }
8445 
8446 /**
8447  * Multiply 128 bit by 128 bit. Unrolled inner loop.
8448  */
8449 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart, Register y, Register z,
8450                                              Register yz_idx, Register idx, Register jdx,
8451                                              Register carry, Register product,
8452                                              Register carry2) {
8453   //   jlong carry, x[], y[], z[];
8454   //   int kdx = ystart+1;
8455   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
8456   //     huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
8457   //     z[kdx+idx+1] = (jlong)product;
8458   //     jlong carry2  = (jlong)(product >>> 64);
8459   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
8460   //     z[kdx+idx] = (jlong)product;
8461   //     carry  = (jlong)(product >>> 64);
8462   //   }
8463   //   idx += 2;
8464   //   if (idx > 0) {
8465   //     product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
8466   //     z[kdx+idx] = (jlong)product;
8467   //     carry  = (jlong)(product >>> 64);
8468   //   }
8469   //
8470 
8471   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
8472 
8473   movl(jdx, idx);
8474   andl(jdx, 0xFFFFFFFC);
8475   shrl(jdx, 2);
8476 
8477   bind(L_third_loop);
8478   subl(jdx, 1);
8479   jcc(Assembler::negative, L_third_loop_exit);
8480   subl(idx, 4);
8481 
8482   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 8);
8483   movq(carry2, rdx);
8484 
8485   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product, 0);
8486   movq(carry, rdx);
8487   jmp(L_third_loop);
8488 
8489   bind (L_third_loop_exit);
8490 
8491   andl (idx, 0x3);
8492   jcc(Assembler::zero, L_post_third_loop_done);
8493 
8494   Label L_check_1;
8495   subl(idx, 2);
8496   jcc(Assembler::negative, L_check_1);
8497 
8498   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product, 0);
8499   movq(carry, rdx);
8500 
8501   bind (L_check_1);
8502   addl (idx, 0x2);
8503   andl (idx, 0x1);
8504   subl(idx, 1);
8505   jcc(Assembler::negative, L_post_third_loop_done);
8506 
8507   movl(yz_idx, Address(y, idx, Address::times_4,  0));
8508   movq(product, x_xstart);
8509   mulq(yz_idx); // product(rax) * yz_idx -> rdx:product(rax)
8510   movl(yz_idx, Address(z, idx, Address::times_4,  0));
8511 
8512   add2_with_carry(rdx, product, yz_idx, carry);
8513 
8514   movl(Address(z, idx, Address::times_4,  0), product);
8515   shrq(product, 32);
8516 
8517   shlq(rdx, 32);
8518   orq(product, rdx);
8519   movq(carry, product);
8520 
8521   bind(L_post_third_loop_done);
8522 }
8523 
8524 /**
8525  * Multiply 128 bit by 128 bit using BMI2. Unrolled inner loop.
8526  *
8527  */
8528 void MacroAssembler::multiply_128_x_128_bmi2_loop(Register y, Register z,
8529                                                   Register carry, Register carry2,
8530                                                   Register idx, Register jdx,
8531                                                   Register yz_idx1, Register yz_idx2,
8532                                                   Register tmp, Register tmp3, Register tmp4) {
8533   assert(UseBMI2Instructions, "should be used only when BMI2 is available");
8534 
8535   //   jlong carry, x[], y[], z[];
8536   //   int kdx = ystart+1;
8537   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
8538   //     huge_128 tmp3 = (y[idx+1] * rdx) + z[kdx+idx+1] + carry;
8539   //     jlong carry2  = (jlong)(tmp3 >>> 64);
8540   //     huge_128 tmp4 = (y[idx]   * rdx) + z[kdx+idx] + carry2;
8541   //     carry  = (jlong)(tmp4 >>> 64);
8542   //     z[kdx+idx+1] = (jlong)tmp3;
8543   //     z[kdx+idx] = (jlong)tmp4;
8544   //   }
8545   //   idx += 2;
8546   //   if (idx > 0) {
8547   //     yz_idx1 = (y[idx] * rdx) + z[kdx+idx] + carry;
8548   //     z[kdx+idx] = (jlong)yz_idx1;
8549   //     carry  = (jlong)(yz_idx1 >>> 64);
8550   //   }
8551   //
8552 
8553   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
8554 
8555   movl(jdx, idx);
8556   andl(jdx, 0xFFFFFFFC);
8557   shrl(jdx, 2);
8558 
8559   bind(L_third_loop);
8560   subl(jdx, 1);
8561   jcc(Assembler::negative, L_third_loop_exit);
8562   subl(idx, 4);
8563 
8564   movq(yz_idx1,  Address(y, idx, Address::times_4,  8));
8565   rorxq(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
8566   movq(yz_idx2, Address(y, idx, Address::times_4,  0));
8567   rorxq(yz_idx2, yz_idx2, 32);
8568 
8569   mulxq(tmp4, tmp3, yz_idx1);  //  yz_idx1 * rdx -> tmp4:tmp3
8570   mulxq(carry2, tmp, yz_idx2); //  yz_idx2 * rdx -> carry2:tmp
8571 
8572   movq(yz_idx1,  Address(z, idx, Address::times_4,  8));
8573   rorxq(yz_idx1, yz_idx1, 32);
8574   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
8575   rorxq(yz_idx2, yz_idx2, 32);
8576 
8577   if (VM_Version::supports_adx()) {
8578     adcxq(tmp3, carry);
8579     adoxq(tmp3, yz_idx1);
8580 
8581     adcxq(tmp4, tmp);
8582     adoxq(tmp4, yz_idx2);
8583 
8584     movl(carry, 0); // does not affect flags
8585     adcxq(carry2, carry);
8586     adoxq(carry2, carry);
8587   } else {
8588     add2_with_carry(tmp4, tmp3, carry, yz_idx1);
8589     add2_with_carry(carry2, tmp4, tmp, yz_idx2);
8590   }
8591   movq(carry, carry2);
8592 
8593   movl(Address(z, idx, Address::times_4, 12), tmp3);
8594   shrq(tmp3, 32);
8595   movl(Address(z, idx, Address::times_4,  8), tmp3);
8596 
8597   movl(Address(z, idx, Address::times_4,  4), tmp4);
8598   shrq(tmp4, 32);
8599   movl(Address(z, idx, Address::times_4,  0), tmp4);
8600 
8601   jmp(L_third_loop);
8602 
8603   bind (L_third_loop_exit);
8604 
8605   andl (idx, 0x3);
8606   jcc(Assembler::zero, L_post_third_loop_done);
8607 
8608   Label L_check_1;
8609   subl(idx, 2);
8610   jcc(Assembler::negative, L_check_1);
8611 
8612   movq(yz_idx1, Address(y, idx, Address::times_4,  0));
8613   rorxq(yz_idx1, yz_idx1, 32);
8614   mulxq(tmp4, tmp3, yz_idx1); //  yz_idx1 * rdx -> tmp4:tmp3
8615   movq(yz_idx2, Address(z, idx, Address::times_4,  0));
8616   rorxq(yz_idx2, yz_idx2, 32);
8617 
8618   add2_with_carry(tmp4, tmp3, carry, yz_idx2);
8619 
8620   movl(Address(z, idx, Address::times_4,  4), tmp3);
8621   shrq(tmp3, 32);
8622   movl(Address(z, idx, Address::times_4,  0), tmp3);
8623   movq(carry, tmp4);
8624 
8625   bind (L_check_1);
8626   addl (idx, 0x2);
8627   andl (idx, 0x1);
8628   subl(idx, 1);
8629   jcc(Assembler::negative, L_post_third_loop_done);
8630   movl(tmp4, Address(y, idx, Address::times_4,  0));
8631   mulxq(carry2, tmp3, tmp4);  //  tmp4 * rdx -> carry2:tmp3
8632   movl(tmp4, Address(z, idx, Address::times_4,  0));
8633 
8634   add2_with_carry(carry2, tmp3, tmp4, carry);
8635 
8636   movl(Address(z, idx, Address::times_4,  0), tmp3);
8637   shrq(tmp3, 32);
8638 
8639   shlq(carry2, 32);
8640   orq(tmp3, carry2);
8641   movq(carry, tmp3);
8642 
8643   bind(L_post_third_loop_done);
8644 }
8645 
8646 /**
8647  * Code for BigInteger::multiplyToLen() instrinsic.
8648  *
8649  * rdi: x
8650  * rax: xlen
8651  * rsi: y
8652  * rcx: ylen
8653  * r8:  z
8654  * r11: zlen
8655  * r12: tmp1
8656  * r13: tmp2
8657  * r14: tmp3
8658  * r15: tmp4
8659  * rbx: tmp5
8660  *
8661  */
8662 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, Register z, Register zlen,
8663                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5) {
8664   ShortBranchVerifier sbv(this);
8665   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx);
8666 
8667   push(tmp1);
8668   push(tmp2);
8669   push(tmp3);
8670   push(tmp4);
8671   push(tmp5);
8672 
8673   push(xlen);
8674   push(zlen);
8675 
8676   const Register idx = tmp1;
8677   const Register kdx = tmp2;
8678   const Register xstart = tmp3;
8679 
8680   const Register y_idx = tmp4;
8681   const Register carry = tmp5;
8682   const Register product  = xlen;
8683   const Register x_xstart = zlen;  // reuse register
8684 
8685   // First Loop.
8686   //
8687   //  final static long LONG_MASK = 0xffffffffL;
8688   //  int xstart = xlen - 1;
8689   //  int ystart = ylen - 1;
8690   //  long carry = 0;
8691   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
8692   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
8693   //    z[kdx] = (int)product;
8694   //    carry = product >>> 32;
8695   //  }
8696   //  z[xstart] = (int)carry;
8697   //
8698 
8699   movl(idx, ylen);      // idx = ylen;
8700   movl(kdx, zlen);      // kdx = xlen+ylen;
8701   xorq(carry, carry);   // carry = 0;
8702 
8703   Label L_done;
8704 
8705   movl(xstart, xlen);
8706   decrementl(xstart);
8707   jcc(Assembler::negative, L_done);
8708 
8709   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
8710 
8711   Label L_second_loop;
8712   testl(kdx, kdx);
8713   jcc(Assembler::zero, L_second_loop);
8714 
8715   Label L_carry;
8716   subl(kdx, 1);
8717   jcc(Assembler::zero, L_carry);
8718 
8719   movl(Address(z, kdx, Address::times_4,  0), carry);
8720   shrq(carry, 32);
8721   subl(kdx, 1);
8722 
8723   bind(L_carry);
8724   movl(Address(z, kdx, Address::times_4,  0), carry);
8725 
8726   // Second and third (nested) loops.
8727   //
8728   // for (int i = xstart-1; i >= 0; i--) { // Second loop
8729   //   carry = 0;
8730   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
8731   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
8732   //                    (z[k] & LONG_MASK) + carry;
8733   //     z[k] = (int)product;
8734   //     carry = product >>> 32;
8735   //   }
8736   //   z[i] = (int)carry;
8737   // }
8738   //
8739   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
8740 
8741   const Register jdx = tmp1;
8742 
8743   bind(L_second_loop);
8744   xorl(carry, carry);    // carry = 0;
8745   movl(jdx, ylen);       // j = ystart+1
8746 
8747   subl(xstart, 1);       // i = xstart-1;
8748   jcc(Assembler::negative, L_done);
8749 
8750   push (z);
8751 
8752   Label L_last_x;
8753   lea(z, Address(z, xstart, Address::times_4, 4)); // z = z + k - j
8754   subl(xstart, 1);       // i = xstart-1;
8755   jcc(Assembler::negative, L_last_x);
8756 
8757   if (UseBMI2Instructions) {
8758     movq(rdx,  Address(x, xstart, Address::times_4,  0));
8759     rorxq(rdx, rdx, 32); // convert big-endian to little-endian
8760   } else {
8761     movq(x_xstart, Address(x, xstart, Address::times_4,  0));
8762     rorq(x_xstart, 32);  // convert big-endian to little-endian
8763   }
8764 
8765   Label L_third_loop_prologue;
8766   bind(L_third_loop_prologue);
8767 
8768   push (x);
8769   push (xstart);
8770   push (ylen);
8771 
8772 
8773   if (UseBMI2Instructions) {
8774     multiply_128_x_128_bmi2_loop(y, z, carry, x, jdx, ylen, product, tmp2, x_xstart, tmp3, tmp4);
8775   } else { // !UseBMI2Instructions
8776     multiply_128_x_128_loop(x_xstart, y, z, y_idx, jdx, ylen, carry, product, x);
8777   }
8778 
8779   pop(ylen);
8780   pop(xlen);
8781   pop(x);
8782   pop(z);
8783 
8784   movl(tmp3, xlen);
8785   addl(tmp3, 1);
8786   movl(Address(z, tmp3, Address::times_4,  0), carry);
8787   subl(tmp3, 1);
8788   jccb(Assembler::negative, L_done);
8789 
8790   shrq(carry, 32);
8791   movl(Address(z, tmp3, Address::times_4,  0), carry);
8792   jmp(L_second_loop);
8793 
8794   // Next infrequent code is moved outside loops.
8795   bind(L_last_x);
8796   if (UseBMI2Instructions) {
8797     movl(rdx, Address(x,  0));
8798   } else {
8799     movl(x_xstart, Address(x,  0));
8800   }
8801   jmp(L_third_loop_prologue);
8802 
8803   bind(L_done);
8804 
8805   pop(zlen);
8806   pop(xlen);
8807 
8808   pop(tmp5);
8809   pop(tmp4);
8810   pop(tmp3);
8811   pop(tmp2);
8812   pop(tmp1);
8813 }
8814 
8815 void MacroAssembler::vectorized_mismatch(Register obja, Register objb, Register length, Register log2_array_indxscale,
8816   Register result, Register tmp1, Register tmp2, XMMRegister rymm0, XMMRegister rymm1, XMMRegister rymm2){
8817   assert(UseSSE42Intrinsics, "SSE4.2 must be enabled.");
8818   Label VECTOR16_LOOP, VECTOR8_LOOP, VECTOR4_LOOP;
8819   Label VECTOR8_TAIL, VECTOR4_TAIL;
8820   Label VECTOR32_NOT_EQUAL, VECTOR16_NOT_EQUAL, VECTOR8_NOT_EQUAL, VECTOR4_NOT_EQUAL;
8821   Label SAME_TILL_END, DONE;
8822   Label BYTES_LOOP, BYTES_TAIL, BYTES_NOT_EQUAL;
8823 
8824   //scale is in rcx in both Win64 and Unix
8825   ShortBranchVerifier sbv(this);
8826 
8827   shlq(length);
8828   xorq(result, result);
8829 
8830   if ((UseAVX > 2) &&
8831       VM_Version::supports_avx512vlbw()) {
8832     Label VECTOR64_LOOP, VECTOR64_NOT_EQUAL, VECTOR32_TAIL;
8833 
8834     set_vector_masking();  // opening of the stub context for programming mask registers
8835     cmpq(length, 64);
8836     jcc(Assembler::less, VECTOR32_TAIL);
8837     movq(tmp1, length);
8838     andq(tmp1, 0x3F);      // tail count
8839     andq(length, ~(0x3F)); //vector count
8840 
8841     bind(VECTOR64_LOOP);
8842     // AVX512 code to compare 64 byte vectors.
8843     evmovdqub(rymm0, Address(obja, result), Assembler::AVX_512bit);
8844     evpcmpeqb(k7, rymm0, Address(objb, result), Assembler::AVX_512bit);
8845     kortestql(k7, k7);
8846     jcc(Assembler::aboveEqual, VECTOR64_NOT_EQUAL);     // mismatch
8847     addq(result, 64);
8848     subq(length, 64);
8849     jccb(Assembler::notZero, VECTOR64_LOOP);
8850 
8851     //bind(VECTOR64_TAIL);
8852     testq(tmp1, tmp1);
8853     jcc(Assembler::zero, SAME_TILL_END);
8854 
8855     //bind(VECTOR64_TAIL);
8856     // AVX512 code to compare upto 63 byte vectors.
8857     // Save k1
8858     kmovql(k3, k1);
8859     mov64(tmp2, 0xFFFFFFFFFFFFFFFF);
8860     shlxq(tmp2, tmp2, tmp1);
8861     notq(tmp2);
8862     kmovql(k1, tmp2);
8863 
8864     evmovdqub(rymm0, k1, Address(obja, result), Assembler::AVX_512bit);
8865     evpcmpeqb(k7, k1, rymm0, Address(objb, result), Assembler::AVX_512bit);
8866 
8867     ktestql(k7, k1);
8868     // Restore k1
8869     kmovql(k1, k3);
8870     jcc(Assembler::below, SAME_TILL_END);     // not mismatch
8871 
8872     bind(VECTOR64_NOT_EQUAL);
8873     kmovql(tmp1, k7);
8874     notq(tmp1);
8875     tzcntq(tmp1, tmp1);
8876     addq(result, tmp1);
8877     shrq(result);
8878     jmp(DONE);
8879     bind(VECTOR32_TAIL);
8880     clear_vector_masking();   // closing of the stub context for programming mask registers
8881   }
8882 
8883   cmpq(length, 8);
8884   jcc(Assembler::equal, VECTOR8_LOOP);
8885   jcc(Assembler::less, VECTOR4_TAIL);
8886 
8887   if (UseAVX >= 2) {
8888     Label VECTOR16_TAIL, VECTOR32_LOOP;
8889 
8890     cmpq(length, 16);
8891     jcc(Assembler::equal, VECTOR16_LOOP);
8892     jcc(Assembler::less, VECTOR8_LOOP);
8893 
8894     cmpq(length, 32);
8895     jccb(Assembler::less, VECTOR16_TAIL);
8896 
8897     subq(length, 32);
8898     bind(VECTOR32_LOOP);
8899     vmovdqu(rymm0, Address(obja, result));
8900     vmovdqu(rymm1, Address(objb, result));
8901     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_256bit);
8902     vptest(rymm2, rymm2);
8903     jcc(Assembler::notZero, VECTOR32_NOT_EQUAL);//mismatch found
8904     addq(result, 32);
8905     subq(length, 32);
8906     jcc(Assembler::greaterEqual, VECTOR32_LOOP);
8907     addq(length, 32);
8908     jcc(Assembler::equal, SAME_TILL_END);
8909     //falling through if less than 32 bytes left //close the branch here.
8910 
8911     bind(VECTOR16_TAIL);
8912     cmpq(length, 16);
8913     jccb(Assembler::less, VECTOR8_TAIL);
8914     bind(VECTOR16_LOOP);
8915     movdqu(rymm0, Address(obja, result));
8916     movdqu(rymm1, Address(objb, result));
8917     vpxor(rymm2, rymm0, rymm1, Assembler::AVX_128bit);
8918     ptest(rymm2, rymm2);
8919     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
8920     addq(result, 16);
8921     subq(length, 16);
8922     jcc(Assembler::equal, SAME_TILL_END);
8923     //falling through if less than 16 bytes left
8924   } else {//regular intrinsics
8925 
8926     cmpq(length, 16);
8927     jccb(Assembler::less, VECTOR8_TAIL);
8928 
8929     subq(length, 16);
8930     bind(VECTOR16_LOOP);
8931     movdqu(rymm0, Address(obja, result));
8932     movdqu(rymm1, Address(objb, result));
8933     pxor(rymm0, rymm1);
8934     ptest(rymm0, rymm0);
8935     jcc(Assembler::notZero, VECTOR16_NOT_EQUAL);//mismatch found
8936     addq(result, 16);
8937     subq(length, 16);
8938     jccb(Assembler::greaterEqual, VECTOR16_LOOP);
8939     addq(length, 16);
8940     jcc(Assembler::equal, SAME_TILL_END);
8941     //falling through if less than 16 bytes left
8942   }
8943 
8944   bind(VECTOR8_TAIL);
8945   cmpq(length, 8);
8946   jccb(Assembler::less, VECTOR4_TAIL);
8947   bind(VECTOR8_LOOP);
8948   movq(tmp1, Address(obja, result));
8949   movq(tmp2, Address(objb, result));
8950   xorq(tmp1, tmp2);
8951   testq(tmp1, tmp1);
8952   jcc(Assembler::notZero, VECTOR8_NOT_EQUAL);//mismatch found
8953   addq(result, 8);
8954   subq(length, 8);
8955   jcc(Assembler::equal, SAME_TILL_END);
8956   //falling through if less than 8 bytes left
8957 
8958   bind(VECTOR4_TAIL);
8959   cmpq(length, 4);
8960   jccb(Assembler::less, BYTES_TAIL);
8961   bind(VECTOR4_LOOP);
8962   movl(tmp1, Address(obja, result));
8963   xorl(tmp1, Address(objb, result));
8964   testl(tmp1, tmp1);
8965   jcc(Assembler::notZero, VECTOR4_NOT_EQUAL);//mismatch found
8966   addq(result, 4);
8967   subq(length, 4);
8968   jcc(Assembler::equal, SAME_TILL_END);
8969   //falling through if less than 4 bytes left
8970 
8971   bind(BYTES_TAIL);
8972   bind(BYTES_LOOP);
8973   load_unsigned_byte(tmp1, Address(obja, result));
8974   load_unsigned_byte(tmp2, Address(objb, result));
8975   xorl(tmp1, tmp2);
8976   testl(tmp1, tmp1);
8977   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
8978   decq(length);
8979   jcc(Assembler::zero, SAME_TILL_END);
8980   incq(result);
8981   load_unsigned_byte(tmp1, Address(obja, result));
8982   load_unsigned_byte(tmp2, Address(objb, result));
8983   xorl(tmp1, tmp2);
8984   testl(tmp1, tmp1);
8985   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
8986   decq(length);
8987   jcc(Assembler::zero, SAME_TILL_END);
8988   incq(result);
8989   load_unsigned_byte(tmp1, Address(obja, result));
8990   load_unsigned_byte(tmp2, Address(objb, result));
8991   xorl(tmp1, tmp2);
8992   testl(tmp1, tmp1);
8993   jcc(Assembler::notZero, BYTES_NOT_EQUAL);//mismatch found
8994   jmp(SAME_TILL_END);
8995 
8996   if (UseAVX >= 2) {
8997     bind(VECTOR32_NOT_EQUAL);
8998     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_256bit);
8999     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_256bit);
9000     vpxor(rymm0, rymm0, rymm2, Assembler::AVX_256bit);
9001     vpmovmskb(tmp1, rymm0);
9002     bsfq(tmp1, tmp1);
9003     addq(result, tmp1);
9004     shrq(result);
9005     jmp(DONE);
9006   }
9007 
9008   bind(VECTOR16_NOT_EQUAL);
9009   if (UseAVX >= 2) {
9010     vpcmpeqb(rymm2, rymm2, rymm2, Assembler::AVX_128bit);
9011     vpcmpeqb(rymm0, rymm0, rymm1, Assembler::AVX_128bit);
9012     pxor(rymm0, rymm2);
9013   } else {
9014     pcmpeqb(rymm2, rymm2);
9015     pxor(rymm0, rymm1);
9016     pcmpeqb(rymm0, rymm1);
9017     pxor(rymm0, rymm2);
9018   }
9019   pmovmskb(tmp1, rymm0);
9020   bsfq(tmp1, tmp1);
9021   addq(result, tmp1);
9022   shrq(result);
9023   jmpb(DONE);
9024 
9025   bind(VECTOR8_NOT_EQUAL);
9026   bind(VECTOR4_NOT_EQUAL);
9027   bsfq(tmp1, tmp1);
9028   shrq(tmp1, 3);
9029   addq(result, tmp1);
9030   bind(BYTES_NOT_EQUAL);
9031   shrq(result);
9032   jmpb(DONE);
9033 
9034   bind(SAME_TILL_END);
9035   mov64(result, -1);
9036 
9037   bind(DONE);
9038 }
9039 
9040 //Helper functions for square_to_len()
9041 
9042 /**
9043  * Store the squares of x[], right shifted one bit (divided by 2) into z[]
9044  * Preserves x and z and modifies rest of the registers.
9045  */
9046 void MacroAssembler::square_rshift(Register x, Register xlen, Register z, Register tmp1, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9047   // Perform square and right shift by 1
9048   // Handle odd xlen case first, then for even xlen do the following
9049   // jlong carry = 0;
9050   // for (int j=0, i=0; j < xlen; j+=2, i+=4) {
9051   //     huge_128 product = x[j:j+1] * x[j:j+1];
9052   //     z[i:i+1] = (carry << 63) | (jlong)(product >>> 65);
9053   //     z[i+2:i+3] = (jlong)(product >>> 1);
9054   //     carry = (jlong)product;
9055   // }
9056 
9057   xorq(tmp5, tmp5);     // carry
9058   xorq(rdxReg, rdxReg);
9059   xorl(tmp1, tmp1);     // index for x
9060   xorl(tmp4, tmp4);     // index for z
9061 
9062   Label L_first_loop, L_first_loop_exit;
9063 
9064   testl(xlen, 1);
9065   jccb(Assembler::zero, L_first_loop); //jump if xlen is even
9066 
9067   // Square and right shift by 1 the odd element using 32 bit multiply
9068   movl(raxReg, Address(x, tmp1, Address::times_4, 0));
9069   imulq(raxReg, raxReg);
9070   shrq(raxReg, 1);
9071   adcq(tmp5, 0);
9072   movq(Address(z, tmp4, Address::times_4, 0), raxReg);
9073   incrementl(tmp1);
9074   addl(tmp4, 2);
9075 
9076   // Square and  right shift by 1 the rest using 64 bit multiply
9077   bind(L_first_loop);
9078   cmpptr(tmp1, xlen);
9079   jccb(Assembler::equal, L_first_loop_exit);
9080 
9081   // Square
9082   movq(raxReg, Address(x, tmp1, Address::times_4,  0));
9083   rorq(raxReg, 32);    // convert big-endian to little-endian
9084   mulq(raxReg);        // 64-bit multiply rax * rax -> rdx:rax
9085 
9086   // Right shift by 1 and save carry
9087   shrq(tmp5, 1);       // rdx:rax:tmp5 = (tmp5:rdx:rax) >>> 1
9088   rcrq(rdxReg, 1);
9089   rcrq(raxReg, 1);
9090   adcq(tmp5, 0);
9091 
9092   // Store result in z
9093   movq(Address(z, tmp4, Address::times_4, 0), rdxReg);
9094   movq(Address(z, tmp4, Address::times_4, 8), raxReg);
9095 
9096   // Update indices for x and z
9097   addl(tmp1, 2);
9098   addl(tmp4, 4);
9099   jmp(L_first_loop);
9100 
9101   bind(L_first_loop_exit);
9102 }
9103 
9104 
9105 /**
9106  * Perform the following multiply add operation using BMI2 instructions
9107  * carry:sum = sum + op1*op2 + carry
9108  * op2 should be in rdx
9109  * op2 is preserved, all other registers are modified
9110  */
9111 void MacroAssembler::multiply_add_64_bmi2(Register sum, Register op1, Register op2, Register carry, Register tmp2) {
9112   // assert op2 is rdx
9113   mulxq(tmp2, op1, op1);  //  op1 * op2 -> tmp2:op1
9114   addq(sum, carry);
9115   adcq(tmp2, 0);
9116   addq(sum, op1);
9117   adcq(tmp2, 0);
9118   movq(carry, tmp2);
9119 }
9120 
9121 /**
9122  * Perform the following multiply add operation:
9123  * carry:sum = sum + op1*op2 + carry
9124  * Preserves op1, op2 and modifies rest of registers
9125  */
9126 void MacroAssembler::multiply_add_64(Register sum, Register op1, Register op2, Register carry, Register rdxReg, Register raxReg) {
9127   // rdx:rax = op1 * op2
9128   movq(raxReg, op2);
9129   mulq(op1);
9130 
9131   //  rdx:rax = sum + carry + rdx:rax
9132   addq(sum, carry);
9133   adcq(rdxReg, 0);
9134   addq(sum, raxReg);
9135   adcq(rdxReg, 0);
9136 
9137   // carry:sum = rdx:sum
9138   movq(carry, rdxReg);
9139 }
9140 
9141 /**
9142  * Add 64 bit long carry into z[] with carry propogation.
9143  * Preserves z and carry register values and modifies rest of registers.
9144  *
9145  */
9146 void MacroAssembler::add_one_64(Register z, Register zlen, Register carry, Register tmp1) {
9147   Label L_fourth_loop, L_fourth_loop_exit;
9148 
9149   movl(tmp1, 1);
9150   subl(zlen, 2);
9151   addq(Address(z, zlen, Address::times_4, 0), carry);
9152 
9153   bind(L_fourth_loop);
9154   jccb(Assembler::carryClear, L_fourth_loop_exit);
9155   subl(zlen, 2);
9156   jccb(Assembler::negative, L_fourth_loop_exit);
9157   addq(Address(z, zlen, Address::times_4, 0), tmp1);
9158   jmp(L_fourth_loop);
9159   bind(L_fourth_loop_exit);
9160 }
9161 
9162 /**
9163  * Shift z[] left by 1 bit.
9164  * Preserves x, len, z and zlen registers and modifies rest of the registers.
9165  *
9166  */
9167 void MacroAssembler::lshift_by_1(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4) {
9168 
9169   Label L_fifth_loop, L_fifth_loop_exit;
9170 
9171   // Fifth loop
9172   // Perform primitiveLeftShift(z, zlen, 1)
9173 
9174   const Register prev_carry = tmp1;
9175   const Register new_carry = tmp4;
9176   const Register value = tmp2;
9177   const Register zidx = tmp3;
9178 
9179   // int zidx, carry;
9180   // long value;
9181   // carry = 0;
9182   // for (zidx = zlen-2; zidx >=0; zidx -= 2) {
9183   //    (carry:value)  = (z[i] << 1) | carry ;
9184   //    z[i] = value;
9185   // }
9186 
9187   movl(zidx, zlen);
9188   xorl(prev_carry, prev_carry); // clear carry flag and prev_carry register
9189 
9190   bind(L_fifth_loop);
9191   decl(zidx);  // Use decl to preserve carry flag
9192   decl(zidx);
9193   jccb(Assembler::negative, L_fifth_loop_exit);
9194 
9195   if (UseBMI2Instructions) {
9196      movq(value, Address(z, zidx, Address::times_4, 0));
9197      rclq(value, 1);
9198      rorxq(value, value, 32);
9199      movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9200   }
9201   else {
9202     // clear new_carry
9203     xorl(new_carry, new_carry);
9204 
9205     // Shift z[i] by 1, or in previous carry and save new carry
9206     movq(value, Address(z, zidx, Address::times_4, 0));
9207     shlq(value, 1);
9208     adcl(new_carry, 0);
9209 
9210     orq(value, prev_carry);
9211     rorq(value, 0x20);
9212     movq(Address(z, zidx, Address::times_4,  0), value);  // Store back in big endian form
9213 
9214     // Set previous carry = new carry
9215     movl(prev_carry, new_carry);
9216   }
9217   jmp(L_fifth_loop);
9218 
9219   bind(L_fifth_loop_exit);
9220 }
9221 
9222 
9223 /**
9224  * Code for BigInteger::squareToLen() intrinsic
9225  *
9226  * rdi: x
9227  * rsi: len
9228  * r8:  z
9229  * rcx: zlen
9230  * r12: tmp1
9231  * r13: tmp2
9232  * r14: tmp3
9233  * r15: tmp4
9234  * rbx: tmp5
9235  *
9236  */
9237 void MacroAssembler::square_to_len(Register x, Register len, Register z, Register zlen, Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9238 
9239   Label L_second_loop, L_second_loop_exit, L_third_loop, L_third_loop_exit, L_last_x, L_multiply;
9240   push(tmp1);
9241   push(tmp2);
9242   push(tmp3);
9243   push(tmp4);
9244   push(tmp5);
9245 
9246   // First loop
9247   // Store the squares, right shifted one bit (i.e., divided by 2).
9248   square_rshift(x, len, z, tmp1, tmp3, tmp4, tmp5, rdxReg, raxReg);
9249 
9250   // Add in off-diagonal sums.
9251   //
9252   // Second, third (nested) and fourth loops.
9253   // zlen +=2;
9254   // for (int xidx=len-2,zidx=zlen-4; xidx > 0; xidx-=2,zidx-=4) {
9255   //    carry = 0;
9256   //    long op2 = x[xidx:xidx+1];
9257   //    for (int j=xidx-2,k=zidx; j >= 0; j-=2) {
9258   //       k -= 2;
9259   //       long op1 = x[j:j+1];
9260   //       long sum = z[k:k+1];
9261   //       carry:sum = multiply_add_64(sum, op1, op2, carry, tmp_regs);
9262   //       z[k:k+1] = sum;
9263   //    }
9264   //    add_one_64(z, k, carry, tmp_regs);
9265   // }
9266 
9267   const Register carry = tmp5;
9268   const Register sum = tmp3;
9269   const Register op1 = tmp4;
9270   Register op2 = tmp2;
9271 
9272   push(zlen);
9273   push(len);
9274   addl(zlen,2);
9275   bind(L_second_loop);
9276   xorq(carry, carry);
9277   subl(zlen, 4);
9278   subl(len, 2);
9279   push(zlen);
9280   push(len);
9281   cmpl(len, 0);
9282   jccb(Assembler::lessEqual, L_second_loop_exit);
9283 
9284   // Multiply an array by one 64 bit long.
9285   if (UseBMI2Instructions) {
9286     op2 = rdxReg;
9287     movq(op2, Address(x, len, Address::times_4,  0));
9288     rorxq(op2, op2, 32);
9289   }
9290   else {
9291     movq(op2, Address(x, len, Address::times_4,  0));
9292     rorq(op2, 32);
9293   }
9294 
9295   bind(L_third_loop);
9296   decrementl(len);
9297   jccb(Assembler::negative, L_third_loop_exit);
9298   decrementl(len);
9299   jccb(Assembler::negative, L_last_x);
9300 
9301   movq(op1, Address(x, len, Address::times_4,  0));
9302   rorq(op1, 32);
9303 
9304   bind(L_multiply);
9305   subl(zlen, 2);
9306   movq(sum, Address(z, zlen, Address::times_4,  0));
9307 
9308   // Multiply 64 bit by 64 bit and add 64 bits lower half and upper 64 bits as carry.
9309   if (UseBMI2Instructions) {
9310     multiply_add_64_bmi2(sum, op1, op2, carry, tmp2);
9311   }
9312   else {
9313     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9314   }
9315 
9316   movq(Address(z, zlen, Address::times_4, 0), sum);
9317 
9318   jmp(L_third_loop);
9319   bind(L_third_loop_exit);
9320 
9321   // Fourth loop
9322   // Add 64 bit long carry into z with carry propogation.
9323   // Uses offsetted zlen.
9324   add_one_64(z, zlen, carry, tmp1);
9325 
9326   pop(len);
9327   pop(zlen);
9328   jmp(L_second_loop);
9329 
9330   // Next infrequent code is moved outside loops.
9331   bind(L_last_x);
9332   movl(op1, Address(x, 0));
9333   jmp(L_multiply);
9334 
9335   bind(L_second_loop_exit);
9336   pop(len);
9337   pop(zlen);
9338   pop(len);
9339   pop(zlen);
9340 
9341   // Fifth loop
9342   // Shift z left 1 bit.
9343   lshift_by_1(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4);
9344 
9345   // z[zlen-1] |= x[len-1] & 1;
9346   movl(tmp3, Address(x, len, Address::times_4, -4));
9347   andl(tmp3, 1);
9348   orl(Address(z, zlen, Address::times_4,  -4), tmp3);
9349 
9350   pop(tmp5);
9351   pop(tmp4);
9352   pop(tmp3);
9353   pop(tmp2);
9354   pop(tmp1);
9355 }
9356 
9357 /**
9358  * Helper function for mul_add()
9359  * Multiply the in[] by int k and add to out[] starting at offset offs using
9360  * 128 bit by 32 bit multiply and return the carry in tmp5.
9361  * Only quad int aligned length of in[] is operated on in this function.
9362  * k is in rdxReg for BMI2Instructions, for others it is in tmp2.
9363  * This function preserves out, in and k registers.
9364  * len and offset point to the appropriate index in "in" & "out" correspondingly
9365  * tmp5 has the carry.
9366  * other registers are temporary and are modified.
9367  *
9368  */
9369 void MacroAssembler::mul_add_128_x_32_loop(Register out, Register in,
9370   Register offset, Register len, Register tmp1, Register tmp2, Register tmp3,
9371   Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9372 
9373   Label L_first_loop, L_first_loop_exit;
9374 
9375   movl(tmp1, len);
9376   shrl(tmp1, 2);
9377 
9378   bind(L_first_loop);
9379   subl(tmp1, 1);
9380   jccb(Assembler::negative, L_first_loop_exit);
9381 
9382   subl(len, 4);
9383   subl(offset, 4);
9384 
9385   Register op2 = tmp2;
9386   const Register sum = tmp3;
9387   const Register op1 = tmp4;
9388   const Register carry = tmp5;
9389 
9390   if (UseBMI2Instructions) {
9391     op2 = rdxReg;
9392   }
9393 
9394   movq(op1, Address(in, len, Address::times_4,  8));
9395   rorq(op1, 32);
9396   movq(sum, Address(out, offset, Address::times_4,  8));
9397   rorq(sum, 32);
9398   if (UseBMI2Instructions) {
9399     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9400   }
9401   else {
9402     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9403   }
9404   // Store back in big endian from little endian
9405   rorq(sum, 0x20);
9406   movq(Address(out, offset, Address::times_4,  8), sum);
9407 
9408   movq(op1, Address(in, len, Address::times_4,  0));
9409   rorq(op1, 32);
9410   movq(sum, Address(out, offset, Address::times_4,  0));
9411   rorq(sum, 32);
9412   if (UseBMI2Instructions) {
9413     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9414   }
9415   else {
9416     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9417   }
9418   // Store back in big endian from little endian
9419   rorq(sum, 0x20);
9420   movq(Address(out, offset, Address::times_4,  0), sum);
9421 
9422   jmp(L_first_loop);
9423   bind(L_first_loop_exit);
9424 }
9425 
9426 /**
9427  * Code for BigInteger::mulAdd() intrinsic
9428  *
9429  * rdi: out
9430  * rsi: in
9431  * r11: offs (out.length - offset)
9432  * rcx: len
9433  * r8:  k
9434  * r12: tmp1
9435  * r13: tmp2
9436  * r14: tmp3
9437  * r15: tmp4
9438  * rbx: tmp5
9439  * Multiply the in[] by word k and add to out[], return the carry in rax
9440  */
9441 void MacroAssembler::mul_add(Register out, Register in, Register offs,
9442    Register len, Register k, Register tmp1, Register tmp2, Register tmp3,
9443    Register tmp4, Register tmp5, Register rdxReg, Register raxReg) {
9444 
9445   Label L_carry, L_last_in, L_done;
9446 
9447 // carry = 0;
9448 // for (int j=len-1; j >= 0; j--) {
9449 //    long product = (in[j] & LONG_MASK) * kLong +
9450 //                   (out[offs] & LONG_MASK) + carry;
9451 //    out[offs--] = (int)product;
9452 //    carry = product >>> 32;
9453 // }
9454 //
9455   push(tmp1);
9456   push(tmp2);
9457   push(tmp3);
9458   push(tmp4);
9459   push(tmp5);
9460 
9461   Register op2 = tmp2;
9462   const Register sum = tmp3;
9463   const Register op1 = tmp4;
9464   const Register carry =  tmp5;
9465 
9466   if (UseBMI2Instructions) {
9467     op2 = rdxReg;
9468     movl(op2, k);
9469   }
9470   else {
9471     movl(op2, k);
9472   }
9473 
9474   xorq(carry, carry);
9475 
9476   //First loop
9477 
9478   //Multiply in[] by k in a 4 way unrolled loop using 128 bit by 32 bit multiply
9479   //The carry is in tmp5
9480   mul_add_128_x_32_loop(out, in, offs, len, tmp1, tmp2, tmp3, tmp4, tmp5, rdxReg, raxReg);
9481 
9482   //Multiply the trailing in[] entry using 64 bit by 32 bit, if any
9483   decrementl(len);
9484   jccb(Assembler::negative, L_carry);
9485   decrementl(len);
9486   jccb(Assembler::negative, L_last_in);
9487 
9488   movq(op1, Address(in, len, Address::times_4,  0));
9489   rorq(op1, 32);
9490 
9491   subl(offs, 2);
9492   movq(sum, Address(out, offs, Address::times_4,  0));
9493   rorq(sum, 32);
9494 
9495   if (UseBMI2Instructions) {
9496     multiply_add_64_bmi2(sum, op1, op2, carry, raxReg);
9497   }
9498   else {
9499     multiply_add_64(sum, op1, op2, carry, rdxReg, raxReg);
9500   }
9501 
9502   // Store back in big endian from little endian
9503   rorq(sum, 0x20);
9504   movq(Address(out, offs, Address::times_4,  0), sum);
9505 
9506   testl(len, len);
9507   jccb(Assembler::zero, L_carry);
9508 
9509   //Multiply the last in[] entry, if any
9510   bind(L_last_in);
9511   movl(op1, Address(in, 0));
9512   movl(sum, Address(out, offs, Address::times_4,  -4));
9513 
9514   movl(raxReg, k);
9515   mull(op1); //tmp4 * eax -> edx:eax
9516   addl(sum, carry);
9517   adcl(rdxReg, 0);
9518   addl(sum, raxReg);
9519   adcl(rdxReg, 0);
9520   movl(carry, rdxReg);
9521 
9522   movl(Address(out, offs, Address::times_4,  -4), sum);
9523 
9524   bind(L_carry);
9525   //return tmp5/carry as carry in rax
9526   movl(rax, carry);
9527 
9528   bind(L_done);
9529   pop(tmp5);
9530   pop(tmp4);
9531   pop(tmp3);
9532   pop(tmp2);
9533   pop(tmp1);
9534 }
9535 #endif
9536 
9537 /**
9538  * Emits code to update CRC-32 with a byte value according to constants in table
9539  *
9540  * @param [in,out]crc   Register containing the crc.
9541  * @param [in]val       Register containing the byte to fold into the CRC.
9542  * @param [in]table     Register containing the table of crc constants.
9543  *
9544  * uint32_t crc;
9545  * val = crc_table[(val ^ crc) & 0xFF];
9546  * crc = val ^ (crc >> 8);
9547  *
9548  */
9549 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
9550   xorl(val, crc);
9551   andl(val, 0xFF);
9552   shrl(crc, 8); // unsigned shift
9553   xorl(crc, Address(table, val, Address::times_4, 0));
9554 }
9555 
9556 /**
9557 * Fold four 128-bit data chunks
9558 */
9559 void MacroAssembler::fold_128bit_crc32_avx512(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
9560   evpclmulhdq(xtmp, xK, xcrc, Assembler::AVX_512bit); // [123:64]
9561   evpclmulldq(xcrc, xK, xcrc, Assembler::AVX_512bit); // [63:0]
9562   evpxorq(xcrc, xcrc, Address(buf, offset), Assembler::AVX_512bit /* vector_len */);
9563   evpxorq(xcrc, xcrc, xtmp, Assembler::AVX_512bit /* vector_len */);
9564 }
9565 
9566 /**
9567  * Fold 128-bit data chunk
9568  */
9569 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, Register buf, int offset) {
9570   if (UseAVX > 0) {
9571     vpclmulhdq(xtmp, xK, xcrc); // [123:64]
9572     vpclmulldq(xcrc, xK, xcrc); // [63:0]
9573     vpxor(xcrc, xcrc, Address(buf, offset), 0 /* vector_len */);
9574     pxor(xcrc, xtmp);
9575   } else {
9576     movdqa(xtmp, xcrc);
9577     pclmulhdq(xtmp, xK);   // [123:64]
9578     pclmulldq(xcrc, xK);   // [63:0]
9579     pxor(xcrc, xtmp);
9580     movdqu(xtmp, Address(buf, offset));
9581     pxor(xcrc, xtmp);
9582   }
9583 }
9584 
9585 void MacroAssembler::fold_128bit_crc32(XMMRegister xcrc, XMMRegister xK, XMMRegister xtmp, XMMRegister xbuf) {
9586   if (UseAVX > 0) {
9587     vpclmulhdq(xtmp, xK, xcrc);
9588     vpclmulldq(xcrc, xK, xcrc);
9589     pxor(xcrc, xbuf);
9590     pxor(xcrc, xtmp);
9591   } else {
9592     movdqa(xtmp, xcrc);
9593     pclmulhdq(xtmp, xK);
9594     pclmulldq(xcrc, xK);
9595     pxor(xcrc, xbuf);
9596     pxor(xcrc, xtmp);
9597   }
9598 }
9599 
9600 /**
9601  * 8-bit folds to compute 32-bit CRC
9602  *
9603  * uint64_t xcrc;
9604  * timesXtoThe32[xcrc & 0xFF] ^ (xcrc >> 8);
9605  */
9606 void MacroAssembler::fold_8bit_crc32(XMMRegister xcrc, Register table, XMMRegister xtmp, Register tmp) {
9607   movdl(tmp, xcrc);
9608   andl(tmp, 0xFF);
9609   movdl(xtmp, Address(table, tmp, Address::times_4, 0));
9610   psrldq(xcrc, 1); // unsigned shift one byte
9611   pxor(xcrc, xtmp);
9612 }
9613 
9614 /**
9615  * uint32_t crc;
9616  * timesXtoThe32[crc & 0xFF] ^ (crc >> 8);
9617  */
9618 void MacroAssembler::fold_8bit_crc32(Register crc, Register table, Register tmp) {
9619   movl(tmp, crc);
9620   andl(tmp, 0xFF);
9621   shrl(crc, 8);
9622   xorl(crc, Address(table, tmp, Address::times_4, 0));
9623 }
9624 
9625 /**
9626  * @param crc   register containing existing CRC (32-bit)
9627  * @param buf   register pointing to input byte buffer (byte*)
9628  * @param len   register containing number of bytes
9629  * @param table register that will contain address of CRC table
9630  * @param tmp   scratch register
9631  */
9632 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, Register table, Register tmp) {
9633   assert_different_registers(crc, buf, len, table, tmp, rax);
9634 
9635   Label L_tail, L_tail_restore, L_tail_loop, L_exit, L_align_loop, L_aligned;
9636   Label L_fold_tail, L_fold_128b, L_fold_512b, L_fold_512b_loop, L_fold_tail_loop;
9637 
9638   // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
9639   // context for the registers used, where all instructions below are using 128-bit mode
9640   // On EVEX without VL and BW, these instructions will all be AVX.
9641   if (VM_Version::supports_avx512vlbw()) {
9642     movl(tmp, 0xffff);
9643     kmovwl(k1, tmp);
9644   }
9645 
9646   lea(table, ExternalAddress(StubRoutines::crc_table_addr()));
9647   notl(crc); // ~crc
9648   cmpl(len, 16);
9649   jcc(Assembler::less, L_tail);
9650 
9651   // Align buffer to 16 bytes
9652   movl(tmp, buf);
9653   andl(tmp, 0xF);
9654   jccb(Assembler::zero, L_aligned);
9655   subl(tmp,  16);
9656   addl(len, tmp);
9657 
9658   align(4);
9659   BIND(L_align_loop);
9660   movsbl(rax, Address(buf, 0)); // load byte with sign extension
9661   update_byte_crc32(crc, rax, table);
9662   increment(buf);
9663   incrementl(tmp);
9664   jccb(Assembler::less, L_align_loop);
9665 
9666   BIND(L_aligned);
9667   movl(tmp, len); // save
9668   shrl(len, 4);
9669   jcc(Assembler::zero, L_tail_restore);
9670 
9671   // Fold total 512 bits of polynomial on each iteration
9672   if (VM_Version::supports_vpclmulqdq()) {
9673     Label Parallel_loop, L_No_Parallel;
9674 
9675     cmpl(len, 8);
9676     jccb(Assembler::less, L_No_Parallel);
9677 
9678     movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
9679     evmovdquq(xmm1, Address(buf, 0), Assembler::AVX_512bit);
9680     movdl(xmm5, crc);
9681     evpxorq(xmm1, xmm1, xmm5, Assembler::AVX_512bit);
9682     addptr(buf, 64);
9683     subl(len, 7);
9684     evshufi64x2(xmm0, xmm0, xmm0, 0x00, Assembler::AVX_512bit); //propagate the mask from 128 bits to 512 bits
9685 
9686     BIND(Parallel_loop);
9687     fold_128bit_crc32_avx512(xmm1, xmm0, xmm5, buf, 0);
9688     addptr(buf, 64);
9689     subl(len, 4);
9690     jcc(Assembler::greater, Parallel_loop);
9691 
9692     vextracti64x2(xmm2, xmm1, 0x01);
9693     vextracti64x2(xmm3, xmm1, 0x02);
9694     vextracti64x2(xmm4, xmm1, 0x03);
9695     jmp(L_fold_512b);
9696 
9697     BIND(L_No_Parallel);
9698   }
9699   // Fold crc into first bytes of vector
9700   movdqa(xmm1, Address(buf, 0));
9701   movdl(rax, xmm1);
9702   xorl(crc, rax);
9703   if (VM_Version::supports_sse4_1()) {
9704     pinsrd(xmm1, crc, 0);
9705   } else {
9706     pinsrw(xmm1, crc, 0);
9707     shrl(crc, 16);
9708     pinsrw(xmm1, crc, 1);
9709   }
9710   addptr(buf, 16);
9711   subl(len, 4); // len > 0
9712   jcc(Assembler::less, L_fold_tail);
9713 
9714   movdqa(xmm2, Address(buf,  0));
9715   movdqa(xmm3, Address(buf, 16));
9716   movdqa(xmm4, Address(buf, 32));
9717   addptr(buf, 48);
9718   subl(len, 3);
9719   jcc(Assembler::lessEqual, L_fold_512b);
9720 
9721   // Fold total 512 bits of polynomial on each iteration,
9722   // 128 bits per each of 4 parallel streams.
9723   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 32));
9724 
9725   align(32);
9726   BIND(L_fold_512b_loop);
9727   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
9728   fold_128bit_crc32(xmm2, xmm0, xmm5, buf, 16);
9729   fold_128bit_crc32(xmm3, xmm0, xmm5, buf, 32);
9730   fold_128bit_crc32(xmm4, xmm0, xmm5, buf, 48);
9731   addptr(buf, 64);
9732   subl(len, 4);
9733   jcc(Assembler::greater, L_fold_512b_loop);
9734 
9735   // Fold 512 bits to 128 bits.
9736   BIND(L_fold_512b);
9737   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
9738   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm2);
9739   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm3);
9740   fold_128bit_crc32(xmm1, xmm0, xmm5, xmm4);
9741 
9742   // Fold the rest of 128 bits data chunks
9743   BIND(L_fold_tail);
9744   addl(len, 3);
9745   jccb(Assembler::lessEqual, L_fold_128b);
9746   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr() + 16));
9747 
9748   BIND(L_fold_tail_loop);
9749   fold_128bit_crc32(xmm1, xmm0, xmm5, buf,  0);
9750   addptr(buf, 16);
9751   decrementl(len);
9752   jccb(Assembler::greater, L_fold_tail_loop);
9753 
9754   // Fold 128 bits in xmm1 down into 32 bits in crc register.
9755   BIND(L_fold_128b);
9756   movdqu(xmm0, ExternalAddress(StubRoutines::x86::crc_by128_masks_addr()));
9757   if (UseAVX > 0) {
9758     vpclmulqdq(xmm2, xmm0, xmm1, 0x1);
9759     vpand(xmm3, xmm0, xmm2, 0 /* vector_len */);
9760     vpclmulqdq(xmm0, xmm0, xmm3, 0x1);
9761   } else {
9762     movdqa(xmm2, xmm0);
9763     pclmulqdq(xmm2, xmm1, 0x1);
9764     movdqa(xmm3, xmm0);
9765     pand(xmm3, xmm2);
9766     pclmulqdq(xmm0, xmm3, 0x1);
9767   }
9768   psrldq(xmm1, 8);
9769   psrldq(xmm2, 4);
9770   pxor(xmm0, xmm1);
9771   pxor(xmm0, xmm2);
9772 
9773   // 8 8-bit folds to compute 32-bit CRC.
9774   for (int j = 0; j < 4; j++) {
9775     fold_8bit_crc32(xmm0, table, xmm1, rax);
9776   }
9777   movdl(crc, xmm0); // mov 32 bits to general register
9778   for (int j = 0; j < 4; j++) {
9779     fold_8bit_crc32(crc, table, rax);
9780   }
9781 
9782   BIND(L_tail_restore);
9783   movl(len, tmp); // restore
9784   BIND(L_tail);
9785   andl(len, 0xf);
9786   jccb(Assembler::zero, L_exit);
9787 
9788   // Fold the rest of bytes
9789   align(4);
9790   BIND(L_tail_loop);
9791   movsbl(rax, Address(buf, 0)); // load byte with sign extension
9792   update_byte_crc32(crc, rax, table);
9793   increment(buf);
9794   decrementl(len);
9795   jccb(Assembler::greater, L_tail_loop);
9796 
9797   BIND(L_exit);
9798   notl(crc); // ~c
9799 }
9800 
9801 #ifdef _LP64
9802 // S. Gueron / Information Processing Letters 112 (2012) 184
9803 // Algorithm 4: Computing carry-less multiplication using a precomputed lookup table.
9804 // Input: A 32 bit value B = [byte3, byte2, byte1, byte0].
9805 // Output: the 64-bit carry-less product of B * CONST
9806 void MacroAssembler::crc32c_ipl_alg4(Register in, uint32_t n,
9807                                      Register tmp1, Register tmp2, Register tmp3) {
9808   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
9809   if (n > 0) {
9810     addq(tmp3, n * 256 * 8);
9811   }
9812   //    Q1 = TABLEExt[n][B & 0xFF];
9813   movl(tmp1, in);
9814   andl(tmp1, 0x000000FF);
9815   shll(tmp1, 3);
9816   addq(tmp1, tmp3);
9817   movq(tmp1, Address(tmp1, 0));
9818 
9819   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
9820   movl(tmp2, in);
9821   shrl(tmp2, 8);
9822   andl(tmp2, 0x000000FF);
9823   shll(tmp2, 3);
9824   addq(tmp2, tmp3);
9825   movq(tmp2, Address(tmp2, 0));
9826 
9827   shlq(tmp2, 8);
9828   xorq(tmp1, tmp2);
9829 
9830   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
9831   movl(tmp2, in);
9832   shrl(tmp2, 16);
9833   andl(tmp2, 0x000000FF);
9834   shll(tmp2, 3);
9835   addq(tmp2, tmp3);
9836   movq(tmp2, Address(tmp2, 0));
9837 
9838   shlq(tmp2, 16);
9839   xorq(tmp1, tmp2);
9840 
9841   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
9842   shrl(in, 24);
9843   andl(in, 0x000000FF);
9844   shll(in, 3);
9845   addq(in, tmp3);
9846   movq(in, Address(in, 0));
9847 
9848   shlq(in, 24);
9849   xorq(in, tmp1);
9850   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
9851 }
9852 
9853 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
9854                                       Register in_out,
9855                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
9856                                       XMMRegister w_xtmp2,
9857                                       Register tmp1,
9858                                       Register n_tmp2, Register n_tmp3) {
9859   if (is_pclmulqdq_supported) {
9860     movdl(w_xtmp1, in_out); // modified blindly
9861 
9862     movl(tmp1, const_or_pre_comp_const_index);
9863     movdl(w_xtmp2, tmp1);
9864     pclmulqdq(w_xtmp1, w_xtmp2, 0);
9865 
9866     movdq(in_out, w_xtmp1);
9867   } else {
9868     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3);
9869   }
9870 }
9871 
9872 // Recombination Alternative 2: No bit-reflections
9873 // T1 = (CRC_A * U1) << 1
9874 // T2 = (CRC_B * U2) << 1
9875 // C1 = T1 >> 32
9876 // C2 = T2 >> 32
9877 // T1 = T1 & 0xFFFFFFFF
9878 // T2 = T2 & 0xFFFFFFFF
9879 // T1 = CRC32(0, T1)
9880 // T2 = CRC32(0, T2)
9881 // C1 = C1 ^ T1
9882 // C2 = C2 ^ T2
9883 // CRC = C1 ^ C2 ^ CRC_C
9884 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
9885                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
9886                                      Register tmp1, Register tmp2,
9887                                      Register n_tmp3) {
9888   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
9889   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
9890   shlq(in_out, 1);
9891   movl(tmp1, in_out);
9892   shrq(in_out, 32);
9893   xorl(tmp2, tmp2);
9894   crc32(tmp2, tmp1, 4);
9895   xorl(in_out, tmp2); // we don't care about upper 32 bit contents here
9896   shlq(in1, 1);
9897   movl(tmp1, in1);
9898   shrq(in1, 32);
9899   xorl(tmp2, tmp2);
9900   crc32(tmp2, tmp1, 4);
9901   xorl(in1, tmp2);
9902   xorl(in_out, in1);
9903   xorl(in_out, in2);
9904 }
9905 
9906 // Set N to predefined value
9907 // Subtract from a lenght of a buffer
9908 // execute in a loop:
9909 // CRC_A = 0xFFFFFFFF, CRC_B = 0, CRC_C = 0
9910 // for i = 1 to N do
9911 //  CRC_A = CRC32(CRC_A, A[i])
9912 //  CRC_B = CRC32(CRC_B, B[i])
9913 //  CRC_C = CRC32(CRC_C, C[i])
9914 // end for
9915 // Recombine
9916 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
9917                                        Register in_out1, Register in_out2, Register in_out3,
9918                                        Register tmp1, Register tmp2, Register tmp3,
9919                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
9920                                        Register tmp4, Register tmp5,
9921                                        Register n_tmp6) {
9922   Label L_processPartitions;
9923   Label L_processPartition;
9924   Label L_exit;
9925 
9926   bind(L_processPartitions);
9927   cmpl(in_out1, 3 * size);
9928   jcc(Assembler::less, L_exit);
9929     xorl(tmp1, tmp1);
9930     xorl(tmp2, tmp2);
9931     movq(tmp3, in_out2);
9932     addq(tmp3, size);
9933 
9934     bind(L_processPartition);
9935       crc32(in_out3, Address(in_out2, 0), 8);
9936       crc32(tmp1, Address(in_out2, size), 8);
9937       crc32(tmp2, Address(in_out2, size * 2), 8);
9938       addq(in_out2, 8);
9939       cmpq(in_out2, tmp3);
9940       jcc(Assembler::less, L_processPartition);
9941     crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
9942             w_xtmp1, w_xtmp2, w_xtmp3,
9943             tmp4, tmp5,
9944             n_tmp6);
9945     addq(in_out2, 2 * size);
9946     subl(in_out1, 3 * size);
9947     jmp(L_processPartitions);
9948 
9949   bind(L_exit);
9950 }
9951 #else
9952 void MacroAssembler::crc32c_ipl_alg4(Register in_out, uint32_t n,
9953                                      Register tmp1, Register tmp2, Register tmp3,
9954                                      XMMRegister xtmp1, XMMRegister xtmp2) {
9955   lea(tmp3, ExternalAddress(StubRoutines::crc32c_table_addr()));
9956   if (n > 0) {
9957     addl(tmp3, n * 256 * 8);
9958   }
9959   //    Q1 = TABLEExt[n][B & 0xFF];
9960   movl(tmp1, in_out);
9961   andl(tmp1, 0x000000FF);
9962   shll(tmp1, 3);
9963   addl(tmp1, tmp3);
9964   movq(xtmp1, Address(tmp1, 0));
9965 
9966   //    Q2 = TABLEExt[n][B >> 8 & 0xFF];
9967   movl(tmp2, in_out);
9968   shrl(tmp2, 8);
9969   andl(tmp2, 0x000000FF);
9970   shll(tmp2, 3);
9971   addl(tmp2, tmp3);
9972   movq(xtmp2, Address(tmp2, 0));
9973 
9974   psllq(xtmp2, 8);
9975   pxor(xtmp1, xtmp2);
9976 
9977   //    Q3 = TABLEExt[n][B >> 16 & 0xFF];
9978   movl(tmp2, in_out);
9979   shrl(tmp2, 16);
9980   andl(tmp2, 0x000000FF);
9981   shll(tmp2, 3);
9982   addl(tmp2, tmp3);
9983   movq(xtmp2, Address(tmp2, 0));
9984 
9985   psllq(xtmp2, 16);
9986   pxor(xtmp1, xtmp2);
9987 
9988   //    Q4 = TABLEExt[n][B >> 24 & 0xFF];
9989   shrl(in_out, 24);
9990   andl(in_out, 0x000000FF);
9991   shll(in_out, 3);
9992   addl(in_out, tmp3);
9993   movq(xtmp2, Address(in_out, 0));
9994 
9995   psllq(xtmp2, 24);
9996   pxor(xtmp1, xtmp2); // Result in CXMM
9997   //    return Q1 ^ Q2 << 8 ^ Q3 << 16 ^ Q4 << 24;
9998 }
9999 
10000 void MacroAssembler::crc32c_pclmulqdq(XMMRegister w_xtmp1,
10001                                       Register in_out,
10002                                       uint32_t const_or_pre_comp_const_index, bool is_pclmulqdq_supported,
10003                                       XMMRegister w_xtmp2,
10004                                       Register tmp1,
10005                                       Register n_tmp2, Register n_tmp3) {
10006   if (is_pclmulqdq_supported) {
10007     movdl(w_xtmp1, in_out);
10008 
10009     movl(tmp1, const_or_pre_comp_const_index);
10010     movdl(w_xtmp2, tmp1);
10011     pclmulqdq(w_xtmp1, w_xtmp2, 0);
10012     // Keep result in XMM since GPR is 32 bit in length
10013   } else {
10014     crc32c_ipl_alg4(in_out, const_or_pre_comp_const_index, tmp1, n_tmp2, n_tmp3, w_xtmp1, w_xtmp2);
10015   }
10016 }
10017 
10018 void MacroAssembler::crc32c_rec_alt2(uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported, Register in_out, Register in1, Register in2,
10019                                      XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10020                                      Register tmp1, Register tmp2,
10021                                      Register n_tmp3) {
10022   crc32c_pclmulqdq(w_xtmp1, in_out, const_or_pre_comp_const_index_u1, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10023   crc32c_pclmulqdq(w_xtmp2, in1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, w_xtmp3, tmp1, tmp2, n_tmp3);
10024 
10025   psllq(w_xtmp1, 1);
10026   movdl(tmp1, w_xtmp1);
10027   psrlq(w_xtmp1, 32);
10028   movdl(in_out, w_xtmp1);
10029 
10030   xorl(tmp2, tmp2);
10031   crc32(tmp2, tmp1, 4);
10032   xorl(in_out, tmp2);
10033 
10034   psllq(w_xtmp2, 1);
10035   movdl(tmp1, w_xtmp2);
10036   psrlq(w_xtmp2, 32);
10037   movdl(in1, w_xtmp2);
10038 
10039   xorl(tmp2, tmp2);
10040   crc32(tmp2, tmp1, 4);
10041   xorl(in1, tmp2);
10042   xorl(in_out, in1);
10043   xorl(in_out, in2);
10044 }
10045 
10046 void MacroAssembler::crc32c_proc_chunk(uint32_t size, uint32_t const_or_pre_comp_const_index_u1, uint32_t const_or_pre_comp_const_index_u2, bool is_pclmulqdq_supported,
10047                                        Register in_out1, Register in_out2, Register in_out3,
10048                                        Register tmp1, Register tmp2, Register tmp3,
10049                                        XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10050                                        Register tmp4, Register tmp5,
10051                                        Register n_tmp6) {
10052   Label L_processPartitions;
10053   Label L_processPartition;
10054   Label L_exit;
10055 
10056   bind(L_processPartitions);
10057   cmpl(in_out1, 3 * size);
10058   jcc(Assembler::less, L_exit);
10059     xorl(tmp1, tmp1);
10060     xorl(tmp2, tmp2);
10061     movl(tmp3, in_out2);
10062     addl(tmp3, size);
10063 
10064     bind(L_processPartition);
10065       crc32(in_out3, Address(in_out2, 0), 4);
10066       crc32(tmp1, Address(in_out2, size), 4);
10067       crc32(tmp2, Address(in_out2, size*2), 4);
10068       crc32(in_out3, Address(in_out2, 0+4), 4);
10069       crc32(tmp1, Address(in_out2, size+4), 4);
10070       crc32(tmp2, Address(in_out2, size*2+4), 4);
10071       addl(in_out2, 8);
10072       cmpl(in_out2, tmp3);
10073       jcc(Assembler::less, L_processPartition);
10074 
10075         push(tmp3);
10076         push(in_out1);
10077         push(in_out2);
10078         tmp4 = tmp3;
10079         tmp5 = in_out1;
10080         n_tmp6 = in_out2;
10081 
10082       crc32c_rec_alt2(const_or_pre_comp_const_index_u1, const_or_pre_comp_const_index_u2, is_pclmulqdq_supported, in_out3, tmp1, tmp2,
10083             w_xtmp1, w_xtmp2, w_xtmp3,
10084             tmp4, tmp5,
10085             n_tmp6);
10086 
10087         pop(in_out2);
10088         pop(in_out1);
10089         pop(tmp3);
10090 
10091     addl(in_out2, 2 * size);
10092     subl(in_out1, 3 * size);
10093     jmp(L_processPartitions);
10094 
10095   bind(L_exit);
10096 }
10097 #endif //LP64
10098 
10099 #ifdef _LP64
10100 // Algorithm 2: Pipelined usage of the CRC32 instruction.
10101 // Input: A buffer I of L bytes.
10102 // Output: the CRC32C value of the buffer.
10103 // Notations:
10104 // Write L = 24N + r, with N = floor (L/24).
10105 // r = L mod 24 (0 <= r < 24).
10106 // Consider I as the concatenation of A|B|C|R, where A, B, C, each,
10107 // N quadwords, and R consists of r bytes.
10108 // A[j] = I [8j+7:8j], j= 0, 1, ..., N-1
10109 // B[j] = I [N + 8j+7:N + 8j], j= 0, 1, ..., N-1
10110 // C[j] = I [2N + 8j+7:2N + 8j], j= 0, 1, ..., N-1
10111 // if r > 0 R[j] = I [3N +j], j= 0, 1, ...,r-1
10112 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10113                                           Register tmp1, Register tmp2, Register tmp3,
10114                                           Register tmp4, Register tmp5, Register tmp6,
10115                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10116                                           bool is_pclmulqdq_supported) {
10117   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10118   Label L_wordByWord;
10119   Label L_byteByByteProlog;
10120   Label L_byteByByte;
10121   Label L_exit;
10122 
10123   if (is_pclmulqdq_supported ) {
10124     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10125     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr+1);
10126 
10127     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10128     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10129 
10130     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10131     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10132     assert((CRC32C_NUM_PRECOMPUTED_CONSTANTS - 1 ) == 5, "Checking whether you declared all of the constants based on the number of \"chunks\"");
10133   } else {
10134     const_or_pre_comp_const_index[0] = 1;
10135     const_or_pre_comp_const_index[1] = 0;
10136 
10137     const_or_pre_comp_const_index[2] = 3;
10138     const_or_pre_comp_const_index[3] = 2;
10139 
10140     const_or_pre_comp_const_index[4] = 5;
10141     const_or_pre_comp_const_index[5] = 4;
10142    }
10143   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10144                     in2, in1, in_out,
10145                     tmp1, tmp2, tmp3,
10146                     w_xtmp1, w_xtmp2, w_xtmp3,
10147                     tmp4, tmp5,
10148                     tmp6);
10149   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10150                     in2, in1, in_out,
10151                     tmp1, tmp2, tmp3,
10152                     w_xtmp1, w_xtmp2, w_xtmp3,
10153                     tmp4, tmp5,
10154                     tmp6);
10155   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10156                     in2, in1, in_out,
10157                     tmp1, tmp2, tmp3,
10158                     w_xtmp1, w_xtmp2, w_xtmp3,
10159                     tmp4, tmp5,
10160                     tmp6);
10161   movl(tmp1, in2);
10162   andl(tmp1, 0x00000007);
10163   negl(tmp1);
10164   addl(tmp1, in2);
10165   addq(tmp1, in1);
10166 
10167   BIND(L_wordByWord);
10168   cmpq(in1, tmp1);
10169   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10170     crc32(in_out, Address(in1, 0), 4);
10171     addq(in1, 4);
10172     jmp(L_wordByWord);
10173 
10174   BIND(L_byteByByteProlog);
10175   andl(in2, 0x00000007);
10176   movl(tmp2, 1);
10177 
10178   BIND(L_byteByByte);
10179   cmpl(tmp2, in2);
10180   jccb(Assembler::greater, L_exit);
10181     crc32(in_out, Address(in1, 0), 1);
10182     incq(in1);
10183     incl(tmp2);
10184     jmp(L_byteByByte);
10185 
10186   BIND(L_exit);
10187 }
10188 #else
10189 void MacroAssembler::crc32c_ipl_alg2_alt2(Register in_out, Register in1, Register in2,
10190                                           Register tmp1, Register  tmp2, Register tmp3,
10191                                           Register tmp4, Register  tmp5, Register tmp6,
10192                                           XMMRegister w_xtmp1, XMMRegister w_xtmp2, XMMRegister w_xtmp3,
10193                                           bool is_pclmulqdq_supported) {
10194   uint32_t const_or_pre_comp_const_index[CRC32C_NUM_PRECOMPUTED_CONSTANTS];
10195   Label L_wordByWord;
10196   Label L_byteByByteProlog;
10197   Label L_byteByByte;
10198   Label L_exit;
10199 
10200   if (is_pclmulqdq_supported) {
10201     const_or_pre_comp_const_index[1] = *(uint32_t *)StubRoutines::_crc32c_table_addr;
10202     const_or_pre_comp_const_index[0] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 1);
10203 
10204     const_or_pre_comp_const_index[3] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 2);
10205     const_or_pre_comp_const_index[2] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 3);
10206 
10207     const_or_pre_comp_const_index[5] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 4);
10208     const_or_pre_comp_const_index[4] = *((uint32_t *)StubRoutines::_crc32c_table_addr + 5);
10209   } else {
10210     const_or_pre_comp_const_index[0] = 1;
10211     const_or_pre_comp_const_index[1] = 0;
10212 
10213     const_or_pre_comp_const_index[2] = 3;
10214     const_or_pre_comp_const_index[3] = 2;
10215 
10216     const_or_pre_comp_const_index[4] = 5;
10217     const_or_pre_comp_const_index[5] = 4;
10218   }
10219   crc32c_proc_chunk(CRC32C_HIGH, const_or_pre_comp_const_index[0], const_or_pre_comp_const_index[1], is_pclmulqdq_supported,
10220                     in2, in1, in_out,
10221                     tmp1, tmp2, tmp3,
10222                     w_xtmp1, w_xtmp2, w_xtmp3,
10223                     tmp4, tmp5,
10224                     tmp6);
10225   crc32c_proc_chunk(CRC32C_MIDDLE, const_or_pre_comp_const_index[2], const_or_pre_comp_const_index[3], is_pclmulqdq_supported,
10226                     in2, in1, in_out,
10227                     tmp1, tmp2, tmp3,
10228                     w_xtmp1, w_xtmp2, w_xtmp3,
10229                     tmp4, tmp5,
10230                     tmp6);
10231   crc32c_proc_chunk(CRC32C_LOW, const_or_pre_comp_const_index[4], const_or_pre_comp_const_index[5], is_pclmulqdq_supported,
10232                     in2, in1, in_out,
10233                     tmp1, tmp2, tmp3,
10234                     w_xtmp1, w_xtmp2, w_xtmp3,
10235                     tmp4, tmp5,
10236                     tmp6);
10237   movl(tmp1, in2);
10238   andl(tmp1, 0x00000007);
10239   negl(tmp1);
10240   addl(tmp1, in2);
10241   addl(tmp1, in1);
10242 
10243   BIND(L_wordByWord);
10244   cmpl(in1, tmp1);
10245   jcc(Assembler::greaterEqual, L_byteByByteProlog);
10246     crc32(in_out, Address(in1,0), 4);
10247     addl(in1, 4);
10248     jmp(L_wordByWord);
10249 
10250   BIND(L_byteByByteProlog);
10251   andl(in2, 0x00000007);
10252   movl(tmp2, 1);
10253 
10254   BIND(L_byteByByte);
10255   cmpl(tmp2, in2);
10256   jccb(Assembler::greater, L_exit);
10257     movb(tmp1, Address(in1, 0));
10258     crc32(in_out, tmp1, 1);
10259     incl(in1);
10260     incl(tmp2);
10261     jmp(L_byteByByte);
10262 
10263   BIND(L_exit);
10264 }
10265 #endif // LP64
10266 #undef BIND
10267 #undef BLOCK_COMMENT
10268 
10269 // Compress char[] array to byte[].
10270 //   ..\jdk\src\java.base\share\classes\java\lang\StringUTF16.java
10271 //   @HotSpotIntrinsicCandidate
10272 //   private static int compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) {
10273 //     for (int i = 0; i < len; i++) {
10274 //       int c = src[srcOff++];
10275 //       if (c >>> 8 != 0) {
10276 //         return 0;
10277 //       }
10278 //       dst[dstOff++] = (byte)c;
10279 //     }
10280 //     return len;
10281 //   }
10282 void MacroAssembler::char_array_compress(Register src, Register dst, Register len,
10283   XMMRegister tmp1Reg, XMMRegister tmp2Reg,
10284   XMMRegister tmp3Reg, XMMRegister tmp4Reg,
10285   Register tmp5, Register result) {
10286   Label copy_chars_loop, return_length, return_zero, done;
10287 
10288   // rsi: src
10289   // rdi: dst
10290   // rdx: len
10291   // rcx: tmp5
10292   // rax: result
10293 
10294   // rsi holds start addr of source char[] to be compressed
10295   // rdi holds start addr of destination byte[]
10296   // rdx holds length
10297 
10298   assert(len != result, "");
10299 
10300   // save length for return
10301   push(len);
10302 
10303   if ((UseAVX > 2) && // AVX512
10304     VM_Version::supports_avx512vlbw() &&
10305     VM_Version::supports_bmi2()) {
10306 
10307     set_vector_masking();  // opening of the stub context for programming mask registers
10308 
10309     Label copy_32_loop, copy_loop_tail, restore_k1_return_zero, below_threshold;
10310 
10311     // alignment
10312     Label post_alignment;
10313 
10314     // if length of the string is less than 16, handle it in an old fashioned way
10315     testl(len, -32);
10316     jcc(Assembler::zero, below_threshold);
10317 
10318     // First check whether a character is compressable ( <= 0xFF).
10319     // Create mask to test for Unicode chars inside zmm vector
10320     movl(result, 0x00FF);
10321     evpbroadcastw(tmp2Reg, result, Assembler::AVX_512bit);
10322 
10323     // Save k1
10324     kmovql(k3, k1);
10325 
10326     testl(len, -64);
10327     jcc(Assembler::zero, post_alignment);
10328 
10329     movl(tmp5, dst);
10330     andl(tmp5, (32 - 1));
10331     negl(tmp5);
10332     andl(tmp5, (32 - 1));
10333 
10334     // bail out when there is nothing to be done
10335     testl(tmp5, 0xFFFFFFFF);
10336     jcc(Assembler::zero, post_alignment);
10337 
10338     // ~(~0 << len), where len is the # of remaining elements to process
10339     movl(result, 0xFFFFFFFF);
10340     shlxl(result, result, tmp5);
10341     notl(result);
10342     kmovdl(k1, result);
10343 
10344     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
10345     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10346     ktestd(k2, k1);
10347     jcc(Assembler::carryClear, restore_k1_return_zero);
10348 
10349     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
10350 
10351     addptr(src, tmp5);
10352     addptr(src, tmp5);
10353     addptr(dst, tmp5);
10354     subl(len, tmp5);
10355 
10356     bind(post_alignment);
10357     // end of alignment
10358 
10359     movl(tmp5, len);
10360     andl(tmp5, (32 - 1));    // tail count (in chars)
10361     andl(len, ~(32 - 1));    // vector count (in chars)
10362     jcc(Assembler::zero, copy_loop_tail);
10363 
10364     lea(src, Address(src, len, Address::times_2));
10365     lea(dst, Address(dst, len, Address::times_1));
10366     negptr(len);
10367 
10368     bind(copy_32_loop);
10369     evmovdquw(tmp1Reg, Address(src, len, Address::times_2), Assembler::AVX_512bit);
10370     evpcmpuw(k2, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10371     kortestdl(k2, k2);
10372     jcc(Assembler::carryClear, restore_k1_return_zero);
10373 
10374     // All elements in current processed chunk are valid candidates for
10375     // compression. Write a truncated byte elements to the memory.
10376     evpmovwb(Address(dst, len, Address::times_1), tmp1Reg, Assembler::AVX_512bit);
10377     addptr(len, 32);
10378     jcc(Assembler::notZero, copy_32_loop);
10379 
10380     bind(copy_loop_tail);
10381     // bail out when there is nothing to be done
10382     testl(tmp5, 0xFFFFFFFF);
10383     // Restore k1
10384     kmovql(k1, k3);
10385     jcc(Assembler::zero, return_length);
10386 
10387     movl(len, tmp5);
10388 
10389     // ~(~0 << len), where len is the # of remaining elements to process
10390     movl(result, 0xFFFFFFFF);
10391     shlxl(result, result, len);
10392     notl(result);
10393 
10394     kmovdl(k1, result);
10395 
10396     evmovdquw(tmp1Reg, k1, Address(src, 0), Assembler::AVX_512bit);
10397     evpcmpuw(k2, k1, tmp1Reg, tmp2Reg, Assembler::le, Assembler::AVX_512bit);
10398     ktestd(k2, k1);
10399     jcc(Assembler::carryClear, restore_k1_return_zero);
10400 
10401     evpmovwb(Address(dst, 0), k1, tmp1Reg, Assembler::AVX_512bit);
10402     // Restore k1
10403     kmovql(k1, k3);
10404     jmp(return_length);
10405 
10406     bind(restore_k1_return_zero);
10407     // Restore k1
10408     kmovql(k1, k3);
10409     jmp(return_zero);
10410 
10411     clear_vector_masking();   // closing of the stub context for programming mask registers
10412 
10413     bind(below_threshold);
10414   }
10415 
10416   if (UseSSE42Intrinsics) {
10417     Label copy_32_loop, copy_16, copy_tail;
10418 
10419     movl(result, len);
10420 
10421     movl(tmp5, 0xff00ff00);   // create mask to test for Unicode chars in vectors
10422 
10423     // vectored compression
10424     andl(len, 0xfffffff0);    // vector count (in chars)
10425     andl(result, 0x0000000f);    // tail count (in chars)
10426     testl(len, len);
10427     jcc(Assembler::zero, copy_16);
10428 
10429     // compress 16 chars per iter
10430     movdl(tmp1Reg, tmp5);
10431     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
10432     pxor(tmp4Reg, tmp4Reg);
10433 
10434     lea(src, Address(src, len, Address::times_2));
10435     lea(dst, Address(dst, len, Address::times_1));
10436     negptr(len);
10437 
10438     bind(copy_32_loop);
10439     movdqu(tmp2Reg, Address(src, len, Address::times_2));     // load 1st 8 characters
10440     por(tmp4Reg, tmp2Reg);
10441     movdqu(tmp3Reg, Address(src, len, Address::times_2, 16)); // load next 8 characters
10442     por(tmp4Reg, tmp3Reg);
10443     ptest(tmp4Reg, tmp1Reg);       // check for Unicode chars in next vector
10444     jcc(Assembler::notZero, return_zero);
10445     packuswb(tmp2Reg, tmp3Reg);    // only ASCII chars; compress each to 1 byte
10446     movdqu(Address(dst, len, Address::times_1), tmp2Reg);
10447     addptr(len, 16);
10448     jcc(Assembler::notZero, copy_32_loop);
10449 
10450     // compress next vector of 8 chars (if any)
10451     bind(copy_16);
10452     movl(len, result);
10453     andl(len, 0xfffffff8);    // vector count (in chars)
10454     andl(result, 0x00000007);    // tail count (in chars)
10455     testl(len, len);
10456     jccb(Assembler::zero, copy_tail);
10457 
10458     movdl(tmp1Reg, tmp5);
10459     pshufd(tmp1Reg, tmp1Reg, 0);   // store Unicode mask in tmp1Reg
10460     pxor(tmp3Reg, tmp3Reg);
10461 
10462     movdqu(tmp2Reg, Address(src, 0));
10463     ptest(tmp2Reg, tmp1Reg);       // check for Unicode chars in vector
10464     jccb(Assembler::notZero, return_zero);
10465     packuswb(tmp2Reg, tmp3Reg);    // only LATIN1 chars; compress each to 1 byte
10466     movq(Address(dst, 0), tmp2Reg);
10467     addptr(src, 16);
10468     addptr(dst, 8);
10469 
10470     bind(copy_tail);
10471     movl(len, result);
10472   }
10473   // compress 1 char per iter
10474   testl(len, len);
10475   jccb(Assembler::zero, return_length);
10476   lea(src, Address(src, len, Address::times_2));
10477   lea(dst, Address(dst, len, Address::times_1));
10478   negptr(len);
10479 
10480   bind(copy_chars_loop);
10481   load_unsigned_short(result, Address(src, len, Address::times_2));
10482   testl(result, 0xff00);      // check if Unicode char
10483   jccb(Assembler::notZero, return_zero);
10484   movb(Address(dst, len, Address::times_1), result);  // ASCII char; compress to 1 byte
10485   increment(len);
10486   jcc(Assembler::notZero, copy_chars_loop);
10487 
10488   // if compression succeeded, return length
10489   bind(return_length);
10490   pop(result);
10491   jmpb(done);
10492 
10493   // if compression failed, return 0
10494   bind(return_zero);
10495   xorl(result, result);
10496   addptr(rsp, wordSize);
10497 
10498   bind(done);
10499 }
10500 
10501 // Inflate byte[] array to char[].
10502 //   ..\jdk\src\java.base\share\classes\java\lang\StringLatin1.java
10503 //   @HotSpotIntrinsicCandidate
10504 //   private static void inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) {
10505 //     for (int i = 0; i < len; i++) {
10506 //       dst[dstOff++] = (char)(src[srcOff++] & 0xff);
10507 //     }
10508 //   }
10509 void MacroAssembler::byte_array_inflate(Register src, Register dst, Register len,
10510   XMMRegister tmp1, Register tmp2) {
10511   Label copy_chars_loop, done, below_threshold;
10512   // rsi: src
10513   // rdi: dst
10514   // rdx: len
10515   // rcx: tmp2
10516 
10517   // rsi holds start addr of source byte[] to be inflated
10518   // rdi holds start addr of destination char[]
10519   // rdx holds length
10520   assert_different_registers(src, dst, len, tmp2);
10521 
10522   if ((UseAVX > 2) && // AVX512
10523     VM_Version::supports_avx512vlbw() &&
10524     VM_Version::supports_bmi2()) {
10525 
10526     set_vector_masking();  // opening of the stub context for programming mask registers
10527 
10528     Label copy_32_loop, copy_tail;
10529     Register tmp3_aliased = len;
10530 
10531     // if length of the string is less than 16, handle it in an old fashioned way
10532     testl(len, -16);
10533     jcc(Assembler::zero, below_threshold);
10534 
10535     // In order to use only one arithmetic operation for the main loop we use
10536     // this pre-calculation
10537     movl(tmp2, len);
10538     andl(tmp2, (32 - 1)); // tail count (in chars), 32 element wide loop
10539     andl(len, -32);     // vector count
10540     jccb(Assembler::zero, copy_tail);
10541 
10542     lea(src, Address(src, len, Address::times_1));
10543     lea(dst, Address(dst, len, Address::times_2));
10544     negptr(len);
10545 
10546 
10547     // inflate 32 chars per iter
10548     bind(copy_32_loop);
10549     vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_512bit);
10550     evmovdquw(Address(dst, len, Address::times_2), tmp1, Assembler::AVX_512bit);
10551     addptr(len, 32);
10552     jcc(Assembler::notZero, copy_32_loop);
10553 
10554     bind(copy_tail);
10555     // bail out when there is nothing to be done
10556     testl(tmp2, -1); // we don't destroy the contents of tmp2 here
10557     jcc(Assembler::zero, done);
10558 
10559     // Save k1
10560     kmovql(k2, k1);
10561 
10562     // ~(~0 << length), where length is the # of remaining elements to process
10563     movl(tmp3_aliased, -1);
10564     shlxl(tmp3_aliased, tmp3_aliased, tmp2);
10565     notl(tmp3_aliased);
10566     kmovdl(k1, tmp3_aliased);
10567     evpmovzxbw(tmp1, k1, Address(src, 0), Assembler::AVX_512bit);
10568     evmovdquw(Address(dst, 0), k1, tmp1, Assembler::AVX_512bit);
10569 
10570     // Restore k1
10571     kmovql(k1, k2);
10572     jmp(done);
10573 
10574     clear_vector_masking();   // closing of the stub context for programming mask registers
10575   }
10576   if (UseSSE42Intrinsics) {
10577     Label copy_16_loop, copy_8_loop, copy_bytes, copy_new_tail, copy_tail;
10578 
10579     movl(tmp2, len);
10580 
10581     if (UseAVX > 1) {
10582       andl(tmp2, (16 - 1));
10583       andl(len, -16);
10584       jccb(Assembler::zero, copy_new_tail);
10585     } else {
10586       andl(tmp2, 0x00000007);   // tail count (in chars)
10587       andl(len, 0xfffffff8);    // vector count (in chars)
10588       jccb(Assembler::zero, copy_tail);
10589     }
10590 
10591     // vectored inflation
10592     lea(src, Address(src, len, Address::times_1));
10593     lea(dst, Address(dst, len, Address::times_2));
10594     negptr(len);
10595 
10596     if (UseAVX > 1) {
10597       bind(copy_16_loop);
10598       vpmovzxbw(tmp1, Address(src, len, Address::times_1), Assembler::AVX_256bit);
10599       vmovdqu(Address(dst, len, Address::times_2), tmp1);
10600       addptr(len, 16);
10601       jcc(Assembler::notZero, copy_16_loop);
10602 
10603       bind(below_threshold);
10604       bind(copy_new_tail);
10605       if ((UseAVX > 2) &&
10606         VM_Version::supports_avx512vlbw() &&
10607         VM_Version::supports_bmi2()) {
10608         movl(tmp2, len);
10609       } else {
10610         movl(len, tmp2);
10611       }
10612       andl(tmp2, 0x00000007);
10613       andl(len, 0xFFFFFFF8);
10614       jccb(Assembler::zero, copy_tail);
10615 
10616       pmovzxbw(tmp1, Address(src, 0));
10617       movdqu(Address(dst, 0), tmp1);
10618       addptr(src, 8);
10619       addptr(dst, 2 * 8);
10620 
10621       jmp(copy_tail, true);
10622     }
10623 
10624     // inflate 8 chars per iter
10625     bind(copy_8_loop);
10626     pmovzxbw(tmp1, Address(src, len, Address::times_1));  // unpack to 8 words
10627     movdqu(Address(dst, len, Address::times_2), tmp1);
10628     addptr(len, 8);
10629     jcc(Assembler::notZero, copy_8_loop);
10630 
10631     bind(copy_tail);
10632     movl(len, tmp2);
10633 
10634     cmpl(len, 4);
10635     jccb(Assembler::less, copy_bytes);
10636 
10637     movdl(tmp1, Address(src, 0));  // load 4 byte chars
10638     pmovzxbw(tmp1, tmp1);
10639     movq(Address(dst, 0), tmp1);
10640     subptr(len, 4);
10641     addptr(src, 4);
10642     addptr(dst, 8);
10643 
10644     bind(copy_bytes);
10645   } else {
10646     bind(below_threshold);
10647   }
10648 
10649   testl(len, len);
10650   jccb(Assembler::zero, done);
10651   lea(src, Address(src, len, Address::times_1));
10652   lea(dst, Address(dst, len, Address::times_2));
10653   negptr(len);
10654 
10655   // inflate 1 char per iter
10656   bind(copy_chars_loop);
10657   load_unsigned_byte(tmp2, Address(src, len, Address::times_1));  // load byte char
10658   movw(Address(dst, len, Address::times_2), tmp2);  // inflate byte char to word
10659   increment(len);
10660   jcc(Assembler::notZero, copy_chars_loop);
10661 
10662   bind(done);
10663 }
10664 
10665 Assembler::Condition MacroAssembler::negate_condition(Assembler::Condition cond) {
10666   switch (cond) {
10667     // Note some conditions are synonyms for others
10668     case Assembler::zero:         return Assembler::notZero;
10669     case Assembler::notZero:      return Assembler::zero;
10670     case Assembler::less:         return Assembler::greaterEqual;
10671     case Assembler::lessEqual:    return Assembler::greater;
10672     case Assembler::greater:      return Assembler::lessEqual;
10673     case Assembler::greaterEqual: return Assembler::less;
10674     case Assembler::below:        return Assembler::aboveEqual;
10675     case Assembler::belowEqual:   return Assembler::above;
10676     case Assembler::above:        return Assembler::belowEqual;
10677     case Assembler::aboveEqual:   return Assembler::below;
10678     case Assembler::overflow:     return Assembler::noOverflow;
10679     case Assembler::noOverflow:   return Assembler::overflow;
10680     case Assembler::negative:     return Assembler::positive;
10681     case Assembler::positive:     return Assembler::negative;
10682     case Assembler::parity:       return Assembler::noParity;
10683     case Assembler::noParity:     return Assembler::parity;
10684   }
10685   ShouldNotReachHere(); return Assembler::overflow;
10686 }
10687 
10688 SkipIfEqual::SkipIfEqual(
10689     MacroAssembler* masm, const bool* flag_addr, bool value) {
10690   _masm = masm;
10691   _masm->cmp8(ExternalAddress((address)flag_addr), value);
10692   _masm->jcc(Assembler::equal, _label);
10693 }
10694 
10695 SkipIfEqual::~SkipIfEqual() {
10696   _masm->bind(_label);
10697 }
10698 
10699 // 32-bit Windows has its own fast-path implementation
10700 // of get_thread
10701 #if !defined(WIN32) || defined(_LP64)
10702 
10703 // This is simply a call to Thread::current()
10704 void MacroAssembler::get_thread(Register thread) {
10705   if (thread != rax) {
10706     push(rax);
10707   }
10708   LP64_ONLY(push(rdi);)
10709   LP64_ONLY(push(rsi);)
10710   push(rdx);
10711   push(rcx);
10712 #ifdef _LP64
10713   push(r8);
10714   push(r9);
10715   push(r10);
10716   push(r11);
10717 #endif
10718 
10719   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
10720 
10721 #ifdef _LP64
10722   pop(r11);
10723   pop(r10);
10724   pop(r9);
10725   pop(r8);
10726 #endif
10727   pop(rcx);
10728   pop(rdx);
10729   LP64_ONLY(pop(rsi);)
10730   LP64_ONLY(pop(rdi);)
10731   if (thread != rax) {
10732     mov(thread, rax);
10733     pop(rax);
10734   }
10735 }
10736 
10737 #endif
--- EOF ---