1 /*
   2  * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "ci/ciUtilities.hpp"
  29 #include "gc/shared/barrierSet.hpp"
  30 #include "gc/shared/barrierSetAssembler.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "nativeInst_x86.hpp"
  33 #include "oops/instanceOop.hpp"
  34 #include "oops/method.hpp"
  35 #include "oops/objArrayKlass.hpp"
  36 #include "oops/oop.inline.hpp"
  37 #include "prims/methodHandles.hpp"
  38 #include "runtime/frame.inline.hpp"
  39 #include "runtime/handles.inline.hpp"
  40 #include "runtime/sharedRuntime.hpp"
  41 #include "runtime/stubCodeGenerator.hpp"
  42 #include "runtime/stubRoutines.hpp"
  43 #include "runtime/thread.inline.hpp"
  44 #ifdef COMPILER2
  45 #include "opto/runtime.hpp"
  46 #endif
  47 
  48 // Declaration and definition of StubGenerator (no .hpp file).
  49 // For a more detailed description of the stub routine structure
  50 // see the comment in stubRoutines.hpp
  51 
  52 #define __ _masm->
  53 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
  54 #define a__ ((Assembler*)_masm)->
  55 
  56 #ifdef PRODUCT
  57 #define BLOCK_COMMENT(str) /* nothing */
  58 #else
  59 #define BLOCK_COMMENT(str) __ block_comment(str)
  60 #endif
  61 
  62 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  63 const int MXCSR_MASK = 0xFFC0;  // Mask out any pending exceptions
  64 
  65 // Stub Code definitions
  66 
  67 class StubGenerator: public StubCodeGenerator {
  68  private:
  69 
  70 #ifdef PRODUCT
  71 #define inc_counter_np(counter) ((void)0)
  72 #else
  73   void inc_counter_np_(int& counter) {
  74     // This can destroy rscratch1 if counter is far from the code cache
  75     __ incrementl(ExternalAddress((address)&counter));
  76   }
  77 #define inc_counter_np(counter) \
  78   BLOCK_COMMENT("inc_counter " #counter); \
  79   inc_counter_np_(counter);
  80 #endif
  81 
  82   // Call stubs are used to call Java from C
  83   //
  84   // Linux Arguments:
  85   //    c_rarg0:   call wrapper address                   address
  86   //    c_rarg1:   result                                 address
  87   //    c_rarg2:   result type                            BasicType
  88   //    c_rarg3:   method                                 Method*
  89   //    c_rarg4:   (interpreter) entry point              address
  90   //    c_rarg5:   parameters                             intptr_t*
  91   //    16(rbp): parameter size (in words)              int
  92   //    24(rbp): thread                                 Thread*
  93   //
  94   //     [ return_from_Java     ] <--- rsp
  95   //     [ argument word n      ]
  96   //      ...
  97   // -12 [ argument word 1      ]
  98   // -11 [ saved r15            ] <--- rsp_after_call
  99   // -10 [ saved r14            ]
 100   //  -9 [ saved r13            ]
 101   //  -8 [ saved r12            ]
 102   //  -7 [ saved rbx            ]
 103   //  -6 [ call wrapper         ]
 104   //  -5 [ result               ]
 105   //  -4 [ result type          ]
 106   //  -3 [ method               ]
 107   //  -2 [ entry point          ]
 108   //  -1 [ parameters           ]
 109   //   0 [ saved rbp            ] <--- rbp
 110   //   1 [ return address       ]
 111   //   2 [ parameter size       ]
 112   //   3 [ thread               ]
 113   //
 114   // Windows Arguments:
 115   //    c_rarg0:   call wrapper address                   address
 116   //    c_rarg1:   result                                 address
 117   //    c_rarg2:   result type                            BasicType
 118   //    c_rarg3:   method                                 Method*
 119   //    48(rbp): (interpreter) entry point              address
 120   //    56(rbp): parameters                             intptr_t*
 121   //    64(rbp): parameter size (in words)              int
 122   //    72(rbp): thread                                 Thread*
 123   //
 124   //     [ return_from_Java     ] <--- rsp
 125   //     [ argument word n      ]
 126   //      ...
 127   // -60 [ argument word 1      ]
 128   // -59 [ saved xmm31          ] <--- rsp after_call
 129   //     [ saved xmm16-xmm30    ] (EVEX enabled, else the space is blank)
 130   // -27 [ saved xmm15          ]
 131   //     [ saved xmm7-xmm14     ]
 132   //  -9 [ saved xmm6           ] (each xmm register takes 2 slots)
 133   //  -7 [ saved r15            ]
 134   //  -6 [ saved r14            ]
 135   //  -5 [ saved r13            ]
 136   //  -4 [ saved r12            ]
 137   //  -3 [ saved rdi            ]
 138   //  -2 [ saved rsi            ]
 139   //  -1 [ saved rbx            ]
 140   //   0 [ saved rbp            ] <--- rbp
 141   //   1 [ return address       ]
 142   //   2 [ call wrapper         ]
 143   //   3 [ result               ]
 144   //   4 [ result type          ]
 145   //   5 [ method               ]
 146   //   6 [ entry point          ]
 147   //   7 [ parameters           ]
 148   //   8 [ parameter size       ]
 149   //   9 [ thread               ]
 150   //
 151   //    Windows reserves the callers stack space for arguments 1-4.
 152   //    We spill c_rarg0-c_rarg3 to this space.
 153 
 154   // Call stub stack layout word offsets from rbp
 155   enum call_stub_layout {
 156 #ifdef _WIN64
 157     xmm_save_first     = 6,  // save from xmm6
 158     xmm_save_last      = 31, // to xmm31
 159     xmm_save_base      = -9,
 160     rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27
 161     r15_off            = -7,
 162     r14_off            = -6,
 163     r13_off            = -5,
 164     r12_off            = -4,
 165     rdi_off            = -3,
 166     rsi_off            = -2,
 167     rbx_off            = -1,
 168     rbp_off            =  0,
 169     retaddr_off        =  1,
 170     call_wrapper_off   =  2,
 171     result_off         =  3,
 172     result_type_off    =  4,
 173     method_off         =  5,
 174     entry_point_off    =  6,
 175     parameters_off     =  7,
 176     parameter_size_off =  8,
 177     thread_off         =  9
 178 #else
 179     rsp_after_call_off = -12,
 180     mxcsr_off          = rsp_after_call_off,
 181     r15_off            = -11,
 182     r14_off            = -10,
 183     r13_off            = -9,
 184     r12_off            = -8,
 185     rbx_off            = -7,
 186     call_wrapper_off   = -6,
 187     result_off         = -5,
 188     result_type_off    = -4,
 189     method_off         = -3,
 190     entry_point_off    = -2,
 191     parameters_off     = -1,
 192     rbp_off            =  0,
 193     retaddr_off        =  1,
 194     parameter_size_off =  2,
 195     thread_off         =  3
 196 #endif
 197   };
 198 
 199 #ifdef _WIN64
 200   Address xmm_save(int reg) {
 201     assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range");
 202     return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize);
 203   }
 204 #endif
 205 
 206   address generate_call_stub(address& return_address) {
 207     assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 &&
 208            (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
 209            "adjust this code");
 210     StubCodeMark mark(this, "StubRoutines", "call_stub");
 211     address start = __ pc();
 212 
 213     // same as in generate_catch_exception()!
 214     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 215 
 216     const Address call_wrapper  (rbp, call_wrapper_off   * wordSize);
 217     const Address result        (rbp, result_off         * wordSize);
 218     const Address result_type   (rbp, result_type_off    * wordSize);
 219     const Address method        (rbp, method_off         * wordSize);
 220     const Address entry_point   (rbp, entry_point_off    * wordSize);
 221     const Address parameters    (rbp, parameters_off     * wordSize);
 222     const Address parameter_size(rbp, parameter_size_off * wordSize);
 223 
 224     // same as in generate_catch_exception()!
 225     const Address thread        (rbp, thread_off         * wordSize);
 226 
 227     const Address r15_save(rbp, r15_off * wordSize);
 228     const Address r14_save(rbp, r14_off * wordSize);
 229     const Address r13_save(rbp, r13_off * wordSize);
 230     const Address r12_save(rbp, r12_off * wordSize);
 231     const Address rbx_save(rbp, rbx_off * wordSize);
 232 
 233     // stub code
 234     __ enter();
 235     __ subptr(rsp, -rsp_after_call_off * wordSize);
 236 
 237     // save register parameters
 238 #ifndef _WIN64
 239     __ movptr(parameters,   c_rarg5); // parameters
 240     __ movptr(entry_point,  c_rarg4); // entry_point
 241 #endif
 242 
 243     __ movptr(method,       c_rarg3); // method
 244     __ movl(result_type,  c_rarg2);   // result type
 245     __ movptr(result,       c_rarg1); // result
 246     __ movptr(call_wrapper, c_rarg0); // call wrapper
 247 
 248     // save regs belonging to calling function
 249     __ movptr(rbx_save, rbx);
 250     __ movptr(r12_save, r12);
 251     __ movptr(r13_save, r13);
 252     __ movptr(r14_save, r14);
 253     __ movptr(r15_save, r15);
 254     if (UseAVX > 2) {
 255       __ movl(rbx, 0xffff);
 256       __ kmovwl(k1, rbx);
 257     }
 258 #ifdef _WIN64
 259     int last_reg = 15;
 260     if (UseAVX > 2) {
 261       last_reg = 31;
 262     }
 263     if (VM_Version::supports_evex()) {
 264       for (int i = xmm_save_first; i <= last_reg; i++) {
 265         __ vextractf32x4(xmm_save(i), as_XMMRegister(i), 0);
 266       }
 267     } else {
 268       for (int i = xmm_save_first; i <= last_reg; i++) {
 269         __ movdqu(xmm_save(i), as_XMMRegister(i));
 270       }
 271     }
 272 
 273     const Address rdi_save(rbp, rdi_off * wordSize);
 274     const Address rsi_save(rbp, rsi_off * wordSize);
 275 
 276     __ movptr(rsi_save, rsi);
 277     __ movptr(rdi_save, rdi);
 278 #else
 279     const Address mxcsr_save(rbp, mxcsr_off * wordSize);
 280     {
 281       Label skip_ldmx;
 282       __ stmxcsr(mxcsr_save);
 283       __ movl(rax, mxcsr_save);
 284       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 285       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 286       __ cmp32(rax, mxcsr_std);
 287       __ jcc(Assembler::equal, skip_ldmx);
 288       __ ldmxcsr(mxcsr_std);
 289       __ bind(skip_ldmx);
 290     }
 291 #endif
 292 
 293     // Load up thread register
 294     __ movptr(r15_thread, thread);
 295     __ reinit_heapbase();
 296 
 297 #ifdef ASSERT
 298     // make sure we have no pending exceptions
 299     {
 300       Label L;
 301       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 302       __ jcc(Assembler::equal, L);
 303       __ stop("StubRoutines::call_stub: entered with pending exception");
 304       __ bind(L);
 305     }
 306 #endif
 307 
 308     // pass parameters if any
 309     BLOCK_COMMENT("pass parameters if any");
 310     Label parameters_done;
 311     __ movl(c_rarg3, parameter_size);
 312     __ testl(c_rarg3, c_rarg3);
 313     __ jcc(Assembler::zero, parameters_done);
 314 
 315     Label loop;
 316     __ movptr(c_rarg2, parameters);       // parameter pointer
 317     __ movl(c_rarg1, c_rarg3);            // parameter counter is in c_rarg1
 318     __ BIND(loop);
 319     __ movptr(rax, Address(c_rarg2, 0));// get parameter
 320     __ addptr(c_rarg2, wordSize);       // advance to next parameter
 321     __ decrementl(c_rarg1);             // decrement counter
 322     __ push(rax);                       // pass parameter
 323     __ jcc(Assembler::notZero, loop);
 324 
 325     // call Java function
 326     __ BIND(parameters_done);
 327     __ movptr(rbx, method);             // get Method*
 328     __ movptr(c_rarg1, entry_point);    // get entry_point
 329     __ mov(r13, rsp);                   // set sender sp
 330     BLOCK_COMMENT("call Java function");
 331     __ call(c_rarg1);
 332 
 333     BLOCK_COMMENT("call_stub_return_address:");
 334     return_address = __ pc();
 335 
 336     // store result depending on type (everything that is not
 337     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 338     __ movptr(c_rarg0, result);
 339     Label is_long, is_float, is_double, exit;
 340     __ movl(c_rarg1, result_type);
 341     __ cmpl(c_rarg1, T_OBJECT);
 342     __ jcc(Assembler::equal, is_long);
 343     __ cmpl(c_rarg1, T_LONG);
 344     __ jcc(Assembler::equal, is_long);
 345     __ cmpl(c_rarg1, T_FLOAT);
 346     __ jcc(Assembler::equal, is_float);
 347     __ cmpl(c_rarg1, T_DOUBLE);
 348     __ jcc(Assembler::equal, is_double);
 349 
 350     // handle T_INT case
 351     __ movl(Address(c_rarg0, 0), rax);
 352 
 353     __ BIND(exit);
 354 
 355     // pop parameters
 356     __ lea(rsp, rsp_after_call);
 357 
 358 #ifdef ASSERT
 359     // verify that threads correspond
 360     {
 361      Label L1, L2, L3;
 362       __ cmpptr(r15_thread, thread);
 363       __ jcc(Assembler::equal, L1);
 364       __ stop("StubRoutines::call_stub: r15_thread is corrupted");
 365       __ bind(L1);
 366       __ get_thread(rbx);
 367       __ cmpptr(r15_thread, thread);
 368       __ jcc(Assembler::equal, L2);
 369       __ stop("StubRoutines::call_stub: r15_thread is modified by call");
 370       __ bind(L2);
 371       __ cmpptr(r15_thread, rbx);
 372       __ jcc(Assembler::equal, L3);
 373       __ stop("StubRoutines::call_stub: threads must correspond");
 374       __ bind(L3);
 375     }
 376 #endif
 377 
 378     // restore regs belonging to calling function
 379 #ifdef _WIN64
 380     // emit the restores for xmm regs
 381     if (VM_Version::supports_evex()) {
 382       for (int i = xmm_save_first; i <= last_reg; i++) {
 383         __ vinsertf32x4(as_XMMRegister(i), as_XMMRegister(i), xmm_save(i), 0);
 384       }
 385     } else {
 386       for (int i = xmm_save_first; i <= last_reg; i++) {
 387         __ movdqu(as_XMMRegister(i), xmm_save(i));
 388       }
 389     }
 390 #endif
 391     __ movptr(r15, r15_save);
 392     __ movptr(r14, r14_save);
 393     __ movptr(r13, r13_save);
 394     __ movptr(r12, r12_save);
 395     __ movptr(rbx, rbx_save);
 396 
 397 #ifdef _WIN64
 398     __ movptr(rdi, rdi_save);
 399     __ movptr(rsi, rsi_save);
 400 #else
 401     __ ldmxcsr(mxcsr_save);
 402 #endif
 403 
 404     // restore rsp
 405     __ addptr(rsp, -rsp_after_call_off * wordSize);
 406 
 407     // return
 408     __ vzeroupper();
 409     __ pop(rbp);
 410     __ ret(0);
 411 
 412     // handle return types different from T_INT
 413     __ BIND(is_long);
 414     __ movq(Address(c_rarg0, 0), rax);
 415     __ jmp(exit);
 416 
 417     __ BIND(is_float);
 418     __ movflt(Address(c_rarg0, 0), xmm0);
 419     __ jmp(exit);
 420 
 421     __ BIND(is_double);
 422     __ movdbl(Address(c_rarg0, 0), xmm0);
 423     __ jmp(exit);
 424 
 425     return start;
 426   }
 427 
 428   // Return point for a Java call if there's an exception thrown in
 429   // Java code.  The exception is caught and transformed into a
 430   // pending exception stored in JavaThread that can be tested from
 431   // within the VM.
 432   //
 433   // Note: Usually the parameters are removed by the callee. In case
 434   // of an exception crossing an activation frame boundary, that is
 435   // not the case if the callee is compiled code => need to setup the
 436   // rsp.
 437   //
 438   // rax: exception oop
 439 
 440   address generate_catch_exception() {
 441     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 442     address start = __ pc();
 443 
 444     // same as in generate_call_stub():
 445     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 446     const Address thread        (rbp, thread_off         * wordSize);
 447 
 448 #ifdef ASSERT
 449     // verify that threads correspond
 450     {
 451       Label L1, L2, L3;
 452       __ cmpptr(r15_thread, thread);
 453       __ jcc(Assembler::equal, L1);
 454       __ stop("StubRoutines::catch_exception: r15_thread is corrupted");
 455       __ bind(L1);
 456       __ get_thread(rbx);
 457       __ cmpptr(r15_thread, thread);
 458       __ jcc(Assembler::equal, L2);
 459       __ stop("StubRoutines::catch_exception: r15_thread is modified by call");
 460       __ bind(L2);
 461       __ cmpptr(r15_thread, rbx);
 462       __ jcc(Assembler::equal, L3);
 463       __ stop("StubRoutines::catch_exception: threads must correspond");
 464       __ bind(L3);
 465     }
 466 #endif
 467 
 468     // set pending exception
 469     __ verify_oop(rax);
 470 
 471     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax);
 472     __ lea(rscratch1, ExternalAddress((address)__FILE__));
 473     __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
 474     __ movl(Address(r15_thread, Thread::exception_line_offset()), (int)  __LINE__);
 475 
 476     // complete return to VM
 477     assert(StubRoutines::_call_stub_return_address != NULL,
 478            "_call_stub_return_address must have been generated before");
 479     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 480 
 481     return start;
 482   }
 483 
 484   // Continuation point for runtime calls returning with a pending
 485   // exception.  The pending exception check happened in the runtime
 486   // or native call stub.  The pending exception in Thread is
 487   // converted into a Java-level exception.
 488   //
 489   // Contract with Java-level exception handlers:
 490   // rax: exception
 491   // rdx: throwing pc
 492   //
 493   // NOTE: At entry of this stub, exception-pc must be on stack !!
 494 
 495   address generate_forward_exception() {
 496     StubCodeMark mark(this, "StubRoutines", "forward exception");
 497     address start = __ pc();
 498 
 499     // Upon entry, the sp points to the return address returning into
 500     // Java (interpreted or compiled) code; i.e., the return address
 501     // becomes the throwing pc.
 502     //
 503     // Arguments pushed before the runtime call are still on the stack
 504     // but the exception handler will reset the stack pointer ->
 505     // ignore them.  A potential result in registers can be ignored as
 506     // well.
 507 
 508 #ifdef ASSERT
 509     // make sure this code is only executed if there is a pending exception
 510     {
 511       Label L;
 512       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL);
 513       __ jcc(Assembler::notEqual, L);
 514       __ stop("StubRoutines::forward exception: no pending exception (1)");
 515       __ bind(L);
 516     }
 517 #endif
 518 
 519     // compute exception handler into rbx
 520     __ movptr(c_rarg0, Address(rsp, 0));
 521     BLOCK_COMMENT("call exception_handler_for_return_address");
 522     __ call_VM_leaf(CAST_FROM_FN_PTR(address,
 523                          SharedRuntime::exception_handler_for_return_address),
 524                     r15_thread, c_rarg0);
 525     __ mov(rbx, rax);
 526 
 527     // setup rax & rdx, remove return address & clear pending exception
 528     __ pop(rdx);
 529     __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
 530     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 531 
 532 #ifdef ASSERT
 533     // make sure exception is set
 534     {
 535       Label L;
 536       __ testptr(rax, rax);
 537       __ jcc(Assembler::notEqual, L);
 538       __ stop("StubRoutines::forward exception: no pending exception (2)");
 539       __ bind(L);
 540     }
 541 #endif
 542 
 543     // continue at exception handler (return address removed)
 544     // rax: exception
 545     // rbx: exception handler
 546     // rdx: throwing pc
 547     __ verify_oop(rax);
 548     __ jmp(rbx);
 549 
 550     return start;
 551   }
 552 
 553   // Support for jint atomic::xchg(jint exchange_value, volatile jint* dest)
 554   //
 555   // Arguments :
 556   //    c_rarg0: exchange_value
 557   //    c_rarg0: dest
 558   //
 559   // Result:
 560   //    *dest <- ex, return (orig *dest)
 561   address generate_atomic_xchg() {
 562     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 563     address start = __ pc();
 564 
 565     __ movl(rax, c_rarg0); // Copy to eax we need a return value anyhow
 566     __ xchgl(rax, Address(c_rarg1, 0)); // automatic LOCK
 567     __ ret(0);
 568 
 569     return start;
 570   }
 571 
 572   // Support for intptr_t atomic::xchg_long(jlong exchange_value, volatile jlong* dest)
 573   //
 574   // Arguments :
 575   //    c_rarg0: exchange_value
 576   //    c_rarg1: dest
 577   //
 578   // Result:
 579   //    *dest <- ex, return (orig *dest)
 580   address generate_atomic_xchg_long() {
 581     StubCodeMark mark(this, "StubRoutines", "atomic_xchg_long");
 582     address start = __ pc();
 583 
 584     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 585     __ xchgptr(rax, Address(c_rarg1, 0)); // automatic LOCK
 586     __ ret(0);
 587 
 588     return start;
 589   }
 590 
 591   // Support for jint atomic::atomic_cmpxchg(jint exchange_value, volatile jint* dest,
 592   //                                         jint compare_value)
 593   //
 594   // Arguments :
 595   //    c_rarg0: exchange_value
 596   //    c_rarg1: dest
 597   //    c_rarg2: compare_value
 598   //
 599   // Result:
 600   //    if ( compare_value == *dest ) {
 601   //       *dest = exchange_value
 602   //       return compare_value;
 603   //    else
 604   //       return *dest;
 605   address generate_atomic_cmpxchg() {
 606     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg");
 607     address start = __ pc();
 608 
 609     __ movl(rax, c_rarg2);
 610    if ( os::is_MP() ) __ lock();
 611     __ cmpxchgl(c_rarg0, Address(c_rarg1, 0));
 612     __ ret(0);
 613 
 614     return start;
 615   }
 616 
 617   // Support for int8_t atomic::atomic_cmpxchg(int8_t exchange_value, volatile int8_t* dest,
 618   //                                           int8_t compare_value)
 619   //
 620   // Arguments :
 621   //    c_rarg0: exchange_value
 622   //    c_rarg1: dest
 623   //    c_rarg2: compare_value
 624   //
 625   // Result:
 626   //    if ( compare_value == *dest ) {
 627   //       *dest = exchange_value
 628   //       return compare_value;
 629   //    else
 630   //       return *dest;
 631   address generate_atomic_cmpxchg_byte() {
 632     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_byte");
 633     address start = __ pc();
 634 
 635     __ movsbq(rax, c_rarg2);
 636    if ( os::is_MP() ) __ lock();
 637     __ cmpxchgb(c_rarg0, Address(c_rarg1, 0));
 638     __ ret(0);
 639 
 640     return start;
 641   }
 642 
 643   // Support for int64_t atomic::atomic_cmpxchg(int64_t exchange_value,
 644   //                                            volatile int64_t* dest,
 645   //                                            int64_t compare_value)
 646   // Arguments :
 647   //    c_rarg0: exchange_value
 648   //    c_rarg1: dest
 649   //    c_rarg2: compare_value
 650   //
 651   // Result:
 652   //    if ( compare_value == *dest ) {
 653   //       *dest = exchange_value
 654   //       return compare_value;
 655   //    else
 656   //       return *dest;
 657   address generate_atomic_cmpxchg_long() {
 658     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long");
 659     address start = __ pc();
 660 
 661     __ movq(rax, c_rarg2);
 662    if ( os::is_MP() ) __ lock();
 663     __ cmpxchgq(c_rarg0, Address(c_rarg1, 0));
 664     __ ret(0);
 665 
 666     return start;
 667   }
 668 
 669   // Support for jint atomic::add(jint add_value, volatile jint* dest)
 670   //
 671   // Arguments :
 672   //    c_rarg0: add_value
 673   //    c_rarg1: dest
 674   //
 675   // Result:
 676   //    *dest += add_value
 677   //    return *dest;
 678   address generate_atomic_add() {
 679     StubCodeMark mark(this, "StubRoutines", "atomic_add");
 680     address start = __ pc();
 681 
 682     __ movl(rax, c_rarg0);
 683    if ( os::is_MP() ) __ lock();
 684     __ xaddl(Address(c_rarg1, 0), c_rarg0);
 685     __ addl(rax, c_rarg0);
 686     __ ret(0);
 687 
 688     return start;
 689   }
 690 
 691   // Support for intptr_t atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest)
 692   //
 693   // Arguments :
 694   //    c_rarg0: add_value
 695   //    c_rarg1: dest
 696   //
 697   // Result:
 698   //    *dest += add_value
 699   //    return *dest;
 700   address generate_atomic_add_long() {
 701     StubCodeMark mark(this, "StubRoutines", "atomic_add_long");
 702     address start = __ pc();
 703 
 704     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 705    if ( os::is_MP() ) __ lock();
 706     __ xaddptr(Address(c_rarg1, 0), c_rarg0);
 707     __ addptr(rax, c_rarg0);
 708     __ ret(0);
 709 
 710     return start;
 711   }
 712 
 713   // Support for intptr_t OrderAccess::fence()
 714   //
 715   // Arguments :
 716   //
 717   // Result:
 718   address generate_orderaccess_fence() {
 719     StubCodeMark mark(this, "StubRoutines", "orderaccess_fence");
 720     address start = __ pc();
 721     __ membar(Assembler::StoreLoad);
 722     __ ret(0);
 723 
 724     return start;
 725   }
 726 
 727   // Support for intptr_t get_previous_fp()
 728   //
 729   // This routine is used to find the previous frame pointer for the
 730   // caller (current_frame_guess). This is used as part of debugging
 731   // ps() is seemingly lost trying to find frames.
 732   // This code assumes that caller current_frame_guess) has a frame.
 733   address generate_get_previous_fp() {
 734     StubCodeMark mark(this, "StubRoutines", "get_previous_fp");
 735     const Address old_fp(rbp, 0);
 736     const Address older_fp(rax, 0);
 737     address start = __ pc();
 738 
 739     __ enter();
 740     __ movptr(rax, old_fp); // callers fp
 741     __ movptr(rax, older_fp); // the frame for ps()
 742     __ pop(rbp);
 743     __ ret(0);
 744 
 745     return start;
 746   }
 747 
 748   // Support for intptr_t get_previous_sp()
 749   //
 750   // This routine is used to find the previous stack pointer for the
 751   // caller.
 752   address generate_get_previous_sp() {
 753     StubCodeMark mark(this, "StubRoutines", "get_previous_sp");
 754     address start = __ pc();
 755 
 756     __ movptr(rax, rsp);
 757     __ addptr(rax, 8); // return address is at the top of the stack.
 758     __ ret(0);
 759 
 760     return start;
 761   }
 762 
 763   //----------------------------------------------------------------------------------------------------
 764   // Support for void verify_mxcsr()
 765   //
 766   // This routine is used with -Xcheck:jni to verify that native
 767   // JNI code does not return to Java code without restoring the
 768   // MXCSR register to our expected state.
 769 
 770   address generate_verify_mxcsr() {
 771     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 772     address start = __ pc();
 773 
 774     const Address mxcsr_save(rsp, 0);
 775 
 776     if (CheckJNICalls) {
 777       Label ok_ret;
 778       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 779       __ push(rax);
 780       __ subptr(rsp, wordSize);      // allocate a temp location
 781       __ stmxcsr(mxcsr_save);
 782       __ movl(rax, mxcsr_save);
 783       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 784       __ cmp32(rax, mxcsr_std);
 785       __ jcc(Assembler::equal, ok_ret);
 786 
 787       __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
 788 
 789       __ ldmxcsr(mxcsr_std);
 790 
 791       __ bind(ok_ret);
 792       __ addptr(rsp, wordSize);
 793       __ pop(rax);
 794     }
 795 
 796     __ ret(0);
 797 
 798     return start;
 799   }
 800 
 801   address generate_f2i_fixup() {
 802     StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
 803     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 804 
 805     address start = __ pc();
 806 
 807     Label L;
 808 
 809     __ push(rax);
 810     __ push(c_rarg3);
 811     __ push(c_rarg2);
 812     __ push(c_rarg1);
 813 
 814     __ movl(rax, 0x7f800000);
 815     __ xorl(c_rarg3, c_rarg3);
 816     __ movl(c_rarg2, inout);
 817     __ movl(c_rarg1, c_rarg2);
 818     __ andl(c_rarg1, 0x7fffffff);
 819     __ cmpl(rax, c_rarg1); // NaN? -> 0
 820     __ jcc(Assembler::negative, L);
 821     __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint
 822     __ movl(c_rarg3, 0x80000000);
 823     __ movl(rax, 0x7fffffff);
 824     __ cmovl(Assembler::positive, c_rarg3, rax);
 825 
 826     __ bind(L);
 827     __ movptr(inout, c_rarg3);
 828 
 829     __ pop(c_rarg1);
 830     __ pop(c_rarg2);
 831     __ pop(c_rarg3);
 832     __ pop(rax);
 833 
 834     __ ret(0);
 835 
 836     return start;
 837   }
 838 
 839   address generate_f2l_fixup() {
 840     StubCodeMark mark(this, "StubRoutines", "f2l_fixup");
 841     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 842     address start = __ pc();
 843 
 844     Label L;
 845 
 846     __ push(rax);
 847     __ push(c_rarg3);
 848     __ push(c_rarg2);
 849     __ push(c_rarg1);
 850 
 851     __ movl(rax, 0x7f800000);
 852     __ xorl(c_rarg3, c_rarg3);
 853     __ movl(c_rarg2, inout);
 854     __ movl(c_rarg1, c_rarg2);
 855     __ andl(c_rarg1, 0x7fffffff);
 856     __ cmpl(rax, c_rarg1); // NaN? -> 0
 857     __ jcc(Assembler::negative, L);
 858     __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong
 859     __ mov64(c_rarg3, 0x8000000000000000);
 860     __ mov64(rax, 0x7fffffffffffffff);
 861     __ cmov(Assembler::positive, c_rarg3, rax);
 862 
 863     __ bind(L);
 864     __ movptr(inout, c_rarg3);
 865 
 866     __ pop(c_rarg1);
 867     __ pop(c_rarg2);
 868     __ pop(c_rarg3);
 869     __ pop(rax);
 870 
 871     __ ret(0);
 872 
 873     return start;
 874   }
 875 
 876   address generate_d2i_fixup() {
 877     StubCodeMark mark(this, "StubRoutines", "d2i_fixup");
 878     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 879 
 880     address start = __ pc();
 881 
 882     Label L;
 883 
 884     __ push(rax);
 885     __ push(c_rarg3);
 886     __ push(c_rarg2);
 887     __ push(c_rarg1);
 888     __ push(c_rarg0);
 889 
 890     __ movl(rax, 0x7ff00000);
 891     __ movq(c_rarg2, inout);
 892     __ movl(c_rarg3, c_rarg2);
 893     __ mov(c_rarg1, c_rarg2);
 894     __ mov(c_rarg0, c_rarg2);
 895     __ negl(c_rarg3);
 896     __ shrptr(c_rarg1, 0x20);
 897     __ orl(c_rarg3, c_rarg2);
 898     __ andl(c_rarg1, 0x7fffffff);
 899     __ xorl(c_rarg2, c_rarg2);
 900     __ shrl(c_rarg3, 0x1f);
 901     __ orl(c_rarg1, c_rarg3);
 902     __ cmpl(rax, c_rarg1);
 903     __ jcc(Assembler::negative, L); // NaN -> 0
 904     __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint
 905     __ movl(c_rarg2, 0x80000000);
 906     __ movl(rax, 0x7fffffff);
 907     __ cmov(Assembler::positive, c_rarg2, rax);
 908 
 909     __ bind(L);
 910     __ movptr(inout, c_rarg2);
 911 
 912     __ pop(c_rarg0);
 913     __ pop(c_rarg1);
 914     __ pop(c_rarg2);
 915     __ pop(c_rarg3);
 916     __ pop(rax);
 917 
 918     __ ret(0);
 919 
 920     return start;
 921   }
 922 
 923   address generate_d2l_fixup() {
 924     StubCodeMark mark(this, "StubRoutines", "d2l_fixup");
 925     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 926 
 927     address start = __ pc();
 928 
 929     Label L;
 930 
 931     __ push(rax);
 932     __ push(c_rarg3);
 933     __ push(c_rarg2);
 934     __ push(c_rarg1);
 935     __ push(c_rarg0);
 936 
 937     __ movl(rax, 0x7ff00000);
 938     __ movq(c_rarg2, inout);
 939     __ movl(c_rarg3, c_rarg2);
 940     __ mov(c_rarg1, c_rarg2);
 941     __ mov(c_rarg0, c_rarg2);
 942     __ negl(c_rarg3);
 943     __ shrptr(c_rarg1, 0x20);
 944     __ orl(c_rarg3, c_rarg2);
 945     __ andl(c_rarg1, 0x7fffffff);
 946     __ xorl(c_rarg2, c_rarg2);
 947     __ shrl(c_rarg3, 0x1f);
 948     __ orl(c_rarg1, c_rarg3);
 949     __ cmpl(rax, c_rarg1);
 950     __ jcc(Assembler::negative, L); // NaN -> 0
 951     __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
 952     __ mov64(c_rarg2, 0x8000000000000000);
 953     __ mov64(rax, 0x7fffffffffffffff);
 954     __ cmovq(Assembler::positive, c_rarg2, rax);
 955 
 956     __ bind(L);
 957     __ movq(inout, c_rarg2);
 958 
 959     __ pop(c_rarg0);
 960     __ pop(c_rarg1);
 961     __ pop(c_rarg2);
 962     __ pop(c_rarg3);
 963     __ pop(rax);
 964 
 965     __ ret(0);
 966 
 967     return start;
 968   }
 969 
 970   address generate_fp_mask(const char *stub_name, int64_t mask) {
 971     __ align(CodeEntryAlignment);
 972     StubCodeMark mark(this, "StubRoutines", stub_name);
 973     address start = __ pc();
 974 
 975     __ emit_data64( mask, relocInfo::none );
 976     __ emit_data64( mask, relocInfo::none );
 977 
 978     return start;
 979   }
 980 
 981   // Non-destructive plausibility checks for oops
 982   //
 983   // Arguments:
 984   //    all args on stack!
 985   //
 986   // Stack after saving c_rarg3:
 987   //    [tos + 0]: saved c_rarg3
 988   //    [tos + 1]: saved c_rarg2
 989   //    [tos + 2]: saved r12 (several TemplateTable methods use it)
 990   //    [tos + 3]: saved flags
 991   //    [tos + 4]: return address
 992   //  * [tos + 5]: error message (char*)
 993   //  * [tos + 6]: object to verify (oop)
 994   //  * [tos + 7]: saved rax - saved by caller and bashed
 995   //  * [tos + 8]: saved r10 (rscratch1) - saved by caller
 996   //  * = popped on exit
 997   address generate_verify_oop() {
 998     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 999     address start = __ pc();
1000 
1001     Label exit, error;
1002 
1003     __ pushf();
1004     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
1005 
1006     __ push(r12);
1007 
1008     // save c_rarg2 and c_rarg3
1009     __ push(c_rarg2);
1010     __ push(c_rarg3);
1011 
1012     enum {
1013            // After previous pushes.
1014            oop_to_verify = 6 * wordSize,
1015            saved_rax     = 7 * wordSize,
1016            saved_r10     = 8 * wordSize,
1017 
1018            // Before the call to MacroAssembler::debug(), see below.
1019            return_addr   = 16 * wordSize,
1020            error_msg     = 17 * wordSize
1021     };
1022 
1023     // get object
1024     __ movptr(rax, Address(rsp, oop_to_verify));
1025 
1026     // make sure object is 'reasonable'
1027     __ testptr(rax, rax);
1028     __ jcc(Assembler::zero, exit); // if obj is NULL it is OK
1029     // Check if the oop is in the right area of memory
1030     __ movptr(c_rarg2, rax);
1031     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask());
1032     __ andptr(c_rarg2, c_rarg3);
1033     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_bits());
1034     __ cmpptr(c_rarg2, c_rarg3);
1035     __ jcc(Assembler::notZero, error);
1036 
1037     // set r12 to heapbase for load_klass()
1038     __ reinit_heapbase();
1039 
1040     // make sure klass is 'reasonable', which is not zero.
1041     __ load_klass(rax, rax);  // get klass
1042     __ testptr(rax, rax);
1043     __ jcc(Assembler::zero, error); // if klass is NULL it is broken
1044 
1045     // return if everything seems ok
1046     __ bind(exit);
1047     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
1048     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1049     __ pop(c_rarg3);                             // restore c_rarg3
1050     __ pop(c_rarg2);                             // restore c_rarg2
1051     __ pop(r12);                                 // restore r12
1052     __ popf();                                   // restore flags
1053     __ ret(4 * wordSize);                        // pop caller saved stuff
1054 
1055     // handle errors
1056     __ bind(error);
1057     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
1058     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1059     __ pop(c_rarg3);                             // get saved c_rarg3 back
1060     __ pop(c_rarg2);                             // get saved c_rarg2 back
1061     __ pop(r12);                                 // get saved r12 back
1062     __ popf();                                   // get saved flags off stack --
1063                                                  // will be ignored
1064 
1065     __ pusha();                                  // push registers
1066                                                  // (rip is already
1067                                                  // already pushed)
1068     // debug(char* msg, int64_t pc, int64_t regs[])
1069     // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and
1070     // pushed all the registers, so now the stack looks like:
1071     //     [tos +  0] 16 saved registers
1072     //     [tos + 16] return address
1073     //   * [tos + 17] error message (char*)
1074     //   * [tos + 18] object to verify (oop)
1075     //   * [tos + 19] saved rax - saved by caller and bashed
1076     //   * [tos + 20] saved r10 (rscratch1) - saved by caller
1077     //   * = popped on exit
1078 
1079     __ movptr(c_rarg0, Address(rsp, error_msg));    // pass address of error message
1080     __ movptr(c_rarg1, Address(rsp, return_addr));  // pass return address
1081     __ movq(c_rarg2, rsp);                          // pass address of regs on stack
1082     __ mov(r12, rsp);                               // remember rsp
1083     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1084     __ andptr(rsp, -16);                            // align stack as required by ABI
1085     BLOCK_COMMENT("call MacroAssembler::debug");
1086     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
1087     __ mov(rsp, r12);                               // restore rsp
1088     __ popa();                                      // pop registers (includes r12)
1089     __ ret(4 * wordSize);                           // pop caller saved stuff
1090 
1091     return start;
1092   }
1093 
1094   //
1095   // Verify that a register contains clean 32-bits positive value
1096   // (high 32-bits are 0) so it could be used in 64-bits shifts.
1097   //
1098   //  Input:
1099   //    Rint  -  32-bits value
1100   //    Rtmp  -  scratch
1101   //
1102   void assert_clean_int(Register Rint, Register Rtmp) {
1103 #ifdef ASSERT
1104     Label L;
1105     assert_different_registers(Rtmp, Rint);
1106     __ movslq(Rtmp, Rint);
1107     __ cmpq(Rtmp, Rint);
1108     __ jcc(Assembler::equal, L);
1109     __ stop("high 32-bits of int value are not 0");
1110     __ bind(L);
1111 #endif
1112   }
1113 
1114   //  Generate overlap test for array copy stubs
1115   //
1116   //  Input:
1117   //     c_rarg0 - from
1118   //     c_rarg1 - to
1119   //     c_rarg2 - element count
1120   //
1121   //  Output:
1122   //     rax   - &from[element count - 1]
1123   //
1124   void array_overlap_test(address no_overlap_target, Address::ScaleFactor sf) {
1125     assert(no_overlap_target != NULL, "must be generated");
1126     array_overlap_test(no_overlap_target, NULL, sf);
1127   }
1128   void array_overlap_test(Label& L_no_overlap, Address::ScaleFactor sf) {
1129     array_overlap_test(NULL, &L_no_overlap, sf);
1130   }
1131   void array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) {
1132     const Register from     = c_rarg0;
1133     const Register to       = c_rarg1;
1134     const Register count    = c_rarg2;
1135     const Register end_from = rax;
1136 
1137     __ cmpptr(to, from);
1138     __ lea(end_from, Address(from, count, sf, 0));
1139     if (NOLp == NULL) {
1140       ExternalAddress no_overlap(no_overlap_target);
1141       __ jump_cc(Assembler::belowEqual, no_overlap);
1142       __ cmpptr(to, end_from);
1143       __ jump_cc(Assembler::aboveEqual, no_overlap);
1144     } else {
1145       __ jcc(Assembler::belowEqual, (*NOLp));
1146       __ cmpptr(to, end_from);
1147       __ jcc(Assembler::aboveEqual, (*NOLp));
1148     }
1149   }
1150 
1151   // Shuffle first three arg regs on Windows into Linux/Solaris locations.
1152   //
1153   // Outputs:
1154   //    rdi - rcx
1155   //    rsi - rdx
1156   //    rdx - r8
1157   //    rcx - r9
1158   //
1159   // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter
1160   // are non-volatile.  r9 and r10 should not be used by the caller.
1161   //
1162   void setup_arg_regs(int nargs = 3) {
1163     const Register saved_rdi = r9;
1164     const Register saved_rsi = r10;
1165     assert(nargs == 3 || nargs == 4, "else fix");
1166 #ifdef _WIN64
1167     assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,
1168            "unexpected argument registers");
1169     if (nargs >= 4)
1170       __ mov(rax, r9);  // r9 is also saved_rdi
1171     __ movptr(saved_rdi, rdi);
1172     __ movptr(saved_rsi, rsi);
1173     __ mov(rdi, rcx); // c_rarg0
1174     __ mov(rsi, rdx); // c_rarg1
1175     __ mov(rdx, r8);  // c_rarg2
1176     if (nargs >= 4)
1177       __ mov(rcx, rax); // c_rarg3 (via rax)
1178 #else
1179     assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,
1180            "unexpected argument registers");
1181 #endif
1182   }
1183 
1184   void restore_arg_regs() {
1185     const Register saved_rdi = r9;
1186     const Register saved_rsi = r10;
1187 #ifdef _WIN64
1188     __ movptr(rdi, saved_rdi);
1189     __ movptr(rsi, saved_rsi);
1190 #endif
1191   }
1192 
1193 
1194   // Copy big chunks forward
1195   //
1196   // Inputs:
1197   //   end_from     - source arrays end address
1198   //   end_to       - destination array end address
1199   //   qword_count  - 64-bits element count, negative
1200   //   to           - scratch
1201   //   L_copy_bytes - entry label
1202   //   L_copy_8_bytes  - exit  label
1203   //
1204   void copy_bytes_forward(Register end_from, Register end_to,
1205                              Register qword_count, Register to,
1206                              Label& L_copy_bytes, Label& L_copy_8_bytes) {
1207     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1208     Label L_loop;
1209     __ align(OptoLoopAlignment);
1210     if (UseUnalignedLoadStores) {
1211       Label L_end;
1212       if (UseAVX > 2) {
1213         __ movl(to, 0xffff);
1214         __ kmovwl(k1, to);
1215       }
1216       // Copy 64-bytes per iteration
1217       __ BIND(L_loop);
1218       if (UseAVX > 2) {
1219         __ evmovdqul(xmm0, Address(end_from, qword_count, Address::times_8, -56), Assembler::AVX_512bit);
1220         __ evmovdqul(Address(end_to, qword_count, Address::times_8, -56), xmm0, Assembler::AVX_512bit);
1221       } else if (UseAVX == 2) {
1222         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1223         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1224         __ vmovdqu(xmm1, Address(end_from, qword_count, Address::times_8, -24));
1225         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm1);
1226       } else {
1227         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1228         __ movdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1229         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, -40));
1230         __ movdqu(Address(end_to, qword_count, Address::times_8, -40), xmm1);
1231         __ movdqu(xmm2, Address(end_from, qword_count, Address::times_8, -24));
1232         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm2);
1233         __ movdqu(xmm3, Address(end_from, qword_count, Address::times_8, - 8));
1234         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm3);
1235       }
1236       __ BIND(L_copy_bytes);
1237       __ addptr(qword_count, 8);
1238       __ jcc(Assembler::lessEqual, L_loop);
1239       __ subptr(qword_count, 4);  // sub(8) and add(4)
1240       __ jccb(Assembler::greater, L_end);
1241       // Copy trailing 32 bytes
1242       if (UseAVX >= 2) {
1243         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1244         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1245       } else {
1246         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1247         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1248         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, - 8));
1249         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm1);
1250       }
1251       __ addptr(qword_count, 4);
1252       __ BIND(L_end);
1253       if (UseAVX >= 2) {
1254         // clean upper bits of YMM registers
1255         __ vpxor(xmm0, xmm0);
1256         __ vpxor(xmm1, xmm1);
1257       }
1258     } else {
1259       // Copy 32-bytes per iteration
1260       __ BIND(L_loop);
1261       __ movq(to, Address(end_from, qword_count, Address::times_8, -24));
1262       __ movq(Address(end_to, qword_count, Address::times_8, -24), to);
1263       __ movq(to, Address(end_from, qword_count, Address::times_8, -16));
1264       __ movq(Address(end_to, qword_count, Address::times_8, -16), to);
1265       __ movq(to, Address(end_from, qword_count, Address::times_8, - 8));
1266       __ movq(Address(end_to, qword_count, Address::times_8, - 8), to);
1267       __ movq(to, Address(end_from, qword_count, Address::times_8, - 0));
1268       __ movq(Address(end_to, qword_count, Address::times_8, - 0), to);
1269 
1270       __ BIND(L_copy_bytes);
1271       __ addptr(qword_count, 4);
1272       __ jcc(Assembler::lessEqual, L_loop);
1273     }
1274     __ subptr(qword_count, 4);
1275     __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords
1276   }
1277 
1278   // Copy big chunks backward
1279   //
1280   // Inputs:
1281   //   from         - source arrays address
1282   //   dest         - destination array address
1283   //   qword_count  - 64-bits element count
1284   //   to           - scratch
1285   //   L_copy_bytes - entry label
1286   //   L_copy_8_bytes  - exit  label
1287   //
1288   void copy_bytes_backward(Register from, Register dest,
1289                               Register qword_count, Register to,
1290                               Label& L_copy_bytes, Label& L_copy_8_bytes) {
1291     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1292     Label L_loop;
1293     __ align(OptoLoopAlignment);
1294     if (UseUnalignedLoadStores) {
1295       Label L_end;
1296       if (UseAVX > 2) {
1297         __ movl(to, 0xffff);
1298         __ kmovwl(k1, to);
1299       }
1300       // Copy 64-bytes per iteration
1301       __ BIND(L_loop);
1302       if (UseAVX > 2) {
1303         __ evmovdqul(xmm0, Address(from, qword_count, Address::times_8, 0), Assembler::AVX_512bit);
1304         __ evmovdqul(Address(dest, qword_count, Address::times_8, 0), xmm0, Assembler::AVX_512bit);
1305       } else if (UseAVX == 2) {
1306         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 32));
1307         __ vmovdqu(Address(dest, qword_count, Address::times_8, 32), xmm0);
1308         __ vmovdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
1309         __ vmovdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
1310       } else {
1311         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 48));
1312         __ movdqu(Address(dest, qword_count, Address::times_8, 48), xmm0);
1313         __ movdqu(xmm1, Address(from, qword_count, Address::times_8, 32));
1314         __ movdqu(Address(dest, qword_count, Address::times_8, 32), xmm1);
1315         __ movdqu(xmm2, Address(from, qword_count, Address::times_8, 16));
1316         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm2);
1317         __ movdqu(xmm3, Address(from, qword_count, Address::times_8,  0));
1318         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm3);
1319       }
1320       __ BIND(L_copy_bytes);
1321       __ subptr(qword_count, 8);
1322       __ jcc(Assembler::greaterEqual, L_loop);
1323 
1324       __ addptr(qword_count, 4);  // add(8) and sub(4)
1325       __ jccb(Assembler::less, L_end);
1326       // Copy trailing 32 bytes
1327       if (UseAVX >= 2) {
1328         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 0));
1329         __ vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm0);
1330       } else {
1331         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 16));
1332         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm0);
1333         __ movdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
1334         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
1335       }
1336       __ subptr(qword_count, 4);
1337       __ BIND(L_end);
1338       if (UseAVX >= 2) {
1339         // clean upper bits of YMM registers
1340         __ vpxor(xmm0, xmm0);
1341         __ vpxor(xmm1, xmm1);
1342       }
1343     } else {
1344       // Copy 32-bytes per iteration
1345       __ BIND(L_loop);
1346       __ movq(to, Address(from, qword_count, Address::times_8, 24));
1347       __ movq(Address(dest, qword_count, Address::times_8, 24), to);
1348       __ movq(to, Address(from, qword_count, Address::times_8, 16));
1349       __ movq(Address(dest, qword_count, Address::times_8, 16), to);
1350       __ movq(to, Address(from, qword_count, Address::times_8,  8));
1351       __ movq(Address(dest, qword_count, Address::times_8,  8), to);
1352       __ movq(to, Address(from, qword_count, Address::times_8,  0));
1353       __ movq(Address(dest, qword_count, Address::times_8,  0), to);
1354 
1355       __ BIND(L_copy_bytes);
1356       __ subptr(qword_count, 4);
1357       __ jcc(Assembler::greaterEqual, L_loop);
1358     }
1359     __ addptr(qword_count, 4);
1360     __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
1361   }
1362 
1363 
1364   // Arguments:
1365   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1366   //             ignored
1367   //   name    - stub name string
1368   //
1369   // Inputs:
1370   //   c_rarg0   - source array address
1371   //   c_rarg1   - destination array address
1372   //   c_rarg2   - element count, treated as ssize_t, can be zero
1373   //
1374   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1375   // we let the hardware handle it.  The one to eight bytes within words,
1376   // dwords or qwords that span cache line boundaries will still be loaded
1377   // and stored atomically.
1378   //
1379   // Side Effects:
1380   //   disjoint_byte_copy_entry is set to the no-overlap entry point
1381   //   used by generate_conjoint_byte_copy().
1382   //
1383   address generate_disjoint_byte_copy(bool aligned, address* entry, const char *name) {
1384     __ align(CodeEntryAlignment);
1385     StubCodeMark mark(this, "StubRoutines", name);
1386     address start = __ pc();
1387 
1388     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1389     Label L_copy_byte, L_exit;
1390     const Register from        = rdi;  // source array address
1391     const Register to          = rsi;  // destination array address
1392     const Register count       = rdx;  // elements count
1393     const Register byte_count  = rcx;
1394     const Register qword_count = count;
1395     const Register end_from    = from; // source array end address
1396     const Register end_to      = to;   // destination array end address
1397     // End pointers are inclusive, and if count is not zero they point
1398     // to the last unit copied:  end_to[0] := end_from[0]
1399 
1400     __ enter(); // required for proper stackwalking of RuntimeStub frame
1401     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1402 
1403     if (entry != NULL) {
1404       *entry = __ pc();
1405        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1406       BLOCK_COMMENT("Entry:");
1407     }
1408 
1409     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1410                       // r9 and r10 may be used to save non-volatile registers
1411 
1412     // 'from', 'to' and 'count' are now valid
1413     __ movptr(byte_count, count);
1414     __ shrptr(count, 3); // count => qword_count
1415 
1416     // Copy from low to high addresses.  Use 'to' as scratch.
1417     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1418     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1419     __ negptr(qword_count); // make the count negative
1420     __ jmp(L_copy_bytes);
1421 
1422     // Copy trailing qwords
1423   __ BIND(L_copy_8_bytes);
1424     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1425     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1426     __ increment(qword_count);
1427     __ jcc(Assembler::notZero, L_copy_8_bytes);
1428 
1429     // Check for and copy trailing dword
1430   __ BIND(L_copy_4_bytes);
1431     __ testl(byte_count, 4);
1432     __ jccb(Assembler::zero, L_copy_2_bytes);
1433     __ movl(rax, Address(end_from, 8));
1434     __ movl(Address(end_to, 8), rax);
1435 
1436     __ addptr(end_from, 4);
1437     __ addptr(end_to, 4);
1438 
1439     // Check for and copy trailing word
1440   __ BIND(L_copy_2_bytes);
1441     __ testl(byte_count, 2);
1442     __ jccb(Assembler::zero, L_copy_byte);
1443     __ movw(rax, Address(end_from, 8));
1444     __ movw(Address(end_to, 8), rax);
1445 
1446     __ addptr(end_from, 2);
1447     __ addptr(end_to, 2);
1448 
1449     // Check for and copy trailing byte
1450   __ BIND(L_copy_byte);
1451     __ testl(byte_count, 1);
1452     __ jccb(Assembler::zero, L_exit);
1453     __ movb(rax, Address(end_from, 8));
1454     __ movb(Address(end_to, 8), rax);
1455 
1456   __ BIND(L_exit);
1457     restore_arg_regs();
1458     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1459     __ xorptr(rax, rax); // return 0
1460     __ vzeroupper();
1461     __ leave(); // required for proper stackwalking of RuntimeStub frame
1462     __ ret(0);
1463 
1464     // Copy in multi-bytes chunks
1465     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1466     __ jmp(L_copy_4_bytes);
1467 
1468     return start;
1469   }
1470 
1471   // Arguments:
1472   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1473   //             ignored
1474   //   name    - stub name string
1475   //
1476   // Inputs:
1477   //   c_rarg0   - source array address
1478   //   c_rarg1   - destination array address
1479   //   c_rarg2   - element count, treated as ssize_t, can be zero
1480   //
1481   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1482   // we let the hardware handle it.  The one to eight bytes within words,
1483   // dwords or qwords that span cache line boundaries will still be loaded
1484   // and stored atomically.
1485   //
1486   address generate_conjoint_byte_copy(bool aligned, address nooverlap_target,
1487                                       address* entry, const char *name) {
1488     __ align(CodeEntryAlignment);
1489     StubCodeMark mark(this, "StubRoutines", name);
1490     address start = __ pc();
1491 
1492     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1493     const Register from        = rdi;  // source array address
1494     const Register to          = rsi;  // destination array address
1495     const Register count       = rdx;  // elements count
1496     const Register byte_count  = rcx;
1497     const Register qword_count = count;
1498 
1499     __ enter(); // required for proper stackwalking of RuntimeStub frame
1500     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1501 
1502     if (entry != NULL) {
1503       *entry = __ pc();
1504       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1505       BLOCK_COMMENT("Entry:");
1506     }
1507 
1508     array_overlap_test(nooverlap_target, Address::times_1);
1509     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1510                       // r9 and r10 may be used to save non-volatile registers
1511 
1512     // 'from', 'to' and 'count' are now valid
1513     __ movptr(byte_count, count);
1514     __ shrptr(count, 3);   // count => qword_count
1515 
1516     // Copy from high to low addresses.
1517 
1518     // Check for and copy trailing byte
1519     __ testl(byte_count, 1);
1520     __ jcc(Assembler::zero, L_copy_2_bytes);
1521     __ movb(rax, Address(from, byte_count, Address::times_1, -1));
1522     __ movb(Address(to, byte_count, Address::times_1, -1), rax);
1523     __ decrement(byte_count); // Adjust for possible trailing word
1524 
1525     // Check for and copy trailing word
1526   __ BIND(L_copy_2_bytes);
1527     __ testl(byte_count, 2);
1528     __ jcc(Assembler::zero, L_copy_4_bytes);
1529     __ movw(rax, Address(from, byte_count, Address::times_1, -2));
1530     __ movw(Address(to, byte_count, Address::times_1, -2), rax);
1531 
1532     // Check for and copy trailing dword
1533   __ BIND(L_copy_4_bytes);
1534     __ testl(byte_count, 4);
1535     __ jcc(Assembler::zero, L_copy_bytes);
1536     __ movl(rax, Address(from, qword_count, Address::times_8));
1537     __ movl(Address(to, qword_count, Address::times_8), rax);
1538     __ jmp(L_copy_bytes);
1539 
1540     // Copy trailing qwords
1541   __ BIND(L_copy_8_bytes);
1542     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1543     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1544     __ decrement(qword_count);
1545     __ jcc(Assembler::notZero, L_copy_8_bytes);
1546 
1547     restore_arg_regs();
1548     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1549     __ xorptr(rax, rax); // return 0
1550     __ vzeroupper();
1551     __ leave(); // required for proper stackwalking of RuntimeStub frame
1552     __ ret(0);
1553 
1554     // Copy in multi-bytes chunks
1555     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1556 
1557     restore_arg_regs();
1558     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1559     __ xorptr(rax, rax); // return 0
1560     __ vzeroupper();
1561     __ leave(); // required for proper stackwalking of RuntimeStub frame
1562     __ ret(0);
1563 
1564     return start;
1565   }
1566 
1567   // Arguments:
1568   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1569   //             ignored
1570   //   name    - stub name string
1571   //
1572   // Inputs:
1573   //   c_rarg0   - source array address
1574   //   c_rarg1   - destination array address
1575   //   c_rarg2   - element count, treated as ssize_t, can be zero
1576   //
1577   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1578   // let the hardware handle it.  The two or four words within dwords
1579   // or qwords that span cache line boundaries will still be loaded
1580   // and stored atomically.
1581   //
1582   // Side Effects:
1583   //   disjoint_short_copy_entry is set to the no-overlap entry point
1584   //   used by generate_conjoint_short_copy().
1585   //
1586   address generate_disjoint_short_copy(bool aligned, address *entry, const char *name) {
1587     __ align(CodeEntryAlignment);
1588     StubCodeMark mark(this, "StubRoutines", name);
1589     address start = __ pc();
1590 
1591     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
1592     const Register from        = rdi;  // source array address
1593     const Register to          = rsi;  // destination array address
1594     const Register count       = rdx;  // elements count
1595     const Register word_count  = rcx;
1596     const Register qword_count = count;
1597     const Register end_from    = from; // source array end address
1598     const Register end_to      = to;   // destination array end address
1599     // End pointers are inclusive, and if count is not zero they point
1600     // to the last unit copied:  end_to[0] := end_from[0]
1601 
1602     __ enter(); // required for proper stackwalking of RuntimeStub frame
1603     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1604 
1605     if (entry != NULL) {
1606       *entry = __ pc();
1607       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1608       BLOCK_COMMENT("Entry:");
1609     }
1610 
1611     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1612                       // r9 and r10 may be used to save non-volatile registers
1613 
1614     // 'from', 'to' and 'count' are now valid
1615     __ movptr(word_count, count);
1616     __ shrptr(count, 2); // count => qword_count
1617 
1618     // Copy from low to high addresses.  Use 'to' as scratch.
1619     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1620     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1621     __ negptr(qword_count);
1622     __ jmp(L_copy_bytes);
1623 
1624     // Copy trailing qwords
1625   __ BIND(L_copy_8_bytes);
1626     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1627     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1628     __ increment(qword_count);
1629     __ jcc(Assembler::notZero, L_copy_8_bytes);
1630 
1631     // Original 'dest' is trashed, so we can't use it as a
1632     // base register for a possible trailing word copy
1633 
1634     // Check for and copy trailing dword
1635   __ BIND(L_copy_4_bytes);
1636     __ testl(word_count, 2);
1637     __ jccb(Assembler::zero, L_copy_2_bytes);
1638     __ movl(rax, Address(end_from, 8));
1639     __ movl(Address(end_to, 8), rax);
1640 
1641     __ addptr(end_from, 4);
1642     __ addptr(end_to, 4);
1643 
1644     // Check for and copy trailing word
1645   __ BIND(L_copy_2_bytes);
1646     __ testl(word_count, 1);
1647     __ jccb(Assembler::zero, L_exit);
1648     __ movw(rax, Address(end_from, 8));
1649     __ movw(Address(end_to, 8), rax);
1650 
1651   __ BIND(L_exit);
1652     restore_arg_regs();
1653     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1654     __ xorptr(rax, rax); // return 0
1655     __ vzeroupper();
1656     __ leave(); // required for proper stackwalking of RuntimeStub frame
1657     __ ret(0);
1658 
1659     // Copy in multi-bytes chunks
1660     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1661     __ jmp(L_copy_4_bytes);
1662 
1663     return start;
1664   }
1665 
1666   address generate_fill(BasicType t, bool aligned, const char *name) {
1667     __ align(CodeEntryAlignment);
1668     StubCodeMark mark(this, "StubRoutines", name);
1669     address start = __ pc();
1670 
1671     BLOCK_COMMENT("Entry:");
1672 
1673     const Register to       = c_rarg0;  // source array address
1674     const Register value    = c_rarg1;  // value
1675     const Register count    = c_rarg2;  // elements count
1676 
1677     __ enter(); // required for proper stackwalking of RuntimeStub frame
1678 
1679     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1680 
1681     __ vzeroupper();
1682     __ leave(); // required for proper stackwalking of RuntimeStub frame
1683     __ ret(0);
1684     return start;
1685   }
1686 
1687   // Arguments:
1688   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1689   //             ignored
1690   //   name    - stub name string
1691   //
1692   // Inputs:
1693   //   c_rarg0   - source array address
1694   //   c_rarg1   - destination array address
1695   //   c_rarg2   - element count, treated as ssize_t, can be zero
1696   //
1697   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1698   // let the hardware handle it.  The two or four words within dwords
1699   // or qwords that span cache line boundaries will still be loaded
1700   // and stored atomically.
1701   //
1702   address generate_conjoint_short_copy(bool aligned, address nooverlap_target,
1703                                        address *entry, const char *name) {
1704     __ align(CodeEntryAlignment);
1705     StubCodeMark mark(this, "StubRoutines", name);
1706     address start = __ pc();
1707 
1708     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes;
1709     const Register from        = rdi;  // source array address
1710     const Register to          = rsi;  // destination array address
1711     const Register count       = rdx;  // elements count
1712     const Register word_count  = rcx;
1713     const Register qword_count = count;
1714 
1715     __ enter(); // required for proper stackwalking of RuntimeStub frame
1716     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1717 
1718     if (entry != NULL) {
1719       *entry = __ pc();
1720       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1721       BLOCK_COMMENT("Entry:");
1722     }
1723 
1724     array_overlap_test(nooverlap_target, Address::times_2);
1725     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1726                       // r9 and r10 may be used to save non-volatile registers
1727 
1728     // 'from', 'to' and 'count' are now valid
1729     __ movptr(word_count, count);
1730     __ shrptr(count, 2); // count => qword_count
1731 
1732     // Copy from high to low addresses.  Use 'to' as scratch.
1733 
1734     // Check for and copy trailing word
1735     __ testl(word_count, 1);
1736     __ jccb(Assembler::zero, L_copy_4_bytes);
1737     __ movw(rax, Address(from, word_count, Address::times_2, -2));
1738     __ movw(Address(to, word_count, Address::times_2, -2), rax);
1739 
1740     // Check for and copy trailing dword
1741   __ BIND(L_copy_4_bytes);
1742     __ testl(word_count, 2);
1743     __ jcc(Assembler::zero, L_copy_bytes);
1744     __ movl(rax, Address(from, qword_count, Address::times_8));
1745     __ movl(Address(to, qword_count, Address::times_8), rax);
1746     __ jmp(L_copy_bytes);
1747 
1748     // Copy trailing qwords
1749   __ BIND(L_copy_8_bytes);
1750     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1751     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1752     __ decrement(qword_count);
1753     __ jcc(Assembler::notZero, L_copy_8_bytes);
1754 
1755     restore_arg_regs();
1756     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1757     __ xorptr(rax, rax); // return 0
1758     __ vzeroupper();
1759     __ leave(); // required for proper stackwalking of RuntimeStub frame
1760     __ ret(0);
1761 
1762     // Copy in multi-bytes chunks
1763     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1764 
1765     restore_arg_regs();
1766     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1767     __ xorptr(rax, rax); // return 0
1768     __ vzeroupper();
1769     __ leave(); // required for proper stackwalking of RuntimeStub frame
1770     __ ret(0);
1771 
1772     return start;
1773   }
1774 
1775   // Arguments:
1776   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1777   //             ignored
1778   //   is_oop  - true => oop array, so generate store check code
1779   //   name    - stub name string
1780   //
1781   // Inputs:
1782   //   c_rarg0   - source array address
1783   //   c_rarg1   - destination array address
1784   //   c_rarg2   - element count, treated as ssize_t, can be zero
1785   //
1786   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1787   // the hardware handle it.  The two dwords within qwords that span
1788   // cache line boundaries will still be loaded and stored atomicly.
1789   //
1790   // Side Effects:
1791   //   disjoint_int_copy_entry is set to the no-overlap entry point
1792   //   used by generate_conjoint_int_oop_copy().
1793   //
1794   address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry,
1795                                          const char *name, bool dest_uninitialized = false) {
1796     __ align(CodeEntryAlignment);
1797     StubCodeMark mark(this, "StubRoutines", name);
1798     address start = __ pc();
1799 
1800     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
1801     const Register from        = rdi;  // source array address
1802     const Register to          = rsi;  // destination array address
1803     const Register count       = rdx;  // elements count
1804     const Register dword_count = rcx;
1805     const Register qword_count = count;
1806     const Register end_from    = from; // source array end address
1807     const Register end_to      = to;   // destination array end address
1808     // End pointers are inclusive, and if count is not zero they point
1809     // to the last unit copied:  end_to[0] := end_from[0]
1810 
1811     __ enter(); // required for proper stackwalking of RuntimeStub frame
1812     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1813 
1814     if (entry != NULL) {
1815       *entry = __ pc();
1816       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1817       BLOCK_COMMENT("Entry:");
1818     }
1819 
1820     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1821                       // r9 and r10 may be used to save non-volatile registers
1822 
1823     DecoratorSet decorators = ARRAYCOPY_DISJOINT;
1824     if (dest_uninitialized) {
1825       decorators |= AS_DEST_NOT_INITIALIZED;
1826     }
1827     if (aligned) {
1828       decorators |= ARRAYCOPY_ALIGNED;
1829     }
1830 
1831     BasicType type = is_oop ? T_OBJECT : T_INT;
1832     BarrierSetAssembler *bs = Universe::heap()->barrier_set()->barrier_set_assembler();
1833     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1834 
1835     // 'from', 'to' and 'count' are now valid
1836     __ movptr(dword_count, count);
1837     __ shrptr(count, 1); // count => qword_count
1838 
1839     // Copy from low to high addresses.  Use 'to' as scratch.
1840     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1841     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1842     __ negptr(qword_count);
1843     __ jmp(L_copy_bytes);
1844 
1845     // Copy trailing qwords
1846   __ BIND(L_copy_8_bytes);
1847     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1848     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1849     __ increment(qword_count);
1850     __ jcc(Assembler::notZero, L_copy_8_bytes);
1851 
1852     // Check for and copy trailing dword
1853   __ BIND(L_copy_4_bytes);
1854     __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
1855     __ jccb(Assembler::zero, L_exit);
1856     __ movl(rax, Address(end_from, 8));
1857     __ movl(Address(end_to, 8), rax);
1858 
1859   __ BIND(L_exit);
1860     bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
1861     restore_arg_regs();
1862     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
1863     __ vzeroupper();
1864     __ xorptr(rax, rax); // return 0
1865     __ leave(); // required for proper stackwalking of RuntimeStub frame
1866     __ ret(0);
1867 
1868     // Copy in multi-bytes chunks
1869     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1870     __ jmp(L_copy_4_bytes);
1871 
1872     return start;
1873   }
1874 
1875   // Arguments:
1876   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1877   //             ignored
1878   //   is_oop  - true => oop array, so generate store check code
1879   //   name    - stub name string
1880   //
1881   // Inputs:
1882   //   c_rarg0   - source array address
1883   //   c_rarg1   - destination array address
1884   //   c_rarg2   - element count, treated as ssize_t, can be zero
1885   //
1886   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1887   // the hardware handle it.  The two dwords within qwords that span
1888   // cache line boundaries will still be loaded and stored atomicly.
1889   //
1890   address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, address nooverlap_target,
1891                                          address *entry, const char *name,
1892                                          bool dest_uninitialized = false) {
1893     __ align(CodeEntryAlignment);
1894     StubCodeMark mark(this, "StubRoutines", name);
1895     address start = __ pc();
1896 
1897     Label L_copy_bytes, L_copy_8_bytes, L_copy_2_bytes, L_exit;
1898     const Register from        = rdi;  // source array address
1899     const Register to          = rsi;  // destination array address
1900     const Register count       = rdx;  // elements count
1901     const Register dword_count = rcx;
1902     const Register qword_count = count;
1903 
1904     __ enter(); // required for proper stackwalking of RuntimeStub frame
1905     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1906 
1907     if (entry != NULL) {
1908       *entry = __ pc();
1909        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1910       BLOCK_COMMENT("Entry:");
1911     }
1912 
1913     array_overlap_test(nooverlap_target, Address::times_4);
1914     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1915                       // r9 and r10 may be used to save non-volatile registers
1916 
1917     DecoratorSet decorators = 0;
1918     if (dest_uninitialized) {
1919       decorators |= AS_DEST_NOT_INITIALIZED;
1920     }
1921     if (aligned) {
1922       decorators |= ARRAYCOPY_ALIGNED;
1923     }
1924 
1925     BasicType type = is_oop ? T_OBJECT : T_INT;
1926     BarrierSetAssembler *bs = Universe::heap()->barrier_set()->barrier_set_assembler();
1927     // no registers are destroyed by this call
1928     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1929 
1930     assert_clean_int(count, rax); // Make sure 'count' is clean int.
1931     // 'from', 'to' and 'count' are now valid
1932     __ movptr(dword_count, count);
1933     __ shrptr(count, 1); // count => qword_count
1934 
1935     // Copy from high to low addresses.  Use 'to' as scratch.
1936 
1937     // Check for and copy trailing dword
1938     __ testl(dword_count, 1);
1939     __ jcc(Assembler::zero, L_copy_bytes);
1940     __ movl(rax, Address(from, dword_count, Address::times_4, -4));
1941     __ movl(Address(to, dword_count, Address::times_4, -4), rax);
1942     __ jmp(L_copy_bytes);
1943 
1944     // Copy trailing qwords
1945   __ BIND(L_copy_8_bytes);
1946     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1947     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1948     __ decrement(qword_count);
1949     __ jcc(Assembler::notZero, L_copy_8_bytes);
1950 
1951     if (is_oop) {
1952       __ jmp(L_exit);
1953     }
1954     restore_arg_regs();
1955     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
1956     __ xorptr(rax, rax); // return 0
1957     __ vzeroupper();
1958     __ leave(); // required for proper stackwalking of RuntimeStub frame
1959     __ ret(0);
1960 
1961     // Copy in multi-bytes chunks
1962     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1963 
1964   __ BIND(L_exit);
1965     bs->arraycopy_epilogue(_masm, decorators, type, from, to, dword_count);
1966     restore_arg_regs();
1967     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
1968     __ xorptr(rax, rax); // return 0
1969     __ vzeroupper();
1970     __ leave(); // required for proper stackwalking of RuntimeStub frame
1971     __ ret(0);
1972 
1973     return start;
1974   }
1975 
1976   // Arguments:
1977   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1978   //             ignored
1979   //   is_oop  - true => oop array, so generate store check code
1980   //   name    - stub name string
1981   //
1982   // Inputs:
1983   //   c_rarg0   - source array address
1984   //   c_rarg1   - destination array address
1985   //   c_rarg2   - element count, treated as ssize_t, can be zero
1986   //
1987  // Side Effects:
1988   //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
1989   //   no-overlap entry point used by generate_conjoint_long_oop_copy().
1990   //
1991   address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry,
1992                                           const char *name, bool dest_uninitialized = false) {
1993     __ align(CodeEntryAlignment);
1994     StubCodeMark mark(this, "StubRoutines", name);
1995     address start = __ pc();
1996 
1997     Label L_copy_bytes, L_copy_8_bytes, L_exit;
1998     const Register from        = rdi;  // source array address
1999     const Register to          = rsi;  // destination array address
2000     const Register qword_count = rdx;  // elements count
2001     const Register end_from    = from; // source array end address
2002     const Register end_to      = rcx;  // destination array end address
2003     const Register saved_count = r11;
2004     // End pointers are inclusive, and if count is not zero they point
2005     // to the last unit copied:  end_to[0] := end_from[0]
2006 
2007     __ enter(); // required for proper stackwalking of RuntimeStub frame
2008     // Save no-overlap entry point for generate_conjoint_long_oop_copy()
2009     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2010 
2011     if (entry != NULL) {
2012       *entry = __ pc();
2013       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2014       BLOCK_COMMENT("Entry:");
2015     }
2016 
2017     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2018                       // r9 and r10 may be used to save non-volatile registers
2019     // 'from', 'to' and 'qword_count' are now valid
2020 
2021     DecoratorSet decorators = ARRAYCOPY_DISJOINT;
2022     if (dest_uninitialized) {
2023       decorators |= AS_DEST_NOT_INITIALIZED;
2024     }
2025     if (aligned) {
2026       decorators |= ARRAYCOPY_ALIGNED;
2027     }
2028 
2029     BasicType type = is_oop ? T_OBJECT : T_LONG;
2030     BarrierSetAssembler *bs = Universe::heap()->barrier_set()->barrier_set_assembler();
2031     bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
2032 
2033     // Copy from low to high addresses.  Use 'to' as scratch.
2034     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
2035     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
2036     __ negptr(qword_count);
2037     __ jmp(L_copy_bytes);
2038 
2039     // Copy trailing qwords
2040   __ BIND(L_copy_8_bytes);
2041     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2042     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2043     __ increment(qword_count);
2044     __ jcc(Assembler::notZero, L_copy_8_bytes);
2045 
2046     if (is_oop) {
2047       __ jmp(L_exit);
2048     } else {
2049       restore_arg_regs();
2050       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2051       __ xorptr(rax, rax); // return 0
2052       __ vzeroupper();
2053       __ leave(); // required for proper stackwalking of RuntimeStub frame
2054       __ ret(0);
2055     }
2056 
2057     // Copy in multi-bytes chunks
2058     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2059 
2060     __ BIND(L_exit);
2061     bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
2062     restore_arg_regs();
2063     if (is_oop) {
2064       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2065     } else {
2066       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2067     }
2068     __ vzeroupper();
2069     __ xorptr(rax, rax); // return 0
2070     __ leave(); // required for proper stackwalking of RuntimeStub frame
2071     __ ret(0);
2072 
2073     return start;
2074   }
2075 
2076   // Arguments:
2077   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2078   //             ignored
2079   //   is_oop  - true => oop array, so generate store check code
2080   //   name    - stub name string
2081   //
2082   // Inputs:
2083   //   c_rarg0   - source array address
2084   //   c_rarg1   - destination array address
2085   //   c_rarg2   - element count, treated as ssize_t, can be zero
2086   //
2087   address generate_conjoint_long_oop_copy(bool aligned, bool is_oop,
2088                                           address nooverlap_target, address *entry,
2089                                           const char *name, bool dest_uninitialized = false) {
2090     __ align(CodeEntryAlignment);
2091     StubCodeMark mark(this, "StubRoutines", name);
2092     address start = __ pc();
2093 
2094     Label L_copy_bytes, L_copy_8_bytes, L_exit;
2095     const Register from        = rdi;  // source array address
2096     const Register to          = rsi;  // destination array address
2097     const Register qword_count = rdx;  // elements count
2098     const Register saved_count = rcx;
2099 
2100     __ enter(); // required for proper stackwalking of RuntimeStub frame
2101     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2102 
2103     if (entry != NULL) {
2104       *entry = __ pc();
2105       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2106       BLOCK_COMMENT("Entry:");
2107     }
2108 
2109     array_overlap_test(nooverlap_target, Address::times_8);
2110     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2111                       // r9 and r10 may be used to save non-volatile registers
2112     // 'from', 'to' and 'qword_count' are now valid
2113 
2114     DecoratorSet decorators = ARRAYCOPY_DISJOINT;
2115     if (dest_uninitialized) {
2116       decorators |= AS_DEST_NOT_INITIALIZED;
2117     }
2118     if (aligned) {
2119       decorators |= ARRAYCOPY_ALIGNED;
2120     }
2121 
2122     BasicType type = is_oop ? T_OBJECT : T_LONG;
2123     BarrierSetAssembler *bs = Universe::heap()->barrier_set()->barrier_set_assembler();
2124     bs->arraycopy_prologue(_masm, decorators, type, from, to, qword_count);
2125 
2126     __ jmp(L_copy_bytes);
2127 
2128     // Copy trailing qwords
2129   __ BIND(L_copy_8_bytes);
2130     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2131     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2132     __ decrement(qword_count);
2133     __ jcc(Assembler::notZero, L_copy_8_bytes);
2134 
2135     if (is_oop) {
2136       __ jmp(L_exit);
2137     } else {
2138       restore_arg_regs();
2139       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2140       __ xorptr(rax, rax); // return 0
2141       __ vzeroupper();
2142       __ leave(); // required for proper stackwalking of RuntimeStub frame
2143       __ ret(0);
2144     }
2145 
2146     // Copy in multi-bytes chunks
2147     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2148 
2149     __ BIND(L_exit);
2150     bs->arraycopy_epilogue(_masm, decorators, type, from, to, qword_count);
2151     restore_arg_regs();
2152     if (is_oop) {
2153       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2154     } else {
2155       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2156     }
2157     __ vzeroupper();
2158     __ xorptr(rax, rax); // return 0
2159     __ leave(); // required for proper stackwalking of RuntimeStub frame
2160     __ ret(0);
2161 
2162     return start;
2163   }
2164 
2165 
2166   // Helper for generating a dynamic type check.
2167   // Smashes no registers.
2168   void generate_type_check(Register sub_klass,
2169                            Register super_check_offset,
2170                            Register super_klass,
2171                            Label& L_success) {
2172     assert_different_registers(sub_klass, super_check_offset, super_klass);
2173 
2174     BLOCK_COMMENT("type_check:");
2175 
2176     Label L_miss;
2177 
2178     __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg,        &L_success, &L_miss, NULL,
2179                                      super_check_offset);
2180     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL);
2181 
2182     // Fall through on failure!
2183     __ BIND(L_miss);
2184   }
2185 
2186   //
2187   //  Generate checkcasting array copy stub
2188   //
2189   //  Input:
2190   //    c_rarg0   - source array address
2191   //    c_rarg1   - destination array address
2192   //    c_rarg2   - element count, treated as ssize_t, can be zero
2193   //    c_rarg3   - size_t ckoff (super_check_offset)
2194   // not Win64
2195   //    c_rarg4   - oop ckval (super_klass)
2196   // Win64
2197   //    rsp+40    - oop ckval (super_klass)
2198   //
2199   //  Output:
2200   //    rax ==  0  -  success
2201   //    rax == -1^K - failure, where K is partial transfer count
2202   //
2203   address generate_checkcast_copy(const char *name, address *entry,
2204                                   bool dest_uninitialized = false) {
2205 
2206     Label L_load_element, L_store_element, L_do_card_marks, L_done;
2207 
2208     // Input registers (after setup_arg_regs)
2209     const Register from        = rdi;   // source array address
2210     const Register to          = rsi;   // destination array address
2211     const Register length      = rdx;   // elements count
2212     const Register ckoff       = rcx;   // super_check_offset
2213     const Register ckval       = r8;    // super_klass
2214 
2215     // Registers used as temps (r13, r14 are save-on-entry)
2216     const Register end_from    = from;  // source array end address
2217     const Register end_to      = r13;   // destination array end address
2218     const Register count       = rdx;   // -(count_remaining)
2219     const Register r14_length  = r14;   // saved copy of length
2220     // End pointers are inclusive, and if length is not zero they point
2221     // to the last unit copied:  end_to[0] := end_from[0]
2222 
2223     const Register rax_oop    = rax;    // actual oop copied
2224     const Register r11_klass  = r11;    // oop._klass
2225 
2226     //---------------------------------------------------------------
2227     // Assembler stub will be used for this call to arraycopy
2228     // if the two arrays are subtypes of Object[] but the
2229     // destination array type is not equal to or a supertype
2230     // of the source type.  Each element must be separately
2231     // checked.
2232 
2233     __ align(CodeEntryAlignment);
2234     StubCodeMark mark(this, "StubRoutines", name);
2235     address start = __ pc();
2236 
2237     __ enter(); // required for proper stackwalking of RuntimeStub frame
2238 
2239 #ifdef ASSERT
2240     // caller guarantees that the arrays really are different
2241     // otherwise, we would have to make conjoint checks
2242     { Label L;
2243       array_overlap_test(L, TIMES_OOP);
2244       __ stop("checkcast_copy within a single array");
2245       __ bind(L);
2246     }
2247 #endif //ASSERT
2248 
2249     setup_arg_regs(4); // from => rdi, to => rsi, length => rdx
2250                        // ckoff => rcx, ckval => r8
2251                        // r9 and r10 may be used to save non-volatile registers
2252 #ifdef _WIN64
2253     // last argument (#4) is on stack on Win64
2254     __ movptr(ckval, Address(rsp, 6 * wordSize));
2255 #endif
2256 
2257     // Caller of this entry point must set up the argument registers.
2258     if (entry != NULL) {
2259       *entry = __ pc();
2260       BLOCK_COMMENT("Entry:");
2261     }
2262 
2263     // allocate spill slots for r13, r14
2264     enum {
2265       saved_r13_offset,
2266       saved_r14_offset,
2267       saved_rbp_offset
2268     };
2269     __ subptr(rsp, saved_rbp_offset * wordSize);
2270     __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
2271     __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
2272 
2273     // check that int operands are properly extended to size_t
2274     assert_clean_int(length, rax);
2275     assert_clean_int(ckoff, rax);
2276 
2277 #ifdef ASSERT
2278     BLOCK_COMMENT("assert consistent ckoff/ckval");
2279     // The ckoff and ckval must be mutually consistent,
2280     // even though caller generates both.
2281     { Label L;
2282       int sco_offset = in_bytes(Klass::super_check_offset_offset());
2283       __ cmpl(ckoff, Address(ckval, sco_offset));
2284       __ jcc(Assembler::equal, L);
2285       __ stop("super_check_offset inconsistent");
2286       __ bind(L);
2287     }
2288 #endif //ASSERT
2289 
2290     // Loop-invariant addresses.  They are exclusive end pointers.
2291     Address end_from_addr(from, length, TIMES_OOP, 0);
2292     Address   end_to_addr(to,   length, TIMES_OOP, 0);
2293     // Loop-variant addresses.  They assume post-incremented count < 0.
2294     Address from_element_addr(end_from, count, TIMES_OOP, 0);
2295     Address   to_element_addr(end_to,   count, TIMES_OOP, 0);
2296 
2297     DecoratorSet decorators = ARRAYCOPY_CHECKCAST;
2298     if (dest_uninitialized) {
2299       decorators |= AS_DEST_NOT_INITIALIZED;
2300     }
2301 
2302     BasicType type = T_OBJECT;
2303     BarrierSetAssembler *bs = Universe::heap()->barrier_set()->barrier_set_assembler();
2304     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
2305 
2306     // Copy from low to high addresses, indexed from the end of each array.
2307     __ lea(end_from, end_from_addr);
2308     __ lea(end_to,   end_to_addr);
2309     __ movptr(r14_length, length);        // save a copy of the length
2310     assert(length == count, "");          // else fix next line:
2311     __ negptr(count);                     // negate and test the length
2312     __ jcc(Assembler::notZero, L_load_element);
2313 
2314     // Empty array:  Nothing to do.
2315     __ xorptr(rax, rax);                  // return 0 on (trivial) success
2316     __ jmp(L_done);
2317 
2318     // ======== begin loop ========
2319     // (Loop is rotated; its entry is L_load_element.)
2320     // Loop control:
2321     //   for (count = -count; count != 0; count++)
2322     // Base pointers src, dst are biased by 8*(count-1),to last element.
2323     __ align(OptoLoopAlignment);
2324 
2325     __ BIND(L_store_element);
2326     __ store_heap_oop(to_element_addr, rax_oop, noreg, noreg, AS_RAW);  // store the oop
2327     __ increment(count);               // increment the count toward zero
2328     __ jcc(Assembler::zero, L_do_card_marks);
2329 
2330     // ======== loop entry is here ========
2331     __ BIND(L_load_element);
2332     __ load_heap_oop(rax_oop, from_element_addr, noreg, noreg, AS_RAW); // load the oop
2333     __ testptr(rax_oop, rax_oop);
2334     __ jcc(Assembler::zero, L_store_element);
2335 
2336     __ load_klass(r11_klass, rax_oop);// query the object klass
2337     generate_type_check(r11_klass, ckoff, ckval, L_store_element);
2338     // ======== end loop ========
2339 
2340     // It was a real error; we must depend on the caller to finish the job.
2341     // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.
2342     // Emit GC store barriers for the oops we have copied (r14 + rdx),
2343     // and report their number to the caller.
2344     assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1);
2345     Label L_post_barrier;
2346     __ addptr(r14_length, count);     // K = (original - remaining) oops
2347     __ movptr(rax, r14_length);       // save the value
2348     __ notptr(rax);                   // report (-1^K) to caller (does not affect flags)
2349     __ jccb(Assembler::notZero, L_post_barrier);
2350     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
2351 
2352     // Come here on success only.
2353     __ BIND(L_do_card_marks);
2354     __ xorptr(rax, rax);              // return 0 on success
2355 
2356     __ BIND(L_post_barrier);
2357     bs->arraycopy_epilogue(_masm, decorators, type, from, to, r14_length);
2358 
2359     // Common exit point (success or failure).
2360     __ BIND(L_done);
2361     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
2362     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
2363     restore_arg_regs();
2364     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); // Update counter after rscratch1 is free
2365     __ leave(); // required for proper stackwalking of RuntimeStub frame
2366     __ ret(0);
2367 
2368     return start;
2369   }
2370 
2371   //
2372   //  Generate 'unsafe' array copy stub
2373   //  Though just as safe as the other stubs, it takes an unscaled
2374   //  size_t argument instead of an element count.
2375   //
2376   //  Input:
2377   //    c_rarg0   - source array address
2378   //    c_rarg1   - destination array address
2379   //    c_rarg2   - byte count, treated as ssize_t, can be zero
2380   //
2381   // Examines the alignment of the operands and dispatches
2382   // to a long, int, short, or byte copy loop.
2383   //
2384   address generate_unsafe_copy(const char *name,
2385                                address byte_copy_entry, address short_copy_entry,
2386                                address int_copy_entry, address long_copy_entry) {
2387 
2388     Label L_long_aligned, L_int_aligned, L_short_aligned;
2389 
2390     // Input registers (before setup_arg_regs)
2391     const Register from        = c_rarg0;  // source array address
2392     const Register to          = c_rarg1;  // destination array address
2393     const Register size        = c_rarg2;  // byte count (size_t)
2394 
2395     // Register used as a temp
2396     const Register bits        = rax;      // test copy of low bits
2397 
2398     __ align(CodeEntryAlignment);
2399     StubCodeMark mark(this, "StubRoutines", name);
2400     address start = __ pc();
2401 
2402     __ enter(); // required for proper stackwalking of RuntimeStub frame
2403 
2404     // bump this on entry, not on exit:
2405     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2406 
2407     __ mov(bits, from);
2408     __ orptr(bits, to);
2409     __ orptr(bits, size);
2410 
2411     __ testb(bits, BytesPerLong-1);
2412     __ jccb(Assembler::zero, L_long_aligned);
2413 
2414     __ testb(bits, BytesPerInt-1);
2415     __ jccb(Assembler::zero, L_int_aligned);
2416 
2417     __ testb(bits, BytesPerShort-1);
2418     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
2419 
2420     __ BIND(L_short_aligned);
2421     __ shrptr(size, LogBytesPerShort); // size => short_count
2422     __ jump(RuntimeAddress(short_copy_entry));
2423 
2424     __ BIND(L_int_aligned);
2425     __ shrptr(size, LogBytesPerInt); // size => int_count
2426     __ jump(RuntimeAddress(int_copy_entry));
2427 
2428     __ BIND(L_long_aligned);
2429     __ shrptr(size, LogBytesPerLong); // size => qword_count
2430     __ jump(RuntimeAddress(long_copy_entry));
2431 
2432     return start;
2433   }
2434 
2435   // Perform range checks on the proposed arraycopy.
2436   // Kills temp, but nothing else.
2437   // Also, clean the sign bits of src_pos and dst_pos.
2438   void arraycopy_range_checks(Register src,     // source array oop (c_rarg0)
2439                               Register src_pos, // source position (c_rarg1)
2440                               Register dst,     // destination array oo (c_rarg2)
2441                               Register dst_pos, // destination position (c_rarg3)
2442                               Register length,
2443                               Register temp,
2444                               Label& L_failed) {
2445     BLOCK_COMMENT("arraycopy_range_checks:");
2446 
2447     //  if (src_pos + length > arrayOop(src)->length())  FAIL;
2448     __ movl(temp, length);
2449     __ addl(temp, src_pos);             // src_pos + length
2450     __ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
2451     __ jcc(Assembler::above, L_failed);
2452 
2453     //  if (dst_pos + length > arrayOop(dst)->length())  FAIL;
2454     __ movl(temp, length);
2455     __ addl(temp, dst_pos);             // dst_pos + length
2456     __ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2457     __ jcc(Assembler::above, L_failed);
2458 
2459     // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
2460     // Move with sign extension can be used since they are positive.
2461     __ movslq(src_pos, src_pos);
2462     __ movslq(dst_pos, dst_pos);
2463 
2464     BLOCK_COMMENT("arraycopy_range_checks done");
2465   }
2466 
2467   //
2468   //  Generate generic array copy stubs
2469   //
2470   //  Input:
2471   //    c_rarg0    -  src oop
2472   //    c_rarg1    -  src_pos (32-bits)
2473   //    c_rarg2    -  dst oop
2474   //    c_rarg3    -  dst_pos (32-bits)
2475   // not Win64
2476   //    c_rarg4    -  element count (32-bits)
2477   // Win64
2478   //    rsp+40     -  element count (32-bits)
2479   //
2480   //  Output:
2481   //    rax ==  0  -  success
2482   //    rax == -1^K - failure, where K is partial transfer count
2483   //
2484   address generate_generic_copy(const char *name,
2485                                 address byte_copy_entry, address short_copy_entry,
2486                                 address int_copy_entry, address oop_copy_entry,
2487                                 address long_copy_entry, address checkcast_copy_entry) {
2488 
2489     Label L_failed, L_failed_0, L_objArray;
2490     Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
2491 
2492     // Input registers
2493     const Register src        = c_rarg0;  // source array oop
2494     const Register src_pos    = c_rarg1;  // source position
2495     const Register dst        = c_rarg2;  // destination array oop
2496     const Register dst_pos    = c_rarg3;  // destination position
2497 #ifndef _WIN64
2498     const Register length     = c_rarg4;
2499 #else
2500     const Address  length(rsp, 6 * wordSize);  // elements count is on stack on Win64
2501 #endif
2502 
2503     { int modulus = CodeEntryAlignment;
2504       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
2505       int advance = target - (__ offset() % modulus);
2506       if (advance < 0)  advance += modulus;
2507       if (advance > 0)  __ nop(advance);
2508     }
2509     StubCodeMark mark(this, "StubRoutines", name);
2510 
2511     // Short-hop target to L_failed.  Makes for denser prologue code.
2512     __ BIND(L_failed_0);
2513     __ jmp(L_failed);
2514     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
2515 
2516     __ align(CodeEntryAlignment);
2517     address start = __ pc();
2518 
2519     __ enter(); // required for proper stackwalking of RuntimeStub frame
2520 
2521     // bump this on entry, not on exit:
2522     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
2523 
2524     //-----------------------------------------------------------------------
2525     // Assembler stub will be used for this call to arraycopy
2526     // if the following conditions are met:
2527     //
2528     // (1) src and dst must not be null.
2529     // (2) src_pos must not be negative.
2530     // (3) dst_pos must not be negative.
2531     // (4) length  must not be negative.
2532     // (5) src klass and dst klass should be the same and not NULL.
2533     // (6) src and dst should be arrays.
2534     // (7) src_pos + length must not exceed length of src.
2535     // (8) dst_pos + length must not exceed length of dst.
2536     //
2537 
2538     //  if (src == NULL) return -1;
2539     __ testptr(src, src);         // src oop
2540     size_t j1off = __ offset();
2541     __ jccb(Assembler::zero, L_failed_0);
2542 
2543     //  if (src_pos < 0) return -1;
2544     __ testl(src_pos, src_pos); // src_pos (32-bits)
2545     __ jccb(Assembler::negative, L_failed_0);
2546 
2547     //  if (dst == NULL) return -1;
2548     __ testptr(dst, dst);         // dst oop
2549     __ jccb(Assembler::zero, L_failed_0);
2550 
2551     //  if (dst_pos < 0) return -1;
2552     __ testl(dst_pos, dst_pos); // dst_pos (32-bits)
2553     size_t j4off = __ offset();
2554     __ jccb(Assembler::negative, L_failed_0);
2555 
2556     // The first four tests are very dense code,
2557     // but not quite dense enough to put four
2558     // jumps in a 16-byte instruction fetch buffer.
2559     // That's good, because some branch predicters
2560     // do not like jumps so close together.
2561     // Make sure of this.
2562     guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
2563 
2564     // registers used as temp
2565     const Register r11_length    = r11; // elements count to copy
2566     const Register r10_src_klass = r10; // array klass
2567 
2568     //  if (length < 0) return -1;
2569     __ movl(r11_length, length);        // length (elements count, 32-bits value)
2570     __ testl(r11_length, r11_length);
2571     __ jccb(Assembler::negative, L_failed_0);
2572 
2573     __ load_klass(r10_src_klass, src);
2574 #ifdef ASSERT
2575     //  assert(src->klass() != NULL);
2576     {
2577       BLOCK_COMMENT("assert klasses not null {");
2578       Label L1, L2;
2579       __ testptr(r10_src_klass, r10_src_klass);
2580       __ jcc(Assembler::notZero, L2);   // it is broken if klass is NULL
2581       __ bind(L1);
2582       __ stop("broken null klass");
2583       __ bind(L2);
2584       __ load_klass(rax, dst);
2585       __ cmpq(rax, 0);
2586       __ jcc(Assembler::equal, L1);     // this would be broken also
2587       BLOCK_COMMENT("} assert klasses not null done");
2588     }
2589 #endif
2590 
2591     // Load layout helper (32-bits)
2592     //
2593     //  |array_tag|     | header_size | element_type |     |log2_element_size|
2594     // 32        30    24            16              8     2                 0
2595     //
2596     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2597     //
2598 
2599     const int lh_offset = in_bytes(Klass::layout_helper_offset());
2600 
2601     // Handle objArrays completely differently...
2602     const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2603     __ cmpl(Address(r10_src_klass, lh_offset), objArray_lh);
2604     __ jcc(Assembler::equal, L_objArray);
2605 
2606     //  if (src->klass() != dst->klass()) return -1;
2607     __ load_klass(rax, dst);
2608     __ cmpq(r10_src_klass, rax);
2609     __ jcc(Assembler::notEqual, L_failed);
2610 
2611     const Register rax_lh = rax;  // layout helper
2612     __ movl(rax_lh, Address(r10_src_klass, lh_offset));
2613 
2614     //  if (!src->is_Array()) return -1;
2615     __ cmpl(rax_lh, Klass::_lh_neutral_value);
2616     __ jcc(Assembler::greaterEqual, L_failed);
2617 
2618     // At this point, it is known to be a typeArray (array_tag 0x3).
2619 #ifdef ASSERT
2620     {
2621       BLOCK_COMMENT("assert primitive array {");
2622       Label L;
2623       __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
2624       __ jcc(Assembler::greaterEqual, L);
2625       __ stop("must be a primitive array");
2626       __ bind(L);
2627       BLOCK_COMMENT("} assert primitive array done");
2628     }
2629 #endif
2630 
2631     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2632                            r10, L_failed);
2633 
2634     // TypeArrayKlass
2635     //
2636     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2637     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2638     //
2639 
2640     const Register r10_offset = r10;    // array offset
2641     const Register rax_elsize = rax_lh; // element size
2642 
2643     __ movl(r10_offset, rax_lh);
2644     __ shrl(r10_offset, Klass::_lh_header_size_shift);
2645     __ andptr(r10_offset, Klass::_lh_header_size_mask);   // array_offset
2646     __ addptr(src, r10_offset);           // src array offset
2647     __ addptr(dst, r10_offset);           // dst array offset
2648     BLOCK_COMMENT("choose copy loop based on element size");
2649     __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
2650 
2651     // next registers should be set before the jump to corresponding stub
2652     const Register from     = c_rarg0;  // source array address
2653     const Register to       = c_rarg1;  // destination array address
2654     const Register count    = c_rarg2;  // elements count
2655 
2656     // 'from', 'to', 'count' registers should be set in such order
2657     // since they are the same as 'src', 'src_pos', 'dst'.
2658 
2659   __ BIND(L_copy_bytes);
2660     __ cmpl(rax_elsize, 0);
2661     __ jccb(Assembler::notEqual, L_copy_shorts);
2662     __ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr
2663     __ lea(to,   Address(dst, dst_pos, Address::times_1, 0));// dst_addr
2664     __ movl2ptr(count, r11_length); // length
2665     __ jump(RuntimeAddress(byte_copy_entry));
2666 
2667   __ BIND(L_copy_shorts);
2668     __ cmpl(rax_elsize, LogBytesPerShort);
2669     __ jccb(Assembler::notEqual, L_copy_ints);
2670     __ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr
2671     __ lea(to,   Address(dst, dst_pos, Address::times_2, 0));// dst_addr
2672     __ movl2ptr(count, r11_length); // length
2673     __ jump(RuntimeAddress(short_copy_entry));
2674 
2675   __ BIND(L_copy_ints);
2676     __ cmpl(rax_elsize, LogBytesPerInt);
2677     __ jccb(Assembler::notEqual, L_copy_longs);
2678     __ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr
2679     __ lea(to,   Address(dst, dst_pos, Address::times_4, 0));// dst_addr
2680     __ movl2ptr(count, r11_length); // length
2681     __ jump(RuntimeAddress(int_copy_entry));
2682 
2683   __ BIND(L_copy_longs);
2684 #ifdef ASSERT
2685     {
2686       BLOCK_COMMENT("assert long copy {");
2687       Label L;
2688       __ cmpl(rax_elsize, LogBytesPerLong);
2689       __ jcc(Assembler::equal, L);
2690       __ stop("must be long copy, but elsize is wrong");
2691       __ bind(L);
2692       BLOCK_COMMENT("} assert long copy done");
2693     }
2694 #endif
2695     __ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr
2696     __ lea(to,   Address(dst, dst_pos, Address::times_8, 0));// dst_addr
2697     __ movl2ptr(count, r11_length); // length
2698     __ jump(RuntimeAddress(long_copy_entry));
2699 
2700     // ObjArrayKlass
2701   __ BIND(L_objArray);
2702     // live at this point:  r10_src_klass, r11_length, src[_pos], dst[_pos]
2703 
2704     Label L_plain_copy, L_checkcast_copy;
2705     //  test array classes for subtyping
2706     __ load_klass(rax, dst);
2707     __ cmpq(r10_src_klass, rax); // usual case is exact equality
2708     __ jcc(Assembler::notEqual, L_checkcast_copy);
2709 
2710     // Identically typed arrays can be copied without element-wise checks.
2711     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2712                            r10, L_failed);
2713 
2714     __ lea(from, Address(src, src_pos, TIMES_OOP,
2715                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
2716     __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2717                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
2718     __ movl2ptr(count, r11_length); // length
2719   __ BIND(L_plain_copy);
2720     __ jump(RuntimeAddress(oop_copy_entry));
2721 
2722   __ BIND(L_checkcast_copy);
2723     // live at this point:  r10_src_klass, r11_length, rax (dst_klass)
2724     {
2725       // Before looking at dst.length, make sure dst is also an objArray.
2726       __ cmpl(Address(rax, lh_offset), objArray_lh);
2727       __ jcc(Assembler::notEqual, L_failed);
2728 
2729       // It is safe to examine both src.length and dst.length.
2730       arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2731                              rax, L_failed);
2732 
2733       const Register r11_dst_klass = r11;
2734       __ load_klass(r11_dst_klass, dst); // reload
2735 
2736       // Marshal the base address arguments now, freeing registers.
2737       __ lea(from, Address(src, src_pos, TIMES_OOP,
2738                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2739       __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2740                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2741       __ movl(count, length);           // length (reloaded)
2742       Register sco_temp = c_rarg3;      // this register is free now
2743       assert_different_registers(from, to, count, sco_temp,
2744                                  r11_dst_klass, r10_src_klass);
2745       assert_clean_int(count, sco_temp);
2746 
2747       // Generate the type check.
2748       const int sco_offset = in_bytes(Klass::super_check_offset_offset());
2749       __ movl(sco_temp, Address(r11_dst_klass, sco_offset));
2750       assert_clean_int(sco_temp, rax);
2751       generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy);
2752 
2753       // Fetch destination element klass from the ObjArrayKlass header.
2754       int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2755       __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
2756       __ movl(  sco_temp,      Address(r11_dst_klass, sco_offset));
2757       assert_clean_int(sco_temp, rax);
2758 
2759       // the checkcast_copy loop needs two extra arguments:
2760       assert(c_rarg3 == sco_temp, "#3 already in place");
2761       // Set up arguments for checkcast_copy_entry.
2762       setup_arg_regs(4);
2763       __ movptr(r8, r11_dst_klass);  // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris
2764       __ jump(RuntimeAddress(checkcast_copy_entry));
2765     }
2766 
2767   __ BIND(L_failed);
2768     __ xorptr(rax, rax);
2769     __ notptr(rax); // return -1
2770     __ leave();   // required for proper stackwalking of RuntimeStub frame
2771     __ ret(0);
2772 
2773     return start;
2774   }
2775 
2776   void generate_arraycopy_stubs() {
2777     address entry;
2778     address entry_jbyte_arraycopy;
2779     address entry_jshort_arraycopy;
2780     address entry_jint_arraycopy;
2781     address entry_oop_arraycopy;
2782     address entry_jlong_arraycopy;
2783     address entry_checkcast_arraycopy;
2784 
2785     StubRoutines::_jbyte_disjoint_arraycopy  = generate_disjoint_byte_copy(false, &entry,
2786                                                                            "jbyte_disjoint_arraycopy");
2787     StubRoutines::_jbyte_arraycopy           = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy,
2788                                                                            "jbyte_arraycopy");
2789 
2790     StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry,
2791                                                                             "jshort_disjoint_arraycopy");
2792     StubRoutines::_jshort_arraycopy          = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy,
2793                                                                             "jshort_arraycopy");
2794 
2795     StubRoutines::_jint_disjoint_arraycopy   = generate_disjoint_int_oop_copy(false, false, &entry,
2796                                                                               "jint_disjoint_arraycopy");
2797     StubRoutines::_jint_arraycopy            = generate_conjoint_int_oop_copy(false, false, entry,
2798                                                                               &entry_jint_arraycopy, "jint_arraycopy");
2799 
2800     StubRoutines::_jlong_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, false, &entry,
2801                                                                                "jlong_disjoint_arraycopy");
2802     StubRoutines::_jlong_arraycopy           = generate_conjoint_long_oop_copy(false, false, entry,
2803                                                                                &entry_jlong_arraycopy, "jlong_arraycopy");
2804 
2805 
2806     if (UseCompressedOops) {
2807       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_int_oop_copy(false, true, &entry,
2808                                                                               "oop_disjoint_arraycopy");
2809       StubRoutines::_oop_arraycopy           = generate_conjoint_int_oop_copy(false, true, entry,
2810                                                                               &entry_oop_arraycopy, "oop_arraycopy");
2811       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_int_oop_copy(false, true, &entry,
2812                                                                                      "oop_disjoint_arraycopy_uninit",
2813                                                                                      /*dest_uninitialized*/true);
2814       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_int_oop_copy(false, true, entry,
2815                                                                                      NULL, "oop_arraycopy_uninit",
2816                                                                                      /*dest_uninitialized*/true);
2817     } else {
2818       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, true, &entry,
2819                                                                                "oop_disjoint_arraycopy");
2820       StubRoutines::_oop_arraycopy           = generate_conjoint_long_oop_copy(false, true, entry,
2821                                                                                &entry_oop_arraycopy, "oop_arraycopy");
2822       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_long_oop_copy(false, true, &entry,
2823                                                                                       "oop_disjoint_arraycopy_uninit",
2824                                                                                       /*dest_uninitialized*/true);
2825       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_long_oop_copy(false, true, entry,
2826                                                                                       NULL, "oop_arraycopy_uninit",
2827                                                                                       /*dest_uninitialized*/true);
2828     }
2829 
2830     StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2831     StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL,
2832                                                                         /*dest_uninitialized*/true);
2833 
2834     StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy("unsafe_arraycopy",
2835                                                               entry_jbyte_arraycopy,
2836                                                               entry_jshort_arraycopy,
2837                                                               entry_jint_arraycopy,
2838                                                               entry_jlong_arraycopy);
2839     StubRoutines::_generic_arraycopy   = generate_generic_copy("generic_arraycopy",
2840                                                                entry_jbyte_arraycopy,
2841                                                                entry_jshort_arraycopy,
2842                                                                entry_jint_arraycopy,
2843                                                                entry_oop_arraycopy,
2844                                                                entry_jlong_arraycopy,
2845                                                                entry_checkcast_arraycopy);
2846 
2847     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2848     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2849     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2850     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2851     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2852     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2853 
2854     // We don't generate specialized code for HeapWord-aligned source
2855     // arrays, so just use the code we've already generated
2856     StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = StubRoutines::_jbyte_disjoint_arraycopy;
2857     StubRoutines::_arrayof_jbyte_arraycopy           = StubRoutines::_jbyte_arraycopy;
2858 
2859     StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy;
2860     StubRoutines::_arrayof_jshort_arraycopy          = StubRoutines::_jshort_arraycopy;
2861 
2862     StubRoutines::_arrayof_jint_disjoint_arraycopy   = StubRoutines::_jint_disjoint_arraycopy;
2863     StubRoutines::_arrayof_jint_arraycopy            = StubRoutines::_jint_arraycopy;
2864 
2865     StubRoutines::_arrayof_jlong_disjoint_arraycopy  = StubRoutines::_jlong_disjoint_arraycopy;
2866     StubRoutines::_arrayof_jlong_arraycopy           = StubRoutines::_jlong_arraycopy;
2867 
2868     StubRoutines::_arrayof_oop_disjoint_arraycopy    = StubRoutines::_oop_disjoint_arraycopy;
2869     StubRoutines::_arrayof_oop_arraycopy             = StubRoutines::_oop_arraycopy;
2870 
2871     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit    = StubRoutines::_oop_disjoint_arraycopy_uninit;
2872     StubRoutines::_arrayof_oop_arraycopy_uninit             = StubRoutines::_oop_arraycopy_uninit;
2873   }
2874 
2875   // AES intrinsic stubs
2876   enum {AESBlockSize = 16};
2877 
2878   address generate_key_shuffle_mask() {
2879     __ align(16);
2880     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
2881     address start = __ pc();
2882     __ emit_data64( 0x0405060700010203, relocInfo::none );
2883     __ emit_data64( 0x0c0d0e0f08090a0b, relocInfo::none );
2884     return start;
2885   }
2886 
2887   address generate_counter_shuffle_mask() {
2888     __ align(16);
2889     StubCodeMark mark(this, "StubRoutines", "counter_shuffle_mask");
2890     address start = __ pc();
2891     __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
2892     __ emit_data64(0x0001020304050607, relocInfo::none);
2893     return start;
2894   }
2895 
2896   // Utility routine for loading a 128-bit key word in little endian format
2897   // can optionally specify that the shuffle mask is already in an xmmregister
2898   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2899     __ movdqu(xmmdst, Address(key, offset));
2900     if (xmm_shuf_mask != NULL) {
2901       __ pshufb(xmmdst, xmm_shuf_mask);
2902     } else {
2903       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2904     }
2905   }
2906 
2907   // Utility routine for increase 128bit counter (iv in CTR mode)
2908   void inc_counter(Register reg, XMMRegister xmmdst, int inc_delta, Label& next_block) {
2909     __ pextrq(reg, xmmdst, 0x0);
2910     __ addq(reg, inc_delta);
2911     __ pinsrq(xmmdst, reg, 0x0);
2912     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2913     __ pextrq(reg, xmmdst, 0x01); // Carry
2914     __ addq(reg, 0x01);
2915     __ pinsrq(xmmdst, reg, 0x01); //Carry end
2916     __ BIND(next_block);          // next instruction
2917   }
2918 
2919   // Arguments:
2920   //
2921   // Inputs:
2922   //   c_rarg0   - source byte array address
2923   //   c_rarg1   - destination byte array address
2924   //   c_rarg2   - K (key) in little endian int array
2925   //
2926   address generate_aescrypt_encryptBlock() {
2927     assert(UseAES, "need AES instructions and misaligned SSE support");
2928     __ align(CodeEntryAlignment);
2929     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
2930     Label L_doLast;
2931     address start = __ pc();
2932 
2933     const Register from        = c_rarg0;  // source array address
2934     const Register to          = c_rarg1;  // destination array address
2935     const Register key         = c_rarg2;  // key array address
2936     const Register keylen      = rax;
2937 
2938     const XMMRegister xmm_result = xmm0;
2939     const XMMRegister xmm_key_shuf_mask = xmm1;
2940     // On win64 xmm6-xmm15 must be preserved so don't use them.
2941     const XMMRegister xmm_temp1  = xmm2;
2942     const XMMRegister xmm_temp2  = xmm3;
2943     const XMMRegister xmm_temp3  = xmm4;
2944     const XMMRegister xmm_temp4  = xmm5;
2945 
2946     __ enter(); // required for proper stackwalking of RuntimeStub frame
2947 
2948     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
2949     // context for the registers used, where all instructions below are using 128-bit mode
2950     // On EVEX without VL and BW, these instructions will all be AVX.
2951     if (VM_Version::supports_avx512vlbw()) {
2952       __ movl(rax, 0xffff);
2953       __ kmovql(k1, rax);
2954     }
2955 
2956     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2957     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2958 
2959     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2960     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
2961 
2962     // For encryption, the java expanded key ordering is just what we need
2963     // we don't know if the key is aligned, hence not using load-execute form
2964 
2965     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
2966     __ pxor(xmm_result, xmm_temp1);
2967 
2968     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2969     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2970     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2971     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2972 
2973     __ aesenc(xmm_result, xmm_temp1);
2974     __ aesenc(xmm_result, xmm_temp2);
2975     __ aesenc(xmm_result, xmm_temp3);
2976     __ aesenc(xmm_result, xmm_temp4);
2977 
2978     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2979     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2980     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2981     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2982 
2983     __ aesenc(xmm_result, xmm_temp1);
2984     __ aesenc(xmm_result, xmm_temp2);
2985     __ aesenc(xmm_result, xmm_temp3);
2986     __ aesenc(xmm_result, xmm_temp4);
2987 
2988     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2989     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2990 
2991     __ cmpl(keylen, 44);
2992     __ jccb(Assembler::equal, L_doLast);
2993 
2994     __ aesenc(xmm_result, xmm_temp1);
2995     __ aesenc(xmm_result, xmm_temp2);
2996 
2997     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2998     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2999 
3000     __ cmpl(keylen, 52);
3001     __ jccb(Assembler::equal, L_doLast);
3002 
3003     __ aesenc(xmm_result, xmm_temp1);
3004     __ aesenc(xmm_result, xmm_temp2);
3005 
3006     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3007     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3008 
3009     __ BIND(L_doLast);
3010     __ aesenc(xmm_result, xmm_temp1);
3011     __ aesenclast(xmm_result, xmm_temp2);
3012     __ movdqu(Address(to, 0), xmm_result);        // store the result
3013     __ xorptr(rax, rax); // return 0
3014     __ leave(); // required for proper stackwalking of RuntimeStub frame
3015     __ ret(0);
3016 
3017     return start;
3018   }
3019 
3020 
3021   // Arguments:
3022   //
3023   // Inputs:
3024   //   c_rarg0   - source byte array address
3025   //   c_rarg1   - destination byte array address
3026   //   c_rarg2   - K (key) in little endian int array
3027   //
3028   address generate_aescrypt_decryptBlock() {
3029     assert(UseAES, "need AES instructions and misaligned SSE support");
3030     __ align(CodeEntryAlignment);
3031     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
3032     Label L_doLast;
3033     address start = __ pc();
3034 
3035     const Register from        = c_rarg0;  // source array address
3036     const Register to          = c_rarg1;  // destination array address
3037     const Register key         = c_rarg2;  // key array address
3038     const Register keylen      = rax;
3039 
3040     const XMMRegister xmm_result = xmm0;
3041     const XMMRegister xmm_key_shuf_mask = xmm1;
3042     // On win64 xmm6-xmm15 must be preserved so don't use them.
3043     const XMMRegister xmm_temp1  = xmm2;
3044     const XMMRegister xmm_temp2  = xmm3;
3045     const XMMRegister xmm_temp3  = xmm4;
3046     const XMMRegister xmm_temp4  = xmm5;
3047 
3048     __ enter(); // required for proper stackwalking of RuntimeStub frame
3049 
3050     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
3051     // context for the registers used, where all instructions below are using 128-bit mode
3052     // On EVEX without VL and BW, these instructions will all be AVX.
3053     if (VM_Version::supports_avx512vlbw()) {
3054       __ movl(rax, 0xffff);
3055       __ kmovql(k1, rax);
3056     }
3057 
3058     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
3059     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3060 
3061     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3062     __ movdqu(xmm_result, Address(from, 0));
3063 
3064     // for decryption java expanded key ordering is rotated one position from what we want
3065     // so we start from 0x10 here and hit 0x00 last
3066     // we don't know if the key is aligned, hence not using load-execute form
3067     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
3068     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
3069     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
3070     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
3071 
3072     __ pxor  (xmm_result, xmm_temp1);
3073     __ aesdec(xmm_result, xmm_temp2);
3074     __ aesdec(xmm_result, xmm_temp3);
3075     __ aesdec(xmm_result, xmm_temp4);
3076 
3077     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
3078     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
3079     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
3080     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
3081 
3082     __ aesdec(xmm_result, xmm_temp1);
3083     __ aesdec(xmm_result, xmm_temp2);
3084     __ aesdec(xmm_result, xmm_temp3);
3085     __ aesdec(xmm_result, xmm_temp4);
3086 
3087     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
3088     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
3089     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
3090 
3091     __ cmpl(keylen, 44);
3092     __ jccb(Assembler::equal, L_doLast);
3093 
3094     __ aesdec(xmm_result, xmm_temp1);
3095     __ aesdec(xmm_result, xmm_temp2);
3096 
3097     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
3098     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
3099 
3100     __ cmpl(keylen, 52);
3101     __ jccb(Assembler::equal, L_doLast);
3102 
3103     __ aesdec(xmm_result, xmm_temp1);
3104     __ aesdec(xmm_result, xmm_temp2);
3105 
3106     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3107     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3108 
3109     __ BIND(L_doLast);
3110     __ aesdec(xmm_result, xmm_temp1);
3111     __ aesdec(xmm_result, xmm_temp2);
3112 
3113     // for decryption the aesdeclast operation is always on key+0x00
3114     __ aesdeclast(xmm_result, xmm_temp3);
3115     __ movdqu(Address(to, 0), xmm_result);  // store the result
3116     __ xorptr(rax, rax); // return 0
3117     __ leave(); // required for proper stackwalking of RuntimeStub frame
3118     __ ret(0);
3119 
3120     return start;
3121   }
3122 
3123 
3124   // Arguments:
3125   //
3126   // Inputs:
3127   //   c_rarg0   - source byte array address
3128   //   c_rarg1   - destination byte array address
3129   //   c_rarg2   - K (key) in little endian int array
3130   //   c_rarg3   - r vector byte array address
3131   //   c_rarg4   - input length
3132   //
3133   // Output:
3134   //   rax       - input length
3135   //
3136   address generate_cipherBlockChaining_encryptAESCrypt() {
3137     assert(UseAES, "need AES instructions and misaligned SSE support");
3138     __ align(CodeEntryAlignment);
3139     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
3140     address start = __ pc();
3141 
3142     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
3143     const Register from        = c_rarg0;  // source array address
3144     const Register to          = c_rarg1;  // destination array address
3145     const Register key         = c_rarg2;  // key array address
3146     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
3147                                            // and left with the results of the last encryption block
3148 #ifndef _WIN64
3149     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
3150 #else
3151     const Address  len_mem(rbp, 6 * wordSize);  // length is on stack on Win64
3152     const Register len_reg     = r11;      // pick the volatile windows register
3153 #endif
3154     const Register pos         = rax;
3155 
3156     // xmm register assignments for the loops below
3157     const XMMRegister xmm_result = xmm0;
3158     const XMMRegister xmm_temp   = xmm1;
3159     // keys 0-10 preloaded into xmm2-xmm12
3160     const int XMM_REG_NUM_KEY_FIRST = 2;
3161     const int XMM_REG_NUM_KEY_LAST  = 15;
3162     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3163     const XMMRegister xmm_key10  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+10);
3164     const XMMRegister xmm_key11  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+11);
3165     const XMMRegister xmm_key12  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+12);
3166     const XMMRegister xmm_key13  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+13);
3167 
3168     __ enter(); // required for proper stackwalking of RuntimeStub frame
3169 
3170     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
3171     // context for the registers used, where all instructions below are using 128-bit mode
3172     // On EVEX without VL and BW, these instructions will all be AVX.
3173     if (VM_Version::supports_avx512vlbw()) {
3174       __ movl(rax, 0xffff);
3175       __ kmovql(k1, rax);
3176     }
3177 
3178 #ifdef _WIN64
3179     // on win64, fill len_reg from stack position
3180     __ movl(len_reg, len_mem);
3181 #else
3182     __ push(len_reg); // Save
3183 #endif
3184 
3185     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
3186     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3187     // load up xmm regs xmm2 thru xmm12 with key 0x00 - 0xa0
3188     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum <= XMM_REG_NUM_KEY_FIRST+10; rnum++) {
3189       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3190       offset += 0x10;
3191     }
3192     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
3193 
3194     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3195     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3196     __ cmpl(rax, 44);
3197     __ jcc(Assembler::notEqual, L_key_192_256);
3198 
3199     // 128 bit code follows here
3200     __ movptr(pos, 0);
3201     __ align(OptoLoopAlignment);
3202 
3203     __ BIND(L_loopTop_128);
3204     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3205     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3206     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3207     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 9; rnum++) {
3208       __ aesenc(xmm_result, as_XMMRegister(rnum));
3209     }
3210     __ aesenclast(xmm_result, xmm_key10);
3211     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3212     // no need to store r to memory until we exit
3213     __ addptr(pos, AESBlockSize);
3214     __ subptr(len_reg, AESBlockSize);
3215     __ jcc(Assembler::notEqual, L_loopTop_128);
3216 
3217     __ BIND(L_exit);
3218     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
3219 
3220 #ifdef _WIN64
3221     __ movl(rax, len_mem);
3222 #else
3223     __ pop(rax); // return length
3224 #endif
3225     __ leave(); // required for proper stackwalking of RuntimeStub frame
3226     __ ret(0);
3227 
3228     __ BIND(L_key_192_256);
3229     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
3230     load_key(xmm_key11, key, 0xb0, xmm_key_shuf_mask);
3231     load_key(xmm_key12, key, 0xc0, xmm_key_shuf_mask);
3232     __ cmpl(rax, 52);
3233     __ jcc(Assembler::notEqual, L_key_256);
3234 
3235     // 192-bit code follows here (could be changed to use more xmm registers)
3236     __ movptr(pos, 0);
3237     __ align(OptoLoopAlignment);
3238 
3239     __ BIND(L_loopTop_192);
3240     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3241     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3242     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3243     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 11; rnum++) {
3244       __ aesenc(xmm_result, as_XMMRegister(rnum));
3245     }
3246     __ aesenclast(xmm_result, xmm_key12);
3247     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3248     // no need to store r to memory until we exit
3249     __ addptr(pos, AESBlockSize);
3250     __ subptr(len_reg, AESBlockSize);
3251     __ jcc(Assembler::notEqual, L_loopTop_192);
3252     __ jmp(L_exit);
3253 
3254     __ BIND(L_key_256);
3255     // 256-bit code follows here (could be changed to use more xmm registers)
3256     load_key(xmm_key13, key, 0xd0, xmm_key_shuf_mask);
3257     __ movptr(pos, 0);
3258     __ align(OptoLoopAlignment);
3259 
3260     __ BIND(L_loopTop_256);
3261     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3262     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3263     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3264     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 13; rnum++) {
3265       __ aesenc(xmm_result, as_XMMRegister(rnum));
3266     }
3267     load_key(xmm_temp, key, 0xe0);
3268     __ aesenclast(xmm_result, xmm_temp);
3269     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3270     // no need to store r to memory until we exit
3271     __ addptr(pos, AESBlockSize);
3272     __ subptr(len_reg, AESBlockSize);
3273     __ jcc(Assembler::notEqual, L_loopTop_256);
3274     __ jmp(L_exit);
3275 
3276     return start;
3277   }
3278 
3279   // Safefetch stubs.
3280   void generate_safefetch(const char* name, int size, address* entry,
3281                           address* fault_pc, address* continuation_pc) {
3282     // safefetch signatures:
3283     //   int      SafeFetch32(int*      adr, int      errValue);
3284     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3285     //
3286     // arguments:
3287     //   c_rarg0 = adr
3288     //   c_rarg1 = errValue
3289     //
3290     // result:
3291     //   PPC_RET  = *adr or errValue
3292 
3293     StubCodeMark mark(this, "StubRoutines", name);
3294 
3295     // Entry point, pc or function descriptor.
3296     *entry = __ pc();
3297 
3298     // Load *adr into c_rarg1, may fault.
3299     *fault_pc = __ pc();
3300     switch (size) {
3301       case 4:
3302         // int32_t
3303         __ movl(c_rarg1, Address(c_rarg0, 0));
3304         break;
3305       case 8:
3306         // int64_t
3307         __ movq(c_rarg1, Address(c_rarg0, 0));
3308         break;
3309       default:
3310         ShouldNotReachHere();
3311     }
3312 
3313     // return errValue or *adr
3314     *continuation_pc = __ pc();
3315     __ movq(rax, c_rarg1);
3316     __ ret(0);
3317   }
3318 
3319   // This is a version of CBC/AES Decrypt which does 4 blocks in a loop at a time
3320   // to hide instruction latency
3321   //
3322   // Arguments:
3323   //
3324   // Inputs:
3325   //   c_rarg0   - source byte array address
3326   //   c_rarg1   - destination byte array address
3327   //   c_rarg2   - K (key) in little endian int array
3328   //   c_rarg3   - r vector byte array address
3329   //   c_rarg4   - input length
3330   //
3331   // Output:
3332   //   rax       - input length
3333   //
3334   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
3335     assert(UseAES, "need AES instructions and misaligned SSE support");
3336     __ align(CodeEntryAlignment);
3337     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
3338     address start = __ pc();
3339 
3340     const Register from        = c_rarg0;  // source array address
3341     const Register to          = c_rarg1;  // destination array address
3342     const Register key         = c_rarg2;  // key array address
3343     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
3344                                            // and left with the results of the last encryption block
3345 #ifndef _WIN64
3346     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
3347 #else
3348     const Address  len_mem(rbp, 6 * wordSize);  // length is on stack on Win64
3349     const Register len_reg     = r11;      // pick the volatile windows register
3350 #endif
3351     const Register pos         = rax;
3352 
3353     const int PARALLEL_FACTOR = 4;
3354     const int ROUNDS[3] = { 10, 12, 14 }; // aes rounds for key128, key192, key256
3355 
3356     Label L_exit;
3357     Label L_singleBlock_loopTopHead[3]; // 128, 192, 256
3358     Label L_singleBlock_loopTopHead2[3]; // 128, 192, 256
3359     Label L_singleBlock_loopTop[3]; // 128, 192, 256
3360     Label L_multiBlock_loopTopHead[3]; // 128, 192, 256
3361     Label L_multiBlock_loopTop[3]; // 128, 192, 256
3362 
3363     // keys 0-10 preloaded into xmm5-xmm15
3364     const int XMM_REG_NUM_KEY_FIRST = 5;
3365     const int XMM_REG_NUM_KEY_LAST  = 15;
3366     const XMMRegister xmm_key_first = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3367     const XMMRegister xmm_key_last  = as_XMMRegister(XMM_REG_NUM_KEY_LAST);
3368 
3369     __ enter(); // required for proper stackwalking of RuntimeStub frame
3370 
3371     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
3372     // context for the registers used, where all instructions below are using 128-bit mode
3373     // On EVEX without VL and BW, these instructions will all be AVX.
3374     if (VM_Version::supports_avx512vlbw()) {
3375       __ movl(rax, 0xffff);
3376       __ kmovql(k1, rax);
3377     }
3378 
3379 #ifdef _WIN64
3380     // on win64, fill len_reg from stack position
3381     __ movl(len_reg, len_mem);
3382 #else
3383     __ push(len_reg); // Save
3384 #endif
3385     __ push(rbx);
3386     // the java expanded key ordering is rotated one position from what we want
3387     // so we start from 0x10 here and hit 0x00 last
3388     const XMMRegister xmm_key_shuf_mask = xmm1;  // used temporarily to swap key bytes up front
3389     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3390     // load up xmm regs 5 thru 15 with key 0x10 - 0xa0 - 0x00
3391     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x10; rnum < XMM_REG_NUM_KEY_LAST; rnum++) {
3392       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3393       offset += 0x10;
3394     }
3395     load_key(xmm_key_last, key, 0x00, xmm_key_shuf_mask);
3396 
3397     const XMMRegister xmm_prev_block_cipher = xmm1;  // holds cipher of previous block
3398 
3399     // registers holding the four results in the parallelized loop
3400     const XMMRegister xmm_result0 = xmm0;
3401     const XMMRegister xmm_result1 = xmm2;
3402     const XMMRegister xmm_result2 = xmm3;
3403     const XMMRegister xmm_result3 = xmm4;
3404 
3405     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));   // initialize with initial rvec
3406 
3407     __ xorptr(pos, pos);
3408 
3409     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3410     __ movl(rbx, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3411     __ cmpl(rbx, 52);
3412     __ jcc(Assembler::equal, L_multiBlock_loopTopHead[1]);
3413     __ cmpl(rbx, 60);
3414     __ jcc(Assembler::equal, L_multiBlock_loopTopHead[2]);
3415 
3416 #define DoFour(opc, src_reg)           \
3417   __ opc(xmm_result0, src_reg);         \
3418   __ opc(xmm_result1, src_reg);         \
3419   __ opc(xmm_result2, src_reg);         \
3420   __ opc(xmm_result3, src_reg);         \
3421 
3422     for (int k = 0; k < 3; ++k) {
3423       __ BIND(L_multiBlock_loopTopHead[k]);
3424       if (k != 0) {
3425         __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
3426         __ jcc(Assembler::less, L_singleBlock_loopTopHead2[k]);
3427       }
3428       if (k == 1) {
3429         __ subptr(rsp, 6 * wordSize);
3430         __ movdqu(Address(rsp, 0), xmm15); //save last_key from xmm15
3431         load_key(xmm15, key, 0xb0); // 0xb0; 192-bit key goes up to 0xc0
3432         __ movdqu(Address(rsp, 2 * wordSize), xmm15);
3433         load_key(xmm1, key, 0xc0);  // 0xc0;
3434         __ movdqu(Address(rsp, 4 * wordSize), xmm1);
3435       } else if (k == 2) {
3436         __ subptr(rsp, 10 * wordSize);
3437         __ movdqu(Address(rsp, 0), xmm15); //save last_key from xmm15
3438         load_key(xmm15, key, 0xd0); // 0xd0; 256-bit key goes upto 0xe0
3439         __ movdqu(Address(rsp, 6 * wordSize), xmm15);
3440         load_key(xmm1, key, 0xe0);  // 0xe0;
3441         __ movdqu(Address(rsp, 8 * wordSize), xmm1);
3442         load_key(xmm15, key, 0xb0); // 0xb0;
3443         __ movdqu(Address(rsp, 2 * wordSize), xmm15);
3444         load_key(xmm1, key, 0xc0);  // 0xc0;
3445         __ movdqu(Address(rsp, 4 * wordSize), xmm1);
3446       }
3447       __ align(OptoLoopAlignment);
3448       __ BIND(L_multiBlock_loopTop[k]);
3449       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
3450       __ jcc(Assembler::less, L_singleBlock_loopTopHead[k]);
3451 
3452       if  (k != 0) {
3453         __ movdqu(xmm15, Address(rsp, 2 * wordSize));
3454         __ movdqu(xmm1, Address(rsp, 4 * wordSize));
3455       }
3456 
3457       __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); // get next 4 blocks into xmmresult registers
3458       __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
3459       __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
3460       __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
3461 
3462       DoFour(pxor, xmm_key_first);
3463       if (k == 0) {
3464         for (int rnum = 1; rnum < ROUNDS[k]; rnum++) {
3465           DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3466         }
3467         DoFour(aesdeclast, xmm_key_last);
3468       } else if (k == 1) {
3469         for (int rnum = 1; rnum <= ROUNDS[k]-2; rnum++) {
3470           DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3471         }
3472         __ movdqu(xmm_key_last, Address(rsp, 0)); // xmm15 needs to be loaded again.
3473         DoFour(aesdec, xmm1);  // key : 0xc0
3474         __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));  // xmm1 needs to be loaded again
3475         DoFour(aesdeclast, xmm_key_last);
3476       } else if (k == 2) {
3477         for (int rnum = 1; rnum <= ROUNDS[k] - 4; rnum++) {
3478           DoFour(aesdec, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3479         }
3480         DoFour(aesdec, xmm1);  // key : 0xc0
3481         __ movdqu(xmm15, Address(rsp, 6 * wordSize));
3482         __ movdqu(xmm1, Address(rsp, 8 * wordSize));
3483         DoFour(aesdec, xmm15);  // key : 0xd0
3484         __ movdqu(xmm_key_last, Address(rsp, 0)); // xmm15 needs to be loaded again.
3485         DoFour(aesdec, xmm1);  // key : 0xe0
3486         __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));  // xmm1 needs to be loaded again
3487         DoFour(aesdeclast, xmm_key_last);
3488       }
3489 
3490       // for each result, xor with the r vector of previous cipher block
3491       __ pxor(xmm_result0, xmm_prev_block_cipher);
3492       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0 * AESBlockSize));
3493       __ pxor(xmm_result1, xmm_prev_block_cipher);
3494       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1 * AESBlockSize));
3495       __ pxor(xmm_result2, xmm_prev_block_cipher);
3496       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2 * AESBlockSize));
3497       __ pxor(xmm_result3, xmm_prev_block_cipher);
3498       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3 * AESBlockSize));   // this will carry over to next set of blocks
3499       if (k != 0) {
3500         __ movdqu(Address(rvec, 0x00), xmm_prev_block_cipher);
3501       }
3502 
3503       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);     // store 4 results into the next 64 bytes of output
3504       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
3505       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
3506       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
3507 
3508       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize);
3509       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize);
3510       __ jmp(L_multiBlock_loopTop[k]);
3511 
3512       // registers used in the non-parallelized loops
3513       // xmm register assignments for the loops below
3514       const XMMRegister xmm_result = xmm0;
3515       const XMMRegister xmm_prev_block_cipher_save = xmm2;
3516       const XMMRegister xmm_key11 = xmm3;
3517       const XMMRegister xmm_key12 = xmm4;
3518       const XMMRegister key_tmp = xmm4;
3519 
3520       __ BIND(L_singleBlock_loopTopHead[k]);
3521       if (k == 1) {
3522         __ addptr(rsp, 6 * wordSize);
3523       } else if (k == 2) {
3524         __ addptr(rsp, 10 * wordSize);
3525       }
3526       __ cmpptr(len_reg, 0); // any blocks left??
3527       __ jcc(Assembler::equal, L_exit);
3528       __ BIND(L_singleBlock_loopTopHead2[k]);
3529       if (k == 1) {
3530         load_key(xmm_key11, key, 0xb0); // 0xb0; 192-bit key goes upto 0xc0
3531         load_key(xmm_key12, key, 0xc0); // 0xc0; 192-bit key goes upto 0xc0
3532       }
3533       if (k == 2) {
3534         load_key(xmm_key11, key, 0xb0); // 0xb0; 256-bit key goes upto 0xe0
3535       }
3536       __ align(OptoLoopAlignment);
3537       __ BIND(L_singleBlock_loopTop[k]);
3538       __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
3539       __ movdqa(xmm_prev_block_cipher_save, xmm_result); // save for next r vector
3540       __ pxor(xmm_result, xmm_key_first); // do the aes dec rounds
3541       for (int rnum = 1; rnum <= 9 ; rnum++) {
3542           __ aesdec(xmm_result, as_XMMRegister(rnum + XMM_REG_NUM_KEY_FIRST));
3543       }
3544       if (k == 1) {
3545         __ aesdec(xmm_result, xmm_key11);
3546         __ aesdec(xmm_result, xmm_key12);
3547       }
3548       if (k == 2) {
3549         __ aesdec(xmm_result, xmm_key11);
3550         load_key(key_tmp, key, 0xc0);
3551         __ aesdec(xmm_result, key_tmp);
3552         load_key(key_tmp, key, 0xd0);
3553         __ aesdec(xmm_result, key_tmp);
3554         load_key(key_tmp, key, 0xe0);
3555         __ aesdec(xmm_result, key_tmp);
3556       }
3557 
3558       __ aesdeclast(xmm_result, xmm_key_last); // xmm15 always came from key+0
3559       __ pxor(xmm_result, xmm_prev_block_cipher); // xor with the current r vector
3560       __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result); // store into the next 16 bytes of output
3561       // no need to store r to memory until we exit
3562       __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save); // set up next r vector with cipher input from this block
3563       __ addptr(pos, AESBlockSize);
3564       __ subptr(len_reg, AESBlockSize);
3565       __ jcc(Assembler::notEqual, L_singleBlock_loopTop[k]);
3566       if (k != 2) {
3567         __ jmp(L_exit);
3568       }
3569     } //for 128/192/256
3570 
3571     __ BIND(L_exit);
3572     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher);     // final value of r stored in rvec of CipherBlockChaining object
3573     __ pop(rbx);
3574 #ifdef _WIN64
3575     __ movl(rax, len_mem);
3576 #else
3577     __ pop(rax); // return length
3578 #endif
3579     __ leave(); // required for proper stackwalking of RuntimeStub frame
3580     __ ret(0);
3581     return start;
3582 }
3583 
3584   address generate_upper_word_mask() {
3585     __ align(64);
3586     StubCodeMark mark(this, "StubRoutines", "upper_word_mask");
3587     address start = __ pc();
3588     __ emit_data64(0x0000000000000000, relocInfo::none);
3589     __ emit_data64(0xFFFFFFFF00000000, relocInfo::none);
3590     return start;
3591   }
3592 
3593   address generate_shuffle_byte_flip_mask() {
3594     __ align(64);
3595     StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask");
3596     address start = __ pc();
3597     __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
3598     __ emit_data64(0x0001020304050607, relocInfo::none);
3599     return start;
3600   }
3601 
3602   // ofs and limit are use for multi-block byte array.
3603   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3604   address generate_sha1_implCompress(bool multi_block, const char *name) {
3605     __ align(CodeEntryAlignment);
3606     StubCodeMark mark(this, "StubRoutines", name);
3607     address start = __ pc();
3608 
3609     Register buf = c_rarg0;
3610     Register state = c_rarg1;
3611     Register ofs = c_rarg2;
3612     Register limit = c_rarg3;
3613 
3614     const XMMRegister abcd = xmm0;
3615     const XMMRegister e0 = xmm1;
3616     const XMMRegister e1 = xmm2;
3617     const XMMRegister msg0 = xmm3;
3618 
3619     const XMMRegister msg1 = xmm4;
3620     const XMMRegister msg2 = xmm5;
3621     const XMMRegister msg3 = xmm6;
3622     const XMMRegister shuf_mask = xmm7;
3623 
3624     __ enter();
3625 
3626     __ subptr(rsp, 4 * wordSize);
3627 
3628     __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask,
3629       buf, state, ofs, limit, rsp, multi_block);
3630 
3631     __ addptr(rsp, 4 * wordSize);
3632 
3633     __ leave();
3634     __ ret(0);
3635     return start;
3636   }
3637 
3638   address generate_pshuffle_byte_flip_mask() {
3639     __ align(64);
3640     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask");
3641     address start = __ pc();
3642     __ emit_data64(0x0405060700010203, relocInfo::none);
3643     __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
3644 
3645     if (VM_Version::supports_avx2()) {
3646       __ emit_data64(0x0405060700010203, relocInfo::none); // second copy
3647       __ emit_data64(0x0c0d0e0f08090a0b, relocInfo::none);
3648       // _SHUF_00BA
3649       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3650       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3651       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3652       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3653       // _SHUF_DC00
3654       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3655       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3656       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3657       __ emit_data64(0x0b0a090803020100, relocInfo::none);
3658     }
3659 
3660     return start;
3661   }
3662 
3663   //Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
3664   address generate_pshuffle_byte_flip_mask_sha512() {
3665     __ align(32);
3666     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask_sha512");
3667     address start = __ pc();
3668     if (VM_Version::supports_avx2()) {
3669       __ emit_data64(0x0001020304050607, relocInfo::none); // PSHUFFLE_BYTE_FLIP_MASK
3670       __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none);
3671       __ emit_data64(0x1011121314151617, relocInfo::none);
3672       __ emit_data64(0x18191a1b1c1d1e1f, relocInfo::none);
3673       __ emit_data64(0x0000000000000000, relocInfo::none); //MASK_YMM_LO
3674       __ emit_data64(0x0000000000000000, relocInfo::none);
3675       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3676       __ emit_data64(0xFFFFFFFFFFFFFFFF, relocInfo::none);
3677     }
3678 
3679     return start;
3680   }
3681 
3682 // ofs and limit are use for multi-block byte array.
3683 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3684   address generate_sha256_implCompress(bool multi_block, const char *name) {
3685     assert(VM_Version::supports_sha() || VM_Version::supports_avx2(), "");
3686     __ align(CodeEntryAlignment);
3687     StubCodeMark mark(this, "StubRoutines", name);
3688     address start = __ pc();
3689 
3690     Register buf = c_rarg0;
3691     Register state = c_rarg1;
3692     Register ofs = c_rarg2;
3693     Register limit = c_rarg3;
3694 
3695     const XMMRegister msg = xmm0;
3696     const XMMRegister state0 = xmm1;
3697     const XMMRegister state1 = xmm2;
3698     const XMMRegister msgtmp0 = xmm3;
3699 
3700     const XMMRegister msgtmp1 = xmm4;
3701     const XMMRegister msgtmp2 = xmm5;
3702     const XMMRegister msgtmp3 = xmm6;
3703     const XMMRegister msgtmp4 = xmm7;
3704 
3705     const XMMRegister shuf_mask = xmm8;
3706 
3707     __ enter();
3708 
3709     __ subptr(rsp, 4 * wordSize);
3710 
3711     if (VM_Version::supports_sha()) {
3712       __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3713         buf, state, ofs, limit, rsp, multi_block, shuf_mask);
3714     } else if (VM_Version::supports_avx2()) {
3715       __ sha256_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3716         buf, state, ofs, limit, rsp, multi_block, shuf_mask);
3717     }
3718     __ addptr(rsp, 4 * wordSize);
3719     __ vzeroupper();
3720     __ leave();
3721     __ ret(0);
3722     return start;
3723   }
3724 
3725   address generate_sha512_implCompress(bool multi_block, const char *name) {
3726     assert(VM_Version::supports_avx2(), "");
3727     assert(VM_Version::supports_bmi2(), "");
3728     __ align(CodeEntryAlignment);
3729     StubCodeMark mark(this, "StubRoutines", name);
3730     address start = __ pc();
3731 
3732     Register buf = c_rarg0;
3733     Register state = c_rarg1;
3734     Register ofs = c_rarg2;
3735     Register limit = c_rarg3;
3736 
3737     const XMMRegister msg = xmm0;
3738     const XMMRegister state0 = xmm1;
3739     const XMMRegister state1 = xmm2;
3740     const XMMRegister msgtmp0 = xmm3;
3741     const XMMRegister msgtmp1 = xmm4;
3742     const XMMRegister msgtmp2 = xmm5;
3743     const XMMRegister msgtmp3 = xmm6;
3744     const XMMRegister msgtmp4 = xmm7;
3745 
3746     const XMMRegister shuf_mask = xmm8;
3747 
3748     __ enter();
3749 
3750     __ sha512_AVX2(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3751     buf, state, ofs, limit, rsp, multi_block, shuf_mask);
3752 
3753     __ vzeroupper();
3754     __ leave();
3755     __ ret(0);
3756     return start;
3757   }
3758 
3759   // This is a version of CTR/AES crypt which does 6 blocks in a loop at a time
3760   // to hide instruction latency
3761   //
3762   // Arguments:
3763   //
3764   // Inputs:
3765   //   c_rarg0   - source byte array address
3766   //   c_rarg1   - destination byte array address
3767   //   c_rarg2   - K (key) in little endian int array
3768   //   c_rarg3   - counter vector byte array address
3769   //   Linux
3770   //     c_rarg4   -          input length
3771   //     c_rarg5   -          saved encryptedCounter start
3772   //     rbp + 6 * wordSize - saved used length
3773   //   Windows
3774   //     rbp + 6 * wordSize - input length
3775   //     rbp + 7 * wordSize - saved encryptedCounter start
3776   //     rbp + 8 * wordSize - saved used length
3777   //
3778   // Output:
3779   //   rax       - input length
3780   //
3781   address generate_counterMode_AESCrypt_Parallel() {
3782     assert(UseAES, "need AES instructions and misaligned SSE support");
3783     __ align(CodeEntryAlignment);
3784     StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
3785     address start = __ pc();
3786     const Register from = c_rarg0; // source array address
3787     const Register to = c_rarg1; // destination array address
3788     const Register key = c_rarg2; // key array address
3789     const Register counter = c_rarg3; // counter byte array initialized from counter array address
3790                                       // and updated with the incremented counter in the end
3791 #ifndef _WIN64
3792     const Register len_reg = c_rarg4;
3793     const Register saved_encCounter_start = c_rarg5;
3794     const Register used_addr = r10;
3795     const Address  used_mem(rbp, 2 * wordSize);
3796     const Register used = r11;
3797 #else
3798     const Address len_mem(rbp, 6 * wordSize); // length is on stack on Win64
3799     const Address saved_encCounter_mem(rbp, 7 * wordSize); // length is on stack on Win64
3800     const Address used_mem(rbp, 8 * wordSize); // length is on stack on Win64
3801     const Register len_reg = r10; // pick the first volatile windows register
3802     const Register saved_encCounter_start = r11;
3803     const Register used_addr = r13;
3804     const Register used = r14;
3805 #endif
3806     const Register pos = rax;
3807 
3808     const int PARALLEL_FACTOR = 6;
3809     const XMMRegister xmm_counter_shuf_mask = xmm0;
3810     const XMMRegister xmm_key_shuf_mask = xmm1; // used temporarily to swap key bytes up front
3811     const XMMRegister xmm_curr_counter = xmm2;
3812 
3813     const XMMRegister xmm_key_tmp0 = xmm3;
3814     const XMMRegister xmm_key_tmp1 = xmm4;
3815 
3816     // registers holding the four results in the parallelized loop
3817     const XMMRegister xmm_result0 = xmm5;
3818     const XMMRegister xmm_result1 = xmm6;
3819     const XMMRegister xmm_result2 = xmm7;
3820     const XMMRegister xmm_result3 = xmm8;
3821     const XMMRegister xmm_result4 = xmm9;
3822     const XMMRegister xmm_result5 = xmm10;
3823 
3824     const XMMRegister xmm_from0 = xmm11;
3825     const XMMRegister xmm_from1 = xmm12;
3826     const XMMRegister xmm_from2 = xmm13;
3827     const XMMRegister xmm_from3 = xmm14; //the last one is xmm14. we have to preserve it on WIN64.
3828     const XMMRegister xmm_from4 = xmm3; //reuse xmm3~4. Because xmm_key_tmp0~1 are useless when loading input text
3829     const XMMRegister xmm_from5 = xmm4;
3830 
3831     //for key_128, key_192, key_256
3832     const int rounds[3] = {10, 12, 14};
3833     Label L_exit_preLoop, L_preLoop_start;
3834     Label L_multiBlock_loopTop[3];
3835     Label L_singleBlockLoopTop[3];
3836     Label L__incCounter[3][6]; //for 6 blocks
3837     Label L__incCounter_single[3]; //for single block, key128, key192, key256
3838     Label L_processTail_insr[3], L_processTail_4_insr[3], L_processTail_2_insr[3], L_processTail_1_insr[3], L_processTail_exit_insr[3];
3839     Label L_processTail_extr[3], L_processTail_4_extr[3], L_processTail_2_extr[3], L_processTail_1_extr[3], L_processTail_exit_extr[3];
3840 
3841     Label L_exit;
3842 
3843     __ enter(); // required for proper stackwalking of RuntimeStub frame
3844 
3845     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
3846     // context for the registers used, where all instructions below are using 128-bit mode
3847     // On EVEX without VL and BW, these instructions will all be AVX.
3848     if (VM_Version::supports_avx512vlbw()) {
3849         __ movl(rax, 0xffff);
3850         __ kmovql(k1, rax);
3851     }
3852 
3853 #ifdef _WIN64
3854     // allocate spill slots for r13, r14
3855     enum {
3856         saved_r13_offset,
3857         saved_r14_offset
3858     };
3859     __ subptr(rsp, 2 * wordSize);
3860     __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
3861     __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
3862 
3863     // on win64, fill len_reg from stack position
3864     __ movl(len_reg, len_mem);
3865     __ movptr(saved_encCounter_start, saved_encCounter_mem);
3866     __ movptr(used_addr, used_mem);
3867     __ movl(used, Address(used_addr, 0));
3868 #else
3869     __ push(len_reg); // Save
3870     __ movptr(used_addr, used_mem);
3871     __ movl(used, Address(used_addr, 0));
3872 #endif
3873 
3874     __ push(rbx); // Save RBX
3875     __ movdqu(xmm_curr_counter, Address(counter, 0x00)); // initialize counter with initial counter
3876     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()), pos); // pos as scratch
3877     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled
3878     __ movptr(pos, 0);
3879 
3880     // Use the partially used encrpyted counter from last invocation
3881     __ BIND(L_preLoop_start);
3882     __ cmpptr(used, 16);
3883     __ jcc(Assembler::aboveEqual, L_exit_preLoop);
3884       __ cmpptr(len_reg, 0);
3885       __ jcc(Assembler::lessEqual, L_exit_preLoop);
3886       __ movb(rbx, Address(saved_encCounter_start, used));
3887       __ xorb(rbx, Address(from, pos));
3888       __ movb(Address(to, pos), rbx);
3889       __ addptr(pos, 1);
3890       __ addptr(used, 1);
3891       __ subptr(len_reg, 1);
3892 
3893     __ jmp(L_preLoop_start);
3894 
3895     __ BIND(L_exit_preLoop);
3896     __ movl(Address(used_addr, 0), used);
3897 
3898     // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
3899     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()), rbx); // rbx as scratch
3900     __ movl(rbx, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3901     __ cmpl(rbx, 52);
3902     __ jcc(Assembler::equal, L_multiBlock_loopTop[1]);
3903     __ cmpl(rbx, 60);
3904     __ jcc(Assembler::equal, L_multiBlock_loopTop[2]);
3905 
3906 #define CTR_DoSix(opc, src_reg)                \
3907     __ opc(xmm_result0, src_reg);              \
3908     __ opc(xmm_result1, src_reg);              \
3909     __ opc(xmm_result2, src_reg);              \
3910     __ opc(xmm_result3, src_reg);              \
3911     __ opc(xmm_result4, src_reg);              \
3912     __ opc(xmm_result5, src_reg);
3913 
3914     // k == 0 :  generate code for key_128
3915     // k == 1 :  generate code for key_192
3916     // k == 2 :  generate code for key_256
3917     for (int k = 0; k < 3; ++k) {
3918       //multi blocks starts here
3919       __ align(OptoLoopAlignment);
3920       __ BIND(L_multiBlock_loopTop[k]);
3921       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least PARALLEL_FACTOR blocks left
3922       __ jcc(Assembler::less, L_singleBlockLoopTop[k]);
3923       load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
3924 
3925       //load, then increase counters
3926       CTR_DoSix(movdqa, xmm_curr_counter);
3927       inc_counter(rbx, xmm_result1, 0x01, L__incCounter[k][0]);
3928       inc_counter(rbx, xmm_result2, 0x02, L__incCounter[k][1]);
3929       inc_counter(rbx, xmm_result3, 0x03, L__incCounter[k][2]);
3930       inc_counter(rbx, xmm_result4, 0x04, L__incCounter[k][3]);
3931       inc_counter(rbx, xmm_result5,  0x05, L__incCounter[k][4]);
3932       inc_counter(rbx, xmm_curr_counter, 0x06, L__incCounter[k][5]);
3933       CTR_DoSix(pshufb, xmm_counter_shuf_mask); // after increased, shuffled counters back for PXOR
3934       CTR_DoSix(pxor, xmm_key_tmp0);   //PXOR with Round 0 key
3935 
3936       //load two ROUND_KEYs at a time
3937       for (int i = 1; i < rounds[k]; ) {
3938         load_key(xmm_key_tmp1, key, (0x10 * i), xmm_key_shuf_mask);
3939         load_key(xmm_key_tmp0, key, (0x10 * (i+1)), xmm_key_shuf_mask);
3940         CTR_DoSix(aesenc, xmm_key_tmp1);
3941         i++;
3942         if (i != rounds[k]) {
3943           CTR_DoSix(aesenc, xmm_key_tmp0);
3944         } else {
3945           CTR_DoSix(aesenclast, xmm_key_tmp0);
3946         }
3947         i++;
3948       }
3949 
3950       // get next PARALLEL_FACTOR blocks into xmm_result registers
3951       __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
3952       __ movdqu(xmm_from1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
3953       __ movdqu(xmm_from2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
3954       __ movdqu(xmm_from3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
3955       __ movdqu(xmm_from4, Address(from, pos, Address::times_1, 4 * AESBlockSize));
3956       __ movdqu(xmm_from5, Address(from, pos, Address::times_1, 5 * AESBlockSize));
3957 
3958       __ pxor(xmm_result0, xmm_from0);
3959       __ pxor(xmm_result1, xmm_from1);
3960       __ pxor(xmm_result2, xmm_from2);
3961       __ pxor(xmm_result3, xmm_from3);
3962       __ pxor(xmm_result4, xmm_from4);
3963       __ pxor(xmm_result5, xmm_from5);
3964 
3965       // store 6 results into the next 64 bytes of output
3966       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
3967       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
3968       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
3969       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
3970       __ movdqu(Address(to, pos, Address::times_1, 4 * AESBlockSize), xmm_result4);
3971       __ movdqu(Address(to, pos, Address::times_1, 5 * AESBlockSize), xmm_result5);
3972 
3973       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize); // increase the length of crypt text
3974       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // decrease the remaining length
3975       __ jmp(L_multiBlock_loopTop[k]);
3976 
3977       // singleBlock starts here
3978       __ align(OptoLoopAlignment);
3979       __ BIND(L_singleBlockLoopTop[k]);
3980       __ cmpptr(len_reg, 0);
3981       __ jcc(Assembler::lessEqual, L_exit);
3982       load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
3983       __ movdqa(xmm_result0, xmm_curr_counter);
3984       inc_counter(rbx, xmm_curr_counter, 0x01, L__incCounter_single[k]);
3985       __ pshufb(xmm_result0, xmm_counter_shuf_mask);
3986       __ pxor(xmm_result0, xmm_key_tmp0);
3987       for (int i = 1; i < rounds[k]; i++) {
3988         load_key(xmm_key_tmp0, key, (0x10 * i), xmm_key_shuf_mask);
3989         __ aesenc(xmm_result0, xmm_key_tmp0);
3990       }
3991       load_key(xmm_key_tmp0, key, (rounds[k] * 0x10), xmm_key_shuf_mask);
3992       __ aesenclast(xmm_result0, xmm_key_tmp0);
3993       __ cmpptr(len_reg, AESBlockSize);
3994       __ jcc(Assembler::less, L_processTail_insr[k]);
3995         __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
3996         __ pxor(xmm_result0, xmm_from0);
3997         __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
3998         __ addptr(pos, AESBlockSize);
3999         __ subptr(len_reg, AESBlockSize);
4000         __ jmp(L_singleBlockLoopTop[k]);
4001       __ BIND(L_processTail_insr[k]);                               // Process the tail part of the input array
4002         __ addptr(pos, len_reg);                                    // 1. Insert bytes from src array into xmm_from0 register
4003         __ testptr(len_reg, 8);
4004         __ jcc(Assembler::zero, L_processTail_4_insr[k]);
4005           __ subptr(pos,8);
4006           __ pinsrq(xmm_from0, Address(from, pos), 0);
4007         __ BIND(L_processTail_4_insr[k]);
4008         __ testptr(len_reg, 4);
4009         __ jcc(Assembler::zero, L_processTail_2_insr[k]);
4010           __ subptr(pos,4);
4011           __ pslldq(xmm_from0, 4);
4012           __ pinsrd(xmm_from0, Address(from, pos), 0);
4013         __ BIND(L_processTail_2_insr[k]);
4014         __ testptr(len_reg, 2);
4015         __ jcc(Assembler::zero, L_processTail_1_insr[k]);
4016           __ subptr(pos, 2);
4017           __ pslldq(xmm_from0, 2);
4018           __ pinsrw(xmm_from0, Address(from, pos), 0);
4019         __ BIND(L_processTail_1_insr[k]);
4020         __ testptr(len_reg, 1);
4021         __ jcc(Assembler::zero, L_processTail_exit_insr[k]);
4022           __ subptr(pos, 1);
4023           __ pslldq(xmm_from0, 1);
4024           __ pinsrb(xmm_from0, Address(from, pos), 0);
4025         __ BIND(L_processTail_exit_insr[k]);
4026 
4027         __ movdqu(Address(saved_encCounter_start, 0), xmm_result0);  // 2. Perform pxor of the encrypted counter and plaintext Bytes.
4028         __ pxor(xmm_result0, xmm_from0);                             //    Also the encrypted counter is saved for next invocation.
4029 
4030         __ testptr(len_reg, 8);
4031         __ jcc(Assembler::zero, L_processTail_4_extr[k]);            // 3. Extract bytes from xmm_result0 into the dest. array
4032           __ pextrq(Address(to, pos), xmm_result0, 0);
4033           __ psrldq(xmm_result0, 8);
4034           __ addptr(pos, 8);
4035         __ BIND(L_processTail_4_extr[k]);
4036         __ testptr(len_reg, 4);
4037         __ jcc(Assembler::zero, L_processTail_2_extr[k]);
4038           __ pextrd(Address(to, pos), xmm_result0, 0);
4039           __ psrldq(xmm_result0, 4);
4040           __ addptr(pos, 4);
4041         __ BIND(L_processTail_2_extr[k]);
4042         __ testptr(len_reg, 2);
4043         __ jcc(Assembler::zero, L_processTail_1_extr[k]);
4044           __ pextrw(Address(to, pos), xmm_result0, 0);
4045           __ psrldq(xmm_result0, 2);
4046           __ addptr(pos, 2);
4047         __ BIND(L_processTail_1_extr[k]);
4048         __ testptr(len_reg, 1);
4049         __ jcc(Assembler::zero, L_processTail_exit_extr[k]);
4050           __ pextrb(Address(to, pos), xmm_result0, 0);
4051 
4052         __ BIND(L_processTail_exit_extr[k]);
4053         __ movl(Address(used_addr, 0), len_reg);
4054         __ jmp(L_exit);
4055 
4056     }
4057 
4058     __ BIND(L_exit);
4059     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled back.
4060     __ movdqu(Address(counter, 0), xmm_curr_counter); //save counter back
4061     __ pop(rbx); // pop the saved RBX.
4062 #ifdef _WIN64
4063     __ movl(rax, len_mem);
4064     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
4065     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
4066     __ addptr(rsp, 2 * wordSize);
4067 #else
4068     __ pop(rax); // return 'len'
4069 #endif
4070     __ leave(); // required for proper stackwalking of RuntimeStub frame
4071     __ ret(0);
4072     return start;
4073   }
4074 
4075   // byte swap x86 long
4076   address generate_ghash_long_swap_mask() {
4077     __ align(CodeEntryAlignment);
4078     StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
4079     address start = __ pc();
4080     __ emit_data64(0x0f0e0d0c0b0a0908, relocInfo::none );
4081     __ emit_data64(0x0706050403020100, relocInfo::none );
4082   return start;
4083   }
4084 
4085   // byte swap x86 byte array
4086   address generate_ghash_byte_swap_mask() {
4087     __ align(CodeEntryAlignment);
4088     StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
4089     address start = __ pc();
4090     __ emit_data64(0x08090a0b0c0d0e0f, relocInfo::none );
4091     __ emit_data64(0x0001020304050607, relocInfo::none );
4092   return start;
4093   }
4094 
4095   /* Single and multi-block ghash operations */
4096   address generate_ghash_processBlocks() {
4097     __ align(CodeEntryAlignment);
4098     Label L_ghash_loop, L_exit;
4099     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
4100     address start = __ pc();
4101 
4102     const Register state        = c_rarg0;
4103     const Register subkeyH      = c_rarg1;
4104     const Register data         = c_rarg2;
4105     const Register blocks       = c_rarg3;
4106 
4107     const XMMRegister xmm_temp0 = xmm0;
4108     const XMMRegister xmm_temp1 = xmm1;
4109     const XMMRegister xmm_temp2 = xmm2;
4110     const XMMRegister xmm_temp3 = xmm3;
4111     const XMMRegister xmm_temp4 = xmm4;
4112     const XMMRegister xmm_temp5 = xmm5;
4113     const XMMRegister xmm_temp6 = xmm6;
4114     const XMMRegister xmm_temp7 = xmm7;
4115     const XMMRegister xmm_temp8 = xmm8;
4116     const XMMRegister xmm_temp9 = xmm9;
4117     const XMMRegister xmm_temp10 = xmm10;
4118 
4119     __ enter();
4120 
4121     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
4122     // context for the registers used, where all instructions below are using 128-bit mode
4123     // On EVEX without VL and BW, these instructions will all be AVX.
4124     if (VM_Version::supports_avx512vlbw()) {
4125       __ movl(rax, 0xffff);
4126       __ kmovql(k1, rax);
4127     }
4128 
4129     __ movdqu(xmm_temp10, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
4130 
4131     __ movdqu(xmm_temp0, Address(state, 0));
4132     __ pshufb(xmm_temp0, xmm_temp10);
4133 
4134 
4135     __ BIND(L_ghash_loop);
4136     __ movdqu(xmm_temp2, Address(data, 0));
4137     __ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
4138 
4139     __ movdqu(xmm_temp1, Address(subkeyH, 0));
4140     __ pshufb(xmm_temp1, xmm_temp10);
4141 
4142     __ pxor(xmm_temp0, xmm_temp2);
4143 
4144     //
4145     // Multiply with the hash key
4146     //
4147     __ movdqu(xmm_temp3, xmm_temp0);
4148     __ pclmulqdq(xmm_temp3, xmm_temp1, 0);      // xmm3 holds a0*b0
4149     __ movdqu(xmm_temp4, xmm_temp0);
4150     __ pclmulqdq(xmm_temp4, xmm_temp1, 16);     // xmm4 holds a0*b1
4151 
4152     __ movdqu(xmm_temp5, xmm_temp0);
4153     __ pclmulqdq(xmm_temp5, xmm_temp1, 1);      // xmm5 holds a1*b0
4154     __ movdqu(xmm_temp6, xmm_temp0);
4155     __ pclmulqdq(xmm_temp6, xmm_temp1, 17);     // xmm6 holds a1*b1
4156 
4157     __ pxor(xmm_temp4, xmm_temp5);      // xmm4 holds a0*b1 + a1*b0
4158 
4159     __ movdqu(xmm_temp5, xmm_temp4);    // move the contents of xmm4 to xmm5
4160     __ psrldq(xmm_temp4, 8);    // shift by xmm4 64 bits to the right
4161     __ pslldq(xmm_temp5, 8);    // shift by xmm5 64 bits to the left
4162     __ pxor(xmm_temp3, xmm_temp5);
4163     __ pxor(xmm_temp6, xmm_temp4);      // Register pair <xmm6:xmm3> holds the result
4164                                         // of the carry-less multiplication of
4165                                         // xmm0 by xmm1.
4166 
4167     // We shift the result of the multiplication by one bit position
4168     // to the left to cope for the fact that the bits are reversed.
4169     __ movdqu(xmm_temp7, xmm_temp3);
4170     __ movdqu(xmm_temp8, xmm_temp6);
4171     __ pslld(xmm_temp3, 1);
4172     __ pslld(xmm_temp6, 1);
4173     __ psrld(xmm_temp7, 31);
4174     __ psrld(xmm_temp8, 31);
4175     __ movdqu(xmm_temp9, xmm_temp7);
4176     __ pslldq(xmm_temp8, 4);
4177     __ pslldq(xmm_temp7, 4);
4178     __ psrldq(xmm_temp9, 12);
4179     __ por(xmm_temp3, xmm_temp7);
4180     __ por(xmm_temp6, xmm_temp8);
4181     __ por(xmm_temp6, xmm_temp9);
4182 
4183     //
4184     // First phase of the reduction
4185     //
4186     // Move xmm3 into xmm7, xmm8, xmm9 in order to perform the shifts
4187     // independently.
4188     __ movdqu(xmm_temp7, xmm_temp3);
4189     __ movdqu(xmm_temp8, xmm_temp3);
4190     __ movdqu(xmm_temp9, xmm_temp3);
4191     __ pslld(xmm_temp7, 31);    // packed right shift shifting << 31
4192     __ pslld(xmm_temp8, 30);    // packed right shift shifting << 30
4193     __ pslld(xmm_temp9, 25);    // packed right shift shifting << 25
4194     __ pxor(xmm_temp7, xmm_temp8);      // xor the shifted versions
4195     __ pxor(xmm_temp7, xmm_temp9);
4196     __ movdqu(xmm_temp8, xmm_temp7);
4197     __ pslldq(xmm_temp7, 12);
4198     __ psrldq(xmm_temp8, 4);
4199     __ pxor(xmm_temp3, xmm_temp7);      // first phase of the reduction complete
4200 
4201     //
4202     // Second phase of the reduction
4203     //
4204     // Make 3 copies of xmm3 in xmm2, xmm4, xmm5 for doing these
4205     // shift operations.
4206     __ movdqu(xmm_temp2, xmm_temp3);
4207     __ movdqu(xmm_temp4, xmm_temp3);
4208     __ movdqu(xmm_temp5, xmm_temp3);
4209     __ psrld(xmm_temp2, 1);     // packed left shifting >> 1
4210     __ psrld(xmm_temp4, 2);     // packed left shifting >> 2
4211     __ psrld(xmm_temp5, 7);     // packed left shifting >> 7
4212     __ pxor(xmm_temp2, xmm_temp4);      // xor the shifted versions
4213     __ pxor(xmm_temp2, xmm_temp5);
4214     __ pxor(xmm_temp2, xmm_temp8);
4215     __ pxor(xmm_temp3, xmm_temp2);
4216     __ pxor(xmm_temp6, xmm_temp3);      // the result is in xmm6
4217 
4218     __ decrement(blocks);
4219     __ jcc(Assembler::zero, L_exit);
4220     __ movdqu(xmm_temp0, xmm_temp6);
4221     __ addptr(data, 16);
4222     __ jmp(L_ghash_loop);
4223 
4224     __ BIND(L_exit);
4225     __ pshufb(xmm_temp6, xmm_temp10);          // Byte swap 16-byte result
4226     __ movdqu(Address(state, 0), xmm_temp6);   // store the result
4227     __ leave();
4228     __ ret(0);
4229     return start;
4230   }
4231 
4232   /**
4233    *  Arguments:
4234    *
4235    * Inputs:
4236    *   c_rarg0   - int crc
4237    *   c_rarg1   - byte* buf
4238    *   c_rarg2   - int length
4239    *
4240    * Ouput:
4241    *       rax   - int crc result
4242    */
4243   address generate_updateBytesCRC32() {
4244     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
4245 
4246     __ align(CodeEntryAlignment);
4247     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
4248 
4249     address start = __ pc();
4250     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
4251     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
4252     // rscratch1: r10
4253     const Register crc   = c_rarg0;  // crc
4254     const Register buf   = c_rarg1;  // source java byte array address
4255     const Register len   = c_rarg2;  // length
4256     const Register table = c_rarg3;  // crc_table address (reuse register)
4257     const Register tmp   = r11;
4258     assert_different_registers(crc, buf, len, table, tmp, rax);
4259 
4260     BLOCK_COMMENT("Entry:");
4261     __ enter(); // required for proper stackwalking of RuntimeStub frame
4262 
4263     __ kernel_crc32(crc, buf, len, table, tmp);
4264 
4265     __ movl(rax, crc);
4266     __ vzeroupper();
4267     __ leave(); // required for proper stackwalking of RuntimeStub frame
4268     __ ret(0);
4269 
4270     return start;
4271   }
4272 
4273   /**
4274   *  Arguments:
4275   *
4276   * Inputs:
4277   *   c_rarg0   - int crc
4278   *   c_rarg1   - byte* buf
4279   *   c_rarg2   - long length
4280   *   c_rarg3   - table_start - optional (present only when doing a library_call,
4281   *              not used by x86 algorithm)
4282   *
4283   * Ouput:
4284   *       rax   - int crc result
4285   */
4286   address generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
4287       assert(UseCRC32CIntrinsics, "need SSE4_2");
4288       __ align(CodeEntryAlignment);
4289       StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
4290       address start = __ pc();
4291       //reg.arg        int#0        int#1        int#2        int#3        int#4        int#5        float regs
4292       //Windows        RCX          RDX          R8           R9           none         none         XMM0..XMM3
4293       //Lin / Sol      RDI          RSI          RDX          RCX          R8           R9           XMM0..XMM7
4294       const Register crc = c_rarg0;  // crc
4295       const Register buf = c_rarg1;  // source java byte array address
4296       const Register len = c_rarg2;  // length
4297       const Register a = rax;
4298       const Register j = r9;
4299       const Register k = r10;
4300       const Register l = r11;
4301 #ifdef _WIN64
4302       const Register y = rdi;
4303       const Register z = rsi;
4304 #else
4305       const Register y = rcx;
4306       const Register z = r8;
4307 #endif
4308       assert_different_registers(crc, buf, len, a, j, k, l, y, z);
4309 
4310       BLOCK_COMMENT("Entry:");
4311       __ enter(); // required for proper stackwalking of RuntimeStub frame
4312 #ifdef _WIN64
4313       __ push(y);
4314       __ push(z);
4315 #endif
4316       __ crc32c_ipl_alg2_alt2(crc, buf, len,
4317                               a, j, k,
4318                               l, y, z,
4319                               c_farg0, c_farg1, c_farg2,
4320                               is_pclmulqdq_supported);
4321       __ movl(rax, crc);
4322 #ifdef _WIN64
4323       __ pop(z);
4324       __ pop(y);
4325 #endif
4326       __ vzeroupper();
4327       __ leave(); // required for proper stackwalking of RuntimeStub frame
4328       __ ret(0);
4329 
4330       return start;
4331   }
4332 
4333   /**
4334    *  Arguments:
4335    *
4336    *  Input:
4337    *    c_rarg0   - x address
4338    *    c_rarg1   - x length
4339    *    c_rarg2   - y address
4340    *    c_rarg3   - y length
4341    * not Win64
4342    *    c_rarg4   - z address
4343    *    c_rarg5   - z length
4344    * Win64
4345    *    rsp+40    - z address
4346    *    rsp+48    - z length
4347    */
4348   address generate_multiplyToLen() {
4349     __ align(CodeEntryAlignment);
4350     StubCodeMark mark(this, "StubRoutines", "multiplyToLen");
4351 
4352     address start = __ pc();
4353     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
4354     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
4355     const Register x     = rdi;
4356     const Register xlen  = rax;
4357     const Register y     = rsi;
4358     const Register ylen  = rcx;
4359     const Register z     = r8;
4360     const Register zlen  = r11;
4361 
4362     // Next registers will be saved on stack in multiply_to_len().
4363     const Register tmp1  = r12;
4364     const Register tmp2  = r13;
4365     const Register tmp3  = r14;
4366     const Register tmp4  = r15;
4367     const Register tmp5  = rbx;
4368 
4369     BLOCK_COMMENT("Entry:");
4370     __ enter(); // required for proper stackwalking of RuntimeStub frame
4371 
4372 #ifndef _WIN64
4373     __ movptr(zlen, r9); // Save r9 in r11 - zlen
4374 #endif
4375     setup_arg_regs(4); // x => rdi, xlen => rsi, y => rdx
4376                        // ylen => rcx, z => r8, zlen => r11
4377                        // r9 and r10 may be used to save non-volatile registers
4378 #ifdef _WIN64
4379     // last 2 arguments (#4, #5) are on stack on Win64
4380     __ movptr(z, Address(rsp, 6 * wordSize));
4381     __ movptr(zlen, Address(rsp, 7 * wordSize));
4382 #endif
4383 
4384     __ movptr(xlen, rsi);
4385     __ movptr(y,    rdx);
4386     __ multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5);
4387 
4388     restore_arg_regs();
4389 
4390     __ leave(); // required for proper stackwalking of RuntimeStub frame
4391     __ ret(0);
4392 
4393     return start;
4394   }
4395 
4396   /**
4397   *  Arguments:
4398   *
4399   *  Input:
4400   *    c_rarg0   - obja     address
4401   *    c_rarg1   - objb     address
4402   *    c_rarg3   - length   length
4403   *    c_rarg4   - scale    log2_array_indxscale
4404   *
4405   *  Output:
4406   *        rax   - int >= mismatched index, < 0 bitwise complement of tail
4407   */
4408   address generate_vectorizedMismatch() {
4409     __ align(CodeEntryAlignment);
4410     StubCodeMark mark(this, "StubRoutines", "vectorizedMismatch");
4411     address start = __ pc();
4412 
4413     BLOCK_COMMENT("Entry:");
4414     __ enter();
4415 
4416 #ifdef _WIN64  // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
4417     const Register scale = c_rarg0;  //rcx, will exchange with r9
4418     const Register objb = c_rarg1;   //rdx
4419     const Register length = c_rarg2; //r8
4420     const Register obja = c_rarg3;   //r9
4421     __ xchgq(obja, scale);  //now obja and scale contains the correct contents
4422 
4423     const Register tmp1 = r10;
4424     const Register tmp2 = r11;
4425 #endif
4426 #ifndef _WIN64 // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
4427     const Register obja = c_rarg0;   //U:rdi
4428     const Register objb = c_rarg1;   //U:rsi
4429     const Register length = c_rarg2; //U:rdx
4430     const Register scale = c_rarg3;  //U:rcx
4431     const Register tmp1 = r8;
4432     const Register tmp2 = r9;
4433 #endif
4434     const Register result = rax; //return value
4435     const XMMRegister vec0 = xmm0;
4436     const XMMRegister vec1 = xmm1;
4437     const XMMRegister vec2 = xmm2;
4438 
4439     __ vectorized_mismatch(obja, objb, length, scale, result, tmp1, tmp2, vec0, vec1, vec2);
4440 
4441     __ vzeroupper();
4442     __ leave();
4443     __ ret(0);
4444 
4445     return start;
4446   }
4447 
4448 /**
4449    *  Arguments:
4450    *
4451   //  Input:
4452   //    c_rarg0   - x address
4453   //    c_rarg1   - x length
4454   //    c_rarg2   - z address
4455   //    c_rarg3   - z lenth
4456    *
4457    */
4458   address generate_squareToLen() {
4459 
4460     __ align(CodeEntryAlignment);
4461     StubCodeMark mark(this, "StubRoutines", "squareToLen");
4462 
4463     address start = __ pc();
4464     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
4465     // Unix:  rdi, rsi, rdx, rcx (c_rarg0, c_rarg1, ...)
4466     const Register x      = rdi;
4467     const Register len    = rsi;
4468     const Register z      = r8;
4469     const Register zlen   = rcx;
4470 
4471    const Register tmp1      = r12;
4472    const Register tmp2      = r13;
4473    const Register tmp3      = r14;
4474    const Register tmp4      = r15;
4475    const Register tmp5      = rbx;
4476 
4477     BLOCK_COMMENT("Entry:");
4478     __ enter(); // required for proper stackwalking of RuntimeStub frame
4479 
4480        setup_arg_regs(4); // x => rdi, len => rsi, z => rdx
4481                           // zlen => rcx
4482                           // r9 and r10 may be used to save non-volatile registers
4483     __ movptr(r8, rdx);
4484     __ square_to_len(x, len, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
4485 
4486     restore_arg_regs();
4487 
4488     __ leave(); // required for proper stackwalking of RuntimeStub frame
4489     __ ret(0);
4490 
4491     return start;
4492   }
4493 
4494    /**
4495    *  Arguments:
4496    *
4497    *  Input:
4498    *    c_rarg0   - out address
4499    *    c_rarg1   - in address
4500    *    c_rarg2   - offset
4501    *    c_rarg3   - len
4502    * not Win64
4503    *    c_rarg4   - k
4504    * Win64
4505    *    rsp+40    - k
4506    */
4507   address generate_mulAdd() {
4508     __ align(CodeEntryAlignment);
4509     StubCodeMark mark(this, "StubRoutines", "mulAdd");
4510 
4511     address start = __ pc();
4512     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
4513     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
4514     const Register out     = rdi;
4515     const Register in      = rsi;
4516     const Register offset  = r11;
4517     const Register len     = rcx;
4518     const Register k       = r8;
4519 
4520     // Next registers will be saved on stack in mul_add().
4521     const Register tmp1  = r12;
4522     const Register tmp2  = r13;
4523     const Register tmp3  = r14;
4524     const Register tmp4  = r15;
4525     const Register tmp5  = rbx;
4526 
4527     BLOCK_COMMENT("Entry:");
4528     __ enter(); // required for proper stackwalking of RuntimeStub frame
4529 
4530     setup_arg_regs(4); // out => rdi, in => rsi, offset => rdx
4531                        // len => rcx, k => r8
4532                        // r9 and r10 may be used to save non-volatile registers
4533 #ifdef _WIN64
4534     // last argument is on stack on Win64
4535     __ movl(k, Address(rsp, 6 * wordSize));
4536 #endif
4537     __ movptr(r11, rdx);  // move offset in rdx to offset(r11)
4538     __ mul_add(out, in, offset, len, k, tmp1, tmp2, tmp3, tmp4, tmp5, rdx, rax);
4539 
4540     restore_arg_regs();
4541 
4542     __ leave(); // required for proper stackwalking of RuntimeStub frame
4543     __ ret(0);
4544 
4545     return start;
4546   }
4547 
4548   address generate_libmExp() {
4549     StubCodeMark mark(this, "StubRoutines", "libmExp");
4550 
4551     address start = __ pc();
4552 
4553     const XMMRegister x0  = xmm0;
4554     const XMMRegister x1  = xmm1;
4555     const XMMRegister x2  = xmm2;
4556     const XMMRegister x3  = xmm3;
4557 
4558     const XMMRegister x4  = xmm4;
4559     const XMMRegister x5  = xmm5;
4560     const XMMRegister x6  = xmm6;
4561     const XMMRegister x7  = xmm7;
4562 
4563     const Register tmp   = r11;
4564 
4565     BLOCK_COMMENT("Entry:");
4566     __ enter(); // required for proper stackwalking of RuntimeStub frame
4567 
4568     __ fast_exp(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
4569 
4570     __ leave(); // required for proper stackwalking of RuntimeStub frame
4571     __ ret(0);
4572 
4573     return start;
4574 
4575   }
4576 
4577   address generate_libmLog() {
4578     StubCodeMark mark(this, "StubRoutines", "libmLog");
4579 
4580     address start = __ pc();
4581 
4582     const XMMRegister x0 = xmm0;
4583     const XMMRegister x1 = xmm1;
4584     const XMMRegister x2 = xmm2;
4585     const XMMRegister x3 = xmm3;
4586 
4587     const XMMRegister x4 = xmm4;
4588     const XMMRegister x5 = xmm5;
4589     const XMMRegister x6 = xmm6;
4590     const XMMRegister x7 = xmm7;
4591 
4592     const Register tmp1 = r11;
4593     const Register tmp2 = r8;
4594 
4595     BLOCK_COMMENT("Entry:");
4596     __ enter(); // required for proper stackwalking of RuntimeStub frame
4597 
4598     __ fast_log(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2);
4599 
4600     __ leave(); // required for proper stackwalking of RuntimeStub frame
4601     __ ret(0);
4602 
4603     return start;
4604 
4605   }
4606 
4607   address generate_libmLog10() {
4608     StubCodeMark mark(this, "StubRoutines", "libmLog10");
4609 
4610     address start = __ pc();
4611 
4612     const XMMRegister x0 = xmm0;
4613     const XMMRegister x1 = xmm1;
4614     const XMMRegister x2 = xmm2;
4615     const XMMRegister x3 = xmm3;
4616 
4617     const XMMRegister x4 = xmm4;
4618     const XMMRegister x5 = xmm5;
4619     const XMMRegister x6 = xmm6;
4620     const XMMRegister x7 = xmm7;
4621 
4622     const Register tmp = r11;
4623 
4624     BLOCK_COMMENT("Entry:");
4625     __ enter(); // required for proper stackwalking of RuntimeStub frame
4626 
4627     __ fast_log10(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
4628 
4629     __ leave(); // required for proper stackwalking of RuntimeStub frame
4630     __ ret(0);
4631 
4632     return start;
4633 
4634   }
4635 
4636   address generate_libmPow() {
4637     StubCodeMark mark(this, "StubRoutines", "libmPow");
4638 
4639     address start = __ pc();
4640 
4641     const XMMRegister x0 = xmm0;
4642     const XMMRegister x1 = xmm1;
4643     const XMMRegister x2 = xmm2;
4644     const XMMRegister x3 = xmm3;
4645 
4646     const XMMRegister x4 = xmm4;
4647     const XMMRegister x5 = xmm5;
4648     const XMMRegister x6 = xmm6;
4649     const XMMRegister x7 = xmm7;
4650 
4651     const Register tmp1 = r8;
4652     const Register tmp2 = r9;
4653     const Register tmp3 = r10;
4654     const Register tmp4 = r11;
4655 
4656     BLOCK_COMMENT("Entry:");
4657     __ enter(); // required for proper stackwalking of RuntimeStub frame
4658 
4659     __ fast_pow(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
4660 
4661     __ leave(); // required for proper stackwalking of RuntimeStub frame
4662     __ ret(0);
4663 
4664     return start;
4665 
4666   }
4667 
4668   address generate_libmSin() {
4669     StubCodeMark mark(this, "StubRoutines", "libmSin");
4670 
4671     address start = __ pc();
4672 
4673     const XMMRegister x0 = xmm0;
4674     const XMMRegister x1 = xmm1;
4675     const XMMRegister x2 = xmm2;
4676     const XMMRegister x3 = xmm3;
4677 
4678     const XMMRegister x4 = xmm4;
4679     const XMMRegister x5 = xmm5;
4680     const XMMRegister x6 = xmm6;
4681     const XMMRegister x7 = xmm7;
4682 
4683     const Register tmp1 = r8;
4684     const Register tmp2 = r9;
4685     const Register tmp3 = r10;
4686     const Register tmp4 = r11;
4687 
4688     BLOCK_COMMENT("Entry:");
4689     __ enter(); // required for proper stackwalking of RuntimeStub frame
4690 
4691 #ifdef _WIN64
4692     __ push(rsi);
4693     __ push(rdi);
4694 #endif
4695     __ fast_sin(x0, x1, x2, x3, x4, x5, x6, x7, rax, rbx, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
4696 
4697 #ifdef _WIN64
4698     __ pop(rdi);
4699     __ pop(rsi);
4700 #endif
4701 
4702     __ leave(); // required for proper stackwalking of RuntimeStub frame
4703     __ ret(0);
4704 
4705     return start;
4706 
4707   }
4708 
4709   address generate_libmCos() {
4710     StubCodeMark mark(this, "StubRoutines", "libmCos");
4711 
4712     address start = __ pc();
4713 
4714     const XMMRegister x0 = xmm0;
4715     const XMMRegister x1 = xmm1;
4716     const XMMRegister x2 = xmm2;
4717     const XMMRegister x3 = xmm3;
4718 
4719     const XMMRegister x4 = xmm4;
4720     const XMMRegister x5 = xmm5;
4721     const XMMRegister x6 = xmm6;
4722     const XMMRegister x7 = xmm7;
4723 
4724     const Register tmp1 = r8;
4725     const Register tmp2 = r9;
4726     const Register tmp3 = r10;
4727     const Register tmp4 = r11;
4728 
4729     BLOCK_COMMENT("Entry:");
4730     __ enter(); // required for proper stackwalking of RuntimeStub frame
4731 
4732 #ifdef _WIN64
4733     __ push(rsi);
4734     __ push(rdi);
4735 #endif
4736     __ fast_cos(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
4737 
4738 #ifdef _WIN64
4739     __ pop(rdi);
4740     __ pop(rsi);
4741 #endif
4742 
4743     __ leave(); // required for proper stackwalking of RuntimeStub frame
4744     __ ret(0);
4745 
4746     return start;
4747 
4748   }
4749 
4750   address generate_libmTan() {
4751     StubCodeMark mark(this, "StubRoutines", "libmTan");
4752 
4753     address start = __ pc();
4754 
4755     const XMMRegister x0 = xmm0;
4756     const XMMRegister x1 = xmm1;
4757     const XMMRegister x2 = xmm2;
4758     const XMMRegister x3 = xmm3;
4759 
4760     const XMMRegister x4 = xmm4;
4761     const XMMRegister x5 = xmm5;
4762     const XMMRegister x6 = xmm6;
4763     const XMMRegister x7 = xmm7;
4764 
4765     const Register tmp1 = r8;
4766     const Register tmp2 = r9;
4767     const Register tmp3 = r10;
4768     const Register tmp4 = r11;
4769 
4770     BLOCK_COMMENT("Entry:");
4771     __ enter(); // required for proper stackwalking of RuntimeStub frame
4772 
4773 #ifdef _WIN64
4774     __ push(rsi);
4775     __ push(rdi);
4776 #endif
4777     __ fast_tan(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp1, tmp2, tmp3, tmp4);
4778 
4779 #ifdef _WIN64
4780     __ pop(rdi);
4781     __ pop(rsi);
4782 #endif
4783 
4784     __ leave(); // required for proper stackwalking of RuntimeStub frame
4785     __ ret(0);
4786 
4787     return start;
4788 
4789   }
4790 
4791 #undef __
4792 #define __ masm->
4793 
4794   // Continuation point for throwing of implicit exceptions that are
4795   // not handled in the current activation. Fabricates an exception
4796   // oop and initiates normal exception dispatching in this
4797   // frame. Since we need to preserve callee-saved values (currently
4798   // only for C2, but done for C1 as well) we need a callee-saved oop
4799   // map and therefore have to make these stubs into RuntimeStubs
4800   // rather than BufferBlobs.  If the compiler needs all registers to
4801   // be preserved between the fault point and the exception handler
4802   // then it must assume responsibility for that in
4803   // AbstractCompiler::continuation_for_implicit_null_exception or
4804   // continuation_for_implicit_division_by_zero_exception. All other
4805   // implicit exceptions (e.g., NullPointerException or
4806   // AbstractMethodError on entry) are either at call sites or
4807   // otherwise assume that stack unwinding will be initiated, so
4808   // caller saved registers were assumed volatile in the compiler.
4809   address generate_throw_exception(const char* name,
4810                                    address runtime_entry,
4811                                    Register arg1 = noreg,
4812                                    Register arg2 = noreg) {
4813     // Information about frame layout at time of blocking runtime call.
4814     // Note that we only have to preserve callee-saved registers since
4815     // the compilers are responsible for supplying a continuation point
4816     // if they expect all registers to be preserved.
4817     enum layout {
4818       rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
4819       rbp_off2,
4820       return_off,
4821       return_off2,
4822       framesize // inclusive of return address
4823     };
4824 
4825     int insts_size = 512;
4826     int locs_size  = 64;
4827 
4828     CodeBuffer code(name, insts_size, locs_size);
4829     OopMapSet* oop_maps  = new OopMapSet();
4830     MacroAssembler* masm = new MacroAssembler(&code);
4831 
4832     address start = __ pc();
4833 
4834     // This is an inlined and slightly modified version of call_VM
4835     // which has the ability to fetch the return PC out of
4836     // thread-local storage and also sets up last_Java_sp slightly
4837     // differently than the real call_VM
4838 
4839     __ enter(); // required for proper stackwalking of RuntimeStub frame
4840 
4841     assert(is_even(framesize/2), "sp not 16-byte aligned");
4842 
4843     // return address and rbp are already in place
4844     __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
4845 
4846     int frame_complete = __ pc() - start;
4847 
4848     // Set up last_Java_sp and last_Java_fp
4849     address the_pc = __ pc();
4850     __ set_last_Java_frame(rsp, rbp, the_pc);
4851     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
4852 
4853     // Call runtime
4854     if (arg1 != noreg) {
4855       assert(arg2 != c_rarg1, "clobbered");
4856       __ movptr(c_rarg1, arg1);
4857     }
4858     if (arg2 != noreg) {
4859       __ movptr(c_rarg2, arg2);
4860     }
4861     __ movptr(c_rarg0, r15_thread);
4862     BLOCK_COMMENT("call runtime_entry");
4863     __ call(RuntimeAddress(runtime_entry));
4864 
4865     // Generate oop map
4866     OopMap* map = new OopMap(framesize, 0);
4867 
4868     oop_maps->add_gc_map(the_pc - start, map);
4869 
4870     __ reset_last_Java_frame(true);
4871 
4872     __ leave(); // required for proper stackwalking of RuntimeStub frame
4873 
4874     // check for pending exceptions
4875 #ifdef ASSERT
4876     Label L;
4877     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()),
4878             (int32_t) NULL_WORD);
4879     __ jcc(Assembler::notEqual, L);
4880     __ should_not_reach_here();
4881     __ bind(L);
4882 #endif // ASSERT
4883     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
4884 
4885 
4886     // codeBlob framesize is in words (not VMRegImpl::slot_size)
4887     RuntimeStub* stub =
4888       RuntimeStub::new_runtime_stub(name,
4889                                     &code,
4890                                     frame_complete,
4891                                     (framesize >> (LogBytesPerWord - LogBytesPerInt)),
4892                                     oop_maps, false);
4893     return stub->entry_point();
4894   }
4895 
4896   void create_control_words() {
4897     // Round to nearest, 53-bit mode, exceptions masked
4898     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
4899     // Round to zero, 53-bit mode, exception mased
4900     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
4901     // Round to nearest, 24-bit mode, exceptions masked
4902     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
4903     // Round to nearest, 64-bit mode, exceptions masked
4904     StubRoutines::_fpu_cntrl_wrd_64    = 0x037F;
4905     // Round to nearest, 64-bit mode, exceptions masked
4906     StubRoutines::_mxcsr_std           = 0x1F80;
4907     // Note: the following two constants are 80-bit values
4908     //       layout is critical for correct loading by FPU.
4909     // Bias for strict fp multiply/divide
4910     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
4911     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
4912     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
4913     // Un-Bias for strict fp multiply/divide
4914     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
4915     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
4916     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
4917   }
4918 
4919   // Initialization
4920   void generate_initial() {
4921     // Generates all stubs and initializes the entry points
4922 
4923     // This platform-specific settings are needed by generate_call_stub()
4924     create_control_words();
4925 
4926     // entry points that exist in all platforms Note: This is code
4927     // that could be shared among different platforms - however the
4928     // benefit seems to be smaller than the disadvantage of having a
4929     // much more complicated generator structure. See also comment in
4930     // stubRoutines.hpp.
4931 
4932     StubRoutines::_forward_exception_entry = generate_forward_exception();
4933 
4934     StubRoutines::_call_stub_entry =
4935       generate_call_stub(StubRoutines::_call_stub_return_address);
4936 
4937     // is referenced by megamorphic call
4938     StubRoutines::_catch_exception_entry = generate_catch_exception();
4939 
4940     // atomic calls
4941     StubRoutines::_atomic_xchg_entry          = generate_atomic_xchg();
4942     StubRoutines::_atomic_xchg_long_entry     = generate_atomic_xchg_long();
4943     StubRoutines::_atomic_cmpxchg_entry       = generate_atomic_cmpxchg();
4944     StubRoutines::_atomic_cmpxchg_byte_entry  = generate_atomic_cmpxchg_byte();
4945     StubRoutines::_atomic_cmpxchg_long_entry  = generate_atomic_cmpxchg_long();
4946     StubRoutines::_atomic_add_entry           = generate_atomic_add();
4947     StubRoutines::_atomic_add_long_entry      = generate_atomic_add_long();
4948     StubRoutines::_fence_entry                = generate_orderaccess_fence();
4949 
4950     // platform dependent
4951     StubRoutines::x86::_get_previous_fp_entry = generate_get_previous_fp();
4952     StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp();
4953 
4954     StubRoutines::x86::_verify_mxcsr_entry    = generate_verify_mxcsr();
4955 
4956     // Build this early so it's available for the interpreter.
4957     StubRoutines::_throw_StackOverflowError_entry =
4958       generate_throw_exception("StackOverflowError throw_exception",
4959                                CAST_FROM_FN_PTR(address,
4960                                                 SharedRuntime::
4961                                                 throw_StackOverflowError));
4962     StubRoutines::_throw_delayed_StackOverflowError_entry =
4963       generate_throw_exception("delayed StackOverflowError throw_exception",
4964                                CAST_FROM_FN_PTR(address,
4965                                                 SharedRuntime::
4966                                                 throw_delayed_StackOverflowError));
4967     if (UseCRC32Intrinsics) {
4968       // set table address before stub generation which use it
4969       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
4970       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
4971     }
4972 
4973     if (UseCRC32CIntrinsics) {
4974       bool supports_clmul = VM_Version::supports_clmul();
4975       StubRoutines::x86::generate_CRC32C_table(supports_clmul);
4976       StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table;
4977       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
4978     }
4979     if (VM_Version::supports_sse2() && UseLibmIntrinsic && InlineIntrinsics) {
4980       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
4981           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
4982           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
4983         StubRoutines::x86::_ONEHALF_adr = (address)StubRoutines::x86::_ONEHALF;
4984         StubRoutines::x86::_P_2_adr = (address)StubRoutines::x86::_P_2;
4985         StubRoutines::x86::_SC_4_adr = (address)StubRoutines::x86::_SC_4;
4986         StubRoutines::x86::_Ctable_adr = (address)StubRoutines::x86::_Ctable;
4987         StubRoutines::x86::_SC_2_adr = (address)StubRoutines::x86::_SC_2;
4988         StubRoutines::x86::_SC_3_adr = (address)StubRoutines::x86::_SC_3;
4989         StubRoutines::x86::_SC_1_adr = (address)StubRoutines::x86::_SC_1;
4990         StubRoutines::x86::_PI_INV_TABLE_adr = (address)StubRoutines::x86::_PI_INV_TABLE;
4991         StubRoutines::x86::_PI_4_adr = (address)StubRoutines::x86::_PI_4;
4992         StubRoutines::x86::_PI32INV_adr = (address)StubRoutines::x86::_PI32INV;
4993         StubRoutines::x86::_SIGN_MASK_adr = (address)StubRoutines::x86::_SIGN_MASK;
4994         StubRoutines::x86::_P_1_adr = (address)StubRoutines::x86::_P_1;
4995         StubRoutines::x86::_P_3_adr = (address)StubRoutines::x86::_P_3;
4996         StubRoutines::x86::_NEG_ZERO_adr = (address)StubRoutines::x86::_NEG_ZERO;
4997       }
4998       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) {
4999         StubRoutines::_dexp = generate_libmExp();
5000       }
5001       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) {
5002         StubRoutines::_dlog = generate_libmLog();
5003       }
5004       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) {
5005         StubRoutines::_dlog10 = generate_libmLog10();
5006       }
5007       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) {
5008         StubRoutines::_dpow = generate_libmPow();
5009       }
5010       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
5011         StubRoutines::_dsin = generate_libmSin();
5012       }
5013       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
5014         StubRoutines::_dcos = generate_libmCos();
5015       }
5016       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
5017         StubRoutines::_dtan = generate_libmTan();
5018       }
5019     }
5020   }
5021 
5022   void generate_all() {
5023     // Generates all stubs and initializes the entry points
5024 
5025     // These entry points require SharedInfo::stack0 to be set up in
5026     // non-core builds and need to be relocatable, so they each
5027     // fabricate a RuntimeStub internally.
5028     StubRoutines::_throw_AbstractMethodError_entry =
5029       generate_throw_exception("AbstractMethodError throw_exception",
5030                                CAST_FROM_FN_PTR(address,
5031                                                 SharedRuntime::
5032                                                 throw_AbstractMethodError));
5033 
5034     StubRoutines::_throw_IncompatibleClassChangeError_entry =
5035       generate_throw_exception("IncompatibleClassChangeError throw_exception",
5036                                CAST_FROM_FN_PTR(address,
5037                                                 SharedRuntime::
5038                                                 throw_IncompatibleClassChangeError));
5039 
5040     StubRoutines::_throw_NullPointerException_at_call_entry =
5041       generate_throw_exception("NullPointerException at call throw_exception",
5042                                CAST_FROM_FN_PTR(address,
5043                                                 SharedRuntime::
5044                                                 throw_NullPointerException_at_call));
5045 
5046     // entry points that are platform specific
5047     StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
5048     StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
5049     StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
5050     StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
5051 
5052     StubRoutines::x86::_float_sign_mask  = generate_fp_mask("float_sign_mask",  0x7FFFFFFF7FFFFFFF);
5053     StubRoutines::x86::_float_sign_flip  = generate_fp_mask("float_sign_flip",  0x8000000080000000);
5054     StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
5055     StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
5056 
5057     // support for verify_oop (must happen after universe_init)
5058     StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
5059 
5060     // arraycopy stubs used by compilers
5061     generate_arraycopy_stubs();
5062 
5063     // don't bother generating these AES intrinsic stubs unless global flag is set
5064     if (UseAESIntrinsics) {
5065       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // needed by the others
5066       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
5067       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
5068       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
5069       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
5070     }
5071     if (UseAESCTRIntrinsics){
5072       StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
5073       StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
5074     }
5075 
5076     if (UseSHA1Intrinsics) {
5077       StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask();
5078       StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask();
5079       StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
5080       StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
5081     }
5082     if (UseSHA256Intrinsics) {
5083       StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
5084       char* dst = (char*)StubRoutines::x86::_k256_W;
5085       char* src = (char*)StubRoutines::x86::_k256;
5086       for (int ii = 0; ii < 16; ++ii) {
5087         memcpy(dst + 32 * ii,      src + 16 * ii, 16);
5088         memcpy(dst + 32 * ii + 16, src + 16 * ii, 16);
5089       }
5090       StubRoutines::x86::_k256_W_adr = (address)StubRoutines::x86::_k256_W;
5091       StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
5092       StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
5093       StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
5094     }
5095     if (UseSHA512Intrinsics) {
5096       StubRoutines::x86::_k512_W_addr = (address)StubRoutines::x86::_k512_W;
5097       StubRoutines::x86::_pshuffle_byte_flip_mask_addr_sha512 = generate_pshuffle_byte_flip_mask_sha512();
5098       StubRoutines::_sha512_implCompress = generate_sha512_implCompress(false, "sha512_implCompress");
5099       StubRoutines::_sha512_implCompressMB = generate_sha512_implCompress(true, "sha512_implCompressMB");
5100     }
5101 
5102     // Generate GHASH intrinsics code
5103     if (UseGHASHIntrinsics) {
5104       StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
5105       StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
5106       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
5107     }
5108 
5109     // Safefetch stubs.
5110     generate_safefetch("SafeFetch32", sizeof(int),     &StubRoutines::_safefetch32_entry,
5111                                                        &StubRoutines::_safefetch32_fault_pc,
5112                                                        &StubRoutines::_safefetch32_continuation_pc);
5113     generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
5114                                                        &StubRoutines::_safefetchN_fault_pc,
5115                                                        &StubRoutines::_safefetchN_continuation_pc);
5116 #ifdef COMPILER2
5117     if (UseMultiplyToLenIntrinsic) {
5118       StubRoutines::_multiplyToLen = generate_multiplyToLen();
5119     }
5120     if (UseSquareToLenIntrinsic) {
5121       StubRoutines::_squareToLen = generate_squareToLen();
5122     }
5123     if (UseMulAddIntrinsic) {
5124       StubRoutines::_mulAdd = generate_mulAdd();
5125     }
5126 #ifndef _WINDOWS
5127     if (UseMontgomeryMultiplyIntrinsic) {
5128       StubRoutines::_montgomeryMultiply
5129         = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_multiply);
5130     }
5131     if (UseMontgomerySquareIntrinsic) {
5132       StubRoutines::_montgomerySquare
5133         = CAST_FROM_FN_PTR(address, SharedRuntime::montgomery_square);
5134     }
5135 #endif // WINDOWS
5136 #endif // COMPILER2
5137 
5138     if (UseVectorizedMismatchIntrinsic) {
5139       StubRoutines::_vectorizedMismatch = generate_vectorizedMismatch();
5140     }
5141   }
5142 
5143  public:
5144   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
5145     if (all) {
5146       generate_all();
5147     } else {
5148       generate_initial();
5149     }
5150   }
5151 }; // end class declaration
5152 
5153 void StubGenerator_generate(CodeBuffer* code, bool all) {
5154   StubGenerator g(code, all);
5155 }