1 /*
   2  * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "gc/shared/cardTable.hpp"
  29 #include "gc/shared/cardTableModRefBS.hpp"
  30 #include "interpreter/interpreter.hpp"
  31 #include "nativeInst_x86.hpp"
  32 #include "oops/instanceOop.hpp"
  33 #include "oops/method.hpp"
  34 #include "oops/objArrayKlass.hpp"
  35 #include "oops/oop.inline.hpp"
  36 #include "prims/methodHandles.hpp"
  37 #include "runtime/frame.inline.hpp"
  38 #include "runtime/handles.inline.hpp"
  39 #include "runtime/sharedRuntime.hpp"
  40 #include "runtime/stubCodeGenerator.hpp"
  41 #include "runtime/stubRoutines.hpp"
  42 #include "runtime/thread.inline.hpp"
  43 #ifdef COMPILER2
  44 #include "opto/runtime.hpp"
  45 #endif
  46 
  47 // Declaration and definition of StubGenerator (no .hpp file).
  48 // For a more detailed description of the stub routine structure
  49 // see the comment in stubRoutines.hpp
  50 
  51 #define __ _masm->
  52 #define a__ ((Assembler*)_masm)->
  53 
  54 #ifdef PRODUCT
  55 #define BLOCK_COMMENT(str) /* nothing */
  56 #else
  57 #define BLOCK_COMMENT(str) __ block_comment(str)
  58 #endif
  59 
  60 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  61 
  62 const int MXCSR_MASK  = 0xFFC0;  // Mask out any pending exceptions
  63 const int FPU_CNTRL_WRD_MASK = 0xFFFF;
  64 
  65 // -------------------------------------------------------------------------------------------------------------------------
  66 // Stub Code definitions
  67 
  68 class StubGenerator: public StubCodeGenerator {
  69  private:
  70 
  71 #ifdef PRODUCT
  72 #define inc_counter_np(counter) ((void)0)
  73 #else
  74   void inc_counter_np_(int& counter) {
  75     __ incrementl(ExternalAddress((address)&counter));
  76   }
  77 #define inc_counter_np(counter) \
  78   BLOCK_COMMENT("inc_counter " #counter); \
  79   inc_counter_np_(counter);
  80 #endif //PRODUCT
  81 
  82   void inc_copy_counter_np(BasicType t) {
  83 #ifndef PRODUCT
  84     switch (t) {
  85     case T_BYTE:    inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;
  86     case T_SHORT:   inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;
  87     case T_INT:     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;
  88     case T_LONG:    inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;
  89     case T_OBJECT:  inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;
  90     }
  91     ShouldNotReachHere();
  92 #endif //PRODUCT
  93   }
  94 
  95   //------------------------------------------------------------------------------------------------------------------------
  96   // Call stubs are used to call Java from C
  97   //
  98   //    [ return_from_Java     ] <--- rsp
  99   //    [ argument word n      ]
 100   //      ...
 101   // -N [ argument word 1      ]
 102   // -7 [ Possible padding for stack alignment ]
 103   // -6 [ Possible padding for stack alignment ]
 104   // -5 [ Possible padding for stack alignment ]
 105   // -4 [ mxcsr save           ] <--- rsp_after_call
 106   // -3 [ saved rbx,            ]
 107   // -2 [ saved rsi            ]
 108   // -1 [ saved rdi            ]
 109   //  0 [ saved rbp,            ] <--- rbp,
 110   //  1 [ return address       ]
 111   //  2 [ ptr. to call wrapper ]
 112   //  3 [ result               ]
 113   //  4 [ result_type          ]
 114   //  5 [ method               ]
 115   //  6 [ entry_point          ]
 116   //  7 [ parameters           ]
 117   //  8 [ parameter_size       ]
 118   //  9 [ thread               ]
 119 
 120 
 121   address generate_call_stub(address& return_address) {
 122     StubCodeMark mark(this, "StubRoutines", "call_stub");
 123     address start = __ pc();
 124 
 125     // stub code parameters / addresses
 126     assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");
 127     bool  sse_save = false;
 128     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!
 129     const int     locals_count_in_bytes  (4*wordSize);
 130     const Address mxcsr_save    (rbp, -4 * wordSize);
 131     const Address saved_rbx     (rbp, -3 * wordSize);
 132     const Address saved_rsi     (rbp, -2 * wordSize);
 133     const Address saved_rdi     (rbp, -1 * wordSize);
 134     const Address result        (rbp,  3 * wordSize);
 135     const Address result_type   (rbp,  4 * wordSize);
 136     const Address method        (rbp,  5 * wordSize);
 137     const Address entry_point   (rbp,  6 * wordSize);
 138     const Address parameters    (rbp,  7 * wordSize);
 139     const Address parameter_size(rbp,  8 * wordSize);
 140     const Address thread        (rbp,  9 * wordSize); // same as in generate_catch_exception()!
 141     sse_save =  UseSSE > 0;
 142 
 143     // stub code
 144     __ enter();
 145     __ movptr(rcx, parameter_size);              // parameter counter
 146     __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes
 147     __ addptr(rcx, locals_count_in_bytes);       // reserve space for register saves
 148     __ subptr(rsp, rcx);
 149     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
 150 
 151     // save rdi, rsi, & rbx, according to C calling conventions
 152     __ movptr(saved_rdi, rdi);
 153     __ movptr(saved_rsi, rsi);
 154     __ movptr(saved_rbx, rbx);
 155 
 156     // provide initial value for required masks
 157     if (UseAVX > 2) {
 158       __ movl(rbx, 0xffff);
 159       __ kmovwl(k1, rbx);
 160     }
 161 
 162     // save and initialize %mxcsr
 163     if (sse_save) {
 164       Label skip_ldmx;
 165       __ stmxcsr(mxcsr_save);
 166       __ movl(rax, mxcsr_save);
 167       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 168       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 169       __ cmp32(rax, mxcsr_std);
 170       __ jcc(Assembler::equal, skip_ldmx);
 171       __ ldmxcsr(mxcsr_std);
 172       __ bind(skip_ldmx);
 173     }
 174 
 175     // make sure the control word is correct.
 176     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 177 
 178 #ifdef ASSERT
 179     // make sure we have no pending exceptions
 180     { Label L;
 181       __ movptr(rcx, thread);
 182       __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 183       __ jcc(Assembler::equal, L);
 184       __ stop("StubRoutines::call_stub: entered with pending exception");
 185       __ bind(L);
 186     }
 187 #endif
 188 
 189     // pass parameters if any
 190     BLOCK_COMMENT("pass parameters if any");
 191     Label parameters_done;
 192     __ movl(rcx, parameter_size);  // parameter counter
 193     __ testl(rcx, rcx);
 194     __ jcc(Assembler::zero, parameters_done);
 195 
 196     // parameter passing loop
 197 
 198     Label loop;
 199     // Copy Java parameters in reverse order (receiver last)
 200     // Note that the argument order is inverted in the process
 201     // source is rdx[rcx: N-1..0]
 202     // dest   is rsp[rbx: 0..N-1]
 203 
 204     __ movptr(rdx, parameters);          // parameter pointer
 205     __ xorptr(rbx, rbx);
 206 
 207     __ BIND(loop);
 208 
 209     // get parameter
 210     __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
 211     __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
 212                     Interpreter::expr_offset_in_bytes(0)), rax);          // store parameter
 213     __ increment(rbx);
 214     __ decrement(rcx);
 215     __ jcc(Assembler::notZero, loop);
 216 
 217     // call Java function
 218     __ BIND(parameters_done);
 219     __ movptr(rbx, method);           // get Method*
 220     __ movptr(rax, entry_point);      // get entry_point
 221     __ mov(rsi, rsp);                 // set sender sp
 222     BLOCK_COMMENT("call Java function");
 223     __ call(rax);
 224 
 225     BLOCK_COMMENT("call_stub_return_address:");
 226     return_address = __ pc();
 227 
 228 #ifdef COMPILER2
 229     {
 230       Label L_skip;
 231       if (UseSSE >= 2) {
 232         __ verify_FPU(0, "call_stub_return");
 233       } else {
 234         for (int i = 1; i < 8; i++) {
 235           __ ffree(i);
 236         }
 237 
 238         // UseSSE <= 1 so double result should be left on TOS
 239         __ movl(rsi, result_type);
 240         __ cmpl(rsi, T_DOUBLE);
 241         __ jcc(Assembler::equal, L_skip);
 242         if (UseSSE == 0) {
 243           // UseSSE == 0 so float result should be left on TOS
 244           __ cmpl(rsi, T_FLOAT);
 245           __ jcc(Assembler::equal, L_skip);
 246         }
 247         __ ffree(0);
 248       }
 249       __ BIND(L_skip);
 250     }
 251 #endif // COMPILER2
 252 
 253     // store result depending on type
 254     // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 255     __ movptr(rdi, result);
 256     Label is_long, is_float, is_double, exit;
 257     __ movl(rsi, result_type);
 258     __ cmpl(rsi, T_LONG);
 259     __ jcc(Assembler::equal, is_long);
 260     __ cmpl(rsi, T_FLOAT);
 261     __ jcc(Assembler::equal, is_float);
 262     __ cmpl(rsi, T_DOUBLE);
 263     __ jcc(Assembler::equal, is_double);
 264 
 265     // handle T_INT case
 266     __ movl(Address(rdi, 0), rax);
 267     __ BIND(exit);
 268 
 269     // check that FPU stack is empty
 270     __ verify_FPU(0, "generate_call_stub");
 271 
 272     // pop parameters
 273     __ lea(rsp, rsp_after_call);
 274 
 275     // restore %mxcsr
 276     if (sse_save) {
 277       __ ldmxcsr(mxcsr_save);
 278     }
 279 
 280     // restore rdi, rsi and rbx,
 281     __ movptr(rbx, saved_rbx);
 282     __ movptr(rsi, saved_rsi);
 283     __ movptr(rdi, saved_rdi);
 284     __ addptr(rsp, 4*wordSize);
 285 
 286     // return
 287     __ pop(rbp);
 288     __ ret(0);
 289 
 290     // handle return types different from T_INT
 291     __ BIND(is_long);
 292     __ movl(Address(rdi, 0 * wordSize), rax);
 293     __ movl(Address(rdi, 1 * wordSize), rdx);
 294     __ jmp(exit);
 295 
 296     __ BIND(is_float);
 297     // interpreter uses xmm0 for return values
 298     if (UseSSE >= 1) {
 299       __ movflt(Address(rdi, 0), xmm0);
 300     } else {
 301       __ fstp_s(Address(rdi, 0));
 302     }
 303     __ jmp(exit);
 304 
 305     __ BIND(is_double);
 306     // interpreter uses xmm0 for return values
 307     if (UseSSE >= 2) {
 308       __ movdbl(Address(rdi, 0), xmm0);
 309     } else {
 310       __ fstp_d(Address(rdi, 0));
 311     }
 312     __ jmp(exit);
 313 
 314     return start;
 315   }
 316 
 317 
 318   //------------------------------------------------------------------------------------------------------------------------
 319   // Return point for a Java call if there's an exception thrown in Java code.
 320   // The exception is caught and transformed into a pending exception stored in
 321   // JavaThread that can be tested from within the VM.
 322   //
 323   // Note: Usually the parameters are removed by the callee. In case of an exception
 324   //       crossing an activation frame boundary, that is not the case if the callee
 325   //       is compiled code => need to setup the rsp.
 326   //
 327   // rax,: exception oop
 328 
 329   address generate_catch_exception() {
 330     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 331     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!
 332     const Address thread        (rbp,  9 * wordSize); // same as in generate_call_stub()!
 333     address start = __ pc();
 334 
 335     // get thread directly
 336     __ movptr(rcx, thread);
 337 #ifdef ASSERT
 338     // verify that threads correspond
 339     { Label L;
 340       __ get_thread(rbx);
 341       __ cmpptr(rbx, rcx);
 342       __ jcc(Assembler::equal, L);
 343       __ stop("StubRoutines::catch_exception: threads must correspond");
 344       __ bind(L);
 345     }
 346 #endif
 347     // set pending exception
 348     __ verify_oop(rax);
 349     __ movptr(Address(rcx, Thread::pending_exception_offset()), rax          );
 350     __ lea(Address(rcx, Thread::exception_file_offset   ()),
 351            ExternalAddress((address)__FILE__));
 352     __ movl(Address(rcx, Thread::exception_line_offset   ()), __LINE__ );
 353     // complete return to VM
 354     assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
 355     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 356 
 357     return start;
 358   }
 359 
 360 
 361   //------------------------------------------------------------------------------------------------------------------------
 362   // Continuation point for runtime calls returning with a pending exception.
 363   // The pending exception check happened in the runtime or native call stub.
 364   // The pending exception in Thread is converted into a Java-level exception.
 365   //
 366   // Contract with Java-level exception handlers:
 367   // rax: exception
 368   // rdx: throwing pc
 369   //
 370   // NOTE: At entry of this stub, exception-pc must be on stack !!
 371 
 372   address generate_forward_exception() {
 373     StubCodeMark mark(this, "StubRoutines", "forward exception");
 374     address start = __ pc();
 375     const Register thread = rcx;
 376 
 377     // other registers used in this stub
 378     const Register exception_oop = rax;
 379     const Register handler_addr  = rbx;
 380     const Register exception_pc  = rdx;
 381 
 382     // Upon entry, the sp points to the return address returning into Java
 383     // (interpreted or compiled) code; i.e., the return address becomes the
 384     // throwing pc.
 385     //
 386     // Arguments pushed before the runtime call are still on the stack but
 387     // the exception handler will reset the stack pointer -> ignore them.
 388     // A potential result in registers can be ignored as well.
 389 
 390 #ifdef ASSERT
 391     // make sure this code is only executed if there is a pending exception
 392     { Label L;
 393       __ get_thread(thread);
 394       __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 395       __ jcc(Assembler::notEqual, L);
 396       __ stop("StubRoutines::forward exception: no pending exception (1)");
 397       __ bind(L);
 398     }
 399 #endif
 400 
 401     // compute exception handler into rbx,
 402     __ get_thread(thread);
 403     __ movptr(exception_pc, Address(rsp, 0));
 404     BLOCK_COMMENT("call exception_handler_for_return_address");
 405     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
 406     __ mov(handler_addr, rax);
 407 
 408     // setup rax & rdx, remove return address & clear pending exception
 409     __ get_thread(thread);
 410     __ pop(exception_pc);
 411     __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
 412     __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
 413 
 414 #ifdef ASSERT
 415     // make sure exception is set
 416     { Label L;
 417       __ testptr(exception_oop, exception_oop);
 418       __ jcc(Assembler::notEqual, L);
 419       __ stop("StubRoutines::forward exception: no pending exception (2)");
 420       __ bind(L);
 421     }
 422 #endif
 423 
 424     // Verify that there is really a valid exception in RAX.
 425     __ verify_oop(exception_oop);
 426 
 427     // continue at exception handler (return address removed)
 428     // rax: exception
 429     // rbx: exception handler
 430     // rdx: throwing pc
 431     __ jmp(handler_addr);
 432 
 433     return start;
 434   }
 435 
 436 
 437   //----------------------------------------------------------------------------------------------------
 438   // Support for int32_t Atomic::xchg(int32_t exchange_value, volatile int32_t* dest)
 439   //
 440   // xchg exists as far back as 8086, lock needed for MP only
 441   // Stack layout immediately after call:
 442   //
 443   // 0 [ret addr ] <--- rsp
 444   // 1 [  ex     ]
 445   // 2 [  dest   ]
 446   //
 447   // Result:   *dest <- ex, return (old *dest)
 448   //
 449   // Note: win32 does not currently use this code
 450 
 451   address generate_atomic_xchg() {
 452     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 453     address start = __ pc();
 454 
 455     __ push(rdx);
 456     Address exchange(rsp, 2 * wordSize);
 457     Address dest_addr(rsp, 3 * wordSize);
 458     __ movl(rax, exchange);
 459     __ movptr(rdx, dest_addr);
 460     __ xchgl(rax, Address(rdx, 0));
 461     __ pop(rdx);
 462     __ ret(0);
 463 
 464     return start;
 465   }
 466 
 467   //----------------------------------------------------------------------------------------------------
 468   // Support for void verify_mxcsr()
 469   //
 470   // This routine is used with -Xcheck:jni to verify that native
 471   // JNI code does not return to Java code without restoring the
 472   // MXCSR register to our expected state.
 473 
 474 
 475   address generate_verify_mxcsr() {
 476     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 477     address start = __ pc();
 478 
 479     const Address mxcsr_save(rsp, 0);
 480 
 481     if (CheckJNICalls && UseSSE > 0 ) {
 482       Label ok_ret;
 483       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 484       __ push(rax);
 485       __ subptr(rsp, wordSize);      // allocate a temp location
 486       __ stmxcsr(mxcsr_save);
 487       __ movl(rax, mxcsr_save);
 488       __ andl(rax, MXCSR_MASK);
 489       __ cmp32(rax, mxcsr_std);
 490       __ jcc(Assembler::equal, ok_ret);
 491 
 492       __ warn("MXCSR changed by native JNI code.");
 493 
 494       __ ldmxcsr(mxcsr_std);
 495 
 496       __ bind(ok_ret);
 497       __ addptr(rsp, wordSize);
 498       __ pop(rax);
 499     }
 500 
 501     __ ret(0);
 502 
 503     return start;
 504   }
 505 
 506 
 507   //---------------------------------------------------------------------------
 508   // Support for void verify_fpu_cntrl_wrd()
 509   //
 510   // This routine is used with -Xcheck:jni to verify that native
 511   // JNI code does not return to Java code without restoring the
 512   // FP control word to our expected state.
 513 
 514   address generate_verify_fpu_cntrl_wrd() {
 515     StubCodeMark mark(this, "StubRoutines", "verify_spcw");
 516     address start = __ pc();
 517 
 518     const Address fpu_cntrl_wrd_save(rsp, 0);
 519 
 520     if (CheckJNICalls) {
 521       Label ok_ret;
 522       __ push(rax);
 523       __ subptr(rsp, wordSize);      // allocate a temp location
 524       __ fnstcw(fpu_cntrl_wrd_save);
 525       __ movl(rax, fpu_cntrl_wrd_save);
 526       __ andl(rax, FPU_CNTRL_WRD_MASK);
 527       ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());
 528       __ cmp32(rax, fpu_std);
 529       __ jcc(Assembler::equal, ok_ret);
 530 
 531       __ warn("Floating point control word changed by native JNI code.");
 532 
 533       __ fldcw(fpu_std);
 534 
 535       __ bind(ok_ret);
 536       __ addptr(rsp, wordSize);
 537       __ pop(rax);
 538     }
 539 
 540     __ ret(0);
 541 
 542     return start;
 543   }
 544 
 545   //---------------------------------------------------------------------------
 546   // Wrapper for slow-case handling of double-to-integer conversion
 547   // d2i or f2i fast case failed either because it is nan or because
 548   // of under/overflow.
 549   // Input:  FPU TOS: float value
 550   // Output: rax, (rdx): integer (long) result
 551 
 552   address generate_d2i_wrapper(BasicType t, address fcn) {
 553     StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");
 554     address start = __ pc();
 555 
 556   // Capture info about frame layout
 557   enum layout { FPUState_off         = 0,
 558                 rbp_off              = FPUStateSizeInWords,
 559                 rdi_off,
 560                 rsi_off,
 561                 rcx_off,
 562                 rbx_off,
 563                 saved_argument_off,
 564                 saved_argument_off2, // 2nd half of double
 565                 framesize
 566   };
 567 
 568   assert(FPUStateSizeInWords == 27, "update stack layout");
 569 
 570     // Save outgoing argument to stack across push_FPU_state()
 571     __ subptr(rsp, wordSize * 2);
 572     __ fstp_d(Address(rsp, 0));
 573 
 574     // Save CPU & FPU state
 575     __ push(rbx);
 576     __ push(rcx);
 577     __ push(rsi);
 578     __ push(rdi);
 579     __ push(rbp);
 580     __ push_FPU_state();
 581 
 582     // push_FPU_state() resets the FP top of stack
 583     // Load original double into FP top of stack
 584     __ fld_d(Address(rsp, saved_argument_off * wordSize));
 585     // Store double into stack as outgoing argument
 586     __ subptr(rsp, wordSize*2);
 587     __ fst_d(Address(rsp, 0));
 588 
 589     // Prepare FPU for doing math in C-land
 590     __ empty_FPU_stack();
 591     // Call the C code to massage the double.  Result in EAX
 592     if (t == T_INT)
 593       { BLOCK_COMMENT("SharedRuntime::d2i"); }
 594     else if (t == T_LONG)
 595       { BLOCK_COMMENT("SharedRuntime::d2l"); }
 596     __ call_VM_leaf( fcn, 2 );
 597 
 598     // Restore CPU & FPU state
 599     __ pop_FPU_state();
 600     __ pop(rbp);
 601     __ pop(rdi);
 602     __ pop(rsi);
 603     __ pop(rcx);
 604     __ pop(rbx);
 605     __ addptr(rsp, wordSize * 2);
 606 
 607     __ ret(0);
 608 
 609     return start;
 610   }
 611 
 612 
 613   //----------------------------------------------------------------------------------------------------
 614   // Non-destructive plausibility checks for oops
 615 
 616   address generate_verify_oop() {
 617     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 618     address start = __ pc();
 619 
 620     // Incoming arguments on stack after saving rax,:
 621     //
 622     // [tos    ]: saved rdx
 623     // [tos + 1]: saved EFLAGS
 624     // [tos + 2]: return address
 625     // [tos + 3]: char* error message
 626     // [tos + 4]: oop   object to verify
 627     // [tos + 5]: saved rax, - saved by caller and bashed
 628 
 629     Label exit, error;
 630     __ pushf();
 631     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 632     __ push(rdx);                                // save rdx
 633     // make sure object is 'reasonable'
 634     __ movptr(rax, Address(rsp, 4 * wordSize));    // get object
 635     __ testptr(rax, rax);
 636     __ jcc(Assembler::zero, exit);               // if obj is NULL it is ok
 637 
 638     // Check if the oop is in the right area of memory
 639     const int oop_mask = Universe::verify_oop_mask();
 640     const int oop_bits = Universe::verify_oop_bits();
 641     __ mov(rdx, rax);
 642     __ andptr(rdx, oop_mask);
 643     __ cmpptr(rdx, oop_bits);
 644     __ jcc(Assembler::notZero, error);
 645 
 646     // make sure klass is 'reasonable', which is not zero.
 647     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
 648     __ testptr(rax, rax);
 649     __ jcc(Assembler::zero, error);              // if klass is NULL it is broken
 650 
 651     // return if everything seems ok
 652     __ bind(exit);
 653     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 654     __ pop(rdx);                                 // restore rdx
 655     __ popf();                                   // restore EFLAGS
 656     __ ret(3 * wordSize);                        // pop arguments
 657 
 658     // handle errors
 659     __ bind(error);
 660     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 661     __ pop(rdx);                                 // get saved rdx back
 662     __ popf();                                   // get saved EFLAGS off stack -- will be ignored
 663     __ pusha();                                  // push registers (eip = return address & msg are already pushed)
 664     BLOCK_COMMENT("call MacroAssembler::debug");
 665     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 666     __ popa();
 667     __ ret(3 * wordSize);                        // pop arguments
 668     return start;
 669   }
 670 
 671   //
 672   //  Generate pre-barrier for array stores
 673   //
 674   //  Input:
 675   //     start   -  starting address
 676   //     count   -  element count
 677   void  gen_write_ref_array_pre_barrier(Register start, Register count, bool uninitialized_target) {
 678     assert_different_registers(start, count);
 679     BarrierSet* bs = Universe::heap()->barrier_set();
 680     switch (bs->kind()) {
 681 #if INCLUDE_ALL_GCS
 682       case BarrierSet::G1SATBCTLogging:
 683         // With G1, don't generate the call if we statically know that the target in uninitialized
 684         if (!uninitialized_target) {
 685           Register thread = rax;
 686           Label filtered;
 687           __ push(thread);
 688           __ get_thread(thread);
 689           Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
 690                                                SATBMarkQueue::byte_offset_of_active()));
 691           // Is marking active?
 692           if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
 693             __ cmpl(in_progress, 0);
 694           } else {
 695             assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
 696             __ cmpb(in_progress, 0);
 697           }
 698           __ pop(thread);
 699           __ jcc(Assembler::equal, filtered);
 700 
 701            __ pusha();                      // push registers
 702            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre),
 703                            start, count);
 704            __ popa();
 705 
 706            __ bind(filtered);
 707          }
 708         break;
 709 #endif // INCLUDE_ALL_GCS
 710       case BarrierSet::CardTableModRef:
 711         break;
 712       default      :
 713         ShouldNotReachHere();
 714 
 715     }
 716   }
 717 
 718 
 719   //
 720   // Generate a post-barrier for an array store
 721   //
 722   //     start    -  starting address
 723   //     count    -  element count
 724   //
 725   //  The two input registers are overwritten.
 726   //
 727   void  gen_write_ref_array_post_barrier(Register start, Register count) {
 728     BarrierSet* bs = Universe::heap()->barrier_set();
 729     assert_different_registers(start, count);
 730     switch (bs->kind()) {
 731 #if INCLUDE_ALL_GCS
 732       case BarrierSet::G1SATBCTLogging:
 733         {
 734           __ pusha();                      // push registers
 735           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post),
 736                           start, count);
 737           __ popa();
 738         }
 739         break;
 740 #endif // INCLUDE_ALL_GCS
 741 
 742       case BarrierSet::CardTableModRef:
 743         {
 744           CardTableModRefBS* ctbs = barrier_set_cast<CardTableModRefBS>(bs);
 745           CardTable* ct = ctbs->card_table();
 746           assert(sizeof(*ct->byte_map_base()) == sizeof(jbyte), "adjust this code");
 747 
 748           Label L_loop;
 749           const Register end = count;  // elements count; end == start+count-1
 750           assert_different_registers(start, end);
 751 
 752           __ lea(end,  Address(start, count, Address::times_ptr, -wordSize));
 753           __ shrptr(start, CardTable::card_shift);
 754           __ shrptr(end,   CardTable::card_shift);
 755           __ subptr(end, start); // end --> count
 756         __ BIND(L_loop);
 757         intptr_t disp = (intptr_t) ct->byte_map_base();
 758           Address cardtable(start, count, Address::times_1, disp);
 759           __ movb(cardtable, 0);
 760           __ decrement(count);
 761           __ jcc(Assembler::greaterEqual, L_loop);
 762         }
 763         break;
 764       case BarrierSet::ModRef:
 765         break;
 766       default      :
 767         ShouldNotReachHere();
 768 
 769     }
 770   }
 771 
 772 
 773   // Copy 64 bytes chunks
 774   //
 775   // Inputs:
 776   //   from        - source array address
 777   //   to_from     - destination array address - from
 778   //   qword_count - 8-bytes element count, negative
 779   //
 780   void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
 781     assert( UseSSE >= 2, "supported cpu only" );
 782     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 783     if (UseAVX > 2) {
 784       __ push(rbx);
 785       __ movl(rbx, 0xffff);
 786       __ kmovwl(k1, rbx);
 787       __ pop(rbx);
 788     }
 789     // Copy 64-byte chunks
 790     __ jmpb(L_copy_64_bytes);
 791     __ align(OptoLoopAlignment);
 792   __ BIND(L_copy_64_bytes_loop);
 793 
 794     if (UseUnalignedLoadStores) {
 795       if (UseAVX > 2) {
 796         __ evmovdqul(xmm0, Address(from, 0), Assembler::AVX_512bit);
 797         __ evmovdqul(Address(from, to_from, Address::times_1, 0), xmm0, Assembler::AVX_512bit);
 798       } else if (UseAVX == 2) {
 799         __ vmovdqu(xmm0, Address(from,  0));
 800         __ vmovdqu(Address(from, to_from, Address::times_1,  0), xmm0);
 801         __ vmovdqu(xmm1, Address(from, 32));
 802         __ vmovdqu(Address(from, to_from, Address::times_1, 32), xmm1);
 803       } else {
 804         __ movdqu(xmm0, Address(from, 0));
 805         __ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
 806         __ movdqu(xmm1, Address(from, 16));
 807         __ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
 808         __ movdqu(xmm2, Address(from, 32));
 809         __ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
 810         __ movdqu(xmm3, Address(from, 48));
 811         __ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
 812       }
 813     } else {
 814       __ movq(xmm0, Address(from, 0));
 815       __ movq(Address(from, to_from, Address::times_1, 0), xmm0);
 816       __ movq(xmm1, Address(from, 8));
 817       __ movq(Address(from, to_from, Address::times_1, 8), xmm1);
 818       __ movq(xmm2, Address(from, 16));
 819       __ movq(Address(from, to_from, Address::times_1, 16), xmm2);
 820       __ movq(xmm3, Address(from, 24));
 821       __ movq(Address(from, to_from, Address::times_1, 24), xmm3);
 822       __ movq(xmm4, Address(from, 32));
 823       __ movq(Address(from, to_from, Address::times_1, 32), xmm4);
 824       __ movq(xmm5, Address(from, 40));
 825       __ movq(Address(from, to_from, Address::times_1, 40), xmm5);
 826       __ movq(xmm6, Address(from, 48));
 827       __ movq(Address(from, to_from, Address::times_1, 48), xmm6);
 828       __ movq(xmm7, Address(from, 56));
 829       __ movq(Address(from, to_from, Address::times_1, 56), xmm7);
 830     }
 831 
 832     __ addl(from, 64);
 833   __ BIND(L_copy_64_bytes);
 834     __ subl(qword_count, 8);
 835     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 836 
 837     if (UseUnalignedLoadStores && (UseAVX == 2)) {
 838       // clean upper bits of YMM registers
 839       __ vpxor(xmm0, xmm0);
 840       __ vpxor(xmm1, xmm1);
 841     }
 842     __ addl(qword_count, 8);
 843     __ jccb(Assembler::zero, L_exit);
 844     //
 845     // length is too short, just copy qwords
 846     //
 847   __ BIND(L_copy_8_bytes);
 848     __ movq(xmm0, Address(from, 0));
 849     __ movq(Address(from, to_from, Address::times_1), xmm0);
 850     __ addl(from, 8);
 851     __ decrement(qword_count);
 852     __ jcc(Assembler::greater, L_copy_8_bytes);
 853   __ BIND(L_exit);
 854   }
 855 
 856   // Copy 64 bytes chunks
 857   //
 858   // Inputs:
 859   //   from        - source array address
 860   //   to_from     - destination array address - from
 861   //   qword_count - 8-bytes element count, negative
 862   //
 863   void mmx_copy_forward(Register from, Register to_from, Register qword_count) {
 864     assert( VM_Version::supports_mmx(), "supported cpu only" );
 865     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 866     // Copy 64-byte chunks
 867     __ jmpb(L_copy_64_bytes);
 868     __ align(OptoLoopAlignment);
 869   __ BIND(L_copy_64_bytes_loop);
 870     __ movq(mmx0, Address(from, 0));
 871     __ movq(mmx1, Address(from, 8));
 872     __ movq(mmx2, Address(from, 16));
 873     __ movq(Address(from, to_from, Address::times_1, 0), mmx0);
 874     __ movq(mmx3, Address(from, 24));
 875     __ movq(Address(from, to_from, Address::times_1, 8), mmx1);
 876     __ movq(mmx4, Address(from, 32));
 877     __ movq(Address(from, to_from, Address::times_1, 16), mmx2);
 878     __ movq(mmx5, Address(from, 40));
 879     __ movq(Address(from, to_from, Address::times_1, 24), mmx3);
 880     __ movq(mmx6, Address(from, 48));
 881     __ movq(Address(from, to_from, Address::times_1, 32), mmx4);
 882     __ movq(mmx7, Address(from, 56));
 883     __ movq(Address(from, to_from, Address::times_1, 40), mmx5);
 884     __ movq(Address(from, to_from, Address::times_1, 48), mmx6);
 885     __ movq(Address(from, to_from, Address::times_1, 56), mmx7);
 886     __ addptr(from, 64);
 887   __ BIND(L_copy_64_bytes);
 888     __ subl(qword_count, 8);
 889     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 890     __ addl(qword_count, 8);
 891     __ jccb(Assembler::zero, L_exit);
 892     //
 893     // length is too short, just copy qwords
 894     //
 895   __ BIND(L_copy_8_bytes);
 896     __ movq(mmx0, Address(from, 0));
 897     __ movq(Address(from, to_from, Address::times_1), mmx0);
 898     __ addptr(from, 8);
 899     __ decrement(qword_count);
 900     __ jcc(Assembler::greater, L_copy_8_bytes);
 901   __ BIND(L_exit);
 902     __ emms();
 903   }
 904 
 905   address generate_disjoint_copy(BasicType t, bool aligned,
 906                                  Address::ScaleFactor sf,
 907                                  address* entry, const char *name,
 908                                  bool dest_uninitialized = false) {
 909     __ align(CodeEntryAlignment);
 910     StubCodeMark mark(this, "StubRoutines", name);
 911     address start = __ pc();
 912 
 913     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
 914     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;
 915 
 916     int shift = Address::times_ptr - sf;
 917 
 918     const Register from     = rsi;  // source array address
 919     const Register to       = rdi;  // destination array address
 920     const Register count    = rcx;  // elements count
 921     const Register to_from  = to;   // (to - from)
 922     const Register saved_to = rdx;  // saved destination array address
 923 
 924     __ enter(); // required for proper stackwalking of RuntimeStub frame
 925     __ push(rsi);
 926     __ push(rdi);
 927     __ movptr(from , Address(rsp, 12+ 4));
 928     __ movptr(to   , Address(rsp, 12+ 8));
 929     __ movl(count, Address(rsp, 12+ 12));
 930 
 931     if (entry != NULL) {
 932       *entry = __ pc(); // Entry point from conjoint arraycopy stub.
 933       BLOCK_COMMENT("Entry:");
 934     }
 935 
 936     if (t == T_OBJECT) {
 937       __ testl(count, count);
 938       __ jcc(Assembler::zero, L_0_count);
 939       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
 940       __ mov(saved_to, to);          // save 'to'
 941     }
 942 
 943     __ subptr(to, from); // to --> to_from
 944     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
 945     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
 946     if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 947       // align source address at 4 bytes address boundary
 948       if (t == T_BYTE) {
 949         // One byte misalignment happens only for byte arrays
 950         __ testl(from, 1);
 951         __ jccb(Assembler::zero, L_skip_align1);
 952         __ movb(rax, Address(from, 0));
 953         __ movb(Address(from, to_from, Address::times_1, 0), rax);
 954         __ increment(from);
 955         __ decrement(count);
 956       __ BIND(L_skip_align1);
 957       }
 958       // Two bytes misalignment happens only for byte and short (char) arrays
 959       __ testl(from, 2);
 960       __ jccb(Assembler::zero, L_skip_align2);
 961       __ movw(rax, Address(from, 0));
 962       __ movw(Address(from, to_from, Address::times_1, 0), rax);
 963       __ addptr(from, 2);
 964       __ subl(count, 1<<(shift-1));
 965     __ BIND(L_skip_align2);
 966     }
 967     if (!VM_Version::supports_mmx()) {
 968       __ mov(rax, count);      // save 'count'
 969       __ shrl(count, shift); // bytes count
 970       __ addptr(to_from, from);// restore 'to'
 971       __ rep_mov();
 972       __ subptr(to_from, from);// restore 'to_from'
 973       __ mov(count, rax);      // restore 'count'
 974       __ jmpb(L_copy_2_bytes); // all dwords were copied
 975     } else {
 976       if (!UseUnalignedLoadStores) {
 977         // align to 8 bytes, we know we are 4 byte aligned to start
 978         __ testptr(from, 4);
 979         __ jccb(Assembler::zero, L_copy_64_bytes);
 980         __ movl(rax, Address(from, 0));
 981         __ movl(Address(from, to_from, Address::times_1, 0), rax);
 982         __ addptr(from, 4);
 983         __ subl(count, 1<<shift);
 984       }
 985     __ BIND(L_copy_64_bytes);
 986       __ mov(rax, count);
 987       __ shrl(rax, shift+1);  // 8 bytes chunk count
 988       //
 989       // Copy 8-byte chunks through MMX registers, 8 per iteration of the loop
 990       //
 991       if (UseXMMForArrayCopy) {
 992         xmm_copy_forward(from, to_from, rax);
 993       } else {
 994         mmx_copy_forward(from, to_from, rax);
 995       }
 996     }
 997     // copy tailing dword
 998   __ BIND(L_copy_4_bytes);
 999     __ testl(count, 1<<shift);
1000     __ jccb(Assembler::zero, L_copy_2_bytes);
1001     __ movl(rax, Address(from, 0));
1002     __ movl(Address(from, to_from, Address::times_1, 0), rax);
1003     if (t == T_BYTE || t == T_SHORT) {
1004       __ addptr(from, 4);
1005     __ BIND(L_copy_2_bytes);
1006       // copy tailing word
1007       __ testl(count, 1<<(shift-1));
1008       __ jccb(Assembler::zero, L_copy_byte);
1009       __ movw(rax, Address(from, 0));
1010       __ movw(Address(from, to_from, Address::times_1, 0), rax);
1011       if (t == T_BYTE) {
1012         __ addptr(from, 2);
1013       __ BIND(L_copy_byte);
1014         // copy tailing byte
1015         __ testl(count, 1);
1016         __ jccb(Assembler::zero, L_exit);
1017         __ movb(rax, Address(from, 0));
1018         __ movb(Address(from, to_from, Address::times_1, 0), rax);
1019       __ BIND(L_exit);
1020       } else {
1021       __ BIND(L_copy_byte);
1022       }
1023     } else {
1024     __ BIND(L_copy_2_bytes);
1025     }
1026 
1027     if (t == T_OBJECT) {
1028       __ movl(count, Address(rsp, 12+12)); // reread 'count'
1029       __ mov(to, saved_to); // restore 'to'
1030       gen_write_ref_array_post_barrier(to, count);
1031     __ BIND(L_0_count);
1032     }
1033     inc_copy_counter_np(t);
1034     __ pop(rdi);
1035     __ pop(rsi);
1036     __ leave(); // required for proper stackwalking of RuntimeStub frame
1037     __ vzeroupper();
1038     __ xorptr(rax, rax); // return 0
1039     __ ret(0);
1040     return start;
1041   }
1042 
1043 
1044   address generate_fill(BasicType t, bool aligned, const char *name) {
1045     __ align(CodeEntryAlignment);
1046     StubCodeMark mark(this, "StubRoutines", name);
1047     address start = __ pc();
1048 
1049     BLOCK_COMMENT("Entry:");
1050 
1051     const Register to       = rdi;  // source array address
1052     const Register value    = rdx;  // value
1053     const Register count    = rsi;  // elements count
1054 
1055     __ enter(); // required for proper stackwalking of RuntimeStub frame
1056     __ push(rsi);
1057     __ push(rdi);
1058     __ movptr(to   , Address(rsp, 12+ 4));
1059     __ movl(value, Address(rsp, 12+ 8));
1060     __ movl(count, Address(rsp, 12+ 12));
1061 
1062     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1063 
1064     __ pop(rdi);
1065     __ pop(rsi);
1066     __ leave(); // required for proper stackwalking of RuntimeStub frame
1067     __ ret(0);
1068     return start;
1069   }
1070 
1071   address generate_conjoint_copy(BasicType t, bool aligned,
1072                                  Address::ScaleFactor sf,
1073                                  address nooverlap_target,
1074                                  address* entry, const char *name,
1075                                  bool dest_uninitialized = false) {
1076     __ align(CodeEntryAlignment);
1077     StubCodeMark mark(this, "StubRoutines", name);
1078     address start = __ pc();
1079 
1080     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
1081     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;
1082 
1083     int shift = Address::times_ptr - sf;
1084 
1085     const Register src   = rax;  // source array address
1086     const Register dst   = rdx;  // destination array address
1087     const Register from  = rsi;  // source array address
1088     const Register to    = rdi;  // destination array address
1089     const Register count = rcx;  // elements count
1090     const Register end   = rax;  // array end address
1091 
1092     __ enter(); // required for proper stackwalking of RuntimeStub frame
1093     __ push(rsi);
1094     __ push(rdi);
1095     __ movptr(src  , Address(rsp, 12+ 4));   // from
1096     __ movptr(dst  , Address(rsp, 12+ 8));   // to
1097     __ movl2ptr(count, Address(rsp, 12+12)); // count
1098 
1099     if (entry != NULL) {
1100       *entry = __ pc(); // Entry point from generic arraycopy stub.
1101       BLOCK_COMMENT("Entry:");
1102     }
1103 
1104     // nooverlap_target expects arguments in rsi and rdi.
1105     __ mov(from, src);
1106     __ mov(to  , dst);
1107 
1108     // arrays overlap test: dispatch to disjoint stub if necessary.
1109     RuntimeAddress nooverlap(nooverlap_target);
1110     __ cmpptr(dst, src);
1111     __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
1112     __ jump_cc(Assembler::belowEqual, nooverlap);
1113     __ cmpptr(dst, end);
1114     __ jump_cc(Assembler::aboveEqual, nooverlap);
1115 
1116     if (t == T_OBJECT) {
1117       __ testl(count, count);
1118       __ jcc(Assembler::zero, L_0_count);
1119       gen_write_ref_array_pre_barrier(dst, count, dest_uninitialized);
1120     }
1121 
1122     // copy from high to low
1123     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1124     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
1125     if (t == T_BYTE || t == T_SHORT) {
1126       // Align the end of destination array at 4 bytes address boundary
1127       __ lea(end, Address(dst, count, sf, 0));
1128       if (t == T_BYTE) {
1129         // One byte misalignment happens only for byte arrays
1130         __ testl(end, 1);
1131         __ jccb(Assembler::zero, L_skip_align1);
1132         __ decrement(count);
1133         __ movb(rdx, Address(from, count, sf, 0));
1134         __ movb(Address(to, count, sf, 0), rdx);
1135       __ BIND(L_skip_align1);
1136       }
1137       // Two bytes misalignment happens only for byte and short (char) arrays
1138       __ testl(end, 2);
1139       __ jccb(Assembler::zero, L_skip_align2);
1140       __ subptr(count, 1<<(shift-1));
1141       __ movw(rdx, Address(from, count, sf, 0));
1142       __ movw(Address(to, count, sf, 0), rdx);
1143     __ BIND(L_skip_align2);
1144       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1145       __ jcc(Assembler::below, L_copy_4_bytes);
1146     }
1147 
1148     if (!VM_Version::supports_mmx()) {
1149       __ std();
1150       __ mov(rax, count); // Save 'count'
1151       __ mov(rdx, to);    // Save 'to'
1152       __ lea(rsi, Address(from, count, sf, -4));
1153       __ lea(rdi, Address(to  , count, sf, -4));
1154       __ shrptr(count, shift); // bytes count
1155       __ rep_mov();
1156       __ cld();
1157       __ mov(count, rax); // restore 'count'
1158       __ andl(count, (1<<shift)-1);      // mask the number of rest elements
1159       __ movptr(from, Address(rsp, 12+4)); // reread 'from'
1160       __ mov(to, rdx);   // restore 'to'
1161       __ jmpb(L_copy_2_bytes); // all dword were copied
1162    } else {
1163       // Align to 8 bytes the end of array. It is aligned to 4 bytes already.
1164       __ testptr(end, 4);
1165       __ jccb(Assembler::zero, L_copy_8_bytes);
1166       __ subl(count, 1<<shift);
1167       __ movl(rdx, Address(from, count, sf, 0));
1168       __ movl(Address(to, count, sf, 0), rdx);
1169       __ jmpb(L_copy_8_bytes);
1170 
1171       __ align(OptoLoopAlignment);
1172       // Move 8 bytes
1173     __ BIND(L_copy_8_bytes_loop);
1174       if (UseXMMForArrayCopy) {
1175         __ movq(xmm0, Address(from, count, sf, 0));
1176         __ movq(Address(to, count, sf, 0), xmm0);
1177       } else {
1178         __ movq(mmx0, Address(from, count, sf, 0));
1179         __ movq(Address(to, count, sf, 0), mmx0);
1180       }
1181     __ BIND(L_copy_8_bytes);
1182       __ subl(count, 2<<shift);
1183       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1184       __ addl(count, 2<<shift);
1185       if (!UseXMMForArrayCopy) {
1186         __ emms();
1187       }
1188     }
1189   __ BIND(L_copy_4_bytes);
1190     // copy prefix qword
1191     __ testl(count, 1<<shift);
1192     __ jccb(Assembler::zero, L_copy_2_bytes);
1193     __ movl(rdx, Address(from, count, sf, -4));
1194     __ movl(Address(to, count, sf, -4), rdx);
1195 
1196     if (t == T_BYTE || t == T_SHORT) {
1197         __ subl(count, (1<<shift));
1198       __ BIND(L_copy_2_bytes);
1199         // copy prefix dword
1200         __ testl(count, 1<<(shift-1));
1201         __ jccb(Assembler::zero, L_copy_byte);
1202         __ movw(rdx, Address(from, count, sf, -2));
1203         __ movw(Address(to, count, sf, -2), rdx);
1204         if (t == T_BYTE) {
1205           __ subl(count, 1<<(shift-1));
1206         __ BIND(L_copy_byte);
1207           // copy prefix byte
1208           __ testl(count, 1);
1209           __ jccb(Assembler::zero, L_exit);
1210           __ movb(rdx, Address(from, 0));
1211           __ movb(Address(to, 0), rdx);
1212         __ BIND(L_exit);
1213         } else {
1214         __ BIND(L_copy_byte);
1215         }
1216     } else {
1217     __ BIND(L_copy_2_bytes);
1218     }
1219     if (t == T_OBJECT) {
1220       __ movl2ptr(count, Address(rsp, 12+12)); // reread count
1221       gen_write_ref_array_post_barrier(to, count);
1222     __ BIND(L_0_count);
1223     }
1224     inc_copy_counter_np(t);
1225     __ pop(rdi);
1226     __ pop(rsi);
1227     __ leave(); // required for proper stackwalking of RuntimeStub frame
1228     __ xorptr(rax, rax); // return 0
1229     __ ret(0);
1230     return start;
1231   }
1232 
1233 
1234   address generate_disjoint_long_copy(address* entry, const char *name) {
1235     __ align(CodeEntryAlignment);
1236     StubCodeMark mark(this, "StubRoutines", name);
1237     address start = __ pc();
1238 
1239     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1240     const Register from       = rax;  // source array address
1241     const Register to         = rdx;  // destination array address
1242     const Register count      = rcx;  // elements count
1243     const Register to_from    = rdx;  // (to - from)
1244 
1245     __ enter(); // required for proper stackwalking of RuntimeStub frame
1246     __ movptr(from , Address(rsp, 8+0));       // from
1247     __ movptr(to   , Address(rsp, 8+4));       // to
1248     __ movl2ptr(count, Address(rsp, 8+8));     // count
1249 
1250     *entry = __ pc(); // Entry point from conjoint arraycopy stub.
1251     BLOCK_COMMENT("Entry:");
1252 
1253     __ subptr(to, from); // to --> to_from
1254     if (VM_Version::supports_mmx()) {
1255       if (UseXMMForArrayCopy) {
1256         xmm_copy_forward(from, to_from, count);
1257       } else {
1258         mmx_copy_forward(from, to_from, count);
1259       }
1260     } else {
1261       __ jmpb(L_copy_8_bytes);
1262       __ align(OptoLoopAlignment);
1263     __ BIND(L_copy_8_bytes_loop);
1264       __ fild_d(Address(from, 0));
1265       __ fistp_d(Address(from, to_from, Address::times_1));
1266       __ addptr(from, 8);
1267     __ BIND(L_copy_8_bytes);
1268       __ decrement(count);
1269       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1270     }
1271     inc_copy_counter_np(T_LONG);
1272     __ leave(); // required for proper stackwalking of RuntimeStub frame
1273     __ vzeroupper();
1274     __ xorptr(rax, rax); // return 0
1275     __ ret(0);
1276     return start;
1277   }
1278 
1279   address generate_conjoint_long_copy(address nooverlap_target,
1280                                       address* entry, const char *name) {
1281     __ align(CodeEntryAlignment);
1282     StubCodeMark mark(this, "StubRoutines", name);
1283     address start = __ pc();
1284 
1285     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1286     const Register from       = rax;  // source array address
1287     const Register to         = rdx;  // destination array address
1288     const Register count      = rcx;  // elements count
1289     const Register end_from   = rax;  // source array end address
1290 
1291     __ enter(); // required for proper stackwalking of RuntimeStub frame
1292     __ movptr(from , Address(rsp, 8+0));       // from
1293     __ movptr(to   , Address(rsp, 8+4));       // to
1294     __ movl2ptr(count, Address(rsp, 8+8));     // count
1295 
1296     *entry = __ pc(); // Entry point from generic arraycopy stub.
1297     BLOCK_COMMENT("Entry:");
1298 
1299     // arrays overlap test
1300     __ cmpptr(to, from);
1301     RuntimeAddress nooverlap(nooverlap_target);
1302     __ jump_cc(Assembler::belowEqual, nooverlap);
1303     __ lea(end_from, Address(from, count, Address::times_8, 0));
1304     __ cmpptr(to, end_from);
1305     __ movptr(from, Address(rsp, 8));  // from
1306     __ jump_cc(Assembler::aboveEqual, nooverlap);
1307 
1308     __ jmpb(L_copy_8_bytes);
1309 
1310     __ align(OptoLoopAlignment);
1311   __ BIND(L_copy_8_bytes_loop);
1312     if (VM_Version::supports_mmx()) {
1313       if (UseXMMForArrayCopy) {
1314         __ movq(xmm0, Address(from, count, Address::times_8));
1315         __ movq(Address(to, count, Address::times_8), xmm0);
1316       } else {
1317         __ movq(mmx0, Address(from, count, Address::times_8));
1318         __ movq(Address(to, count, Address::times_8), mmx0);
1319       }
1320     } else {
1321       __ fild_d(Address(from, count, Address::times_8));
1322       __ fistp_d(Address(to, count, Address::times_8));
1323     }
1324   __ BIND(L_copy_8_bytes);
1325     __ decrement(count);
1326     __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1327 
1328     if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
1329       __ emms();
1330     }
1331     inc_copy_counter_np(T_LONG);
1332     __ leave(); // required for proper stackwalking of RuntimeStub frame
1333     __ xorptr(rax, rax); // return 0
1334     __ ret(0);
1335     return start;
1336   }
1337 
1338 
1339   // Helper for generating a dynamic type check.
1340   // The sub_klass must be one of {rbx, rdx, rsi}.
1341   // The temp is killed.
1342   void generate_type_check(Register sub_klass,
1343                            Address& super_check_offset_addr,
1344                            Address& super_klass_addr,
1345                            Register temp,
1346                            Label* L_success, Label* L_failure) {
1347     BLOCK_COMMENT("type_check:");
1348 
1349     Label L_fallthrough;
1350 #define LOCAL_JCC(assembler_con, label_ptr)                             \
1351     if (label_ptr != NULL)  __ jcc(assembler_con, *(label_ptr));        \
1352     else                    __ jcc(assembler_con, L_fallthrough) /*omit semi*/
1353 
1354     // The following is a strange variation of the fast path which requires
1355     // one less register, because needed values are on the argument stack.
1356     // __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
1357     //                                  L_success, L_failure, NULL);
1358     assert_different_registers(sub_klass, temp);
1359 
1360     int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1361 
1362     // if the pointers are equal, we are done (e.g., String[] elements)
1363     __ cmpptr(sub_klass, super_klass_addr);
1364     LOCAL_JCC(Assembler::equal, L_success);
1365 
1366     // check the supertype display:
1367     __ movl2ptr(temp, super_check_offset_addr);
1368     Address super_check_addr(sub_klass, temp, Address::times_1, 0);
1369     __ movptr(temp, super_check_addr); // load displayed supertype
1370     __ cmpptr(temp, super_klass_addr); // test the super type
1371     LOCAL_JCC(Assembler::equal, L_success);
1372 
1373     // if it was a primary super, we can just fail immediately
1374     __ cmpl(super_check_offset_addr, sc_offset);
1375     LOCAL_JCC(Assembler::notEqual, L_failure);
1376 
1377     // The repne_scan instruction uses fixed registers, which will get spilled.
1378     // We happen to know this works best when super_klass is in rax.
1379     Register super_klass = temp;
1380     __ movptr(super_klass, super_klass_addr);
1381     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
1382                                      L_success, L_failure);
1383 
1384     __ bind(L_fallthrough);
1385 
1386     if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }
1387     if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }
1388 
1389 #undef LOCAL_JCC
1390   }
1391 
1392   //
1393   //  Generate checkcasting array copy stub
1394   //
1395   //  Input:
1396   //    4(rsp)   - source array address
1397   //    8(rsp)   - destination array address
1398   //   12(rsp)   - element count, can be zero
1399   //   16(rsp)   - size_t ckoff (super_check_offset)
1400   //   20(rsp)   - oop ckval (super_klass)
1401   //
1402   //  Output:
1403   //    rax, ==  0  -  success
1404   //    rax, == -1^K - failure, where K is partial transfer count
1405   //
1406   address generate_checkcast_copy(const char *name, address* entry, bool dest_uninitialized = false) {
1407     __ align(CodeEntryAlignment);
1408     StubCodeMark mark(this, "StubRoutines", name);
1409     address start = __ pc();
1410 
1411     Label L_load_element, L_store_element, L_do_card_marks, L_done;
1412 
1413     // register use:
1414     //  rax, rdx, rcx -- loop control (end_from, end_to, count)
1415     //  rdi, rsi      -- element access (oop, klass)
1416     //  rbx,           -- temp
1417     const Register from       = rax;    // source array address
1418     const Register to         = rdx;    // destination array address
1419     const Register length     = rcx;    // elements count
1420     const Register elem       = rdi;    // each oop copied
1421     const Register elem_klass = rsi;    // each elem._klass (sub_klass)
1422     const Register temp       = rbx;    // lone remaining temp
1423 
1424     __ enter(); // required for proper stackwalking of RuntimeStub frame
1425 
1426     __ push(rsi);
1427     __ push(rdi);
1428     __ push(rbx);
1429 
1430     Address   from_arg(rsp, 16+ 4);     // from
1431     Address     to_arg(rsp, 16+ 8);     // to
1432     Address length_arg(rsp, 16+12);     // elements count
1433     Address  ckoff_arg(rsp, 16+16);     // super_check_offset
1434     Address  ckval_arg(rsp, 16+20);     // super_klass
1435 
1436     // Load up:
1437     __ movptr(from,     from_arg);
1438     __ movptr(to,         to_arg);
1439     __ movl2ptr(length, length_arg);
1440 
1441     if (entry != NULL) {
1442       *entry = __ pc(); // Entry point from generic arraycopy stub.
1443       BLOCK_COMMENT("Entry:");
1444     }
1445 
1446     //---------------------------------------------------------------
1447     // Assembler stub will be used for this call to arraycopy
1448     // if the two arrays are subtypes of Object[] but the
1449     // destination array type is not equal to or a supertype
1450     // of the source type.  Each element must be separately
1451     // checked.
1452 
1453     // Loop-invariant addresses.  They are exclusive end pointers.
1454     Address end_from_addr(from, length, Address::times_ptr, 0);
1455     Address   end_to_addr(to,   length, Address::times_ptr, 0);
1456 
1457     Register end_from = from;           // re-use
1458     Register end_to   = to;             // re-use
1459     Register count    = length;         // re-use
1460 
1461     // Loop-variant addresses.  They assume post-incremented count < 0.
1462     Address from_element_addr(end_from, count, Address::times_ptr, 0);
1463     Address   to_element_addr(end_to,   count, Address::times_ptr, 0);
1464     Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
1465 
1466     // Copy from low to high addresses, indexed from the end of each array.
1467     gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
1468     __ lea(end_from, end_from_addr);
1469     __ lea(end_to,   end_to_addr);
1470     assert(length == count, "");        // else fix next line:
1471     __ negptr(count);                   // negate and test the length
1472     __ jccb(Assembler::notZero, L_load_element);
1473 
1474     // Empty array:  Nothing to do.
1475     __ xorptr(rax, rax);                  // return 0 on (trivial) success
1476     __ jmp(L_done);
1477 
1478     // ======== begin loop ========
1479     // (Loop is rotated; its entry is L_load_element.)
1480     // Loop control:
1481     //   for (count = -count; count != 0; count++)
1482     // Base pointers src, dst are biased by 8*count,to last element.
1483     __ align(OptoLoopAlignment);
1484 
1485     __ BIND(L_store_element);
1486     __ movptr(to_element_addr, elem);     // store the oop
1487     __ increment(count);                // increment the count toward zero
1488     __ jccb(Assembler::zero, L_do_card_marks);
1489 
1490     // ======== loop entry is here ========
1491     __ BIND(L_load_element);
1492     __ movptr(elem, from_element_addr);   // load the oop
1493     __ testptr(elem, elem);
1494     __ jccb(Assembler::zero, L_store_element);
1495 
1496     // (Could do a trick here:  Remember last successful non-null
1497     // element stored and make a quick oop equality check on it.)
1498 
1499     __ movptr(elem_klass, elem_klass_addr); // query the object klass
1500     generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,
1501                         &L_store_element, NULL);
1502     // (On fall-through, we have failed the element type check.)
1503     // ======== end loop ========
1504 
1505     // It was a real error; we must depend on the caller to finish the job.
1506     // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
1507     // Emit GC store barriers for the oops we have copied (length_arg + count),
1508     // and report their number to the caller.
1509     assert_different_registers(to, count, rax);
1510     Label L_post_barrier;
1511     __ addl(count, length_arg);         // transfers = (length - remaining)
1512     __ movl2ptr(rax, count);            // save the value
1513     __ notptr(rax);                     // report (-1^K) to caller (does not affect flags)
1514     __ jccb(Assembler::notZero, L_post_barrier);
1515     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
1516 
1517     // Come here on success only.
1518     __ BIND(L_do_card_marks);
1519     __ xorptr(rax, rax);                // return 0 on success
1520     __ movl2ptr(count, length_arg);
1521 
1522     __ BIND(L_post_barrier);
1523     __ movptr(to, to_arg);              // reload
1524     gen_write_ref_array_post_barrier(to, count);
1525 
1526     // Common exit point (success or failure).
1527     __ BIND(L_done);
1528     __ pop(rbx);
1529     __ pop(rdi);
1530     __ pop(rsi);
1531     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1532     __ leave(); // required for proper stackwalking of RuntimeStub frame
1533     __ ret(0);
1534 
1535     return start;
1536   }
1537 
1538   //
1539   //  Generate 'unsafe' array copy stub
1540   //  Though just as safe as the other stubs, it takes an unscaled
1541   //  size_t argument instead of an element count.
1542   //
1543   //  Input:
1544   //    4(rsp)   - source array address
1545   //    8(rsp)   - destination array address
1546   //   12(rsp)   - byte count, can be zero
1547   //
1548   //  Output:
1549   //    rax, ==  0  -  success
1550   //    rax, == -1  -  need to call System.arraycopy
1551   //
1552   // Examines the alignment of the operands and dispatches
1553   // to a long, int, short, or byte copy loop.
1554   //
1555   address generate_unsafe_copy(const char *name,
1556                                address byte_copy_entry,
1557                                address short_copy_entry,
1558                                address int_copy_entry,
1559                                address long_copy_entry) {
1560 
1561     Label L_long_aligned, L_int_aligned, L_short_aligned;
1562 
1563     __ align(CodeEntryAlignment);
1564     StubCodeMark mark(this, "StubRoutines", name);
1565     address start = __ pc();
1566 
1567     const Register from       = rax;  // source array address
1568     const Register to         = rdx;  // destination array address
1569     const Register count      = rcx;  // elements count
1570 
1571     __ enter(); // required for proper stackwalking of RuntimeStub frame
1572     __ push(rsi);
1573     __ push(rdi);
1574     Address  from_arg(rsp, 12+ 4);      // from
1575     Address    to_arg(rsp, 12+ 8);      // to
1576     Address count_arg(rsp, 12+12);      // byte count
1577 
1578     // Load up:
1579     __ movptr(from ,  from_arg);
1580     __ movptr(to   ,    to_arg);
1581     __ movl2ptr(count, count_arg);
1582 
1583     // bump this on entry, not on exit:
1584     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
1585 
1586     const Register bits = rsi;
1587     __ mov(bits, from);
1588     __ orptr(bits, to);
1589     __ orptr(bits, count);
1590 
1591     __ testl(bits, BytesPerLong-1);
1592     __ jccb(Assembler::zero, L_long_aligned);
1593 
1594     __ testl(bits, BytesPerInt-1);
1595     __ jccb(Assembler::zero, L_int_aligned);
1596 
1597     __ testl(bits, BytesPerShort-1);
1598     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
1599 
1600     __ BIND(L_short_aligned);
1601     __ shrptr(count, LogBytesPerShort); // size => short_count
1602     __ movl(count_arg, count);          // update 'count'
1603     __ jump(RuntimeAddress(short_copy_entry));
1604 
1605     __ BIND(L_int_aligned);
1606     __ shrptr(count, LogBytesPerInt); // size => int_count
1607     __ movl(count_arg, count);          // update 'count'
1608     __ jump(RuntimeAddress(int_copy_entry));
1609 
1610     __ BIND(L_long_aligned);
1611     __ shrptr(count, LogBytesPerLong); // size => qword_count
1612     __ movl(count_arg, count);          // update 'count'
1613     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1614     __ pop(rsi);
1615     __ jump(RuntimeAddress(long_copy_entry));
1616 
1617     return start;
1618   }
1619 
1620 
1621   // Perform range checks on the proposed arraycopy.
1622   // Smashes src_pos and dst_pos.  (Uses them up for temps.)
1623   void arraycopy_range_checks(Register src,
1624                               Register src_pos,
1625                               Register dst,
1626                               Register dst_pos,
1627                               Address& length,
1628                               Label& L_failed) {
1629     BLOCK_COMMENT("arraycopy_range_checks:");
1630     const Register src_end = src_pos;   // source array end position
1631     const Register dst_end = dst_pos;   // destination array end position
1632     __ addl(src_end, length); // src_pos + length
1633     __ addl(dst_end, length); // dst_pos + length
1634 
1635     //  if (src_pos + length > arrayOop(src)->length() ) FAIL;
1636     __ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
1637     __ jcc(Assembler::above, L_failed);
1638 
1639     //  if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
1640     __ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
1641     __ jcc(Assembler::above, L_failed);
1642 
1643     BLOCK_COMMENT("arraycopy_range_checks done");
1644   }
1645 
1646 
1647   //
1648   //  Generate generic array copy stubs
1649   //
1650   //  Input:
1651   //     4(rsp)    -  src oop
1652   //     8(rsp)    -  src_pos
1653   //    12(rsp)    -  dst oop
1654   //    16(rsp)    -  dst_pos
1655   //    20(rsp)    -  element count
1656   //
1657   //  Output:
1658   //    rax, ==  0  -  success
1659   //    rax, == -1^K - failure, where K is partial transfer count
1660   //
1661   address generate_generic_copy(const char *name,
1662                                 address entry_jbyte_arraycopy,
1663                                 address entry_jshort_arraycopy,
1664                                 address entry_jint_arraycopy,
1665                                 address entry_oop_arraycopy,
1666                                 address entry_jlong_arraycopy,
1667                                 address entry_checkcast_arraycopy) {
1668     Label L_failed, L_failed_0, L_objArray;
1669 
1670     { int modulus = CodeEntryAlignment;
1671       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
1672       int advance = target - (__ offset() % modulus);
1673       if (advance < 0)  advance += modulus;
1674       if (advance > 0)  __ nop(advance);
1675     }
1676     StubCodeMark mark(this, "StubRoutines", name);
1677 
1678     // Short-hop target to L_failed.  Makes for denser prologue code.
1679     __ BIND(L_failed_0);
1680     __ jmp(L_failed);
1681     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
1682 
1683     __ align(CodeEntryAlignment);
1684     address start = __ pc();
1685 
1686     __ enter(); // required for proper stackwalking of RuntimeStub frame
1687     __ push(rsi);
1688     __ push(rdi);
1689 
1690     // bump this on entry, not on exit:
1691     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
1692 
1693     // Input values
1694     Address SRC     (rsp, 12+ 4);
1695     Address SRC_POS (rsp, 12+ 8);
1696     Address DST     (rsp, 12+12);
1697     Address DST_POS (rsp, 12+16);
1698     Address LENGTH  (rsp, 12+20);
1699 
1700     //-----------------------------------------------------------------------
1701     // Assembler stub will be used for this call to arraycopy
1702     // if the following conditions are met:
1703     //
1704     // (1) src and dst must not be null.
1705     // (2) src_pos must not be negative.
1706     // (3) dst_pos must not be negative.
1707     // (4) length  must not be negative.
1708     // (5) src klass and dst klass should be the same and not NULL.
1709     // (6) src and dst should be arrays.
1710     // (7) src_pos + length must not exceed length of src.
1711     // (8) dst_pos + length must not exceed length of dst.
1712     //
1713 
1714     const Register src     = rax;       // source array oop
1715     const Register src_pos = rsi;
1716     const Register dst     = rdx;       // destination array oop
1717     const Register dst_pos = rdi;
1718     const Register length  = rcx;       // transfer count
1719 
1720     //  if (src == NULL) return -1;
1721     __ movptr(src, SRC);      // src oop
1722     __ testptr(src, src);
1723     __ jccb(Assembler::zero, L_failed_0);
1724 
1725     //  if (src_pos < 0) return -1;
1726     __ movl2ptr(src_pos, SRC_POS);  // src_pos
1727     __ testl(src_pos, src_pos);
1728     __ jccb(Assembler::negative, L_failed_0);
1729 
1730     //  if (dst == NULL) return -1;
1731     __ movptr(dst, DST);      // dst oop
1732     __ testptr(dst, dst);
1733     __ jccb(Assembler::zero, L_failed_0);
1734 
1735     //  if (dst_pos < 0) return -1;
1736     __ movl2ptr(dst_pos, DST_POS);  // dst_pos
1737     __ testl(dst_pos, dst_pos);
1738     __ jccb(Assembler::negative, L_failed_0);
1739 
1740     //  if (length < 0) return -1;
1741     __ movl2ptr(length, LENGTH);   // length
1742     __ testl(length, length);
1743     __ jccb(Assembler::negative, L_failed_0);
1744 
1745     //  if (src->klass() == NULL) return -1;
1746     Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());
1747     Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());
1748     const Register rcx_src_klass = rcx;    // array klass
1749     __ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
1750 
1751 #ifdef ASSERT
1752     //  assert(src->klass() != NULL);
1753     BLOCK_COMMENT("assert klasses not null");
1754     { Label L1, L2;
1755       __ testptr(rcx_src_klass, rcx_src_klass);
1756       __ jccb(Assembler::notZero, L2);   // it is broken if klass is NULL
1757       __ bind(L1);
1758       __ stop("broken null klass");
1759       __ bind(L2);
1760       __ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
1761       __ jccb(Assembler::equal, L1);      // this would be broken also
1762       BLOCK_COMMENT("assert done");
1763     }
1764 #endif //ASSERT
1765 
1766     // Load layout helper (32-bits)
1767     //
1768     //  |array_tag|     | header_size | element_type |     |log2_element_size|
1769     // 32        30    24            16              8     2                 0
1770     //
1771     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
1772     //
1773 
1774     int lh_offset = in_bytes(Klass::layout_helper_offset());
1775     Address src_klass_lh_addr(rcx_src_klass, lh_offset);
1776 
1777     // Handle objArrays completely differently...
1778     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
1779     __ cmpl(src_klass_lh_addr, objArray_lh);
1780     __ jcc(Assembler::equal, L_objArray);
1781 
1782     //  if (src->klass() != dst->klass()) return -1;
1783     __ cmpptr(rcx_src_klass, dst_klass_addr);
1784     __ jccb(Assembler::notEqual, L_failed_0);
1785 
1786     const Register rcx_lh = rcx;  // layout helper
1787     assert(rcx_lh == rcx_src_klass, "known alias");
1788     __ movl(rcx_lh, src_klass_lh_addr);
1789 
1790     //  if (!src->is_Array()) return -1;
1791     __ cmpl(rcx_lh, Klass::_lh_neutral_value);
1792     __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
1793 
1794     // At this point, it is known to be a typeArray (array_tag 0x3).
1795 #ifdef ASSERT
1796     { Label L;
1797       __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
1798       __ jcc(Assembler::greaterEqual, L); // signed cmp
1799       __ stop("must be a primitive array");
1800       __ bind(L);
1801     }
1802 #endif
1803 
1804     assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);
1805     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1806 
1807     // TypeArrayKlass
1808     //
1809     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
1810     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
1811     //
1812     const Register rsi_offset = rsi; // array offset
1813     const Register src_array  = src; // src array offset
1814     const Register dst_array  = dst; // dst array offset
1815     const Register rdi_elsize = rdi; // log2 element size
1816 
1817     __ mov(rsi_offset, rcx_lh);
1818     __ shrptr(rsi_offset, Klass::_lh_header_size_shift);
1819     __ andptr(rsi_offset, Klass::_lh_header_size_mask);   // array_offset
1820     __ addptr(src_array, rsi_offset);  // src array offset
1821     __ addptr(dst_array, rsi_offset);  // dst array offset
1822     __ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
1823 
1824     // next registers should be set before the jump to corresponding stub
1825     const Register from       = src; // source array address
1826     const Register to         = dst; // destination array address
1827     const Register count      = rcx; // elements count
1828     // some of them should be duplicated on stack
1829 #define FROM   Address(rsp, 12+ 4)
1830 #define TO     Address(rsp, 12+ 8)   // Not used now
1831 #define COUNT  Address(rsp, 12+12)   // Only for oop arraycopy
1832 
1833     BLOCK_COMMENT("scale indexes to element size");
1834     __ movl2ptr(rsi, SRC_POS);  // src_pos
1835     __ shlptr(rsi);             // src_pos << rcx (log2 elsize)
1836     assert(src_array == from, "");
1837     __ addptr(from, rsi);       // from = src_array + SRC_POS << log2 elsize
1838     __ movl2ptr(rdi, DST_POS);  // dst_pos
1839     __ shlptr(rdi);             // dst_pos << rcx (log2 elsize)
1840     assert(dst_array == to, "");
1841     __ addptr(to,  rdi);        // to   = dst_array + DST_POS << log2 elsize
1842     __ movptr(FROM, from);      // src_addr
1843     __ mov(rdi_elsize, rcx_lh); // log2 elsize
1844     __ movl2ptr(count, LENGTH); // elements count
1845 
1846     BLOCK_COMMENT("choose copy loop based on element size");
1847     __ cmpl(rdi_elsize, 0);
1848 
1849     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
1850     __ cmpl(rdi_elsize, LogBytesPerShort);
1851     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
1852     __ cmpl(rdi_elsize, LogBytesPerInt);
1853     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
1854 #ifdef ASSERT
1855     __ cmpl(rdi_elsize, LogBytesPerLong);
1856     __ jccb(Assembler::notEqual, L_failed);
1857 #endif
1858     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1859     __ pop(rsi);
1860     __ jump(RuntimeAddress(entry_jlong_arraycopy));
1861 
1862   __ BIND(L_failed);
1863     __ xorptr(rax, rax);
1864     __ notptr(rax); // return -1
1865     __ pop(rdi);
1866     __ pop(rsi);
1867     __ leave(); // required for proper stackwalking of RuntimeStub frame
1868     __ ret(0);
1869 
1870     // ObjArrayKlass
1871   __ BIND(L_objArray);
1872     // live at this point:  rcx_src_klass, src[_pos], dst[_pos]
1873 
1874     Label L_plain_copy, L_checkcast_copy;
1875     //  test array classes for subtyping
1876     __ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
1877     __ jccb(Assembler::notEqual, L_checkcast_copy);
1878 
1879     // Identically typed arrays can be copied without element-wise checks.
1880     assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);
1881     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1882 
1883   __ BIND(L_plain_copy);
1884     __ movl2ptr(count, LENGTH); // elements count
1885     __ movl2ptr(src_pos, SRC_POS);  // reload src_pos
1886     __ lea(from, Address(src, src_pos, Address::times_ptr,
1887                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
1888     __ movl2ptr(dst_pos, DST_POS);  // reload dst_pos
1889     __ lea(to,   Address(dst, dst_pos, Address::times_ptr,
1890                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
1891     __ movptr(FROM,  from);   // src_addr
1892     __ movptr(TO,    to);     // dst_addr
1893     __ movl(COUNT, count);  // count
1894     __ jump(RuntimeAddress(entry_oop_arraycopy));
1895 
1896   __ BIND(L_checkcast_copy);
1897     // live at this point:  rcx_src_klass, dst[_pos], src[_pos]
1898     {
1899       // Handy offsets:
1900       int  ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
1901       int sco_offset = in_bytes(Klass::super_check_offset_offset());
1902 
1903       Register rsi_dst_klass = rsi;
1904       Register rdi_temp      = rdi;
1905       assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");
1906       assert(rdi_temp      == dst_pos, "expected alias w/ dst_pos");
1907       Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);
1908 
1909       // Before looking at dst.length, make sure dst is also an objArray.
1910       __ movptr(rsi_dst_klass, dst_klass_addr);
1911       __ cmpl(dst_klass_lh_addr, objArray_lh);
1912       __ jccb(Assembler::notEqual, L_failed);
1913 
1914       // It is safe to examine both src.length and dst.length.
1915       __ movl2ptr(src_pos, SRC_POS);        // reload rsi
1916       arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1917       // (Now src_pos and dst_pos are killed, but not src and dst.)
1918 
1919       // We'll need this temp (don't forget to pop it after the type check).
1920       __ push(rbx);
1921       Register rbx_src_klass = rbx;
1922 
1923       __ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
1924       __ movptr(rsi_dst_klass, dst_klass_addr);
1925       Address super_check_offset_addr(rsi_dst_klass, sco_offset);
1926       Label L_fail_array_check;
1927       generate_type_check(rbx_src_klass,
1928                           super_check_offset_addr, dst_klass_addr,
1929                           rdi_temp, NULL, &L_fail_array_check);
1930       // (On fall-through, we have passed the array type check.)
1931       __ pop(rbx);
1932       __ jmp(L_plain_copy);
1933 
1934       __ BIND(L_fail_array_check);
1935       // Reshuffle arguments so we can call checkcast_arraycopy:
1936 
1937       // match initial saves for checkcast_arraycopy
1938       // push(rsi);    // already done; see above
1939       // push(rdi);    // already done; see above
1940       // push(rbx);    // already done; see above
1941 
1942       // Marshal outgoing arguments now, freeing registers.
1943       Address   from_arg(rsp, 16+ 4);   // from
1944       Address     to_arg(rsp, 16+ 8);   // to
1945       Address length_arg(rsp, 16+12);   // elements count
1946       Address  ckoff_arg(rsp, 16+16);   // super_check_offset
1947       Address  ckval_arg(rsp, 16+20);   // super_klass
1948 
1949       Address SRC_POS_arg(rsp, 16+ 8);
1950       Address DST_POS_arg(rsp, 16+16);
1951       Address  LENGTH_arg(rsp, 16+20);
1952       // push rbx, changed the incoming offsets (why not just use rbp,??)
1953       // assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");
1954 
1955       __ movptr(rbx, Address(rsi_dst_klass, ek_offset));
1956       __ movl2ptr(length, LENGTH_arg);    // reload elements count
1957       __ movl2ptr(src_pos, SRC_POS_arg);  // reload src_pos
1958       __ movl2ptr(dst_pos, DST_POS_arg);  // reload dst_pos
1959 
1960       __ movptr(ckval_arg, rbx);          // destination element type
1961       __ movl(rbx, Address(rbx, sco_offset));
1962       __ movl(ckoff_arg, rbx);          // corresponding class check offset
1963 
1964       __ movl(length_arg, length);      // outgoing length argument
1965 
1966       __ lea(from, Address(src, src_pos, Address::times_ptr,
1967                             arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1968       __ movptr(from_arg, from);
1969 
1970       __ lea(to, Address(dst, dst_pos, Address::times_ptr,
1971                           arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1972       __ movptr(to_arg, to);
1973       __ jump(RuntimeAddress(entry_checkcast_arraycopy));
1974     }
1975 
1976     return start;
1977   }
1978 
1979   void generate_arraycopy_stubs() {
1980     address entry;
1981     address entry_jbyte_arraycopy;
1982     address entry_jshort_arraycopy;
1983     address entry_jint_arraycopy;
1984     address entry_oop_arraycopy;
1985     address entry_jlong_arraycopy;
1986     address entry_checkcast_arraycopy;
1987 
1988     StubRoutines::_arrayof_jbyte_disjoint_arraycopy =
1989         generate_disjoint_copy(T_BYTE,  true, Address::times_1, &entry,
1990                                "arrayof_jbyte_disjoint_arraycopy");
1991     StubRoutines::_arrayof_jbyte_arraycopy =
1992         generate_conjoint_copy(T_BYTE,  true, Address::times_1,  entry,
1993                                NULL, "arrayof_jbyte_arraycopy");
1994     StubRoutines::_jbyte_disjoint_arraycopy =
1995         generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,
1996                                "jbyte_disjoint_arraycopy");
1997     StubRoutines::_jbyte_arraycopy =
1998         generate_conjoint_copy(T_BYTE, false, Address::times_1,  entry,
1999                                &entry_jbyte_arraycopy, "jbyte_arraycopy");
2000 
2001     StubRoutines::_arrayof_jshort_disjoint_arraycopy =
2002         generate_disjoint_copy(T_SHORT,  true, Address::times_2, &entry,
2003                                "arrayof_jshort_disjoint_arraycopy");
2004     StubRoutines::_arrayof_jshort_arraycopy =
2005         generate_conjoint_copy(T_SHORT,  true, Address::times_2,  entry,
2006                                NULL, "arrayof_jshort_arraycopy");
2007     StubRoutines::_jshort_disjoint_arraycopy =
2008         generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,
2009                                "jshort_disjoint_arraycopy");
2010     StubRoutines::_jshort_arraycopy =
2011         generate_conjoint_copy(T_SHORT, false, Address::times_2,  entry,
2012                                &entry_jshort_arraycopy, "jshort_arraycopy");
2013 
2014     // Next arrays are always aligned on 4 bytes at least.
2015     StubRoutines::_jint_disjoint_arraycopy =
2016         generate_disjoint_copy(T_INT, true, Address::times_4, &entry,
2017                                "jint_disjoint_arraycopy");
2018     StubRoutines::_jint_arraycopy =
2019         generate_conjoint_copy(T_INT, true, Address::times_4,  entry,
2020                                &entry_jint_arraycopy, "jint_arraycopy");
2021 
2022     StubRoutines::_oop_disjoint_arraycopy =
2023         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
2024                                "oop_disjoint_arraycopy");
2025     StubRoutines::_oop_arraycopy =
2026         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
2027                                &entry_oop_arraycopy, "oop_arraycopy");
2028 
2029     StubRoutines::_oop_disjoint_arraycopy_uninit =
2030         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
2031                                "oop_disjoint_arraycopy_uninit",
2032                                /*dest_uninitialized*/true);
2033     StubRoutines::_oop_arraycopy_uninit =
2034         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
2035                                NULL, "oop_arraycopy_uninit",
2036                                /*dest_uninitialized*/true);
2037 
2038     StubRoutines::_jlong_disjoint_arraycopy =
2039         generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");
2040     StubRoutines::_jlong_arraycopy =
2041         generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,
2042                                     "jlong_arraycopy");
2043 
2044     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2045     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2046     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2047     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2048     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2049     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2050 
2051     StubRoutines::_arrayof_jint_disjoint_arraycopy       = StubRoutines::_jint_disjoint_arraycopy;
2052     StubRoutines::_arrayof_oop_disjoint_arraycopy        = StubRoutines::_oop_disjoint_arraycopy;
2053     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit;
2054     StubRoutines::_arrayof_jlong_disjoint_arraycopy      = StubRoutines::_jlong_disjoint_arraycopy;
2055 
2056     StubRoutines::_arrayof_jint_arraycopy       = StubRoutines::_jint_arraycopy;
2057     StubRoutines::_arrayof_oop_arraycopy        = StubRoutines::_oop_arraycopy;
2058     StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit;
2059     StubRoutines::_arrayof_jlong_arraycopy      = StubRoutines::_jlong_arraycopy;
2060 
2061     StubRoutines::_checkcast_arraycopy =
2062         generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2063     StubRoutines::_checkcast_arraycopy_uninit =
2064         generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, /*dest_uninitialized*/true);
2065 
2066     StubRoutines::_unsafe_arraycopy =
2067         generate_unsafe_copy("unsafe_arraycopy",
2068                                entry_jbyte_arraycopy,
2069                                entry_jshort_arraycopy,
2070                                entry_jint_arraycopy,
2071                                entry_jlong_arraycopy);
2072 
2073     StubRoutines::_generic_arraycopy =
2074         generate_generic_copy("generic_arraycopy",
2075                                entry_jbyte_arraycopy,
2076                                entry_jshort_arraycopy,
2077                                entry_jint_arraycopy,
2078                                entry_oop_arraycopy,
2079                                entry_jlong_arraycopy,
2080                                entry_checkcast_arraycopy);
2081   }
2082 
2083   // AES intrinsic stubs
2084   enum {AESBlockSize = 16};
2085 
2086   address generate_key_shuffle_mask() {
2087     __ align(16);
2088     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
2089     address start = __ pc();
2090     __ emit_data(0x00010203, relocInfo::none, 0 );
2091     __ emit_data(0x04050607, relocInfo::none, 0 );
2092     __ emit_data(0x08090a0b, relocInfo::none, 0 );
2093     __ emit_data(0x0c0d0e0f, relocInfo::none, 0 );
2094     return start;
2095   }
2096 
2097   address generate_counter_shuffle_mask() {
2098     __ align(16);
2099     StubCodeMark mark(this, "StubRoutines", "counter_shuffle_mask");
2100     address start = __ pc();
2101     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
2102     __ emit_data(0x08090a0b, relocInfo::none, 0);
2103     __ emit_data(0x04050607, relocInfo::none, 0);
2104     __ emit_data(0x00010203, relocInfo::none, 0);
2105     return start;
2106   }
2107 
2108   // Utility routine for loading a 128-bit key word in little endian format
2109   // can optionally specify that the shuffle mask is already in an xmmregister
2110   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2111     __ movdqu(xmmdst, Address(key, offset));
2112     if (xmm_shuf_mask != NULL) {
2113       __ pshufb(xmmdst, xmm_shuf_mask);
2114     } else {
2115       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2116     }
2117   }
2118 
2119   // aesenc using specified key+offset
2120   // can optionally specify that the shuffle mask is already in an xmmregister
2121   void aes_enc_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2122     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2123     __ aesenc(xmmdst, xmmtmp);
2124   }
2125 
2126   // aesdec using specified key+offset
2127   // can optionally specify that the shuffle mask is already in an xmmregister
2128   void aes_dec_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2129     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2130     __ aesdec(xmmdst, xmmtmp);
2131   }
2132 
2133   // Utility routine for increase 128bit counter (iv in CTR mode)
2134   //  XMM_128bit,  D3, D2, D1, D0
2135   void inc_counter(Register reg, XMMRegister xmmdst, int inc_delta, Label& next_block) {
2136     __ pextrd(reg, xmmdst, 0x0);
2137     __ addl(reg, inc_delta);
2138     __ pinsrd(xmmdst, reg, 0x0);
2139     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2140 
2141     __ pextrd(reg, xmmdst, 0x01); // Carry-> D1
2142     __ addl(reg, 0x01);
2143     __ pinsrd(xmmdst, reg, 0x01);
2144     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2145 
2146     __ pextrd(reg, xmmdst, 0x02); // Carry-> D2
2147     __ addl(reg, 0x01);
2148     __ pinsrd(xmmdst, reg, 0x02);
2149     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2150 
2151     __ pextrd(reg, xmmdst, 0x03); // Carry -> D3
2152     __ addl(reg, 0x01);
2153     __ pinsrd(xmmdst, reg, 0x03);
2154 
2155     __ BIND(next_block);          // next instruction
2156   }
2157 
2158 
2159   // Arguments:
2160   //
2161   // Inputs:
2162   //   c_rarg0   - source byte array address
2163   //   c_rarg1   - destination byte array address
2164   //   c_rarg2   - K (key) in little endian int array
2165   //
2166   address generate_aescrypt_encryptBlock() {
2167     assert(UseAES, "need AES instructions and misaligned SSE support");
2168     __ align(CodeEntryAlignment);
2169     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
2170     Label L_doLast;
2171     address start = __ pc();
2172 
2173     const Register from        = rdx;      // source array address
2174     const Register to          = rdx;      // destination array address
2175     const Register key         = rcx;      // key array address
2176     const Register keylen      = rax;
2177     const Address  from_param(rbp, 8+0);
2178     const Address  to_param  (rbp, 8+4);
2179     const Address  key_param (rbp, 8+8);
2180 
2181     const XMMRegister xmm_result = xmm0;
2182     const XMMRegister xmm_key_shuf_mask = xmm1;
2183     const XMMRegister xmm_temp1  = xmm2;
2184     const XMMRegister xmm_temp2  = xmm3;
2185     const XMMRegister xmm_temp3  = xmm4;
2186     const XMMRegister xmm_temp4  = xmm5;
2187 
2188     __ enter();   // required for proper stackwalking of RuntimeStub frame
2189 
2190     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
2191     // context for the registers used, where all instructions below are using 128-bit mode
2192     // On EVEX without VL and BW, these instructions will all be AVX.
2193     if (VM_Version::supports_avx512vlbw()) {
2194       __ movl(rdx, 0xffff);
2195       __ kmovdl(k1, rdx);
2196     }
2197 
2198     __ movptr(from, from_param);
2199     __ movptr(key, key_param);
2200 
2201     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2202     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2203 
2204     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2205     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
2206     __ movptr(to, to_param);
2207 
2208     // For encryption, the java expanded key ordering is just what we need
2209 
2210     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
2211     __ pxor(xmm_result, xmm_temp1);
2212 
2213     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2214     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2215     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2216     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2217 
2218     __ aesenc(xmm_result, xmm_temp1);
2219     __ aesenc(xmm_result, xmm_temp2);
2220     __ aesenc(xmm_result, xmm_temp3);
2221     __ aesenc(xmm_result, xmm_temp4);
2222 
2223     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2224     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2225     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2226     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2227 
2228     __ aesenc(xmm_result, xmm_temp1);
2229     __ aesenc(xmm_result, xmm_temp2);
2230     __ aesenc(xmm_result, xmm_temp3);
2231     __ aesenc(xmm_result, xmm_temp4);
2232 
2233     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2234     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2235 
2236     __ cmpl(keylen, 44);
2237     __ jccb(Assembler::equal, L_doLast);
2238 
2239     __ aesenc(xmm_result, xmm_temp1);
2240     __ aesenc(xmm_result, xmm_temp2);
2241 
2242     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2243     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2244 
2245     __ cmpl(keylen, 52);
2246     __ jccb(Assembler::equal, L_doLast);
2247 
2248     __ aesenc(xmm_result, xmm_temp1);
2249     __ aesenc(xmm_result, xmm_temp2);
2250 
2251     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2252     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2253 
2254     __ BIND(L_doLast);
2255     __ aesenc(xmm_result, xmm_temp1);
2256     __ aesenclast(xmm_result, xmm_temp2);
2257     __ movdqu(Address(to, 0), xmm_result);        // store the result
2258     __ xorptr(rax, rax); // return 0
2259     __ leave(); // required for proper stackwalking of RuntimeStub frame
2260     __ ret(0);
2261 
2262     return start;
2263   }
2264 
2265 
2266   // Arguments:
2267   //
2268   // Inputs:
2269   //   c_rarg0   - source byte array address
2270   //   c_rarg1   - destination byte array address
2271   //   c_rarg2   - K (key) in little endian int array
2272   //
2273   address generate_aescrypt_decryptBlock() {
2274     assert(UseAES, "need AES instructions and misaligned SSE support");
2275     __ align(CodeEntryAlignment);
2276     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
2277     Label L_doLast;
2278     address start = __ pc();
2279 
2280     const Register from        = rdx;      // source array address
2281     const Register to          = rdx;      // destination array address
2282     const Register key         = rcx;      // key array address
2283     const Register keylen      = rax;
2284     const Address  from_param(rbp, 8+0);
2285     const Address  to_param  (rbp, 8+4);
2286     const Address  key_param (rbp, 8+8);
2287 
2288     const XMMRegister xmm_result = xmm0;
2289     const XMMRegister xmm_key_shuf_mask = xmm1;
2290     const XMMRegister xmm_temp1  = xmm2;
2291     const XMMRegister xmm_temp2  = xmm3;
2292     const XMMRegister xmm_temp3  = xmm4;
2293     const XMMRegister xmm_temp4  = xmm5;
2294 
2295     __ enter(); // required for proper stackwalking of RuntimeStub frame
2296 
2297     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
2298     // context for the registers used, where all instructions below are using 128-bit mode
2299     // On EVEX without VL and BW, these instructions will all be AVX.
2300     if (VM_Version::supports_avx512vlbw()) {
2301       __ movl(rdx, 0xffff);
2302       __ kmovdl(k1, rdx);
2303     }
2304 
2305     __ movptr(from, from_param);
2306     __ movptr(key, key_param);
2307 
2308     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2309     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2310 
2311     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2312     __ movdqu(xmm_result, Address(from, 0));
2313     __ movptr(to, to_param);
2314 
2315     // for decryption java expanded key ordering is rotated one position from what we want
2316     // so we start from 0x10 here and hit 0x00 last
2317     // we don't know if the key is aligned, hence not using load-execute form
2318     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2319     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2320     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2321     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2322 
2323     __ pxor  (xmm_result, xmm_temp1);
2324     __ aesdec(xmm_result, xmm_temp2);
2325     __ aesdec(xmm_result, xmm_temp3);
2326     __ aesdec(xmm_result, xmm_temp4);
2327 
2328     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2329     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2330     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2331     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2332 
2333     __ aesdec(xmm_result, xmm_temp1);
2334     __ aesdec(xmm_result, xmm_temp2);
2335     __ aesdec(xmm_result, xmm_temp3);
2336     __ aesdec(xmm_result, xmm_temp4);
2337 
2338     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2339     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2340     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
2341 
2342     __ cmpl(keylen, 44);
2343     __ jccb(Assembler::equal, L_doLast);
2344 
2345     __ aesdec(xmm_result, xmm_temp1);
2346     __ aesdec(xmm_result, xmm_temp2);
2347 
2348     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2349     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2350 
2351     __ cmpl(keylen, 52);
2352     __ jccb(Assembler::equal, L_doLast);
2353 
2354     __ aesdec(xmm_result, xmm_temp1);
2355     __ aesdec(xmm_result, xmm_temp2);
2356 
2357     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2358     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2359 
2360     __ BIND(L_doLast);
2361     __ aesdec(xmm_result, xmm_temp1);
2362     __ aesdec(xmm_result, xmm_temp2);
2363 
2364     // for decryption the aesdeclast operation is always on key+0x00
2365     __ aesdeclast(xmm_result, xmm_temp3);
2366     __ movdqu(Address(to, 0), xmm_result);  // store the result
2367     __ xorptr(rax, rax); // return 0
2368     __ leave(); // required for proper stackwalking of RuntimeStub frame
2369     __ ret(0);
2370 
2371     return start;
2372   }
2373 
2374   void handleSOERegisters(bool saving) {
2375     const int saveFrameSizeInBytes = 4 * wordSize;
2376     const Address saved_rbx     (rbp, -3 * wordSize);
2377     const Address saved_rsi     (rbp, -2 * wordSize);
2378     const Address saved_rdi     (rbp, -1 * wordSize);
2379 
2380     if (saving) {
2381       __ subptr(rsp, saveFrameSizeInBytes);
2382       __ movptr(saved_rsi, rsi);
2383       __ movptr(saved_rdi, rdi);
2384       __ movptr(saved_rbx, rbx);
2385     } else {
2386       // restoring
2387       __ movptr(rsi, saved_rsi);
2388       __ movptr(rdi, saved_rdi);
2389       __ movptr(rbx, saved_rbx);
2390     }
2391   }
2392 
2393   // Arguments:
2394   //
2395   // Inputs:
2396   //   c_rarg0   - source byte array address
2397   //   c_rarg1   - destination byte array address
2398   //   c_rarg2   - K (key) in little endian int array
2399   //   c_rarg3   - r vector byte array address
2400   //   c_rarg4   - input length
2401   //
2402   // Output:
2403   //   rax       - input length
2404   //
2405   address generate_cipherBlockChaining_encryptAESCrypt() {
2406     assert(UseAES, "need AES instructions and misaligned SSE support");
2407     __ align(CodeEntryAlignment);
2408     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
2409     address start = __ pc();
2410 
2411     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
2412     const Register from        = rsi;      // source array address
2413     const Register to          = rdx;      // destination array address
2414     const Register key         = rcx;      // key array address
2415     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2416                                            // and left with the results of the last encryption block
2417     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2418     const Register pos         = rax;
2419 
2420     // xmm register assignments for the loops below
2421     const XMMRegister xmm_result = xmm0;
2422     const XMMRegister xmm_temp   = xmm1;
2423     // first 6 keys preloaded into xmm2-xmm7
2424     const int XMM_REG_NUM_KEY_FIRST = 2;
2425     const int XMM_REG_NUM_KEY_LAST  = 7;
2426     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
2427 
2428     __ enter(); // required for proper stackwalking of RuntimeStub frame
2429     handleSOERegisters(true /*saving*/);
2430 
2431     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
2432     // context for the registers used, where all instructions below are using 128-bit mode
2433     // On EVEX without VL and BW, these instructions will all be AVX.
2434     if (VM_Version::supports_avx512vlbw()) {
2435       __ movl(rdx, 0xffff);
2436       __ kmovdl(k1, rdx);
2437     }
2438 
2439     // load registers from incoming parameters
2440     const Address  from_param(rbp, 8+0);
2441     const Address  to_param  (rbp, 8+4);
2442     const Address  key_param (rbp, 8+8);
2443     const Address  rvec_param (rbp, 8+12);
2444     const Address  len_param  (rbp, 8+16);
2445     __ movptr(from , from_param);
2446     __ movptr(to   , to_param);
2447     __ movptr(key  , key_param);
2448     __ movptr(rvec , rvec_param);
2449     __ movptr(len_reg , len_param);
2450 
2451     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
2452     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2453     // load up xmm regs 2 thru 7 with keys 0-5
2454     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2455       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
2456       offset += 0x10;
2457     }
2458 
2459     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
2460 
2461     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2462     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2463     __ cmpl(rax, 44);
2464     __ jcc(Assembler::notEqual, L_key_192_256);
2465 
2466     // 128 bit code follows here
2467     __ movl(pos, 0);
2468     __ align(OptoLoopAlignment);
2469     __ BIND(L_loopTop_128);
2470     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2471     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2472 
2473     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2474     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2475       __ aesenc(xmm_result, as_XMMRegister(rnum));
2476     }
2477     for (int key_offset = 0x60; key_offset <= 0x90; key_offset += 0x10) {
2478       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2479     }
2480     load_key(xmm_temp, key, 0xa0);
2481     __ aesenclast(xmm_result, xmm_temp);
2482 
2483     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
2484     // no need to store r to memory until we exit
2485     __ addptr(pos, AESBlockSize);
2486     __ subptr(len_reg, AESBlockSize);
2487     __ jcc(Assembler::notEqual, L_loopTop_128);
2488 
2489     __ BIND(L_exit);
2490     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
2491 
2492     handleSOERegisters(false /*restoring*/);
2493     __ movptr(rax, len_param); // return length
2494     __ leave();                                  // required for proper stackwalking of RuntimeStub frame
2495     __ ret(0);
2496 
2497     __ BIND(L_key_192_256);
2498     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
2499     __ cmpl(rax, 52);
2500     __ jcc(Assembler::notEqual, L_key_256);
2501 
2502     // 192-bit code follows here (could be changed to use more xmm registers)
2503     __ movl(pos, 0);
2504     __ align(OptoLoopAlignment);
2505     __ BIND(L_loopTop_192);
2506     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2507     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2508 
2509     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2510     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2511       __ aesenc(xmm_result, as_XMMRegister(rnum));
2512     }
2513     for (int key_offset = 0x60; key_offset <= 0xb0; key_offset += 0x10) {
2514       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2515     }
2516     load_key(xmm_temp, key, 0xc0);
2517     __ aesenclast(xmm_result, xmm_temp);
2518 
2519     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2520     // no need to store r to memory until we exit
2521     __ addptr(pos, AESBlockSize);
2522     __ subptr(len_reg, AESBlockSize);
2523     __ jcc(Assembler::notEqual, L_loopTop_192);
2524     __ jmp(L_exit);
2525 
2526     __ BIND(L_key_256);
2527     // 256-bit code follows here (could be changed to use more xmm registers)
2528     __ movl(pos, 0);
2529     __ align(OptoLoopAlignment);
2530     __ BIND(L_loopTop_256);
2531     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2532     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2533 
2534     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2535     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2536       __ aesenc(xmm_result, as_XMMRegister(rnum));
2537     }
2538     for (int key_offset = 0x60; key_offset <= 0xd0; key_offset += 0x10) {
2539       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2540     }
2541     load_key(xmm_temp, key, 0xe0);
2542     __ aesenclast(xmm_result, xmm_temp);
2543 
2544     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2545     // no need to store r to memory until we exit
2546     __ addptr(pos, AESBlockSize);
2547     __ subptr(len_reg, AESBlockSize);
2548     __ jcc(Assembler::notEqual, L_loopTop_256);
2549     __ jmp(L_exit);
2550 
2551     return start;
2552   }
2553 
2554 
2555   // CBC AES Decryption.
2556   // In 32-bit stub, because of lack of registers we do not try to parallelize 4 blocks at a time.
2557   //
2558   // Arguments:
2559   //
2560   // Inputs:
2561   //   c_rarg0   - source byte array address
2562   //   c_rarg1   - destination byte array address
2563   //   c_rarg2   - K (key) in little endian int array
2564   //   c_rarg3   - r vector byte array address
2565   //   c_rarg4   - input length
2566   //
2567   // Output:
2568   //   rax       - input length
2569   //
2570 
2571   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
2572     assert(UseAES, "need AES instructions and misaligned SSE support");
2573     __ align(CodeEntryAlignment);
2574     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
2575     address start = __ pc();
2576 
2577     const Register from        = rsi;      // source array address
2578     const Register to          = rdx;      // destination array address
2579     const Register key         = rcx;      // key array address
2580     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2581                                            // and left with the results of the last encryption block
2582     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2583     const Register pos         = rax;
2584 
2585     const int PARALLEL_FACTOR = 4;
2586     const int ROUNDS[3] = { 10, 12, 14 }; //aes rounds for key128, key192, key256
2587 
2588     Label L_exit;
2589     Label L_singleBlock_loopTop[3]; //128, 192, 256
2590     Label L_multiBlock_loopTop[3]; //128, 192, 256
2591 
2592     const XMMRegister xmm_prev_block_cipher = xmm0; // holds cipher of previous block
2593     const XMMRegister xmm_key_shuf_mask = xmm1;
2594 
2595     const XMMRegister xmm_key_tmp0 = xmm2;
2596     const XMMRegister xmm_key_tmp1 = xmm3;
2597 
2598     // registers holding the six results in the parallelized loop
2599     const XMMRegister xmm_result0 = xmm4;
2600     const XMMRegister xmm_result1 = xmm5;
2601     const XMMRegister xmm_result2 = xmm6;
2602     const XMMRegister xmm_result3 = xmm7;
2603 
2604     __ enter(); // required for proper stackwalking of RuntimeStub frame
2605     handleSOERegisters(true /*saving*/);
2606 
2607     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
2608     // context for the registers used, where all instructions below are using 128-bit mode
2609     // On EVEX without VL and BW, these instructions will all be AVX.
2610     if (VM_Version::supports_avx512vlbw()) {
2611       __ movl(rdx, 0xffff);
2612       __ kmovdl(k1, rdx);
2613     }
2614 
2615     // load registers from incoming parameters
2616     const Address  from_param(rbp, 8+0);
2617     const Address  to_param  (rbp, 8+4);
2618     const Address  key_param (rbp, 8+8);
2619     const Address  rvec_param (rbp, 8+12);
2620     const Address  len_param  (rbp, 8+16);
2621 
2622     __ movptr(from , from_param);
2623     __ movptr(to   , to_param);
2624     __ movptr(key  , key_param);
2625     __ movptr(rvec , rvec_param);
2626     __ movptr(len_reg , len_param);
2627 
2628     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2629     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // initialize with initial rvec
2630 
2631     __ xorptr(pos, pos);
2632 
2633     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2634     // rvec is reused
2635     __ movl(rvec, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2636     __ cmpl(rvec, 52);
2637     __ jcc(Assembler::equal, L_multiBlock_loopTop[1]);
2638     __ cmpl(rvec, 60);
2639     __ jcc(Assembler::equal, L_multiBlock_loopTop[2]);
2640 
2641 #define DoFour(opc, src_reg)           \
2642   __ opc(xmm_result0, src_reg);         \
2643   __ opc(xmm_result1, src_reg);         \
2644   __ opc(xmm_result2, src_reg);         \
2645   __ opc(xmm_result3, src_reg);         \
2646 
2647     for (int k = 0; k < 3; ++k) {
2648       __ align(OptoLoopAlignment);
2649       __ BIND(L_multiBlock_loopTop[k]);
2650       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
2651       __ jcc(Assembler::less, L_singleBlock_loopTop[k]);
2652 
2653       __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); // get next 4 blocks into xmmresult registers
2654       __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2655       __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2656       __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2657 
2658       // the java expanded key ordering is rotated one position from what we want
2659       // so we start from 0x10 here and hit 0x00 last
2660       load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2661       DoFour(pxor, xmm_key_tmp0); //xor with first key
2662       // do the aes dec rounds
2663       for (int rnum = 1; rnum <= ROUNDS[k];) {
2664         //load two keys at a time
2665         //k1->0x20, ..., k9->0xa0, k10->0x00
2666         load_key(xmm_key_tmp1, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2667         load_key(xmm_key_tmp0, key, ((rnum + 2) % (ROUNDS[k] + 1)) * 0x10, xmm_key_shuf_mask); // hit 0x00 last!
2668         DoFour(aesdec, xmm_key_tmp1);
2669         rnum++;
2670         if (rnum != ROUNDS[k]) {
2671           DoFour(aesdec, xmm_key_tmp0);
2672         }
2673         else {
2674           DoFour(aesdeclast, xmm_key_tmp0);
2675         }
2676         rnum++;
2677       }
2678 
2679       // for each result, xor with the r vector of previous cipher block
2680       __ pxor(xmm_result0, xmm_prev_block_cipher);
2681       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2682       __ pxor(xmm_result1, xmm_prev_block_cipher);
2683       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2684       __ pxor(xmm_result2, xmm_prev_block_cipher);
2685       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2686       __ pxor(xmm_result3, xmm_prev_block_cipher);
2687       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3 * AESBlockSize)); // this will carry over to next set of blocks
2688 
2689             // store 4 results into the next 64 bytes of output
2690        __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2691        __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2692        __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2693        __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2694 
2695        __ addptr(pos, 4 * AESBlockSize);
2696        __ subptr(len_reg, 4 * AESBlockSize);
2697        __ jmp(L_multiBlock_loopTop[k]);
2698 
2699        //singleBlock starts here
2700        __ align(OptoLoopAlignment);
2701        __ BIND(L_singleBlock_loopTop[k]);
2702        __ cmpptr(len_reg, 0); // any blocks left?
2703        __ jcc(Assembler::equal, L_exit);
2704        __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
2705        __ movdqa(xmm_result1, xmm_result0);
2706 
2707        load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2708        __ pxor(xmm_result0, xmm_key_tmp0);
2709        // do the aes dec rounds
2710        for (int rnum = 1; rnum < ROUNDS[k]; rnum++) {
2711          // the java expanded key ordering is rotated one position from what we want
2712          load_key(xmm_key_tmp0, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2713          __ aesdec(xmm_result0, xmm_key_tmp0);
2714        }
2715        load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
2716        __ aesdeclast(xmm_result0, xmm_key_tmp0);
2717        __ pxor(xmm_result0, xmm_prev_block_cipher); // xor with the current r vector
2718        __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result0); // store into the next 16 bytes of output
2719        // no need to store r to memory until we exit
2720        __ movdqa(xmm_prev_block_cipher, xmm_result1); // set up next r vector with cipher input from this block
2721 
2722        __ addptr(pos, AESBlockSize);
2723        __ subptr(len_reg, AESBlockSize);
2724        __ jmp(L_singleBlock_loopTop[k]);
2725     }//for 128/192/256
2726 
2727     __ BIND(L_exit);
2728     __ movptr(rvec, rvec_param);                        // restore this since reused earlier
2729     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher); // final value of r stored in rvec of CipherBlockChaining object
2730     handleSOERegisters(false /*restoring*/);
2731     __ movptr(rax, len_param);                          // return length
2732     __ leave();                                         // required for proper stackwalking of RuntimeStub frame
2733     __ ret(0);
2734 
2735     return start;
2736   }
2737 
2738   // CTR AES crypt.
2739   // In 32-bit stub, parallelize 4 blocks at a time
2740   // Arguments:
2741   //
2742   // Inputs:
2743   //   c_rarg0   - source byte array address
2744   //   c_rarg1   - destination byte array address
2745   //   c_rarg2   - K (key) in little endian int array
2746   //   c_rarg3   - counter vector byte array address
2747   //   c_rarg4   - input length
2748   //
2749   // Output:
2750   //   rax       - input length
2751   //
2752   address generate_counterMode_AESCrypt_Parallel() {
2753     assert(UseAES, "need AES instructions and misaligned SSE support");
2754     __ align(CodeEntryAlignment);
2755     StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
2756     address start = __ pc();
2757     const Register from        = rsi;      // source array address
2758     const Register to          = rdx;      // destination array address
2759     const Register key         = rcx;      // key array address
2760     const Register counter     = rdi;      // counter byte array initialized from initvector array address
2761                                            // and updated with the incremented counter in the end
2762     const Register len_reg     = rbx;
2763     const Register pos         = rax;
2764 
2765     __ enter(); // required for proper stackwalking of RuntimeStub frame
2766     handleSOERegisters(true /*saving*/); // save rbx, rsi, rdi
2767 
2768     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
2769     // context for the registers used, where all instructions below are using 128-bit mode
2770     // On EVEX without VL and BW, these instructions will all be AVX.
2771     if (VM_Version::supports_avx512vlbw()) {
2772       __ movl(rdx, 0xffff);
2773       __ kmovdl(k1, rdx);
2774     }
2775 
2776     // load registers from incoming parameters
2777     const Address  from_param(rbp, 8+0);
2778     const Address  to_param  (rbp, 8+4);
2779     const Address  key_param (rbp, 8+8);
2780     const Address  rvec_param (rbp, 8+12);
2781     const Address  len_param  (rbp, 8+16);
2782     const Address  saved_counter_param(rbp, 8 + 20);
2783     const Address  used_addr_param(rbp, 8 + 24);
2784 
2785     __ movptr(from , from_param);
2786     __ movptr(to   , to_param);
2787     __ movptr(len_reg , len_param);
2788 
2789     // Use the partially used encrpyted counter from last invocation
2790     Label L_exit_preLoop, L_preLoop_start;
2791 
2792     // Use the registers 'counter' and 'key' here in this preloop
2793     // to hold of last 2 params 'used' and 'saved_encCounter_start'
2794     Register used = counter;
2795     Register saved_encCounter_start = key;
2796     Register used_addr = saved_encCounter_start;
2797 
2798     __ movptr(used_addr, used_addr_param);
2799     __ movptr(used, Address(used_addr, 0));
2800     __ movptr(saved_encCounter_start, saved_counter_param);
2801 
2802     __ BIND(L_preLoop_start);
2803     __ cmpptr(used, 16);
2804     __ jcc(Assembler::aboveEqual, L_exit_preLoop);
2805     __ cmpptr(len_reg, 0);
2806     __ jcc(Assembler::lessEqual, L_exit_preLoop);
2807     __ movb(rax, Address(saved_encCounter_start, used));
2808     __ xorb(rax, Address(from, 0));
2809     __ movb(Address(to, 0), rax);
2810     __ addptr(from, 1);
2811     __ addptr(to, 1);
2812     __ addptr(used, 1);
2813     __ subptr(len_reg, 1);
2814 
2815     __ jmp(L_preLoop_start);
2816 
2817     __ BIND(L_exit_preLoop);
2818     __ movptr(used_addr, used_addr_param);
2819     __ movptr(used_addr, used_addr_param);
2820     __ movl(Address(used_addr, 0), used);
2821 
2822     // load the parameters 'key' and 'counter'
2823     __ movptr(key, key_param);
2824     __ movptr(counter, rvec_param);
2825 
2826     // xmm register assignments for the loops below
2827     const XMMRegister xmm_curr_counter      = xmm0;
2828     const XMMRegister xmm_counter_shuf_mask = xmm1;  // need to be reloaded
2829     const XMMRegister xmm_key_shuf_mask     = xmm2;  // need to be reloaded
2830     const XMMRegister xmm_key               = xmm3;
2831     const XMMRegister xmm_result0           = xmm4;
2832     const XMMRegister xmm_result1           = xmm5;
2833     const XMMRegister xmm_result2           = xmm6;
2834     const XMMRegister xmm_result3           = xmm7;
2835     const XMMRegister xmm_from0             = xmm1;   //reuse XMM register
2836     const XMMRegister xmm_from1             = xmm2;
2837     const XMMRegister xmm_from2             = xmm3;
2838     const XMMRegister xmm_from3             = xmm4;
2839 
2840     //for key_128, key_192, key_256
2841     const int rounds[3] = {10, 12, 14};
2842     Label L_singleBlockLoopTop[3];
2843     Label L_multiBlock_loopTop[3];
2844     Label L_key192_top, L_key256_top;
2845     Label L_incCounter[3][4]; // 3: different key length,  4: 4 blocks at a time
2846     Label L_incCounter_single[3]; //for single block, key128, key192, key256
2847     Label L_processTail_insr[3], L_processTail_4_insr[3], L_processTail_2_insr[3], L_processTail_1_insr[3], L_processTail_exit_insr[3];
2848     Label L_processTail_extr[3], L_processTail_4_extr[3], L_processTail_2_extr[3], L_processTail_1_extr[3], L_processTail_exit_extr[3];
2849 
2850     Label L_exit;
2851     const int PARALLEL_FACTOR = 4;  //because of the limited register number
2852 
2853     // initialize counter with initial counter
2854     __ movdqu(xmm_curr_counter, Address(counter, 0x00));
2855     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2856     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled for increase
2857 
2858     // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
2859     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2860     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2861     __ cmpl(rax, 52);
2862     __ jcc(Assembler::equal, L_key192_top);
2863     __ cmpl(rax, 60);
2864     __ jcc(Assembler::equal, L_key256_top);
2865 
2866     //key128 begins here
2867     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2868 
2869 #define CTR_DoFour(opc, src_reg)               \
2870     __ opc(xmm_result0, src_reg);              \
2871     __ opc(xmm_result1, src_reg);              \
2872     __ opc(xmm_result2, src_reg);              \
2873     __ opc(xmm_result3, src_reg);
2874 
2875     // k == 0 :  generate code for key_128
2876     // k == 1 :  generate code for key_192
2877     // k == 2 :  generate code for key_256
2878     for (int k = 0; k < 3; ++k) {
2879       //multi blocks starts here
2880       __ align(OptoLoopAlignment);
2881       __ BIND(L_multiBlock_loopTop[k]);
2882       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least PARALLEL_FACTOR blocks left
2883       __ jcc(Assembler::less, L_singleBlockLoopTop[k]);
2884 
2885       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2886       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2887 
2888       //load, then increase counters
2889       CTR_DoFour(movdqa, xmm_curr_counter);
2890       __ push(rbx);
2891       inc_counter(rbx, xmm_result1, 0x01, L_incCounter[k][0]);
2892       inc_counter(rbx, xmm_result2, 0x02, L_incCounter[k][1]);
2893       inc_counter(rbx, xmm_result3, 0x03, L_incCounter[k][2]);
2894       inc_counter(rbx, xmm_curr_counter, 0x04, L_incCounter[k][3]);
2895       __ pop (rbx);
2896 
2897       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask); // load Round 0 key. interleaving for better performance
2898 
2899       CTR_DoFour(pshufb, xmm_counter_shuf_mask); // after increased, shuffled counters back for PXOR
2900       CTR_DoFour(pxor, xmm_key);   //PXOR with Round 0 key
2901 
2902       for (int i = 1; i < rounds[k]; ++i) {
2903         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2904         CTR_DoFour(aesenc, xmm_key);
2905       }
2906       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2907       CTR_DoFour(aesenclast, xmm_key);
2908 
2909       // get next PARALLEL_FACTOR blocks into xmm_from registers
2910       __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2911       __ movdqu(xmm_from1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2912       __ movdqu(xmm_from2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2913 
2914       // PXOR with input text
2915       __ pxor(xmm_result0, xmm_from0); //result0 is xmm4
2916       __ pxor(xmm_result1, xmm_from1);
2917       __ pxor(xmm_result2, xmm_from2);
2918 
2919       // store PARALLEL_FACTOR results into the next 64 bytes of output
2920       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2921       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2922       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2923 
2924       // do it here after xmm_result0 is saved, because xmm_from3 reuse the same register of xmm_result0.
2925       __ movdqu(xmm_from3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2926       __ pxor(xmm_result3, xmm_from3);
2927       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2928 
2929       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize); // increase the length of crypt text
2930       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // decrease the remaining length
2931       __ jmp(L_multiBlock_loopTop[k]);
2932 
2933       // singleBlock starts here
2934       __ align(OptoLoopAlignment);
2935       __ BIND(L_singleBlockLoopTop[k]);
2936       __ cmpptr(len_reg, 0);
2937       __ jcc(Assembler::equal, L_exit);
2938       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2939       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2940       __ movdqa(xmm_result0, xmm_curr_counter);
2941       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask);
2942       __ push(rbx);//rbx is used for increasing counter
2943       inc_counter(rbx, xmm_curr_counter, 0x01, L_incCounter_single[k]);
2944       __ pop (rbx);
2945       __ pshufb(xmm_result0, xmm_counter_shuf_mask);
2946       __ pxor(xmm_result0, xmm_key);
2947       for (int i = 1; i < rounds[k]; i++) {
2948         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2949         __ aesenc(xmm_result0, xmm_key);
2950       }
2951       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2952       __ aesenclast(xmm_result0, xmm_key);
2953       __ cmpptr(len_reg, AESBlockSize);
2954       __ jcc(Assembler::less, L_processTail_insr[k]);
2955         __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2956         __ pxor(xmm_result0, xmm_from0);
2957         __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2958         __ addptr(pos, AESBlockSize);
2959         __ subptr(len_reg, AESBlockSize);
2960         __ jmp(L_singleBlockLoopTop[k]);
2961 
2962       __ BIND(L_processTail_insr[k]);                                               // Process the tail part of the input array
2963         __ addptr(pos, len_reg);                                                    // 1. Insert bytes from src array into xmm_from0 register
2964         __ testptr(len_reg, 8);
2965         __ jcc(Assembler::zero, L_processTail_4_insr[k]);
2966           __ subptr(pos,8);
2967           __ pinsrd(xmm_from0, Address(from, pos), 0);
2968           __ pinsrd(xmm_from0, Address(from, pos, Address::times_1, 4), 1);
2969         __ BIND(L_processTail_4_insr[k]);
2970         __ testptr(len_reg, 4);
2971         __ jcc(Assembler::zero, L_processTail_2_insr[k]);
2972           __ subptr(pos,4);
2973           __ pslldq(xmm_from0, 4);
2974           __ pinsrd(xmm_from0, Address(from, pos), 0);
2975         __ BIND(L_processTail_2_insr[k]);
2976         __ testptr(len_reg, 2);
2977         __ jcc(Assembler::zero, L_processTail_1_insr[k]);
2978           __ subptr(pos, 2);
2979           __ pslldq(xmm_from0, 2);
2980           __ pinsrw(xmm_from0, Address(from, pos), 0);
2981         __ BIND(L_processTail_1_insr[k]);
2982         __ testptr(len_reg, 1);
2983         __ jcc(Assembler::zero, L_processTail_exit_insr[k]);
2984           __ subptr(pos, 1);
2985           __ pslldq(xmm_from0, 1);
2986           __ pinsrb(xmm_from0, Address(from, pos), 0);
2987         __ BIND(L_processTail_exit_insr[k]);
2988 
2989         __ movptr(saved_encCounter_start, saved_counter_param);
2990         __ movdqu(Address(saved_encCounter_start, 0), xmm_result0);               // 2. Perform pxor of the encrypted counter and plaintext Bytes.
2991         __ pxor(xmm_result0, xmm_from0);                                          //    Also the encrypted counter is saved for next invocation.
2992 
2993         __ testptr(len_reg, 8);
2994         __ jcc(Assembler::zero, L_processTail_4_extr[k]);                        // 3. Extract bytes from xmm_result0 into the dest. array
2995           __ pextrd(Address(to, pos), xmm_result0, 0);
2996           __ pextrd(Address(to, pos, Address::times_1, 4), xmm_result0, 1);
2997           __ psrldq(xmm_result0, 8);
2998           __ addptr(pos, 8);
2999         __ BIND(L_processTail_4_extr[k]);
3000         __ testptr(len_reg, 4);
3001         __ jcc(Assembler::zero, L_processTail_2_extr[k]);
3002           __ pextrd(Address(to, pos), xmm_result0, 0);
3003           __ psrldq(xmm_result0, 4);
3004           __ addptr(pos, 4);
3005         __ BIND(L_processTail_2_extr[k]);
3006         __ testptr(len_reg, 2);
3007         __ jcc(Assembler::zero, L_processTail_1_extr[k]);
3008           __ pextrb(Address(to, pos), xmm_result0, 0);
3009           __ pextrb(Address(to, pos, Address::times_1, 1), xmm_result0, 1);
3010           __ psrldq(xmm_result0, 2);
3011           __ addptr(pos, 2);
3012         __ BIND(L_processTail_1_extr[k]);
3013         __ testptr(len_reg, 1);
3014         __ jcc(Assembler::zero, L_processTail_exit_extr[k]);
3015           __ pextrb(Address(to, pos), xmm_result0, 0);
3016 
3017         __ BIND(L_processTail_exit_extr[k]);
3018         __ movptr(used_addr, used_addr_param);
3019         __ movl(Address(used_addr, 0), len_reg);
3020         __ jmp(L_exit);
3021     }
3022 
3023     __ BIND(L_exit);
3024     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
3025     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled back.
3026     __ movdqu(Address(counter, 0), xmm_curr_counter); //save counter back
3027     handleSOERegisters(false /*restoring*/);
3028     __ movptr(rax, len_param); // return length
3029     __ leave();                // required for proper stackwalking of RuntimeStub frame
3030     __ ret(0);
3031 
3032     __ BIND (L_key192_top);
3033     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
3034     __ jmp(L_multiBlock_loopTop[1]); //key192
3035 
3036     __ BIND (L_key256_top);
3037     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
3038     __ jmp(L_multiBlock_loopTop[2]); //key192
3039 
3040     return start;
3041   }
3042 
3043   address generate_upper_word_mask() {
3044     __ align(64);
3045     StubCodeMark mark(this, "StubRoutines", "upper_word_mask");
3046     address start = __ pc();
3047     __ emit_data(0x00000000, relocInfo::none, 0);
3048     __ emit_data(0x00000000, relocInfo::none, 0);
3049     __ emit_data(0x00000000, relocInfo::none, 0);
3050     __ emit_data(0xFFFFFFFF, relocInfo::none, 0);
3051     return start;
3052   }
3053 
3054   address generate_shuffle_byte_flip_mask() {
3055     __ align(64);
3056     StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask");
3057     address start = __ pc();
3058     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3059     __ emit_data(0x08090a0b, relocInfo::none, 0);
3060     __ emit_data(0x04050607, relocInfo::none, 0);
3061     __ emit_data(0x00010203, relocInfo::none, 0);
3062     return start;
3063   }
3064 
3065   // ofs and limit are use for multi-block byte array.
3066   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3067   address generate_sha1_implCompress(bool multi_block, const char *name) {
3068     __ align(CodeEntryAlignment);
3069     StubCodeMark mark(this, "StubRoutines", name);
3070     address start = __ pc();
3071 
3072     Register buf   = rax;
3073     Register state = rdx;
3074     Register ofs   = rcx;
3075     Register limit = rdi;
3076 
3077     const Address  buf_param(rbp, 8 + 0);
3078     const Address  state_param(rbp, 8 + 4);
3079     const Address  ofs_param(rbp, 8 + 8);
3080     const Address  limit_param(rbp, 8 + 12);
3081 
3082     const XMMRegister abcd = xmm0;
3083     const XMMRegister e0 = xmm1;
3084     const XMMRegister e1 = xmm2;
3085     const XMMRegister msg0 = xmm3;
3086 
3087     const XMMRegister msg1 = xmm4;
3088     const XMMRegister msg2 = xmm5;
3089     const XMMRegister msg3 = xmm6;
3090     const XMMRegister shuf_mask = xmm7;
3091 
3092     __ enter();
3093     __ subptr(rsp, 8 * wordSize);
3094     if (multi_block) {
3095       __ push(limit);
3096     }
3097     __ movptr(buf, buf_param);
3098     __ movptr(state, state_param);
3099     if (multi_block) {
3100       __ movptr(ofs, ofs_param);
3101       __ movptr(limit, limit_param);
3102     }
3103 
3104     __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask,
3105       buf, state, ofs, limit, rsp, multi_block);
3106 
3107     if (multi_block) {
3108       __ pop(limit);
3109     }
3110     __ addptr(rsp, 8 * wordSize);
3111     __ leave();
3112     __ ret(0);
3113     return start;
3114   }
3115 
3116   address generate_pshuffle_byte_flip_mask() {
3117     __ align(64);
3118     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask");
3119     address start = __ pc();
3120     __ emit_data(0x00010203, relocInfo::none, 0);
3121     __ emit_data(0x04050607, relocInfo::none, 0);
3122     __ emit_data(0x08090a0b, relocInfo::none, 0);
3123     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3124     return start;
3125   }
3126 
3127   // ofs and limit are use for multi-block byte array.
3128   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3129  address generate_sha256_implCompress(bool multi_block, const char *name) {
3130     __ align(CodeEntryAlignment);
3131     StubCodeMark mark(this, "StubRoutines", name);
3132     address start = __ pc();
3133 
3134     Register buf = rbx;
3135     Register state = rsi;
3136     Register ofs = rdx;
3137     Register limit = rcx;
3138 
3139     const Address  buf_param(rbp, 8 + 0);
3140     const Address  state_param(rbp, 8 + 4);
3141     const Address  ofs_param(rbp, 8 + 8);
3142     const Address  limit_param(rbp, 8 + 12);
3143 
3144     const XMMRegister msg = xmm0;
3145     const XMMRegister state0 = xmm1;
3146     const XMMRegister state1 = xmm2;
3147     const XMMRegister msgtmp0 = xmm3;
3148 
3149     const XMMRegister msgtmp1 = xmm4;
3150     const XMMRegister msgtmp2 = xmm5;
3151     const XMMRegister msgtmp3 = xmm6;
3152     const XMMRegister msgtmp4 = xmm7;
3153 
3154     __ enter();
3155     __ subptr(rsp, 8 * wordSize);
3156     handleSOERegisters(true /*saving*/);
3157     __ movptr(buf, buf_param);
3158     __ movptr(state, state_param);
3159     if (multi_block) {
3160      __ movptr(ofs, ofs_param);
3161      __ movptr(limit, limit_param);
3162     }
3163 
3164     __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3165       buf, state, ofs, limit, rsp, multi_block);
3166 
3167     handleSOERegisters(false);
3168     __ addptr(rsp, 8 * wordSize);
3169     __ leave();
3170     __ ret(0);
3171     return start;
3172   }
3173 
3174   // byte swap x86 long
3175   address generate_ghash_long_swap_mask() {
3176     __ align(CodeEntryAlignment);
3177     StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
3178     address start = __ pc();
3179     __ emit_data(0x0b0a0908, relocInfo::none, 0);
3180     __ emit_data(0x0f0e0d0c, relocInfo::none, 0);
3181     __ emit_data(0x03020100, relocInfo::none, 0);
3182     __ emit_data(0x07060504, relocInfo::none, 0);
3183 
3184   return start;
3185   }
3186 
3187   // byte swap x86 byte array
3188   address generate_ghash_byte_swap_mask() {
3189     __ align(CodeEntryAlignment);
3190     StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
3191     address start = __ pc();
3192     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3193     __ emit_data(0x08090a0b, relocInfo::none, 0);
3194     __ emit_data(0x04050607, relocInfo::none, 0);
3195     __ emit_data(0x00010203, relocInfo::none, 0);
3196   return start;
3197   }
3198 
3199   /* Single and multi-block ghash operations */
3200   address generate_ghash_processBlocks() {
3201     assert(UseGHASHIntrinsics, "need GHASH intrinsics and CLMUL support");
3202     __ align(CodeEntryAlignment);
3203     Label L_ghash_loop, L_exit;
3204     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
3205     address start = __ pc();
3206 
3207     const Register state        = rdi;
3208     const Register subkeyH      = rsi;
3209     const Register data         = rdx;
3210     const Register blocks       = rcx;
3211 
3212     const Address  state_param(rbp, 8+0);
3213     const Address  subkeyH_param(rbp, 8+4);
3214     const Address  data_param(rbp, 8+8);
3215     const Address  blocks_param(rbp, 8+12);
3216 
3217     const XMMRegister xmm_temp0 = xmm0;
3218     const XMMRegister xmm_temp1 = xmm1;
3219     const XMMRegister xmm_temp2 = xmm2;
3220     const XMMRegister xmm_temp3 = xmm3;
3221     const XMMRegister xmm_temp4 = xmm4;
3222     const XMMRegister xmm_temp5 = xmm5;
3223     const XMMRegister xmm_temp6 = xmm6;
3224     const XMMRegister xmm_temp7 = xmm7;
3225 
3226     __ enter();
3227     handleSOERegisters(true);  // Save registers
3228 
3229     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
3230     // context for the registers used, where all instructions below are using 128-bit mode
3231     // On EVEX without VL and BW, these instructions will all be AVX.
3232     if (VM_Version::supports_avx512vlbw()) {
3233       __ movl(rdx, 0xffff);
3234       __ kmovdl(k1, rdx);
3235     }
3236 
3237     __ movptr(state, state_param);
3238     __ movptr(subkeyH, subkeyH_param);
3239     __ movptr(data, data_param);
3240     __ movptr(blocks, blocks_param);
3241 
3242     __ movdqu(xmm_temp0, Address(state, 0));
3243     __ pshufb(xmm_temp0, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3244 
3245     __ movdqu(xmm_temp1, Address(subkeyH, 0));
3246     __ pshufb(xmm_temp1, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3247 
3248     __ BIND(L_ghash_loop);
3249     __ movdqu(xmm_temp2, Address(data, 0));
3250     __ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
3251 
3252     __ pxor(xmm_temp0, xmm_temp2);
3253 
3254     //
3255     // Multiply with the hash key
3256     //
3257     __ movdqu(xmm_temp3, xmm_temp0);
3258     __ pclmulqdq(xmm_temp3, xmm_temp1, 0);      // xmm3 holds a0*b0
3259     __ movdqu(xmm_temp4, xmm_temp0);
3260     __ pclmulqdq(xmm_temp4, xmm_temp1, 16);     // xmm4 holds a0*b1
3261 
3262     __ movdqu(xmm_temp5, xmm_temp0);
3263     __ pclmulqdq(xmm_temp5, xmm_temp1, 1);      // xmm5 holds a1*b0
3264     __ movdqu(xmm_temp6, xmm_temp0);
3265     __ pclmulqdq(xmm_temp6, xmm_temp1, 17);     // xmm6 holds a1*b1
3266 
3267     __ pxor(xmm_temp4, xmm_temp5);      // xmm4 holds a0*b1 + a1*b0
3268 
3269     __ movdqu(xmm_temp5, xmm_temp4);    // move the contents of xmm4 to xmm5
3270     __ psrldq(xmm_temp4, 8);    // shift by xmm4 64 bits to the right
3271     __ pslldq(xmm_temp5, 8);    // shift by xmm5 64 bits to the left
3272     __ pxor(xmm_temp3, xmm_temp5);
3273     __ pxor(xmm_temp6, xmm_temp4);      // Register pair <xmm6:xmm3> holds the result
3274                                         // of the carry-less multiplication of
3275                                         // xmm0 by xmm1.
3276 
3277     // We shift the result of the multiplication by one bit position
3278     // to the left to cope for the fact that the bits are reversed.
3279     __ movdqu(xmm_temp7, xmm_temp3);
3280     __ movdqu(xmm_temp4, xmm_temp6);
3281     __ pslld (xmm_temp3, 1);
3282     __ pslld(xmm_temp6, 1);
3283     __ psrld(xmm_temp7, 31);
3284     __ psrld(xmm_temp4, 31);
3285     __ movdqu(xmm_temp5, xmm_temp7);
3286     __ pslldq(xmm_temp4, 4);
3287     __ pslldq(xmm_temp7, 4);
3288     __ psrldq(xmm_temp5, 12);
3289     __ por(xmm_temp3, xmm_temp7);
3290     __ por(xmm_temp6, xmm_temp4);
3291     __ por(xmm_temp6, xmm_temp5);
3292 
3293     //
3294     // First phase of the reduction
3295     //
3296     // Move xmm3 into xmm4, xmm5, xmm7 in order to perform the shifts
3297     // independently.
3298     __ movdqu(xmm_temp7, xmm_temp3);
3299     __ movdqu(xmm_temp4, xmm_temp3);
3300     __ movdqu(xmm_temp5, xmm_temp3);
3301     __ pslld(xmm_temp7, 31);    // packed right shift shifting << 31
3302     __ pslld(xmm_temp4, 30);    // packed right shift shifting << 30
3303     __ pslld(xmm_temp5, 25);    // packed right shift shifting << 25
3304     __ pxor(xmm_temp7, xmm_temp4);      // xor the shifted versions
3305     __ pxor(xmm_temp7, xmm_temp5);
3306     __ movdqu(xmm_temp4, xmm_temp7);
3307     __ pslldq(xmm_temp7, 12);
3308     __ psrldq(xmm_temp4, 4);
3309     __ pxor(xmm_temp3, xmm_temp7);      // first phase of the reduction complete
3310 
3311     //
3312     // Second phase of the reduction
3313     //
3314     // Make 3 copies of xmm3 in xmm2, xmm5, xmm7 for doing these
3315     // shift operations.
3316     __ movdqu(xmm_temp2, xmm_temp3);
3317     __ movdqu(xmm_temp7, xmm_temp3);
3318     __ movdqu(xmm_temp5, xmm_temp3);
3319     __ psrld(xmm_temp2, 1);     // packed left shifting >> 1
3320     __ psrld(xmm_temp7, 2);     // packed left shifting >> 2
3321     __ psrld(xmm_temp5, 7);     // packed left shifting >> 7
3322     __ pxor(xmm_temp2, xmm_temp7);      // xor the shifted versions
3323     __ pxor(xmm_temp2, xmm_temp5);
3324     __ pxor(xmm_temp2, xmm_temp4);
3325     __ pxor(xmm_temp3, xmm_temp2);
3326     __ pxor(xmm_temp6, xmm_temp3);      // the result is in xmm6
3327 
3328     __ decrement(blocks);
3329     __ jcc(Assembler::zero, L_exit);
3330     __ movdqu(xmm_temp0, xmm_temp6);
3331     __ addptr(data, 16);
3332     __ jmp(L_ghash_loop);
3333 
3334     __ BIND(L_exit);
3335        // Byte swap 16-byte result
3336     __ pshufb(xmm_temp6, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3337     __ movdqu(Address(state, 0), xmm_temp6);   // store the result
3338 
3339     handleSOERegisters(false);  // restore registers
3340     __ leave();
3341     __ ret(0);
3342     return start;
3343   }
3344 
3345   /**
3346    *  Arguments:
3347    *
3348    * Inputs:
3349    *   rsp(4)   - int crc
3350    *   rsp(8)   - byte* buf
3351    *   rsp(12)  - int length
3352    *
3353    * Ouput:
3354    *       rax   - int crc result
3355    */
3356   address generate_updateBytesCRC32() {
3357     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
3358 
3359     __ align(CodeEntryAlignment);
3360     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
3361 
3362     address start = __ pc();
3363 
3364     const Register crc   = rdx;  // crc
3365     const Register buf   = rsi;  // source java byte array address
3366     const Register len   = rcx;  // length
3367     const Register table = rdi;  // crc_table address (reuse register)
3368     const Register tmp   = rbx;
3369     assert_different_registers(crc, buf, len, table, tmp, rax);
3370 
3371     BLOCK_COMMENT("Entry:");
3372     __ enter(); // required for proper stackwalking of RuntimeStub frame
3373     __ push(rsi);
3374     __ push(rdi);
3375     __ push(rbx);
3376 
3377     Address crc_arg(rbp, 8 + 0);
3378     Address buf_arg(rbp, 8 + 4);
3379     Address len_arg(rbp, 8 + 8);
3380 
3381     // Load up:
3382     __ movl(crc,   crc_arg);
3383     __ movptr(buf, buf_arg);
3384     __ movl(len,   len_arg);
3385 
3386     __ kernel_crc32(crc, buf, len, table, tmp);
3387 
3388     __ movl(rax, crc);
3389     __ pop(rbx);
3390     __ pop(rdi);
3391     __ pop(rsi);
3392     __ vzeroupper();
3393     __ leave(); // required for proper stackwalking of RuntimeStub frame
3394     __ ret(0);
3395 
3396     return start;
3397   }
3398 
3399   /**
3400   *  Arguments:
3401   *
3402   * Inputs:
3403   *   rsp(4)   - int crc
3404   *   rsp(8)   - byte* buf
3405   *   rsp(12)  - int length
3406   *   rsp(16)  - table_start - optional (present only when doing a library_calll,
3407   *              not used by x86 algorithm)
3408   *
3409   * Ouput:
3410   *       rax  - int crc result
3411   */
3412   address generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
3413     assert(UseCRC32CIntrinsics, "need SSE4_2");
3414     __ align(CodeEntryAlignment);
3415     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
3416     address start = __ pc();
3417     const Register crc = rax;  // crc
3418     const Register buf = rcx;  // source java byte array address
3419     const Register len = rdx;  // length
3420     const Register d = rbx;
3421     const Register g = rsi;
3422     const Register h = rdi;
3423     const Register empty = 0; // will never be used, in order not
3424                               // to change a signature for crc32c_IPL_Alg2_Alt2
3425                               // between 64/32 I'm just keeping it here
3426     assert_different_registers(crc, buf, len, d, g, h);
3427 
3428     BLOCK_COMMENT("Entry:");
3429     __ enter(); // required for proper stackwalking of RuntimeStub frame
3430     Address crc_arg(rsp, 4 + 4 + 0); // ESP+4 +
3431                                      // we need to add additional 4 because __ enter
3432                                      // have just pushed ebp on a stack
3433     Address buf_arg(rsp, 4 + 4 + 4);
3434     Address len_arg(rsp, 4 + 4 + 8);
3435       // Load up:
3436       __ movl(crc, crc_arg);
3437       __ movl(buf, buf_arg);
3438       __ movl(len, len_arg);
3439       __ push(d);
3440       __ push(g);
3441       __ push(h);
3442       __ crc32c_ipl_alg2_alt2(crc, buf, len,
3443                               d, g, h,
3444                               empty, empty, empty,
3445                               xmm0, xmm1, xmm2,
3446                               is_pclmulqdq_supported);
3447       __ pop(h);
3448       __ pop(g);
3449       __ pop(d);
3450     __ vzeroupper();
3451     __ leave(); // required for proper stackwalking of RuntimeStub frame
3452     __ ret(0);
3453 
3454     return start;
3455   }
3456 
3457  address generate_libmExp() {
3458     StubCodeMark mark(this, "StubRoutines", "libmExp");
3459 
3460     address start = __ pc();
3461 
3462     const XMMRegister x0  = xmm0;
3463     const XMMRegister x1  = xmm1;
3464     const XMMRegister x2  = xmm2;
3465     const XMMRegister x3  = xmm3;
3466 
3467     const XMMRegister x4  = xmm4;
3468     const XMMRegister x5  = xmm5;
3469     const XMMRegister x6  = xmm6;
3470     const XMMRegister x7  = xmm7;
3471 
3472     const Register tmp   = rbx;
3473 
3474     BLOCK_COMMENT("Entry:");
3475     __ enter(); // required for proper stackwalking of RuntimeStub frame
3476     __ fast_exp(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3477     __ leave(); // required for proper stackwalking of RuntimeStub frame
3478     __ ret(0);
3479 
3480     return start;
3481 
3482   }
3483 
3484  address generate_libmLog() {
3485    StubCodeMark mark(this, "StubRoutines", "libmLog");
3486 
3487    address start = __ pc();
3488 
3489    const XMMRegister x0 = xmm0;
3490    const XMMRegister x1 = xmm1;
3491    const XMMRegister x2 = xmm2;
3492    const XMMRegister x3 = xmm3;
3493 
3494    const XMMRegister x4 = xmm4;
3495    const XMMRegister x5 = xmm5;
3496    const XMMRegister x6 = xmm6;
3497    const XMMRegister x7 = xmm7;
3498 
3499    const Register tmp = rbx;
3500 
3501    BLOCK_COMMENT("Entry:");
3502    __ enter(); // required for proper stackwalking of RuntimeStub frame
3503    __ fast_log(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3504    __ leave(); // required for proper stackwalking of RuntimeStub frame
3505    __ ret(0);
3506 
3507    return start;
3508 
3509  }
3510 
3511  address generate_libmLog10() {
3512    StubCodeMark mark(this, "StubRoutines", "libmLog10");
3513 
3514    address start = __ pc();
3515 
3516    const XMMRegister x0 = xmm0;
3517    const XMMRegister x1 = xmm1;
3518    const XMMRegister x2 = xmm2;
3519    const XMMRegister x3 = xmm3;
3520 
3521    const XMMRegister x4 = xmm4;
3522    const XMMRegister x5 = xmm5;
3523    const XMMRegister x6 = xmm6;
3524    const XMMRegister x7 = xmm7;
3525 
3526    const Register tmp = rbx;
3527 
3528    BLOCK_COMMENT("Entry:");
3529    __ enter(); // required for proper stackwalking of RuntimeStub frame
3530    __ fast_log10(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3531    __ leave(); // required for proper stackwalking of RuntimeStub frame
3532    __ ret(0);
3533 
3534    return start;
3535 
3536  }
3537 
3538  address generate_libmPow() {
3539    StubCodeMark mark(this, "StubRoutines", "libmPow");
3540 
3541    address start = __ pc();
3542 
3543    const XMMRegister x0 = xmm0;
3544    const XMMRegister x1 = xmm1;
3545    const XMMRegister x2 = xmm2;
3546    const XMMRegister x3 = xmm3;
3547 
3548    const XMMRegister x4 = xmm4;
3549    const XMMRegister x5 = xmm5;
3550    const XMMRegister x6 = xmm6;
3551    const XMMRegister x7 = xmm7;
3552 
3553    const Register tmp = rbx;
3554 
3555    BLOCK_COMMENT("Entry:");
3556    __ enter(); // required for proper stackwalking of RuntimeStub frame
3557    __ fast_pow(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3558    __ leave(); // required for proper stackwalking of RuntimeStub frame
3559    __ ret(0);
3560 
3561    return start;
3562 
3563  }
3564 
3565  address generate_libm_reduce_pi04l() {
3566    StubCodeMark mark(this, "StubRoutines", "libm_reduce_pi04l");
3567 
3568    address start = __ pc();
3569 
3570    BLOCK_COMMENT("Entry:");
3571    __ libm_reduce_pi04l(rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3572 
3573    return start;
3574 
3575  }
3576 
3577  address generate_libm_sin_cos_huge() {
3578    StubCodeMark mark(this, "StubRoutines", "libm_sin_cos_huge");
3579 
3580    address start = __ pc();
3581 
3582    const XMMRegister x0 = xmm0;
3583    const XMMRegister x1 = xmm1;
3584 
3585    BLOCK_COMMENT("Entry:");
3586    __ libm_sincos_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3587 
3588    return start;
3589 
3590  }
3591 
3592  address generate_libmSin() {
3593    StubCodeMark mark(this, "StubRoutines", "libmSin");
3594 
3595    address start = __ pc();
3596 
3597    const XMMRegister x0 = xmm0;
3598    const XMMRegister x1 = xmm1;
3599    const XMMRegister x2 = xmm2;
3600    const XMMRegister x3 = xmm3;
3601 
3602    const XMMRegister x4 = xmm4;
3603    const XMMRegister x5 = xmm5;
3604    const XMMRegister x6 = xmm6;
3605    const XMMRegister x7 = xmm7;
3606 
3607    BLOCK_COMMENT("Entry:");
3608    __ enter(); // required for proper stackwalking of RuntimeStub frame
3609    __ fast_sin(x0, x1, x2, x3, x4, x5, x6, x7, rax, rbx, rdx);
3610    __ leave(); // required for proper stackwalking of RuntimeStub frame
3611    __ ret(0);
3612 
3613    return start;
3614 
3615  }
3616 
3617  address generate_libmCos() {
3618    StubCodeMark mark(this, "StubRoutines", "libmCos");
3619 
3620    address start = __ pc();
3621 
3622    const XMMRegister x0 = xmm0;
3623    const XMMRegister x1 = xmm1;
3624    const XMMRegister x2 = xmm2;
3625    const XMMRegister x3 = xmm3;
3626 
3627    const XMMRegister x4 = xmm4;
3628    const XMMRegister x5 = xmm5;
3629    const XMMRegister x6 = xmm6;
3630    const XMMRegister x7 = xmm7;
3631 
3632    const Register tmp = rbx;
3633 
3634    BLOCK_COMMENT("Entry:");
3635    __ enter(); // required for proper stackwalking of RuntimeStub frame
3636    __ fast_cos(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3637    __ leave(); // required for proper stackwalking of RuntimeStub frame
3638    __ ret(0);
3639 
3640    return start;
3641 
3642  }
3643 
3644  address generate_libm_tan_cot_huge() {
3645    StubCodeMark mark(this, "StubRoutines", "libm_tan_cot_huge");
3646 
3647    address start = __ pc();
3648 
3649    const XMMRegister x0 = xmm0;
3650    const XMMRegister x1 = xmm1;
3651 
3652    BLOCK_COMMENT("Entry:");
3653    __ libm_tancot_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3654 
3655    return start;
3656 
3657  }
3658 
3659  address generate_libmTan() {
3660    StubCodeMark mark(this, "StubRoutines", "libmTan");
3661 
3662    address start = __ pc();
3663 
3664    const XMMRegister x0 = xmm0;
3665    const XMMRegister x1 = xmm1;
3666    const XMMRegister x2 = xmm2;
3667    const XMMRegister x3 = xmm3;
3668 
3669    const XMMRegister x4 = xmm4;
3670    const XMMRegister x5 = xmm5;
3671    const XMMRegister x6 = xmm6;
3672    const XMMRegister x7 = xmm7;
3673 
3674    const Register tmp = rbx;
3675 
3676    BLOCK_COMMENT("Entry:");
3677    __ enter(); // required for proper stackwalking of RuntimeStub frame
3678    __ fast_tan(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3679    __ leave(); // required for proper stackwalking of RuntimeStub frame
3680    __ ret(0);
3681 
3682    return start;
3683 
3684  }
3685 
3686   // Safefetch stubs.
3687   void generate_safefetch(const char* name, int size, address* entry,
3688                           address* fault_pc, address* continuation_pc) {
3689     // safefetch signatures:
3690     //   int      SafeFetch32(int*      adr, int      errValue);
3691     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3692 
3693     StubCodeMark mark(this, "StubRoutines", name);
3694 
3695     // Entry point, pc or function descriptor.
3696     *entry = __ pc();
3697 
3698     __ movl(rax, Address(rsp, 0x8));
3699     __ movl(rcx, Address(rsp, 0x4));
3700     // Load *adr into eax, may fault.
3701     *fault_pc = __ pc();
3702     switch (size) {
3703       case 4:
3704         // int32_t
3705         __ movl(rax, Address(rcx, 0));
3706         break;
3707       case 8:
3708         // int64_t
3709         Unimplemented();
3710         break;
3711       default:
3712         ShouldNotReachHere();
3713     }
3714 
3715     // Return errValue or *adr.
3716     *continuation_pc = __ pc();
3717     __ ret(0);
3718   }
3719 
3720  public:
3721   // Information about frame layout at time of blocking runtime call.
3722   // Note that we only have to preserve callee-saved registers since
3723   // the compilers are responsible for supplying a continuation point
3724   // if they expect all registers to be preserved.
3725   enum layout {
3726     thread_off,    // last_java_sp
3727     arg1_off,
3728     arg2_off,
3729     rbp_off,       // callee saved register
3730     ret_pc,
3731     framesize
3732   };
3733 
3734  private:
3735 
3736 #undef  __
3737 #define __ masm->
3738 
3739   //------------------------------------------------------------------------------------------------------------------------
3740   // Continuation point for throwing of implicit exceptions that are not handled in
3741   // the current activation. Fabricates an exception oop and initiates normal
3742   // exception dispatching in this frame.
3743   //
3744   // Previously the compiler (c2) allowed for callee save registers on Java calls.
3745   // This is no longer true after adapter frames were removed but could possibly
3746   // be brought back in the future if the interpreter code was reworked and it
3747   // was deemed worthwhile. The comment below was left to describe what must
3748   // happen here if callee saves were resurrected. As it stands now this stub
3749   // could actually be a vanilla BufferBlob and have now oopMap at all.
3750   // Since it doesn't make much difference we've chosen to leave it the
3751   // way it was in the callee save days and keep the comment.
3752 
3753   // If we need to preserve callee-saved values we need a callee-saved oop map and
3754   // therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.
3755   // If the compiler needs all registers to be preserved between the fault
3756   // point and the exception handler then it must assume responsibility for that in
3757   // AbstractCompiler::continuation_for_implicit_null_exception or
3758   // continuation_for_implicit_division_by_zero_exception. All other implicit
3759   // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
3760   // either at call sites or otherwise assume that stack unwinding will be initiated,
3761   // so caller saved registers were assumed volatile in the compiler.
3762   address generate_throw_exception(const char* name, address runtime_entry,
3763                                    Register arg1 = noreg, Register arg2 = noreg) {
3764 
3765     int insts_size = 256;
3766     int locs_size  = 32;
3767 
3768     CodeBuffer code(name, insts_size, locs_size);
3769     OopMapSet* oop_maps  = new OopMapSet();
3770     MacroAssembler* masm = new MacroAssembler(&code);
3771 
3772     address start = __ pc();
3773 
3774     // This is an inlined and slightly modified version of call_VM
3775     // which has the ability to fetch the return PC out of
3776     // thread-local storage and also sets up last_Java_sp slightly
3777     // differently than the real call_VM
3778     Register java_thread = rbx;
3779     __ get_thread(java_thread);
3780 
3781     __ enter(); // required for proper stackwalking of RuntimeStub frame
3782 
3783     // pc and rbp, already pushed
3784     __ subptr(rsp, (framesize-2) * wordSize); // prolog
3785 
3786     // Frame is now completed as far as size and linkage.
3787 
3788     int frame_complete = __ pc() - start;
3789 
3790     // push java thread (becomes first argument of C function)
3791     __ movptr(Address(rsp, thread_off * wordSize), java_thread);
3792     if (arg1 != noreg) {
3793       __ movptr(Address(rsp, arg1_off * wordSize), arg1);
3794     }
3795     if (arg2 != noreg) {
3796       assert(arg1 != noreg, "missing reg arg");
3797       __ movptr(Address(rsp, arg2_off * wordSize), arg2);
3798     }
3799 
3800     // Set up last_Java_sp and last_Java_fp
3801     __ set_last_Java_frame(java_thread, rsp, rbp, NULL);
3802 
3803     // Call runtime
3804     BLOCK_COMMENT("call runtime_entry");
3805     __ call(RuntimeAddress(runtime_entry));
3806     // Generate oop map
3807     OopMap* map =  new OopMap(framesize, 0);
3808     oop_maps->add_gc_map(__ pc() - start, map);
3809 
3810     // restore the thread (cannot use the pushed argument since arguments
3811     // may be overwritten by C code generated by an optimizing compiler);
3812     // however can use the register value directly if it is callee saved.
3813     __ get_thread(java_thread);
3814 
3815     __ reset_last_Java_frame(java_thread, true);
3816 
3817     __ leave(); // required for proper stackwalking of RuntimeStub frame
3818 
3819     // check for pending exceptions
3820 #ifdef ASSERT
3821     Label L;
3822     __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3823     __ jcc(Assembler::notEqual, L);
3824     __ should_not_reach_here();
3825     __ bind(L);
3826 #endif /* ASSERT */
3827     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3828 
3829 
3830     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);
3831     return stub->entry_point();
3832   }
3833 
3834 
3835   void create_control_words() {
3836     // Round to nearest, 53-bit mode, exceptions masked
3837     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
3838     // Round to zero, 53-bit mode, exception mased
3839     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
3840     // Round to nearest, 24-bit mode, exceptions masked
3841     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
3842     // Round to nearest, 64-bit mode, exceptions masked
3843     StubRoutines::_fpu_cntrl_wrd_64    = 0x037F;
3844     // Round to nearest, 64-bit mode, exceptions masked
3845     StubRoutines::_mxcsr_std           = 0x1F80;
3846     // Note: the following two constants are 80-bit values
3847     //       layout is critical for correct loading by FPU.
3848     // Bias for strict fp multiply/divide
3849     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
3850     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
3851     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
3852     // Un-Bias for strict fp multiply/divide
3853     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
3854     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
3855     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
3856   }
3857 
3858   //---------------------------------------------------------------------------
3859   // Initialization
3860 
3861   void generate_initial() {
3862     // Generates all stubs and initializes the entry points
3863 
3864     //------------------------------------------------------------------------------------------------------------------------
3865     // entry points that exist in all platforms
3866     // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
3867     //       the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
3868     StubRoutines::_forward_exception_entry      = generate_forward_exception();
3869 
3870     StubRoutines::_call_stub_entry              =
3871       generate_call_stub(StubRoutines::_call_stub_return_address);
3872     // is referenced by megamorphic call
3873     StubRoutines::_catch_exception_entry        = generate_catch_exception();
3874 
3875     // These are currently used by Solaris/Intel
3876     StubRoutines::_atomic_xchg_entry            = generate_atomic_xchg();
3877 
3878     // platform dependent
3879     create_control_words();
3880 
3881     StubRoutines::x86::_verify_mxcsr_entry                 = generate_verify_mxcsr();
3882     StubRoutines::x86::_verify_fpu_cntrl_wrd_entry         = generate_verify_fpu_cntrl_wrd();
3883     StubRoutines::_d2i_wrapper                              = generate_d2i_wrapper(T_INT,
3884                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
3885     StubRoutines::_d2l_wrapper                              = generate_d2i_wrapper(T_LONG,
3886                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
3887 
3888     // Build this early so it's available for the interpreter
3889     StubRoutines::_throw_StackOverflowError_entry          = generate_throw_exception("StackOverflowError throw_exception",
3890                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
3891     StubRoutines::_throw_delayed_StackOverflowError_entry  = generate_throw_exception("delayed StackOverflowError throw_exception",
3892                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_delayed_StackOverflowError));
3893 
3894     if (UseCRC32Intrinsics) {
3895       // set table address before stub generation which use it
3896       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
3897       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
3898     }
3899 
3900     if (UseCRC32CIntrinsics) {
3901       bool supports_clmul = VM_Version::supports_clmul();
3902       StubRoutines::x86::generate_CRC32C_table(supports_clmul);
3903       StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table;
3904       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
3905     }
3906     if (VM_Version::supports_sse2() && UseLibmIntrinsic && InlineIntrinsics) {
3907       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3908           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3909           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3910         StubRoutines::x86::_L_2il0floatpacket_0_adr = (address)StubRoutines::x86::_L_2il0floatpacket_0;
3911         StubRoutines::x86::_Pi4Inv_adr = (address)StubRoutines::x86::_Pi4Inv;
3912         StubRoutines::x86::_Pi4x3_adr = (address)StubRoutines::x86::_Pi4x3;
3913         StubRoutines::x86::_Pi4x4_adr = (address)StubRoutines::x86::_Pi4x4;
3914         StubRoutines::x86::_ones_adr = (address)StubRoutines::x86::_ones;
3915       }
3916       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) {
3917         StubRoutines::_dexp = generate_libmExp();
3918       }
3919       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) {
3920         StubRoutines::_dlog = generate_libmLog();
3921       }
3922       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) {
3923         StubRoutines::_dlog10 = generate_libmLog10();
3924       }
3925       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) {
3926         StubRoutines::_dpow = generate_libmPow();
3927       }
3928       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3929         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3930         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3931         StubRoutines::_dlibm_reduce_pi04l = generate_libm_reduce_pi04l();
3932       }
3933       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3934         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3935         StubRoutines::_dlibm_sin_cos_huge = generate_libm_sin_cos_huge();
3936       }
3937       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
3938         StubRoutines::_dsin = generate_libmSin();
3939       }
3940       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3941         StubRoutines::_dcos = generate_libmCos();
3942       }
3943       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3944         StubRoutines::_dlibm_tan_cot_huge = generate_libm_tan_cot_huge();
3945         StubRoutines::_dtan = generate_libmTan();
3946       }
3947     }
3948   }
3949 
3950   void generate_all() {
3951     // Generates all stubs and initializes the entry points
3952 
3953     // These entry points require SharedInfo::stack0 to be set up in non-core builds
3954     // and need to be relocatable, so they each fabricate a RuntimeStub internally.
3955     StubRoutines::_throw_AbstractMethodError_entry         = generate_throw_exception("AbstractMethodError throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
3956     StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
3957     StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
3958 
3959     //------------------------------------------------------------------------------------------------------------------------
3960     // entry points that are platform specific
3961 
3962     // support for verify_oop (must happen after universe_init)
3963     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
3964 
3965     // arraycopy stubs used by compilers
3966     generate_arraycopy_stubs();
3967 
3968     // don't bother generating these AES intrinsic stubs unless global flag is set
3969     if (UseAESIntrinsics) {
3970       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // might be needed by the others
3971 
3972       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
3973       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
3974       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
3975       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
3976     }
3977 
3978     if (UseAESCTRIntrinsics) {
3979       StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
3980       StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
3981     }
3982 
3983     if (UseSHA1Intrinsics) {
3984       StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask();
3985       StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask();
3986       StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
3987       StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
3988     }
3989     if (UseSHA256Intrinsics) {
3990       StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
3991       StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
3992       StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
3993       StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
3994     }
3995 
3996     // Generate GHASH intrinsics code
3997     if (UseGHASHIntrinsics) {
3998       StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
3999       StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
4000       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
4001     }
4002 
4003     // Safefetch stubs.
4004     generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
4005                                                    &StubRoutines::_safefetch32_fault_pc,
4006                                                    &StubRoutines::_safefetch32_continuation_pc);
4007     StubRoutines::_safefetchN_entry           = StubRoutines::_safefetch32_entry;
4008     StubRoutines::_safefetchN_fault_pc        = StubRoutines::_safefetch32_fault_pc;
4009     StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc;
4010   }
4011 
4012 
4013  public:
4014   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
4015     if (all) {
4016       generate_all();
4017     } else {
4018       generate_initial();
4019     }
4020   }
4021 }; // end class declaration
4022 
4023 
4024 void StubGenerator_generate(CodeBuffer* code, bool all) {
4025   StubGenerator g(code, all);
4026 }