1 /*
   2  * Copyright (c) 1999, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "nativeInst_x86.hpp"
  30 #include "oops/instanceOop.hpp"
  31 #include "oops/method.hpp"
  32 #include "oops/objArrayKlass.hpp"
  33 #include "oops/oop.inline.hpp"
  34 #include "prims/methodHandles.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/handles.inline.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubCodeGenerator.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/thread.inline.hpp"
  41 #include "utilities/top.hpp"
  42 #ifdef COMPILER2
  43 #include "opto/runtime.hpp"
  44 #endif
  45 
  46 // Declaration and definition of StubGenerator (no .hpp file).
  47 // For a more detailed description of the stub routine structure
  48 // see the comment in stubRoutines.hpp
  49 
  50 #define __ _masm->
  51 #define a__ ((Assembler*)_masm)->
  52 
  53 #ifdef PRODUCT
  54 #define BLOCK_COMMENT(str) /* nothing */
  55 #else
  56 #define BLOCK_COMMENT(str) __ block_comment(str)
  57 #endif
  58 
  59 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  60 
  61 const int MXCSR_MASK  = 0xFFC0;  // Mask out any pending exceptions
  62 const int FPU_CNTRL_WRD_MASK = 0xFFFF;
  63 
  64 // -------------------------------------------------------------------------------------------------------------------------
  65 // Stub Code definitions
  66 
  67 static address handle_unsafe_access() {
  68   JavaThread* thread = JavaThread::current();
  69   address pc  = thread->saved_exception_pc();
  70   // pc is the instruction which we must emulate
  71   // doing a no-op is fine:  return garbage from the load
  72   // therefore, compute npc
  73   address npc = Assembler::locate_next_instruction(pc);
  74 
  75   // request an async exception
  76   thread->set_pending_unsafe_access_error();
  77 
  78   // return address of next instruction to execute
  79   return npc;
  80 }
  81 
  82 class StubGenerator: public StubCodeGenerator {
  83  private:
  84 
  85 #ifdef PRODUCT
  86 #define inc_counter_np(counter) ((void)0)
  87 #else
  88   void inc_counter_np_(int& counter) {
  89     __ incrementl(ExternalAddress((address)&counter));
  90   }
  91 #define inc_counter_np(counter) \
  92   BLOCK_COMMENT("inc_counter " #counter); \
  93   inc_counter_np_(counter);
  94 #endif //PRODUCT
  95 
  96   void inc_copy_counter_np(BasicType t) {
  97 #ifndef PRODUCT
  98     switch (t) {
  99     case T_BYTE:    inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;
 100     case T_SHORT:   inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;
 101     case T_INT:     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;
 102     case T_LONG:    inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;
 103     case T_OBJECT:  inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;
 104     }
 105     ShouldNotReachHere();
 106 #endif //PRODUCT
 107   }
 108 
 109   //------------------------------------------------------------------------------------------------------------------------
 110   // Call stubs are used to call Java from C
 111   //
 112   //    [ return_from_Java     ] <--- rsp
 113   //    [ argument word n      ]
 114   //      ...
 115   // -N [ argument word 1      ]
 116   // -7 [ Possible padding for stack alignment ]
 117   // -6 [ Possible padding for stack alignment ]
 118   // -5 [ Possible padding for stack alignment ]
 119   // -4 [ mxcsr save           ] <--- rsp_after_call
 120   // -3 [ saved rbx,            ]
 121   // -2 [ saved rsi            ]
 122   // -1 [ saved rdi            ]
 123   //  0 [ saved rbp,            ] <--- rbp,
 124   //  1 [ return address       ]
 125   //  2 [ ptr. to call wrapper ]
 126   //  3 [ result               ]
 127   //  4 [ result_type          ]
 128   //  5 [ method               ]
 129   //  6 [ entry_point          ]
 130   //  7 [ parameters           ]
 131   //  8 [ parameter_size       ]
 132   //  9 [ thread               ]
 133 
 134 
 135   address generate_call_stub(address& return_address) {
 136     StubCodeMark mark(this, "StubRoutines", "call_stub");
 137     address start = __ pc();
 138 
 139     // stub code parameters / addresses
 140     assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");
 141     bool  sse_save = false;
 142     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!
 143     const int     locals_count_in_bytes  (4*wordSize);
 144     const Address mxcsr_save    (rbp, -4 * wordSize);
 145     const Address saved_rbx     (rbp, -3 * wordSize);
 146     const Address saved_rsi     (rbp, -2 * wordSize);
 147     const Address saved_rdi     (rbp, -1 * wordSize);
 148     const Address result        (rbp,  3 * wordSize);
 149     const Address result_type   (rbp,  4 * wordSize);
 150     const Address method        (rbp,  5 * wordSize);
 151     const Address entry_point   (rbp,  6 * wordSize);
 152     const Address parameters    (rbp,  7 * wordSize);
 153     const Address parameter_size(rbp,  8 * wordSize);
 154     const Address thread        (rbp,  9 * wordSize); // same as in generate_catch_exception()!
 155     sse_save =  UseSSE > 0;
 156 
 157     // stub code
 158     __ enter();
 159     __ movptr(rcx, parameter_size);              // parameter counter
 160     __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes
 161     __ addptr(rcx, locals_count_in_bytes);       // reserve space for register saves
 162     __ subptr(rsp, rcx);
 163     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
 164 
 165     // save rdi, rsi, & rbx, according to C calling conventions
 166     __ movptr(saved_rdi, rdi);
 167     __ movptr(saved_rsi, rsi);
 168     __ movptr(saved_rbx, rbx);
 169 
 170     // provide initial value for required masks
 171     if (UseAVX > 2) {
 172       __ movl(rbx, 0xffff);
 173       __ kmovwl(k1, rbx);
 174     }
 175 
 176     // save and initialize %mxcsr
 177     if (sse_save) {
 178       Label skip_ldmx;
 179       __ stmxcsr(mxcsr_save);
 180       __ movl(rax, mxcsr_save);
 181       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 182       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 183       __ cmp32(rax, mxcsr_std);
 184       __ jcc(Assembler::equal, skip_ldmx);
 185       __ ldmxcsr(mxcsr_std);
 186       __ bind(skip_ldmx);
 187     }
 188 
 189     // make sure the control word is correct.
 190     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 191 
 192 #ifdef ASSERT
 193     // make sure we have no pending exceptions
 194     { Label L;
 195       __ movptr(rcx, thread);
 196       __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 197       __ jcc(Assembler::equal, L);
 198       __ stop("StubRoutines::call_stub: entered with pending exception");
 199       __ bind(L);
 200     }
 201 #endif
 202 
 203     // pass parameters if any
 204     BLOCK_COMMENT("pass parameters if any");
 205     Label parameters_done;
 206     __ movl(rcx, parameter_size);  // parameter counter
 207     __ testl(rcx, rcx);
 208     __ jcc(Assembler::zero, parameters_done);
 209 
 210     // parameter passing loop
 211 
 212     Label loop;
 213     // Copy Java parameters in reverse order (receiver last)
 214     // Note that the argument order is inverted in the process
 215     // source is rdx[rcx: N-1..0]
 216     // dest   is rsp[rbx: 0..N-1]
 217 
 218     __ movptr(rdx, parameters);          // parameter pointer
 219     __ xorptr(rbx, rbx);
 220 
 221     __ BIND(loop);
 222 
 223     // get parameter
 224     __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
 225     __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
 226                     Interpreter::expr_offset_in_bytes(0)), rax);          // store parameter
 227     __ increment(rbx);
 228     __ decrement(rcx);
 229     __ jcc(Assembler::notZero, loop);
 230 
 231     // call Java function
 232     __ BIND(parameters_done);
 233     __ movptr(rbx, method);           // get Method*
 234     __ movptr(rax, entry_point);      // get entry_point
 235     __ mov(rsi, rsp);                 // set sender sp
 236     BLOCK_COMMENT("call Java function");
 237     __ call(rax);
 238 
 239     BLOCK_COMMENT("call_stub_return_address:");
 240     return_address = __ pc();
 241 
 242 #ifdef COMPILER2
 243     {
 244       Label L_skip;
 245       if (UseSSE >= 2) {
 246         __ verify_FPU(0, "call_stub_return");
 247       } else {
 248         for (int i = 1; i < 8; i++) {
 249           __ ffree(i);
 250         }
 251 
 252         // UseSSE <= 1 so double result should be left on TOS
 253         __ movl(rsi, result_type);
 254         __ cmpl(rsi, T_DOUBLE);
 255         __ jcc(Assembler::equal, L_skip);
 256         if (UseSSE == 0) {
 257           // UseSSE == 0 so float result should be left on TOS
 258           __ cmpl(rsi, T_FLOAT);
 259           __ jcc(Assembler::equal, L_skip);
 260         }
 261         __ ffree(0);
 262       }
 263       __ BIND(L_skip);
 264     }
 265 #endif // COMPILER2
 266 
 267     // store result depending on type
 268     // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 269     __ movptr(rdi, result);
 270     Label is_long, is_float, is_double, exit;
 271     __ movl(rsi, result_type);
 272     __ cmpl(rsi, T_LONG);
 273     __ jcc(Assembler::equal, is_long);
 274     __ cmpl(rsi, T_FLOAT);
 275     __ jcc(Assembler::equal, is_float);
 276     __ cmpl(rsi, T_DOUBLE);
 277     __ jcc(Assembler::equal, is_double);
 278 
 279     // handle T_INT case
 280     __ movl(Address(rdi, 0), rax);
 281     __ BIND(exit);
 282 
 283     // check that FPU stack is empty
 284     __ verify_FPU(0, "generate_call_stub");
 285 
 286     // pop parameters
 287     __ lea(rsp, rsp_after_call);
 288 
 289     // restore %mxcsr
 290     if (sse_save) {
 291       __ ldmxcsr(mxcsr_save);
 292     }
 293 
 294     // restore rdi, rsi and rbx,
 295     __ movptr(rbx, saved_rbx);
 296     __ movptr(rsi, saved_rsi);
 297     __ movptr(rdi, saved_rdi);
 298     __ addptr(rsp, 4*wordSize);
 299 
 300     // return
 301     __ pop(rbp);
 302     __ ret(0);
 303 
 304     // handle return types different from T_INT
 305     __ BIND(is_long);
 306     __ movl(Address(rdi, 0 * wordSize), rax);
 307     __ movl(Address(rdi, 1 * wordSize), rdx);
 308     __ jmp(exit);
 309 
 310     __ BIND(is_float);
 311     // interpreter uses xmm0 for return values
 312     if (UseSSE >= 1) {
 313       __ movflt(Address(rdi, 0), xmm0);
 314     } else {
 315       __ fstp_s(Address(rdi, 0));
 316     }
 317     __ jmp(exit);
 318 
 319     __ BIND(is_double);
 320     // interpreter uses xmm0 for return values
 321     if (UseSSE >= 2) {
 322       __ movdbl(Address(rdi, 0), xmm0);
 323     } else {
 324       __ fstp_d(Address(rdi, 0));
 325     }
 326     __ jmp(exit);
 327 
 328     return start;
 329   }
 330 
 331 
 332   //------------------------------------------------------------------------------------------------------------------------
 333   // Return point for a Java call if there's an exception thrown in Java code.
 334   // The exception is caught and transformed into a pending exception stored in
 335   // JavaThread that can be tested from within the VM.
 336   //
 337   // Note: Usually the parameters are removed by the callee. In case of an exception
 338   //       crossing an activation frame boundary, that is not the case if the callee
 339   //       is compiled code => need to setup the rsp.
 340   //
 341   // rax,: exception oop
 342 
 343   address generate_catch_exception() {
 344     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 345     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!
 346     const Address thread        (rbp,  9 * wordSize); // same as in generate_call_stub()!
 347     address start = __ pc();
 348 
 349     // get thread directly
 350     __ movptr(rcx, thread);
 351 #ifdef ASSERT
 352     // verify that threads correspond
 353     { Label L;
 354       __ get_thread(rbx);
 355       __ cmpptr(rbx, rcx);
 356       __ jcc(Assembler::equal, L);
 357       __ stop("StubRoutines::catch_exception: threads must correspond");
 358       __ bind(L);
 359     }
 360 #endif
 361     // set pending exception
 362     __ verify_oop(rax);
 363     __ movptr(Address(rcx, Thread::pending_exception_offset()), rax          );
 364     __ lea(Address(rcx, Thread::exception_file_offset   ()),
 365            ExternalAddress((address)__FILE__));
 366     __ movl(Address(rcx, Thread::exception_line_offset   ()), __LINE__ );
 367     // complete return to VM
 368     assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
 369     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 370 
 371     return start;
 372   }
 373 
 374 
 375   //------------------------------------------------------------------------------------------------------------------------
 376   // Continuation point for runtime calls returning with a pending exception.
 377   // The pending exception check happened in the runtime or native call stub.
 378   // The pending exception in Thread is converted into a Java-level exception.
 379   //
 380   // Contract with Java-level exception handlers:
 381   // rax: exception
 382   // rdx: throwing pc
 383   //
 384   // NOTE: At entry of this stub, exception-pc must be on stack !!
 385 
 386   address generate_forward_exception() {
 387     StubCodeMark mark(this, "StubRoutines", "forward exception");
 388     address start = __ pc();
 389     const Register thread = rcx;
 390 
 391     // other registers used in this stub
 392     const Register exception_oop = rax;
 393     const Register handler_addr  = rbx;
 394     const Register exception_pc  = rdx;
 395 
 396     // Upon entry, the sp points to the return address returning into Java
 397     // (interpreted or compiled) code; i.e., the return address becomes the
 398     // throwing pc.
 399     //
 400     // Arguments pushed before the runtime call are still on the stack but
 401     // the exception handler will reset the stack pointer -> ignore them.
 402     // A potential result in registers can be ignored as well.
 403 
 404 #ifdef ASSERT
 405     // make sure this code is only executed if there is a pending exception
 406     { Label L;
 407       __ get_thread(thread);
 408       __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 409       __ jcc(Assembler::notEqual, L);
 410       __ stop("StubRoutines::forward exception: no pending exception (1)");
 411       __ bind(L);
 412     }
 413 #endif
 414 
 415     // compute exception handler into rbx,
 416     __ get_thread(thread);
 417     __ movptr(exception_pc, Address(rsp, 0));
 418     BLOCK_COMMENT("call exception_handler_for_return_address");
 419     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
 420     __ mov(handler_addr, rax);
 421 
 422     // setup rax & rdx, remove return address & clear pending exception
 423     __ get_thread(thread);
 424     __ pop(exception_pc);
 425     __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
 426     __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
 427 
 428 #ifdef ASSERT
 429     // make sure exception is set
 430     { Label L;
 431       __ testptr(exception_oop, exception_oop);
 432       __ jcc(Assembler::notEqual, L);
 433       __ stop("StubRoutines::forward exception: no pending exception (2)");
 434       __ bind(L);
 435     }
 436 #endif
 437 
 438     // Verify that there is really a valid exception in RAX.
 439     __ verify_oop(exception_oop);
 440 
 441     // continue at exception handler (return address removed)
 442     // rax: exception
 443     // rbx: exception handler
 444     // rdx: throwing pc
 445     __ jmp(handler_addr);
 446 
 447     return start;
 448   }
 449 
 450 
 451   //----------------------------------------------------------------------------------------------------
 452   // Support for jint Atomic::xchg(jint exchange_value, volatile jint* dest)
 453   //
 454   // xchg exists as far back as 8086, lock needed for MP only
 455   // Stack layout immediately after call:
 456   //
 457   // 0 [ret addr ] <--- rsp
 458   // 1 [  ex     ]
 459   // 2 [  dest   ]
 460   //
 461   // Result:   *dest <- ex, return (old *dest)
 462   //
 463   // Note: win32 does not currently use this code
 464 
 465   address generate_atomic_xchg() {
 466     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 467     address start = __ pc();
 468 
 469     __ push(rdx);
 470     Address exchange(rsp, 2 * wordSize);
 471     Address dest_addr(rsp, 3 * wordSize);
 472     __ movl(rax, exchange);
 473     __ movptr(rdx, dest_addr);
 474     __ xchgl(rax, Address(rdx, 0));
 475     __ pop(rdx);
 476     __ ret(0);
 477 
 478     return start;
 479   }
 480 
 481   //----------------------------------------------------------------------------------------------------
 482   // Support for void verify_mxcsr()
 483   //
 484   // This routine is used with -Xcheck:jni to verify that native
 485   // JNI code does not return to Java code without restoring the
 486   // MXCSR register to our expected state.
 487 
 488 
 489   address generate_verify_mxcsr() {
 490     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 491     address start = __ pc();
 492 
 493     const Address mxcsr_save(rsp, 0);
 494 
 495     if (CheckJNICalls && UseSSE > 0 ) {
 496       Label ok_ret;
 497       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 498       __ push(rax);
 499       __ subptr(rsp, wordSize);      // allocate a temp location
 500       __ stmxcsr(mxcsr_save);
 501       __ movl(rax, mxcsr_save);
 502       __ andl(rax, MXCSR_MASK);
 503       __ cmp32(rax, mxcsr_std);
 504       __ jcc(Assembler::equal, ok_ret);
 505 
 506       __ warn("MXCSR changed by native JNI code.");
 507 
 508       __ ldmxcsr(mxcsr_std);
 509 
 510       __ bind(ok_ret);
 511       __ addptr(rsp, wordSize);
 512       __ pop(rax);
 513     }
 514 
 515     __ ret(0);
 516 
 517     return start;
 518   }
 519 
 520 
 521   //---------------------------------------------------------------------------
 522   // Support for void verify_fpu_cntrl_wrd()
 523   //
 524   // This routine is used with -Xcheck:jni to verify that native
 525   // JNI code does not return to Java code without restoring the
 526   // FP control word to our expected state.
 527 
 528   address generate_verify_fpu_cntrl_wrd() {
 529     StubCodeMark mark(this, "StubRoutines", "verify_spcw");
 530     address start = __ pc();
 531 
 532     const Address fpu_cntrl_wrd_save(rsp, 0);
 533 
 534     if (CheckJNICalls) {
 535       Label ok_ret;
 536       __ push(rax);
 537       __ subptr(rsp, wordSize);      // allocate a temp location
 538       __ fnstcw(fpu_cntrl_wrd_save);
 539       __ movl(rax, fpu_cntrl_wrd_save);
 540       __ andl(rax, FPU_CNTRL_WRD_MASK);
 541       ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());
 542       __ cmp32(rax, fpu_std);
 543       __ jcc(Assembler::equal, ok_ret);
 544 
 545       __ warn("Floating point control word changed by native JNI code.");
 546 
 547       __ fldcw(fpu_std);
 548 
 549       __ bind(ok_ret);
 550       __ addptr(rsp, wordSize);
 551       __ pop(rax);
 552     }
 553 
 554     __ ret(0);
 555 
 556     return start;
 557   }
 558 
 559   //---------------------------------------------------------------------------
 560   // Wrapper for slow-case handling of double-to-integer conversion
 561   // d2i or f2i fast case failed either because it is nan or because
 562   // of under/overflow.
 563   // Input:  FPU TOS: float value
 564   // Output: rax, (rdx): integer (long) result
 565 
 566   address generate_d2i_wrapper(BasicType t, address fcn) {
 567     StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");
 568     address start = __ pc();
 569 
 570   // Capture info about frame layout
 571   enum layout { FPUState_off         = 0,
 572                 rbp_off              = FPUStateSizeInWords,
 573                 rdi_off,
 574                 rsi_off,
 575                 rcx_off,
 576                 rbx_off,
 577                 saved_argument_off,
 578                 saved_argument_off2, // 2nd half of double
 579                 framesize
 580   };
 581 
 582   assert(FPUStateSizeInWords == 27, "update stack layout");
 583 
 584     // Save outgoing argument to stack across push_FPU_state()
 585     __ subptr(rsp, wordSize * 2);
 586     __ fstp_d(Address(rsp, 0));
 587 
 588     // Save CPU & FPU state
 589     __ push(rbx);
 590     __ push(rcx);
 591     __ push(rsi);
 592     __ push(rdi);
 593     __ push(rbp);
 594     __ push_FPU_state();
 595 
 596     // push_FPU_state() resets the FP top of stack
 597     // Load original double into FP top of stack
 598     __ fld_d(Address(rsp, saved_argument_off * wordSize));
 599     // Store double into stack as outgoing argument
 600     __ subptr(rsp, wordSize*2);
 601     __ fst_d(Address(rsp, 0));
 602 
 603     // Prepare FPU for doing math in C-land
 604     __ empty_FPU_stack();
 605     // Call the C code to massage the double.  Result in EAX
 606     if (t == T_INT)
 607       { BLOCK_COMMENT("SharedRuntime::d2i"); }
 608     else if (t == T_LONG)
 609       { BLOCK_COMMENT("SharedRuntime::d2l"); }
 610     __ call_VM_leaf( fcn, 2 );
 611 
 612     // Restore CPU & FPU state
 613     __ pop_FPU_state();
 614     __ pop(rbp);
 615     __ pop(rdi);
 616     __ pop(rsi);
 617     __ pop(rcx);
 618     __ pop(rbx);
 619     __ addptr(rsp, wordSize * 2);
 620 
 621     __ ret(0);
 622 
 623     return start;
 624   }
 625 
 626 
 627   //---------------------------------------------------------------------------
 628   // The following routine generates a subroutine to throw an asynchronous
 629   // UnknownError when an unsafe access gets a fault that could not be
 630   // reasonably prevented by the programmer.  (Example: SIGBUS/OBJERR.)
 631   address generate_handler_for_unsafe_access() {
 632     StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
 633     address start = __ pc();
 634 
 635     __ push(0);                       // hole for return address-to-be
 636     __ pusha();                       // push registers
 637     Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);
 638     BLOCK_COMMENT("call handle_unsafe_access");
 639     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
 640     __ movptr(next_pc, rax);          // stuff next address
 641     __ popa();
 642     __ ret(0);                        // jump to next address
 643 
 644     return start;
 645   }
 646 
 647 
 648   //----------------------------------------------------------------------------------------------------
 649   // Non-destructive plausibility checks for oops
 650 
 651   address generate_verify_oop() {
 652     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 653     address start = __ pc();
 654 
 655     // Incoming arguments on stack after saving rax,:
 656     //
 657     // [tos    ]: saved rdx
 658     // [tos + 1]: saved EFLAGS
 659     // [tos + 2]: return address
 660     // [tos + 3]: char* error message
 661     // [tos + 4]: oop   object to verify
 662     // [tos + 5]: saved rax, - saved by caller and bashed
 663 
 664     Label exit, error;
 665     __ pushf();
 666     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 667     __ push(rdx);                                // save rdx
 668     // make sure object is 'reasonable'
 669     __ movptr(rax, Address(rsp, 4 * wordSize));    // get object
 670     __ testptr(rax, rax);
 671     __ jcc(Assembler::zero, exit);               // if obj is NULL it is ok
 672 
 673     // Check if the oop is in the right area of memory
 674     const int oop_mask = Universe::verify_oop_mask();
 675     const int oop_bits = Universe::verify_oop_bits();
 676     __ mov(rdx, rax);
 677     __ andptr(rdx, oop_mask);
 678     __ cmpptr(rdx, oop_bits);
 679     __ jcc(Assembler::notZero, error);
 680 
 681     // make sure klass is 'reasonable', which is not zero.
 682     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
 683     __ testptr(rax, rax);
 684     __ jcc(Assembler::zero, error);              // if klass is NULL it is broken
 685 
 686     // return if everything seems ok
 687     __ bind(exit);
 688     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 689     __ pop(rdx);                                 // restore rdx
 690     __ popf();                                   // restore EFLAGS
 691     __ ret(3 * wordSize);                        // pop arguments
 692 
 693     // handle errors
 694     __ bind(error);
 695     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 696     __ pop(rdx);                                 // get saved rdx back
 697     __ popf();                                   // get saved EFLAGS off stack -- will be ignored
 698     __ pusha();                                  // push registers (eip = return address & msg are already pushed)
 699     BLOCK_COMMENT("call MacroAssembler::debug");
 700     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 701     __ popa();
 702     __ ret(3 * wordSize);                        // pop arguments
 703     return start;
 704   }
 705 
 706   //
 707   //  Generate pre-barrier for array stores
 708   //
 709   //  Input:
 710   //     start   -  starting address
 711   //     count   -  element count
 712   void  gen_write_ref_array_pre_barrier(Register start, Register count, bool uninitialized_target) {
 713     assert_different_registers(start, count);
 714     BarrierSet* bs = Universe::heap()->barrier_set();
 715     switch (bs->kind()) {
 716       case BarrierSet::G1SATBCTLogging:
 717         // With G1, don't generate the call if we statically know that the target in uninitialized
 718         if (!uninitialized_target) {
 719            __ pusha();                      // push registers
 720            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre),
 721                            start, count);
 722            __ popa();
 723          }
 724         break;
 725       case BarrierSet::CardTableForRS:
 726       case BarrierSet::CardTableExtension:
 727       case BarrierSet::ModRef:
 728         break;
 729       default      :
 730         ShouldNotReachHere();
 731 
 732     }
 733   }
 734 
 735 
 736   //
 737   // Generate a post-barrier for an array store
 738   //
 739   //     start    -  starting address
 740   //     count    -  element count
 741   //
 742   //  The two input registers are overwritten.
 743   //
 744   void  gen_write_ref_array_post_barrier(Register start, Register count) {
 745     BarrierSet* bs = Universe::heap()->barrier_set();
 746     assert_different_registers(start, count);
 747     switch (bs->kind()) {
 748       case BarrierSet::G1SATBCTLogging:
 749         {
 750           __ pusha();                      // push registers
 751           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post),
 752                           start, count);
 753           __ popa();
 754         }
 755         break;
 756 
 757       case BarrierSet::CardTableForRS:
 758       case BarrierSet::CardTableExtension:
 759         {
 760           CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs);
 761           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
 762 
 763           Label L_loop;
 764           const Register end = count;  // elements count; end == start+count-1
 765           assert_different_registers(start, end);
 766 
 767           __ lea(end,  Address(start, count, Address::times_ptr, -wordSize));
 768           __ shrptr(start, CardTableModRefBS::card_shift);
 769           __ shrptr(end,   CardTableModRefBS::card_shift);
 770           __ subptr(end, start); // end --> count
 771         __ BIND(L_loop);
 772           intptr_t disp = (intptr_t) ct->byte_map_base;
 773           Address cardtable(start, count, Address::times_1, disp);
 774           __ movb(cardtable, 0);
 775           __ decrement(count);
 776           __ jcc(Assembler::greaterEqual, L_loop);
 777         }
 778         break;
 779       case BarrierSet::ModRef:
 780         break;
 781       default      :
 782         ShouldNotReachHere();
 783 
 784     }
 785   }
 786 
 787 
 788   // Copy 64 bytes chunks
 789   //
 790   // Inputs:
 791   //   from        - source array address
 792   //   to_from     - destination array address - from
 793   //   qword_count - 8-bytes element count, negative
 794   //
 795   void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
 796     assert( UseSSE >= 2, "supported cpu only" );
 797     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 798     if (UseAVX > 2) {
 799       __ push(rbx);
 800       __ movl(rbx, 0xffff);
 801       __ kmovwl(k1, rbx);
 802       __ pop(rbx);
 803     }
 804     // Copy 64-byte chunks
 805     __ jmpb(L_copy_64_bytes);
 806     __ align(OptoLoopAlignment);
 807   __ BIND(L_copy_64_bytes_loop);
 808 
 809     if (UseUnalignedLoadStores) {
 810       if (UseAVX > 2) {
 811         __ evmovdqul(xmm0, Address(from, 0), Assembler::AVX_512bit);
 812         __ evmovdqul(Address(from, to_from, Address::times_1, 0), xmm0, Assembler::AVX_512bit);
 813       } else if (UseAVX == 2) {
 814         __ vmovdqu(xmm0, Address(from,  0));
 815         __ vmovdqu(Address(from, to_from, Address::times_1,  0), xmm0);
 816         __ vmovdqu(xmm1, Address(from, 32));
 817         __ vmovdqu(Address(from, to_from, Address::times_1, 32), xmm1);
 818       } else {
 819         __ movdqu(xmm0, Address(from, 0));
 820         __ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
 821         __ movdqu(xmm1, Address(from, 16));
 822         __ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
 823         __ movdqu(xmm2, Address(from, 32));
 824         __ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
 825         __ movdqu(xmm3, Address(from, 48));
 826         __ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
 827       }
 828     } else {
 829       __ movq(xmm0, Address(from, 0));
 830       __ movq(Address(from, to_from, Address::times_1, 0), xmm0);
 831       __ movq(xmm1, Address(from, 8));
 832       __ movq(Address(from, to_from, Address::times_1, 8), xmm1);
 833       __ movq(xmm2, Address(from, 16));
 834       __ movq(Address(from, to_from, Address::times_1, 16), xmm2);
 835       __ movq(xmm3, Address(from, 24));
 836       __ movq(Address(from, to_from, Address::times_1, 24), xmm3);
 837       __ movq(xmm4, Address(from, 32));
 838       __ movq(Address(from, to_from, Address::times_1, 32), xmm4);
 839       __ movq(xmm5, Address(from, 40));
 840       __ movq(Address(from, to_from, Address::times_1, 40), xmm5);
 841       __ movq(xmm6, Address(from, 48));
 842       __ movq(Address(from, to_from, Address::times_1, 48), xmm6);
 843       __ movq(xmm7, Address(from, 56));
 844       __ movq(Address(from, to_from, Address::times_1, 56), xmm7);
 845     }
 846 
 847     __ addl(from, 64);
 848   __ BIND(L_copy_64_bytes);
 849     __ subl(qword_count, 8);
 850     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 851 
 852     if (UseUnalignedLoadStores && (UseAVX == 2)) {
 853       // clean upper bits of YMM registers
 854       __ vpxor(xmm0, xmm0);
 855       __ vpxor(xmm1, xmm1);
 856     }
 857     __ addl(qword_count, 8);
 858     __ jccb(Assembler::zero, L_exit);
 859     //
 860     // length is too short, just copy qwords
 861     //
 862   __ BIND(L_copy_8_bytes);
 863     __ movq(xmm0, Address(from, 0));
 864     __ movq(Address(from, to_from, Address::times_1), xmm0);
 865     __ addl(from, 8);
 866     __ decrement(qword_count);
 867     __ jcc(Assembler::greater, L_copy_8_bytes);
 868   __ BIND(L_exit);
 869   }
 870 
 871   // Copy 64 bytes chunks
 872   //
 873   // Inputs:
 874   //   from        - source array address
 875   //   to_from     - destination array address - from
 876   //   qword_count - 8-bytes element count, negative
 877   //
 878   void mmx_copy_forward(Register from, Register to_from, Register qword_count) {
 879     assert( VM_Version::supports_mmx(), "supported cpu only" );
 880     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 881     // Copy 64-byte chunks
 882     __ jmpb(L_copy_64_bytes);
 883     __ align(OptoLoopAlignment);
 884   __ BIND(L_copy_64_bytes_loop);
 885     __ movq(mmx0, Address(from, 0));
 886     __ movq(mmx1, Address(from, 8));
 887     __ movq(mmx2, Address(from, 16));
 888     __ movq(Address(from, to_from, Address::times_1, 0), mmx0);
 889     __ movq(mmx3, Address(from, 24));
 890     __ movq(Address(from, to_from, Address::times_1, 8), mmx1);
 891     __ movq(mmx4, Address(from, 32));
 892     __ movq(Address(from, to_from, Address::times_1, 16), mmx2);
 893     __ movq(mmx5, Address(from, 40));
 894     __ movq(Address(from, to_from, Address::times_1, 24), mmx3);
 895     __ movq(mmx6, Address(from, 48));
 896     __ movq(Address(from, to_from, Address::times_1, 32), mmx4);
 897     __ movq(mmx7, Address(from, 56));
 898     __ movq(Address(from, to_from, Address::times_1, 40), mmx5);
 899     __ movq(Address(from, to_from, Address::times_1, 48), mmx6);
 900     __ movq(Address(from, to_from, Address::times_1, 56), mmx7);
 901     __ addptr(from, 64);
 902   __ BIND(L_copy_64_bytes);
 903     __ subl(qword_count, 8);
 904     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 905     __ addl(qword_count, 8);
 906     __ jccb(Assembler::zero, L_exit);
 907     //
 908     // length is too short, just copy qwords
 909     //
 910   __ BIND(L_copy_8_bytes);
 911     __ movq(mmx0, Address(from, 0));
 912     __ movq(Address(from, to_from, Address::times_1), mmx0);
 913     __ addptr(from, 8);
 914     __ decrement(qword_count);
 915     __ jcc(Assembler::greater, L_copy_8_bytes);
 916   __ BIND(L_exit);
 917     __ emms();
 918   }
 919 
 920   address generate_disjoint_copy(BasicType t, bool aligned,
 921                                  Address::ScaleFactor sf,
 922                                  address* entry, const char *name,
 923                                  bool dest_uninitialized = false) {
 924     __ align(CodeEntryAlignment);
 925     StubCodeMark mark(this, "StubRoutines", name);
 926     address start = __ pc();
 927 
 928     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
 929     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;
 930 
 931     int shift = Address::times_ptr - sf;
 932 
 933     const Register from     = rsi;  // source array address
 934     const Register to       = rdi;  // destination array address
 935     const Register count    = rcx;  // elements count
 936     const Register to_from  = to;   // (to - from)
 937     const Register saved_to = rdx;  // saved destination array address
 938 
 939     __ enter(); // required for proper stackwalking of RuntimeStub frame
 940     __ push(rsi);
 941     __ push(rdi);
 942     __ movptr(from , Address(rsp, 12+ 4));
 943     __ movptr(to   , Address(rsp, 12+ 8));
 944     __ movl(count, Address(rsp, 12+ 12));
 945 
 946     if (entry != NULL) {
 947       *entry = __ pc(); // Entry point from conjoint arraycopy stub.
 948       BLOCK_COMMENT("Entry:");
 949     }
 950 
 951     if (t == T_OBJECT) {
 952       __ testl(count, count);
 953       __ jcc(Assembler::zero, L_0_count);
 954       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
 955       __ mov(saved_to, to);          // save 'to'
 956     }
 957 
 958     __ subptr(to, from); // to --> to_from
 959     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
 960     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
 961     if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 962       // align source address at 4 bytes address boundary
 963       if (t == T_BYTE) {
 964         // One byte misalignment happens only for byte arrays
 965         __ testl(from, 1);
 966         __ jccb(Assembler::zero, L_skip_align1);
 967         __ movb(rax, Address(from, 0));
 968         __ movb(Address(from, to_from, Address::times_1, 0), rax);
 969         __ increment(from);
 970         __ decrement(count);
 971       __ BIND(L_skip_align1);
 972       }
 973       // Two bytes misalignment happens only for byte and short (char) arrays
 974       __ testl(from, 2);
 975       __ jccb(Assembler::zero, L_skip_align2);
 976       __ movw(rax, Address(from, 0));
 977       __ movw(Address(from, to_from, Address::times_1, 0), rax);
 978       __ addptr(from, 2);
 979       __ subl(count, 1<<(shift-1));
 980     __ BIND(L_skip_align2);
 981     }
 982     if (!VM_Version::supports_mmx()) {
 983       __ mov(rax, count);      // save 'count'
 984       __ shrl(count, shift); // bytes count
 985       __ addptr(to_from, from);// restore 'to'
 986       __ rep_mov();
 987       __ subptr(to_from, from);// restore 'to_from'
 988       __ mov(count, rax);      // restore 'count'
 989       __ jmpb(L_copy_2_bytes); // all dwords were copied
 990     } else {
 991       if (!UseUnalignedLoadStores) {
 992         // align to 8 bytes, we know we are 4 byte aligned to start
 993         __ testptr(from, 4);
 994         __ jccb(Assembler::zero, L_copy_64_bytes);
 995         __ movl(rax, Address(from, 0));
 996         __ movl(Address(from, to_from, Address::times_1, 0), rax);
 997         __ addptr(from, 4);
 998         __ subl(count, 1<<shift);
 999       }
1000     __ BIND(L_copy_64_bytes);
1001       __ mov(rax, count);
1002       __ shrl(rax, shift+1);  // 8 bytes chunk count
1003       //
1004       // Copy 8-byte chunks through MMX registers, 8 per iteration of the loop
1005       //
1006       if (UseXMMForArrayCopy) {
1007         xmm_copy_forward(from, to_from, rax);
1008       } else {
1009         mmx_copy_forward(from, to_from, rax);
1010       }
1011     }
1012     // copy tailing dword
1013   __ BIND(L_copy_4_bytes);
1014     __ testl(count, 1<<shift);
1015     __ jccb(Assembler::zero, L_copy_2_bytes);
1016     __ movl(rax, Address(from, 0));
1017     __ movl(Address(from, to_from, Address::times_1, 0), rax);
1018     if (t == T_BYTE || t == T_SHORT) {
1019       __ addptr(from, 4);
1020     __ BIND(L_copy_2_bytes);
1021       // copy tailing word
1022       __ testl(count, 1<<(shift-1));
1023       __ jccb(Assembler::zero, L_copy_byte);
1024       __ movw(rax, Address(from, 0));
1025       __ movw(Address(from, to_from, Address::times_1, 0), rax);
1026       if (t == T_BYTE) {
1027         __ addptr(from, 2);
1028       __ BIND(L_copy_byte);
1029         // copy tailing byte
1030         __ testl(count, 1);
1031         __ jccb(Assembler::zero, L_exit);
1032         __ movb(rax, Address(from, 0));
1033         __ movb(Address(from, to_from, Address::times_1, 0), rax);
1034       __ BIND(L_exit);
1035       } else {
1036       __ BIND(L_copy_byte);
1037       }
1038     } else {
1039     __ BIND(L_copy_2_bytes);
1040     }
1041 
1042     if (t == T_OBJECT) {
1043       __ movl(count, Address(rsp, 12+12)); // reread 'count'
1044       __ mov(to, saved_to); // restore 'to'
1045       gen_write_ref_array_post_barrier(to, count);
1046     __ BIND(L_0_count);
1047     }
1048     inc_copy_counter_np(t);
1049     __ pop(rdi);
1050     __ pop(rsi);
1051     __ leave(); // required for proper stackwalking of RuntimeStub frame
1052     __ xorptr(rax, rax); // return 0
1053     __ ret(0);
1054     return start;
1055   }
1056 
1057 
1058   address generate_fill(BasicType t, bool aligned, const char *name) {
1059     __ align(CodeEntryAlignment);
1060     StubCodeMark mark(this, "StubRoutines", name);
1061     address start = __ pc();
1062 
1063     BLOCK_COMMENT("Entry:");
1064 
1065     const Register to       = rdi;  // source array address
1066     const Register value    = rdx;  // value
1067     const Register count    = rsi;  // elements count
1068 
1069     __ enter(); // required for proper stackwalking of RuntimeStub frame
1070     __ push(rsi);
1071     __ push(rdi);
1072     __ movptr(to   , Address(rsp, 12+ 4));
1073     __ movl(value, Address(rsp, 12+ 8));
1074     __ movl(count, Address(rsp, 12+ 12));
1075 
1076     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1077 
1078     __ pop(rdi);
1079     __ pop(rsi);
1080     __ leave(); // required for proper stackwalking of RuntimeStub frame
1081     __ ret(0);
1082     return start;
1083   }
1084 
1085   address generate_conjoint_copy(BasicType t, bool aligned,
1086                                  Address::ScaleFactor sf,
1087                                  address nooverlap_target,
1088                                  address* entry, const char *name,
1089                                  bool dest_uninitialized = false) {
1090     __ align(CodeEntryAlignment);
1091     StubCodeMark mark(this, "StubRoutines", name);
1092     address start = __ pc();
1093 
1094     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
1095     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;
1096 
1097     int shift = Address::times_ptr - sf;
1098 
1099     const Register src   = rax;  // source array address
1100     const Register dst   = rdx;  // destination array address
1101     const Register from  = rsi;  // source array address
1102     const Register to    = rdi;  // destination array address
1103     const Register count = rcx;  // elements count
1104     const Register end   = rax;  // array end address
1105 
1106     __ enter(); // required for proper stackwalking of RuntimeStub frame
1107     __ push(rsi);
1108     __ push(rdi);
1109     __ movptr(src  , Address(rsp, 12+ 4));   // from
1110     __ movptr(dst  , Address(rsp, 12+ 8));   // to
1111     __ movl2ptr(count, Address(rsp, 12+12)); // count
1112 
1113     if (entry != NULL) {
1114       *entry = __ pc(); // Entry point from generic arraycopy stub.
1115       BLOCK_COMMENT("Entry:");
1116     }
1117 
1118     // nooverlap_target expects arguments in rsi and rdi.
1119     __ mov(from, src);
1120     __ mov(to  , dst);
1121 
1122     // arrays overlap test: dispatch to disjoint stub if necessary.
1123     RuntimeAddress nooverlap(nooverlap_target);
1124     __ cmpptr(dst, src);
1125     __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
1126     __ jump_cc(Assembler::belowEqual, nooverlap);
1127     __ cmpptr(dst, end);
1128     __ jump_cc(Assembler::aboveEqual, nooverlap);
1129 
1130     if (t == T_OBJECT) {
1131       __ testl(count, count);
1132       __ jcc(Assembler::zero, L_0_count);
1133       gen_write_ref_array_pre_barrier(dst, count, dest_uninitialized);
1134     }
1135 
1136     // copy from high to low
1137     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1138     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
1139     if (t == T_BYTE || t == T_SHORT) {
1140       // Align the end of destination array at 4 bytes address boundary
1141       __ lea(end, Address(dst, count, sf, 0));
1142       if (t == T_BYTE) {
1143         // One byte misalignment happens only for byte arrays
1144         __ testl(end, 1);
1145         __ jccb(Assembler::zero, L_skip_align1);
1146         __ decrement(count);
1147         __ movb(rdx, Address(from, count, sf, 0));
1148         __ movb(Address(to, count, sf, 0), rdx);
1149       __ BIND(L_skip_align1);
1150       }
1151       // Two bytes misalignment happens only for byte and short (char) arrays
1152       __ testl(end, 2);
1153       __ jccb(Assembler::zero, L_skip_align2);
1154       __ subptr(count, 1<<(shift-1));
1155       __ movw(rdx, Address(from, count, sf, 0));
1156       __ movw(Address(to, count, sf, 0), rdx);
1157     __ BIND(L_skip_align2);
1158       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1159       __ jcc(Assembler::below, L_copy_4_bytes);
1160     }
1161 
1162     if (!VM_Version::supports_mmx()) {
1163       __ std();
1164       __ mov(rax, count); // Save 'count'
1165       __ mov(rdx, to);    // Save 'to'
1166       __ lea(rsi, Address(from, count, sf, -4));
1167       __ lea(rdi, Address(to  , count, sf, -4));
1168       __ shrptr(count, shift); // bytes count
1169       __ rep_mov();
1170       __ cld();
1171       __ mov(count, rax); // restore 'count'
1172       __ andl(count, (1<<shift)-1);      // mask the number of rest elements
1173       __ movptr(from, Address(rsp, 12+4)); // reread 'from'
1174       __ mov(to, rdx);   // restore 'to'
1175       __ jmpb(L_copy_2_bytes); // all dword were copied
1176    } else {
1177       // Align to 8 bytes the end of array. It is aligned to 4 bytes already.
1178       __ testptr(end, 4);
1179       __ jccb(Assembler::zero, L_copy_8_bytes);
1180       __ subl(count, 1<<shift);
1181       __ movl(rdx, Address(from, count, sf, 0));
1182       __ movl(Address(to, count, sf, 0), rdx);
1183       __ jmpb(L_copy_8_bytes);
1184 
1185       __ align(OptoLoopAlignment);
1186       // Move 8 bytes
1187     __ BIND(L_copy_8_bytes_loop);
1188       if (UseXMMForArrayCopy) {
1189         __ movq(xmm0, Address(from, count, sf, 0));
1190         __ movq(Address(to, count, sf, 0), xmm0);
1191       } else {
1192         __ movq(mmx0, Address(from, count, sf, 0));
1193         __ movq(Address(to, count, sf, 0), mmx0);
1194       }
1195     __ BIND(L_copy_8_bytes);
1196       __ subl(count, 2<<shift);
1197       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1198       __ addl(count, 2<<shift);
1199       if (!UseXMMForArrayCopy) {
1200         __ emms();
1201       }
1202     }
1203   __ BIND(L_copy_4_bytes);
1204     // copy prefix qword
1205     __ testl(count, 1<<shift);
1206     __ jccb(Assembler::zero, L_copy_2_bytes);
1207     __ movl(rdx, Address(from, count, sf, -4));
1208     __ movl(Address(to, count, sf, -4), rdx);
1209 
1210     if (t == T_BYTE || t == T_SHORT) {
1211         __ subl(count, (1<<shift));
1212       __ BIND(L_copy_2_bytes);
1213         // copy prefix dword
1214         __ testl(count, 1<<(shift-1));
1215         __ jccb(Assembler::zero, L_copy_byte);
1216         __ movw(rdx, Address(from, count, sf, -2));
1217         __ movw(Address(to, count, sf, -2), rdx);
1218         if (t == T_BYTE) {
1219           __ subl(count, 1<<(shift-1));
1220         __ BIND(L_copy_byte);
1221           // copy prefix byte
1222           __ testl(count, 1);
1223           __ jccb(Assembler::zero, L_exit);
1224           __ movb(rdx, Address(from, 0));
1225           __ movb(Address(to, 0), rdx);
1226         __ BIND(L_exit);
1227         } else {
1228         __ BIND(L_copy_byte);
1229         }
1230     } else {
1231     __ BIND(L_copy_2_bytes);
1232     }
1233     if (t == T_OBJECT) {
1234       __ movl2ptr(count, Address(rsp, 12+12)); // reread count
1235       gen_write_ref_array_post_barrier(to, count);
1236     __ BIND(L_0_count);
1237     }
1238     inc_copy_counter_np(t);
1239     __ pop(rdi);
1240     __ pop(rsi);
1241     __ leave(); // required for proper stackwalking of RuntimeStub frame
1242     __ xorptr(rax, rax); // return 0
1243     __ ret(0);
1244     return start;
1245   }
1246 
1247 
1248   address generate_disjoint_long_copy(address* entry, const char *name) {
1249     __ align(CodeEntryAlignment);
1250     StubCodeMark mark(this, "StubRoutines", name);
1251     address start = __ pc();
1252 
1253     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1254     const Register from       = rax;  // source array address
1255     const Register to         = rdx;  // destination array address
1256     const Register count      = rcx;  // elements count
1257     const Register to_from    = rdx;  // (to - from)
1258 
1259     __ enter(); // required for proper stackwalking of RuntimeStub frame
1260     __ movptr(from , Address(rsp, 8+0));       // from
1261     __ movptr(to   , Address(rsp, 8+4));       // to
1262     __ movl2ptr(count, Address(rsp, 8+8));     // count
1263 
1264     *entry = __ pc(); // Entry point from conjoint arraycopy stub.
1265     BLOCK_COMMENT("Entry:");
1266 
1267     __ subptr(to, from); // to --> to_from
1268     if (VM_Version::supports_mmx()) {
1269       if (UseXMMForArrayCopy) {
1270         xmm_copy_forward(from, to_from, count);
1271       } else {
1272         mmx_copy_forward(from, to_from, count);
1273       }
1274     } else {
1275       __ jmpb(L_copy_8_bytes);
1276       __ align(OptoLoopAlignment);
1277     __ BIND(L_copy_8_bytes_loop);
1278       __ fild_d(Address(from, 0));
1279       __ fistp_d(Address(from, to_from, Address::times_1));
1280       __ addptr(from, 8);
1281     __ BIND(L_copy_8_bytes);
1282       __ decrement(count);
1283       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1284     }
1285     inc_copy_counter_np(T_LONG);
1286     __ leave(); // required for proper stackwalking of RuntimeStub frame
1287     __ xorptr(rax, rax); // return 0
1288     __ ret(0);
1289     return start;
1290   }
1291 
1292   address generate_conjoint_long_copy(address nooverlap_target,
1293                                       address* entry, const char *name) {
1294     __ align(CodeEntryAlignment);
1295     StubCodeMark mark(this, "StubRoutines", name);
1296     address start = __ pc();
1297 
1298     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1299     const Register from       = rax;  // source array address
1300     const Register to         = rdx;  // destination array address
1301     const Register count      = rcx;  // elements count
1302     const Register end_from   = rax;  // source array end address
1303 
1304     __ enter(); // required for proper stackwalking of RuntimeStub frame
1305     __ movptr(from , Address(rsp, 8+0));       // from
1306     __ movptr(to   , Address(rsp, 8+4));       // to
1307     __ movl2ptr(count, Address(rsp, 8+8));     // count
1308 
1309     *entry = __ pc(); // Entry point from generic arraycopy stub.
1310     BLOCK_COMMENT("Entry:");
1311 
1312     // arrays overlap test
1313     __ cmpptr(to, from);
1314     RuntimeAddress nooverlap(nooverlap_target);
1315     __ jump_cc(Assembler::belowEqual, nooverlap);
1316     __ lea(end_from, Address(from, count, Address::times_8, 0));
1317     __ cmpptr(to, end_from);
1318     __ movptr(from, Address(rsp, 8));  // from
1319     __ jump_cc(Assembler::aboveEqual, nooverlap);
1320 
1321     __ jmpb(L_copy_8_bytes);
1322 
1323     __ align(OptoLoopAlignment);
1324   __ BIND(L_copy_8_bytes_loop);
1325     if (VM_Version::supports_mmx()) {
1326       if (UseXMMForArrayCopy) {
1327         __ movq(xmm0, Address(from, count, Address::times_8));
1328         __ movq(Address(to, count, Address::times_8), xmm0);
1329       } else {
1330         __ movq(mmx0, Address(from, count, Address::times_8));
1331         __ movq(Address(to, count, Address::times_8), mmx0);
1332       }
1333     } else {
1334       __ fild_d(Address(from, count, Address::times_8));
1335       __ fistp_d(Address(to, count, Address::times_8));
1336     }
1337   __ BIND(L_copy_8_bytes);
1338     __ decrement(count);
1339     __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1340 
1341     if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
1342       __ emms();
1343     }
1344     inc_copy_counter_np(T_LONG);
1345     __ leave(); // required for proper stackwalking of RuntimeStub frame
1346     __ xorptr(rax, rax); // return 0
1347     __ ret(0);
1348     return start;
1349   }
1350 
1351 
1352   // Helper for generating a dynamic type check.
1353   // The sub_klass must be one of {rbx, rdx, rsi}.
1354   // The temp is killed.
1355   void generate_type_check(Register sub_klass,
1356                            Address& super_check_offset_addr,
1357                            Address& super_klass_addr,
1358                            Register temp,
1359                            Label* L_success, Label* L_failure) {
1360     BLOCK_COMMENT("type_check:");
1361 
1362     Label L_fallthrough;
1363 #define LOCAL_JCC(assembler_con, label_ptr)                             \
1364     if (label_ptr != NULL)  __ jcc(assembler_con, *(label_ptr));        \
1365     else                    __ jcc(assembler_con, L_fallthrough) /*omit semi*/
1366 
1367     // The following is a strange variation of the fast path which requires
1368     // one less register, because needed values are on the argument stack.
1369     // __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
1370     //                                  L_success, L_failure, NULL);
1371     assert_different_registers(sub_klass, temp);
1372 
1373     int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1374 
1375     // if the pointers are equal, we are done (e.g., String[] elements)
1376     __ cmpptr(sub_klass, super_klass_addr);
1377     LOCAL_JCC(Assembler::equal, L_success);
1378 
1379     // check the supertype display:
1380     __ movl2ptr(temp, super_check_offset_addr);
1381     Address super_check_addr(sub_klass, temp, Address::times_1, 0);
1382     __ movptr(temp, super_check_addr); // load displayed supertype
1383     __ cmpptr(temp, super_klass_addr); // test the super type
1384     LOCAL_JCC(Assembler::equal, L_success);
1385 
1386     // if it was a primary super, we can just fail immediately
1387     __ cmpl(super_check_offset_addr, sc_offset);
1388     LOCAL_JCC(Assembler::notEqual, L_failure);
1389 
1390     // The repne_scan instruction uses fixed registers, which will get spilled.
1391     // We happen to know this works best when super_klass is in rax.
1392     Register super_klass = temp;
1393     __ movptr(super_klass, super_klass_addr);
1394     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
1395                                      L_success, L_failure);
1396 
1397     __ bind(L_fallthrough);
1398 
1399     if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }
1400     if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }
1401 
1402 #undef LOCAL_JCC
1403   }
1404 
1405   //
1406   //  Generate checkcasting array copy stub
1407   //
1408   //  Input:
1409   //    4(rsp)   - source array address
1410   //    8(rsp)   - destination array address
1411   //   12(rsp)   - element count, can be zero
1412   //   16(rsp)   - size_t ckoff (super_check_offset)
1413   //   20(rsp)   - oop ckval (super_klass)
1414   //
1415   //  Output:
1416   //    rax, ==  0  -  success
1417   //    rax, == -1^K - failure, where K is partial transfer count
1418   //
1419   address generate_checkcast_copy(const char *name, address* entry, bool dest_uninitialized = false) {
1420     __ align(CodeEntryAlignment);
1421     StubCodeMark mark(this, "StubRoutines", name);
1422     address start = __ pc();
1423 
1424     Label L_load_element, L_store_element, L_do_card_marks, L_done;
1425 
1426     // register use:
1427     //  rax, rdx, rcx -- loop control (end_from, end_to, count)
1428     //  rdi, rsi      -- element access (oop, klass)
1429     //  rbx,           -- temp
1430     const Register from       = rax;    // source array address
1431     const Register to         = rdx;    // destination array address
1432     const Register length     = rcx;    // elements count
1433     const Register elem       = rdi;    // each oop copied
1434     const Register elem_klass = rsi;    // each elem._klass (sub_klass)
1435     const Register temp       = rbx;    // lone remaining temp
1436 
1437     __ enter(); // required for proper stackwalking of RuntimeStub frame
1438 
1439     __ push(rsi);
1440     __ push(rdi);
1441     __ push(rbx);
1442 
1443     Address   from_arg(rsp, 16+ 4);     // from
1444     Address     to_arg(rsp, 16+ 8);     // to
1445     Address length_arg(rsp, 16+12);     // elements count
1446     Address  ckoff_arg(rsp, 16+16);     // super_check_offset
1447     Address  ckval_arg(rsp, 16+20);     // super_klass
1448 
1449     // Load up:
1450     __ movptr(from,     from_arg);
1451     __ movptr(to,         to_arg);
1452     __ movl2ptr(length, length_arg);
1453 
1454     if (entry != NULL) {
1455       *entry = __ pc(); // Entry point from generic arraycopy stub.
1456       BLOCK_COMMENT("Entry:");
1457     }
1458 
1459     //---------------------------------------------------------------
1460     // Assembler stub will be used for this call to arraycopy
1461     // if the two arrays are subtypes of Object[] but the
1462     // destination array type is not equal to or a supertype
1463     // of the source type.  Each element must be separately
1464     // checked.
1465 
1466     // Loop-invariant addresses.  They are exclusive end pointers.
1467     Address end_from_addr(from, length, Address::times_ptr, 0);
1468     Address   end_to_addr(to,   length, Address::times_ptr, 0);
1469 
1470     Register end_from = from;           // re-use
1471     Register end_to   = to;             // re-use
1472     Register count    = length;         // re-use
1473 
1474     // Loop-variant addresses.  They assume post-incremented count < 0.
1475     Address from_element_addr(end_from, count, Address::times_ptr, 0);
1476     Address   to_element_addr(end_to,   count, Address::times_ptr, 0);
1477     Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
1478 
1479     // Copy from low to high addresses, indexed from the end of each array.
1480     gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
1481     __ lea(end_from, end_from_addr);
1482     __ lea(end_to,   end_to_addr);
1483     assert(length == count, "");        // else fix next line:
1484     __ negptr(count);                   // negate and test the length
1485     __ jccb(Assembler::notZero, L_load_element);
1486 
1487     // Empty array:  Nothing to do.
1488     __ xorptr(rax, rax);                  // return 0 on (trivial) success
1489     __ jmp(L_done);
1490 
1491     // ======== begin loop ========
1492     // (Loop is rotated; its entry is L_load_element.)
1493     // Loop control:
1494     //   for (count = -count; count != 0; count++)
1495     // Base pointers src, dst are biased by 8*count,to last element.
1496     __ align(OptoLoopAlignment);
1497 
1498     __ BIND(L_store_element);
1499     __ movptr(to_element_addr, elem);     // store the oop
1500     __ increment(count);                // increment the count toward zero
1501     __ jccb(Assembler::zero, L_do_card_marks);
1502 
1503     // ======== loop entry is here ========
1504     __ BIND(L_load_element);
1505     __ movptr(elem, from_element_addr);   // load the oop
1506     __ testptr(elem, elem);
1507     __ jccb(Assembler::zero, L_store_element);
1508 
1509     // (Could do a trick here:  Remember last successful non-null
1510     // element stored and make a quick oop equality check on it.)
1511 
1512     __ movptr(elem_klass, elem_klass_addr); // query the object klass
1513     generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,
1514                         &L_store_element, NULL);
1515     // (On fall-through, we have failed the element type check.)
1516     // ======== end loop ========
1517 
1518     // It was a real error; we must depend on the caller to finish the job.
1519     // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
1520     // Emit GC store barriers for the oops we have copied (length_arg + count),
1521     // and report their number to the caller.
1522     assert_different_registers(to, count, rax);
1523     Label L_post_barrier;
1524     __ addl(count, length_arg);         // transfers = (length - remaining)
1525     __ movl2ptr(rax, count);            // save the value
1526     __ notptr(rax);                     // report (-1^K) to caller (does not affect flags)
1527     __ jccb(Assembler::notZero, L_post_barrier);
1528     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
1529 
1530     // Come here on success only.
1531     __ BIND(L_do_card_marks);
1532     __ xorptr(rax, rax);                // return 0 on success
1533     __ movl2ptr(count, length_arg);
1534 
1535     __ BIND(L_post_barrier);
1536     __ movptr(to, to_arg);              // reload
1537     gen_write_ref_array_post_barrier(to, count);
1538 
1539     // Common exit point (success or failure).
1540     __ BIND(L_done);
1541     __ pop(rbx);
1542     __ pop(rdi);
1543     __ pop(rsi);
1544     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1545     __ leave(); // required for proper stackwalking of RuntimeStub frame
1546     __ ret(0);
1547 
1548     return start;
1549   }
1550 
1551   //
1552   //  Generate 'unsafe' array copy stub
1553   //  Though just as safe as the other stubs, it takes an unscaled
1554   //  size_t argument instead of an element count.
1555   //
1556   //  Input:
1557   //    4(rsp)   - source array address
1558   //    8(rsp)   - destination array address
1559   //   12(rsp)   - byte count, can be zero
1560   //
1561   //  Output:
1562   //    rax, ==  0  -  success
1563   //    rax, == -1  -  need to call System.arraycopy
1564   //
1565   // Examines the alignment of the operands and dispatches
1566   // to a long, int, short, or byte copy loop.
1567   //
1568   address generate_unsafe_copy(const char *name,
1569                                address byte_copy_entry,
1570                                address short_copy_entry,
1571                                address int_copy_entry,
1572                                address long_copy_entry) {
1573 
1574     Label L_long_aligned, L_int_aligned, L_short_aligned;
1575 
1576     __ align(CodeEntryAlignment);
1577     StubCodeMark mark(this, "StubRoutines", name);
1578     address start = __ pc();
1579 
1580     const Register from       = rax;  // source array address
1581     const Register to         = rdx;  // destination array address
1582     const Register count      = rcx;  // elements count
1583 
1584     __ enter(); // required for proper stackwalking of RuntimeStub frame
1585     __ push(rsi);
1586     __ push(rdi);
1587     Address  from_arg(rsp, 12+ 4);      // from
1588     Address    to_arg(rsp, 12+ 8);      // to
1589     Address count_arg(rsp, 12+12);      // byte count
1590 
1591     // Load up:
1592     __ movptr(from ,  from_arg);
1593     __ movptr(to   ,    to_arg);
1594     __ movl2ptr(count, count_arg);
1595 
1596     // bump this on entry, not on exit:
1597     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
1598 
1599     const Register bits = rsi;
1600     __ mov(bits, from);
1601     __ orptr(bits, to);
1602     __ orptr(bits, count);
1603 
1604     __ testl(bits, BytesPerLong-1);
1605     __ jccb(Assembler::zero, L_long_aligned);
1606 
1607     __ testl(bits, BytesPerInt-1);
1608     __ jccb(Assembler::zero, L_int_aligned);
1609 
1610     __ testl(bits, BytesPerShort-1);
1611     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
1612 
1613     __ BIND(L_short_aligned);
1614     __ shrptr(count, LogBytesPerShort); // size => short_count
1615     __ movl(count_arg, count);          // update 'count'
1616     __ jump(RuntimeAddress(short_copy_entry));
1617 
1618     __ BIND(L_int_aligned);
1619     __ shrptr(count, LogBytesPerInt); // size => int_count
1620     __ movl(count_arg, count);          // update 'count'
1621     __ jump(RuntimeAddress(int_copy_entry));
1622 
1623     __ BIND(L_long_aligned);
1624     __ shrptr(count, LogBytesPerLong); // size => qword_count
1625     __ movl(count_arg, count);          // update 'count'
1626     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1627     __ pop(rsi);
1628     __ jump(RuntimeAddress(long_copy_entry));
1629 
1630     return start;
1631   }
1632 
1633 
1634   // Perform range checks on the proposed arraycopy.
1635   // Smashes src_pos and dst_pos.  (Uses them up for temps.)
1636   void arraycopy_range_checks(Register src,
1637                               Register src_pos,
1638                               Register dst,
1639                               Register dst_pos,
1640                               Address& length,
1641                               Label& L_failed) {
1642     BLOCK_COMMENT("arraycopy_range_checks:");
1643     const Register src_end = src_pos;   // source array end position
1644     const Register dst_end = dst_pos;   // destination array end position
1645     __ addl(src_end, length); // src_pos + length
1646     __ addl(dst_end, length); // dst_pos + length
1647 
1648     //  if (src_pos + length > arrayOop(src)->length() ) FAIL;
1649     __ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
1650     __ jcc(Assembler::above, L_failed);
1651 
1652     //  if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
1653     __ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
1654     __ jcc(Assembler::above, L_failed);
1655 
1656     BLOCK_COMMENT("arraycopy_range_checks done");
1657   }
1658 
1659 
1660   //
1661   //  Generate generic array copy stubs
1662   //
1663   //  Input:
1664   //     4(rsp)    -  src oop
1665   //     8(rsp)    -  src_pos
1666   //    12(rsp)    -  dst oop
1667   //    16(rsp)    -  dst_pos
1668   //    20(rsp)    -  element count
1669   //
1670   //  Output:
1671   //    rax, ==  0  -  success
1672   //    rax, == -1^K - failure, where K is partial transfer count
1673   //
1674   address generate_generic_copy(const char *name,
1675                                 address entry_jbyte_arraycopy,
1676                                 address entry_jshort_arraycopy,
1677                                 address entry_jint_arraycopy,
1678                                 address entry_oop_arraycopy,
1679                                 address entry_jlong_arraycopy,
1680                                 address entry_checkcast_arraycopy) {
1681     Label L_failed, L_failed_0, L_objArray;
1682 
1683     { int modulus = CodeEntryAlignment;
1684       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
1685       int advance = target - (__ offset() % modulus);
1686       if (advance < 0)  advance += modulus;
1687       if (advance > 0)  __ nop(advance);
1688     }
1689     StubCodeMark mark(this, "StubRoutines", name);
1690 
1691     // Short-hop target to L_failed.  Makes for denser prologue code.
1692     __ BIND(L_failed_0);
1693     __ jmp(L_failed);
1694     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
1695 
1696     __ align(CodeEntryAlignment);
1697     address start = __ pc();
1698 
1699     __ enter(); // required for proper stackwalking of RuntimeStub frame
1700     __ push(rsi);
1701     __ push(rdi);
1702 
1703     // bump this on entry, not on exit:
1704     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
1705 
1706     // Input values
1707     Address SRC     (rsp, 12+ 4);
1708     Address SRC_POS (rsp, 12+ 8);
1709     Address DST     (rsp, 12+12);
1710     Address DST_POS (rsp, 12+16);
1711     Address LENGTH  (rsp, 12+20);
1712 
1713     //-----------------------------------------------------------------------
1714     // Assembler stub will be used for this call to arraycopy
1715     // if the following conditions are met:
1716     //
1717     // (1) src and dst must not be null.
1718     // (2) src_pos must not be negative.
1719     // (3) dst_pos must not be negative.
1720     // (4) length  must not be negative.
1721     // (5) src klass and dst klass should be the same and not NULL.
1722     // (6) src and dst should be arrays.
1723     // (7) src_pos + length must not exceed length of src.
1724     // (8) dst_pos + length must not exceed length of dst.
1725     //
1726 
1727     const Register src     = rax;       // source array oop
1728     const Register src_pos = rsi;
1729     const Register dst     = rdx;       // destination array oop
1730     const Register dst_pos = rdi;
1731     const Register length  = rcx;       // transfer count
1732 
1733     //  if (src == NULL) return -1;
1734     __ movptr(src, SRC);      // src oop
1735     __ testptr(src, src);
1736     __ jccb(Assembler::zero, L_failed_0);
1737 
1738     //  if (src_pos < 0) return -1;
1739     __ movl2ptr(src_pos, SRC_POS);  // src_pos
1740     __ testl(src_pos, src_pos);
1741     __ jccb(Assembler::negative, L_failed_0);
1742 
1743     //  if (dst == NULL) return -1;
1744     __ movptr(dst, DST);      // dst oop
1745     __ testptr(dst, dst);
1746     __ jccb(Assembler::zero, L_failed_0);
1747 
1748     //  if (dst_pos < 0) return -1;
1749     __ movl2ptr(dst_pos, DST_POS);  // dst_pos
1750     __ testl(dst_pos, dst_pos);
1751     __ jccb(Assembler::negative, L_failed_0);
1752 
1753     //  if (length < 0) return -1;
1754     __ movl2ptr(length, LENGTH);   // length
1755     __ testl(length, length);
1756     __ jccb(Assembler::negative, L_failed_0);
1757 
1758     //  if (src->klass() == NULL) return -1;
1759     Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());
1760     Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());
1761     const Register rcx_src_klass = rcx;    // array klass
1762     __ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
1763 
1764 #ifdef ASSERT
1765     //  assert(src->klass() != NULL);
1766     BLOCK_COMMENT("assert klasses not null");
1767     { Label L1, L2;
1768       __ testptr(rcx_src_klass, rcx_src_klass);
1769       __ jccb(Assembler::notZero, L2);   // it is broken if klass is NULL
1770       __ bind(L1);
1771       __ stop("broken null klass");
1772       __ bind(L2);
1773       __ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
1774       __ jccb(Assembler::equal, L1);      // this would be broken also
1775       BLOCK_COMMENT("assert done");
1776     }
1777 #endif //ASSERT
1778 
1779     // Load layout helper (32-bits)
1780     //
1781     //  |array_tag|     | header_size | element_type |     |log2_element_size|
1782     // 32        30    24            16              8     2                 0
1783     //
1784     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
1785     //
1786 
1787     int lh_offset = in_bytes(Klass::layout_helper_offset());
1788     Address src_klass_lh_addr(rcx_src_klass, lh_offset);
1789 
1790     // Handle objArrays completely differently...
1791     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
1792     __ cmpl(src_klass_lh_addr, objArray_lh);
1793     __ jcc(Assembler::equal, L_objArray);
1794 
1795     //  if (src->klass() != dst->klass()) return -1;
1796     __ cmpptr(rcx_src_klass, dst_klass_addr);
1797     __ jccb(Assembler::notEqual, L_failed_0);
1798 
1799     const Register rcx_lh = rcx;  // layout helper
1800     assert(rcx_lh == rcx_src_klass, "known alias");
1801     __ movl(rcx_lh, src_klass_lh_addr);
1802 
1803     //  if (!src->is_Array()) return -1;
1804     __ cmpl(rcx_lh, Klass::_lh_neutral_value);
1805     __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
1806 
1807     // At this point, it is known to be a typeArray (array_tag 0x3).
1808 #ifdef ASSERT
1809     { Label L;
1810       __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
1811       __ jcc(Assembler::greaterEqual, L); // signed cmp
1812       __ stop("must be a primitive array");
1813       __ bind(L);
1814     }
1815 #endif
1816 
1817     assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);
1818     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1819 
1820     // TypeArrayKlass
1821     //
1822     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
1823     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
1824     //
1825     const Register rsi_offset = rsi; // array offset
1826     const Register src_array  = src; // src array offset
1827     const Register dst_array  = dst; // dst array offset
1828     const Register rdi_elsize = rdi; // log2 element size
1829 
1830     __ mov(rsi_offset, rcx_lh);
1831     __ shrptr(rsi_offset, Klass::_lh_header_size_shift);
1832     __ andptr(rsi_offset, Klass::_lh_header_size_mask);   // array_offset
1833     __ addptr(src_array, rsi_offset);  // src array offset
1834     __ addptr(dst_array, rsi_offset);  // dst array offset
1835     __ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
1836 
1837     // next registers should be set before the jump to corresponding stub
1838     const Register from       = src; // source array address
1839     const Register to         = dst; // destination array address
1840     const Register count      = rcx; // elements count
1841     // some of them should be duplicated on stack
1842 #define FROM   Address(rsp, 12+ 4)
1843 #define TO     Address(rsp, 12+ 8)   // Not used now
1844 #define COUNT  Address(rsp, 12+12)   // Only for oop arraycopy
1845 
1846     BLOCK_COMMENT("scale indexes to element size");
1847     __ movl2ptr(rsi, SRC_POS);  // src_pos
1848     __ shlptr(rsi);             // src_pos << rcx (log2 elsize)
1849     assert(src_array == from, "");
1850     __ addptr(from, rsi);       // from = src_array + SRC_POS << log2 elsize
1851     __ movl2ptr(rdi, DST_POS);  // dst_pos
1852     __ shlptr(rdi);             // dst_pos << rcx (log2 elsize)
1853     assert(dst_array == to, "");
1854     __ addptr(to,  rdi);        // to   = dst_array + DST_POS << log2 elsize
1855     __ movptr(FROM, from);      // src_addr
1856     __ mov(rdi_elsize, rcx_lh); // log2 elsize
1857     __ movl2ptr(count, LENGTH); // elements count
1858 
1859     BLOCK_COMMENT("choose copy loop based on element size");
1860     __ cmpl(rdi_elsize, 0);
1861 
1862     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
1863     __ cmpl(rdi_elsize, LogBytesPerShort);
1864     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
1865     __ cmpl(rdi_elsize, LogBytesPerInt);
1866     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
1867 #ifdef ASSERT
1868     __ cmpl(rdi_elsize, LogBytesPerLong);
1869     __ jccb(Assembler::notEqual, L_failed);
1870 #endif
1871     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1872     __ pop(rsi);
1873     __ jump(RuntimeAddress(entry_jlong_arraycopy));
1874 
1875   __ BIND(L_failed);
1876     __ xorptr(rax, rax);
1877     __ notptr(rax); // return -1
1878     __ pop(rdi);
1879     __ pop(rsi);
1880     __ leave(); // required for proper stackwalking of RuntimeStub frame
1881     __ ret(0);
1882 
1883     // ObjArrayKlass
1884   __ BIND(L_objArray);
1885     // live at this point:  rcx_src_klass, src[_pos], dst[_pos]
1886 
1887     Label L_plain_copy, L_checkcast_copy;
1888     //  test array classes for subtyping
1889     __ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
1890     __ jccb(Assembler::notEqual, L_checkcast_copy);
1891 
1892     // Identically typed arrays can be copied without element-wise checks.
1893     assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);
1894     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1895 
1896   __ BIND(L_plain_copy);
1897     __ movl2ptr(count, LENGTH); // elements count
1898     __ movl2ptr(src_pos, SRC_POS);  // reload src_pos
1899     __ lea(from, Address(src, src_pos, Address::times_ptr,
1900                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
1901     __ movl2ptr(dst_pos, DST_POS);  // reload dst_pos
1902     __ lea(to,   Address(dst, dst_pos, Address::times_ptr,
1903                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
1904     __ movptr(FROM,  from);   // src_addr
1905     __ movptr(TO,    to);     // dst_addr
1906     __ movl(COUNT, count);  // count
1907     __ jump(RuntimeAddress(entry_oop_arraycopy));
1908 
1909   __ BIND(L_checkcast_copy);
1910     // live at this point:  rcx_src_klass, dst[_pos], src[_pos]
1911     {
1912       // Handy offsets:
1913       int  ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
1914       int sco_offset = in_bytes(Klass::super_check_offset_offset());
1915 
1916       Register rsi_dst_klass = rsi;
1917       Register rdi_temp      = rdi;
1918       assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");
1919       assert(rdi_temp      == dst_pos, "expected alias w/ dst_pos");
1920       Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);
1921 
1922       // Before looking at dst.length, make sure dst is also an objArray.
1923       __ movptr(rsi_dst_klass, dst_klass_addr);
1924       __ cmpl(dst_klass_lh_addr, objArray_lh);
1925       __ jccb(Assembler::notEqual, L_failed);
1926 
1927       // It is safe to examine both src.length and dst.length.
1928       __ movl2ptr(src_pos, SRC_POS);        // reload rsi
1929       arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1930       // (Now src_pos and dst_pos are killed, but not src and dst.)
1931 
1932       // We'll need this temp (don't forget to pop it after the type check).
1933       __ push(rbx);
1934       Register rbx_src_klass = rbx;
1935 
1936       __ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
1937       __ movptr(rsi_dst_klass, dst_klass_addr);
1938       Address super_check_offset_addr(rsi_dst_klass, sco_offset);
1939       Label L_fail_array_check;
1940       generate_type_check(rbx_src_klass,
1941                           super_check_offset_addr, dst_klass_addr,
1942                           rdi_temp, NULL, &L_fail_array_check);
1943       // (On fall-through, we have passed the array type check.)
1944       __ pop(rbx);
1945       __ jmp(L_plain_copy);
1946 
1947       __ BIND(L_fail_array_check);
1948       // Reshuffle arguments so we can call checkcast_arraycopy:
1949 
1950       // match initial saves for checkcast_arraycopy
1951       // push(rsi);    // already done; see above
1952       // push(rdi);    // already done; see above
1953       // push(rbx);    // already done; see above
1954 
1955       // Marshal outgoing arguments now, freeing registers.
1956       Address   from_arg(rsp, 16+ 4);   // from
1957       Address     to_arg(rsp, 16+ 8);   // to
1958       Address length_arg(rsp, 16+12);   // elements count
1959       Address  ckoff_arg(rsp, 16+16);   // super_check_offset
1960       Address  ckval_arg(rsp, 16+20);   // super_klass
1961 
1962       Address SRC_POS_arg(rsp, 16+ 8);
1963       Address DST_POS_arg(rsp, 16+16);
1964       Address  LENGTH_arg(rsp, 16+20);
1965       // push rbx, changed the incoming offsets (why not just use rbp,??)
1966       // assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");
1967 
1968       __ movptr(rbx, Address(rsi_dst_klass, ek_offset));
1969       __ movl2ptr(length, LENGTH_arg);    // reload elements count
1970       __ movl2ptr(src_pos, SRC_POS_arg);  // reload src_pos
1971       __ movl2ptr(dst_pos, DST_POS_arg);  // reload dst_pos
1972 
1973       __ movptr(ckval_arg, rbx);          // destination element type
1974       __ movl(rbx, Address(rbx, sco_offset));
1975       __ movl(ckoff_arg, rbx);          // corresponding class check offset
1976 
1977       __ movl(length_arg, length);      // outgoing length argument
1978 
1979       __ lea(from, Address(src, src_pos, Address::times_ptr,
1980                             arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1981       __ movptr(from_arg, from);
1982 
1983       __ lea(to, Address(dst, dst_pos, Address::times_ptr,
1984                           arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1985       __ movptr(to_arg, to);
1986       __ jump(RuntimeAddress(entry_checkcast_arraycopy));
1987     }
1988 
1989     return start;
1990   }
1991 
1992   void generate_arraycopy_stubs() {
1993     address entry;
1994     address entry_jbyte_arraycopy;
1995     address entry_jshort_arraycopy;
1996     address entry_jint_arraycopy;
1997     address entry_oop_arraycopy;
1998     address entry_jlong_arraycopy;
1999     address entry_checkcast_arraycopy;
2000 
2001     StubRoutines::_arrayof_jbyte_disjoint_arraycopy =
2002         generate_disjoint_copy(T_BYTE,  true, Address::times_1, &entry,
2003                                "arrayof_jbyte_disjoint_arraycopy");
2004     StubRoutines::_arrayof_jbyte_arraycopy =
2005         generate_conjoint_copy(T_BYTE,  true, Address::times_1,  entry,
2006                                NULL, "arrayof_jbyte_arraycopy");
2007     StubRoutines::_jbyte_disjoint_arraycopy =
2008         generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,
2009                                "jbyte_disjoint_arraycopy");
2010     StubRoutines::_jbyte_arraycopy =
2011         generate_conjoint_copy(T_BYTE, false, Address::times_1,  entry,
2012                                &entry_jbyte_arraycopy, "jbyte_arraycopy");
2013 
2014     StubRoutines::_arrayof_jshort_disjoint_arraycopy =
2015         generate_disjoint_copy(T_SHORT,  true, Address::times_2, &entry,
2016                                "arrayof_jshort_disjoint_arraycopy");
2017     StubRoutines::_arrayof_jshort_arraycopy =
2018         generate_conjoint_copy(T_SHORT,  true, Address::times_2,  entry,
2019                                NULL, "arrayof_jshort_arraycopy");
2020     StubRoutines::_jshort_disjoint_arraycopy =
2021         generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,
2022                                "jshort_disjoint_arraycopy");
2023     StubRoutines::_jshort_arraycopy =
2024         generate_conjoint_copy(T_SHORT, false, Address::times_2,  entry,
2025                                &entry_jshort_arraycopy, "jshort_arraycopy");
2026 
2027     // Next arrays are always aligned on 4 bytes at least.
2028     StubRoutines::_jint_disjoint_arraycopy =
2029         generate_disjoint_copy(T_INT, true, Address::times_4, &entry,
2030                                "jint_disjoint_arraycopy");
2031     StubRoutines::_jint_arraycopy =
2032         generate_conjoint_copy(T_INT, true, Address::times_4,  entry,
2033                                &entry_jint_arraycopy, "jint_arraycopy");
2034 
2035     StubRoutines::_oop_disjoint_arraycopy =
2036         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
2037                                "oop_disjoint_arraycopy");
2038     StubRoutines::_oop_arraycopy =
2039         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
2040                                &entry_oop_arraycopy, "oop_arraycopy");
2041 
2042     StubRoutines::_oop_disjoint_arraycopy_uninit =
2043         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
2044                                "oop_disjoint_arraycopy_uninit",
2045                                /*dest_uninitialized*/true);
2046     StubRoutines::_oop_arraycopy_uninit =
2047         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
2048                                NULL, "oop_arraycopy_uninit",
2049                                /*dest_uninitialized*/true);
2050 
2051     StubRoutines::_jlong_disjoint_arraycopy =
2052         generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");
2053     StubRoutines::_jlong_arraycopy =
2054         generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,
2055                                     "jlong_arraycopy");
2056 
2057     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2058     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2059     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2060     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2061     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2062     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2063 
2064     StubRoutines::_arrayof_jint_disjoint_arraycopy       = StubRoutines::_jint_disjoint_arraycopy;
2065     StubRoutines::_arrayof_oop_disjoint_arraycopy        = StubRoutines::_oop_disjoint_arraycopy;
2066     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit;
2067     StubRoutines::_arrayof_jlong_disjoint_arraycopy      = StubRoutines::_jlong_disjoint_arraycopy;
2068 
2069     StubRoutines::_arrayof_jint_arraycopy       = StubRoutines::_jint_arraycopy;
2070     StubRoutines::_arrayof_oop_arraycopy        = StubRoutines::_oop_arraycopy;
2071     StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit;
2072     StubRoutines::_arrayof_jlong_arraycopy      = StubRoutines::_jlong_arraycopy;
2073 
2074     StubRoutines::_checkcast_arraycopy =
2075         generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2076     StubRoutines::_checkcast_arraycopy_uninit =
2077         generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, /*dest_uninitialized*/true);
2078 
2079     StubRoutines::_unsafe_arraycopy =
2080         generate_unsafe_copy("unsafe_arraycopy",
2081                                entry_jbyte_arraycopy,
2082                                entry_jshort_arraycopy,
2083                                entry_jint_arraycopy,
2084                                entry_jlong_arraycopy);
2085 
2086     StubRoutines::_generic_arraycopy =
2087         generate_generic_copy("generic_arraycopy",
2088                                entry_jbyte_arraycopy,
2089                                entry_jshort_arraycopy,
2090                                entry_jint_arraycopy,
2091                                entry_oop_arraycopy,
2092                                entry_jlong_arraycopy,
2093                                entry_checkcast_arraycopy);
2094   }
2095 
2096   // AES intrinsic stubs
2097   enum {AESBlockSize = 16};
2098 
2099   address generate_key_shuffle_mask() {
2100     __ align(16);
2101     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
2102     address start = __ pc();
2103     __ emit_data(0x00010203, relocInfo::none, 0 );
2104     __ emit_data(0x04050607, relocInfo::none, 0 );
2105     __ emit_data(0x08090a0b, relocInfo::none, 0 );
2106     __ emit_data(0x0c0d0e0f, relocInfo::none, 0 );
2107     return start;
2108   }
2109 
2110   address generate_counter_shuffle_mask() {
2111     __ align(16);
2112     StubCodeMark mark(this, "StubRoutines", "counter_shuffle_mask");
2113     address start = __ pc();
2114     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
2115     __ emit_data(0x08090a0b, relocInfo::none, 0);
2116     __ emit_data(0x04050607, relocInfo::none, 0);
2117     __ emit_data(0x00010203, relocInfo::none, 0);
2118     return start;
2119   }
2120 
2121   // Utility routine for loading a 128-bit key word in little endian format
2122   // can optionally specify that the shuffle mask is already in an xmmregister
2123   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2124     __ movdqu(xmmdst, Address(key, offset));
2125     if (xmm_shuf_mask != NULL) {
2126       __ pshufb(xmmdst, xmm_shuf_mask);
2127     } else {
2128       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2129     }
2130   }
2131 
2132   // aesenc using specified key+offset
2133   // can optionally specify that the shuffle mask is already in an xmmregister
2134   void aes_enc_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2135     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2136     __ aesenc(xmmdst, xmmtmp);
2137   }
2138 
2139   // aesdec using specified key+offset
2140   // can optionally specify that the shuffle mask is already in an xmmregister
2141   void aes_dec_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2142     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2143     __ aesdec(xmmdst, xmmtmp);
2144   }
2145 
2146   // Utility routine for increase 128bit counter (iv in CTR mode)
2147   //  XMM_128bit,  D3, D2, D1, D0
2148   void inc_counter(Register reg, XMMRegister xmmdst, int inc_delta, Label& next_block) {
2149     __ pextrd(reg, xmmdst, 0x0);
2150     __ addl(reg, inc_delta);
2151     __ pinsrd(xmmdst, reg, 0x0);
2152     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2153 
2154     __ pextrd(reg, xmmdst, 0x01); // Carry-> D1
2155     __ addl(reg, 0x01);
2156     __ pinsrd(xmmdst, reg, 0x01);
2157     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2158 
2159     __ pextrd(reg, xmmdst, 0x02); // Carry-> D2
2160     __ addl(reg, 0x01);
2161     __ pinsrd(xmmdst, reg, 0x02);
2162     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2163 
2164     __ pextrd(reg, xmmdst, 0x03); // Carry -> D3
2165     __ addl(reg, 0x01);
2166     __ pinsrd(xmmdst, reg, 0x03);
2167 
2168     __ BIND(next_block);          // next instruction
2169   }
2170 
2171 
2172   // Arguments:
2173   //
2174   // Inputs:
2175   //   c_rarg0   - source byte array address
2176   //   c_rarg1   - destination byte array address
2177   //   c_rarg2   - K (key) in little endian int array
2178   //
2179   address generate_aescrypt_encryptBlock() {
2180     assert(UseAES, "need AES instructions and misaligned SSE support");
2181     __ align(CodeEntryAlignment);
2182     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
2183     Label L_doLast;
2184     address start = __ pc();
2185 
2186     const Register from        = rdx;      // source array address
2187     const Register to          = rdx;      // destination array address
2188     const Register key         = rcx;      // key array address
2189     const Register keylen      = rax;
2190     const Address  from_param(rbp, 8+0);
2191     const Address  to_param  (rbp, 8+4);
2192     const Address  key_param (rbp, 8+8);
2193 
2194     const XMMRegister xmm_result = xmm0;
2195     const XMMRegister xmm_key_shuf_mask = xmm1;
2196     const XMMRegister xmm_temp1  = xmm2;
2197     const XMMRegister xmm_temp2  = xmm3;
2198     const XMMRegister xmm_temp3  = xmm4;
2199     const XMMRegister xmm_temp4  = xmm5;
2200 
2201     __ enter();   // required for proper stackwalking of RuntimeStub frame
2202 
2203     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
2204     // context for the registers used, where all instructions below are using 128-bit mode
2205     // On EVEX without VL and BW, these instructions will all be AVX.
2206     if (VM_Version::supports_avx512vlbw()) {
2207       __ movl(rdx, 0xffff);
2208       __ kmovdl(k1, rdx);
2209     }
2210 
2211     __ movptr(from, from_param);
2212     __ movptr(key, key_param);
2213 
2214     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2215     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2216 
2217     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2218     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
2219     __ movptr(to, to_param);
2220 
2221     // For encryption, the java expanded key ordering is just what we need
2222 
2223     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
2224     __ pxor(xmm_result, xmm_temp1);
2225 
2226     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2227     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2228     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2229     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2230 
2231     __ aesenc(xmm_result, xmm_temp1);
2232     __ aesenc(xmm_result, xmm_temp2);
2233     __ aesenc(xmm_result, xmm_temp3);
2234     __ aesenc(xmm_result, xmm_temp4);
2235 
2236     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2237     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2238     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2239     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2240 
2241     __ aesenc(xmm_result, xmm_temp1);
2242     __ aesenc(xmm_result, xmm_temp2);
2243     __ aesenc(xmm_result, xmm_temp3);
2244     __ aesenc(xmm_result, xmm_temp4);
2245 
2246     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2247     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2248 
2249     __ cmpl(keylen, 44);
2250     __ jccb(Assembler::equal, L_doLast);
2251 
2252     __ aesenc(xmm_result, xmm_temp1);
2253     __ aesenc(xmm_result, xmm_temp2);
2254 
2255     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2256     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2257 
2258     __ cmpl(keylen, 52);
2259     __ jccb(Assembler::equal, L_doLast);
2260 
2261     __ aesenc(xmm_result, xmm_temp1);
2262     __ aesenc(xmm_result, xmm_temp2);
2263 
2264     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2265     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2266 
2267     __ BIND(L_doLast);
2268     __ aesenc(xmm_result, xmm_temp1);
2269     __ aesenclast(xmm_result, xmm_temp2);
2270     __ movdqu(Address(to, 0), xmm_result);        // store the result
2271     __ xorptr(rax, rax); // return 0
2272     __ leave(); // required for proper stackwalking of RuntimeStub frame
2273     __ ret(0);
2274 
2275     return start;
2276   }
2277 
2278 
2279   // Arguments:
2280   //
2281   // Inputs:
2282   //   c_rarg0   - source byte array address
2283   //   c_rarg1   - destination byte array address
2284   //   c_rarg2   - K (key) in little endian int array
2285   //
2286   address generate_aescrypt_decryptBlock() {
2287     assert(UseAES, "need AES instructions and misaligned SSE support");
2288     __ align(CodeEntryAlignment);
2289     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
2290     Label L_doLast;
2291     address start = __ pc();
2292 
2293     const Register from        = rdx;      // source array address
2294     const Register to          = rdx;      // destination array address
2295     const Register key         = rcx;      // key array address
2296     const Register keylen      = rax;
2297     const Address  from_param(rbp, 8+0);
2298     const Address  to_param  (rbp, 8+4);
2299     const Address  key_param (rbp, 8+8);
2300 
2301     const XMMRegister xmm_result = xmm0;
2302     const XMMRegister xmm_key_shuf_mask = xmm1;
2303     const XMMRegister xmm_temp1  = xmm2;
2304     const XMMRegister xmm_temp2  = xmm3;
2305     const XMMRegister xmm_temp3  = xmm4;
2306     const XMMRegister xmm_temp4  = xmm5;
2307 
2308     __ enter(); // required for proper stackwalking of RuntimeStub frame
2309 
2310     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
2311     // context for the registers used, where all instructions below are using 128-bit mode
2312     // On EVEX without VL and BW, these instructions will all be AVX.
2313     if (VM_Version::supports_avx512vlbw()) {
2314       __ movl(rdx, 0xffff);
2315       __ kmovdl(k1, rdx);
2316     }
2317 
2318     __ movptr(from, from_param);
2319     __ movptr(key, key_param);
2320 
2321     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2322     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2323 
2324     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2325     __ movdqu(xmm_result, Address(from, 0));
2326     __ movptr(to, to_param);
2327 
2328     // for decryption java expanded key ordering is rotated one position from what we want
2329     // so we start from 0x10 here and hit 0x00 last
2330     // we don't know if the key is aligned, hence not using load-execute form
2331     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2332     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2333     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2334     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2335 
2336     __ pxor  (xmm_result, xmm_temp1);
2337     __ aesdec(xmm_result, xmm_temp2);
2338     __ aesdec(xmm_result, xmm_temp3);
2339     __ aesdec(xmm_result, xmm_temp4);
2340 
2341     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2342     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2343     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2344     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2345 
2346     __ aesdec(xmm_result, xmm_temp1);
2347     __ aesdec(xmm_result, xmm_temp2);
2348     __ aesdec(xmm_result, xmm_temp3);
2349     __ aesdec(xmm_result, xmm_temp4);
2350 
2351     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2352     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2353     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
2354 
2355     __ cmpl(keylen, 44);
2356     __ jccb(Assembler::equal, L_doLast);
2357 
2358     __ aesdec(xmm_result, xmm_temp1);
2359     __ aesdec(xmm_result, xmm_temp2);
2360 
2361     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2362     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2363 
2364     __ cmpl(keylen, 52);
2365     __ jccb(Assembler::equal, L_doLast);
2366 
2367     __ aesdec(xmm_result, xmm_temp1);
2368     __ aesdec(xmm_result, xmm_temp2);
2369 
2370     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2371     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2372 
2373     __ BIND(L_doLast);
2374     __ aesdec(xmm_result, xmm_temp1);
2375     __ aesdec(xmm_result, xmm_temp2);
2376 
2377     // for decryption the aesdeclast operation is always on key+0x00
2378     __ aesdeclast(xmm_result, xmm_temp3);
2379     __ movdqu(Address(to, 0), xmm_result);  // store the result
2380     __ xorptr(rax, rax); // return 0
2381     __ leave(); // required for proper stackwalking of RuntimeStub frame
2382     __ ret(0);
2383 
2384     return start;
2385   }
2386 
2387   void handleSOERegisters(bool saving) {
2388     const int saveFrameSizeInBytes = 4 * wordSize;
2389     const Address saved_rbx     (rbp, -3 * wordSize);
2390     const Address saved_rsi     (rbp, -2 * wordSize);
2391     const Address saved_rdi     (rbp, -1 * wordSize);
2392 
2393     if (saving) {
2394       __ subptr(rsp, saveFrameSizeInBytes);
2395       __ movptr(saved_rsi, rsi);
2396       __ movptr(saved_rdi, rdi);
2397       __ movptr(saved_rbx, rbx);
2398     } else {
2399       // restoring
2400       __ movptr(rsi, saved_rsi);
2401       __ movptr(rdi, saved_rdi);
2402       __ movptr(rbx, saved_rbx);
2403     }
2404   }
2405 
2406   // Arguments:
2407   //
2408   // Inputs:
2409   //   c_rarg0   - source byte array address
2410   //   c_rarg1   - destination byte array address
2411   //   c_rarg2   - K (key) in little endian int array
2412   //   c_rarg3   - r vector byte array address
2413   //   c_rarg4   - input length
2414   //
2415   // Output:
2416   //   rax       - input length
2417   //
2418   address generate_cipherBlockChaining_encryptAESCrypt() {
2419     assert(UseAES, "need AES instructions and misaligned SSE support");
2420     __ align(CodeEntryAlignment);
2421     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
2422     address start = __ pc();
2423 
2424     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
2425     const Register from        = rsi;      // source array address
2426     const Register to          = rdx;      // destination array address
2427     const Register key         = rcx;      // key array address
2428     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2429                                            // and left with the results of the last encryption block
2430     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2431     const Register pos         = rax;
2432 
2433     // xmm register assignments for the loops below
2434     const XMMRegister xmm_result = xmm0;
2435     const XMMRegister xmm_temp   = xmm1;
2436     // first 6 keys preloaded into xmm2-xmm7
2437     const int XMM_REG_NUM_KEY_FIRST = 2;
2438     const int XMM_REG_NUM_KEY_LAST  = 7;
2439     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
2440 
2441     __ enter(); // required for proper stackwalking of RuntimeStub frame
2442     handleSOERegisters(true /*saving*/);
2443 
2444     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
2445     // context for the registers used, where all instructions below are using 128-bit mode
2446     // On EVEX without VL and BW, these instructions will all be AVX.
2447     if (VM_Version::supports_avx512vlbw()) {
2448       __ movl(rdx, 0xffff);
2449       __ kmovdl(k1, rdx);
2450     }
2451 
2452     // load registers from incoming parameters
2453     const Address  from_param(rbp, 8+0);
2454     const Address  to_param  (rbp, 8+4);
2455     const Address  key_param (rbp, 8+8);
2456     const Address  rvec_param (rbp, 8+12);
2457     const Address  len_param  (rbp, 8+16);
2458     __ movptr(from , from_param);
2459     __ movptr(to   , to_param);
2460     __ movptr(key  , key_param);
2461     __ movptr(rvec , rvec_param);
2462     __ movptr(len_reg , len_param);
2463 
2464     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
2465     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2466     // load up xmm regs 2 thru 7 with keys 0-5
2467     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2468       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
2469       offset += 0x10;
2470     }
2471 
2472     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
2473 
2474     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2475     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2476     __ cmpl(rax, 44);
2477     __ jcc(Assembler::notEqual, L_key_192_256);
2478 
2479     // 128 bit code follows here
2480     __ movl(pos, 0);
2481     __ align(OptoLoopAlignment);
2482     __ BIND(L_loopTop_128);
2483     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2484     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2485 
2486     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2487     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2488       __ aesenc(xmm_result, as_XMMRegister(rnum));
2489     }
2490     for (int key_offset = 0x60; key_offset <= 0x90; key_offset += 0x10) {
2491       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2492     }
2493     load_key(xmm_temp, key, 0xa0);
2494     __ aesenclast(xmm_result, xmm_temp);
2495 
2496     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
2497     // no need to store r to memory until we exit
2498     __ addptr(pos, AESBlockSize);
2499     __ subptr(len_reg, AESBlockSize);
2500     __ jcc(Assembler::notEqual, L_loopTop_128);
2501 
2502     __ BIND(L_exit);
2503     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
2504 
2505     handleSOERegisters(false /*restoring*/);
2506     __ movptr(rax, len_param); // return length
2507     __ leave();                                  // required for proper stackwalking of RuntimeStub frame
2508     __ ret(0);
2509 
2510     __ BIND(L_key_192_256);
2511     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
2512     __ cmpl(rax, 52);
2513     __ jcc(Assembler::notEqual, L_key_256);
2514 
2515     // 192-bit code follows here (could be changed to use more xmm registers)
2516     __ movl(pos, 0);
2517     __ align(OptoLoopAlignment);
2518     __ BIND(L_loopTop_192);
2519     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2520     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2521 
2522     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2523     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2524       __ aesenc(xmm_result, as_XMMRegister(rnum));
2525     }
2526     for (int key_offset = 0x60; key_offset <= 0xb0; key_offset += 0x10) {
2527       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2528     }
2529     load_key(xmm_temp, key, 0xc0);
2530     __ aesenclast(xmm_result, xmm_temp);
2531 
2532     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2533     // no need to store r to memory until we exit
2534     __ addptr(pos, AESBlockSize);
2535     __ subptr(len_reg, AESBlockSize);
2536     __ jcc(Assembler::notEqual, L_loopTop_192);
2537     __ jmp(L_exit);
2538 
2539     __ BIND(L_key_256);
2540     // 256-bit code follows here (could be changed to use more xmm registers)
2541     __ movl(pos, 0);
2542     __ align(OptoLoopAlignment);
2543     __ BIND(L_loopTop_256);
2544     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2545     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2546 
2547     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2548     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2549       __ aesenc(xmm_result, as_XMMRegister(rnum));
2550     }
2551     for (int key_offset = 0x60; key_offset <= 0xd0; key_offset += 0x10) {
2552       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2553     }
2554     load_key(xmm_temp, key, 0xe0);
2555     __ aesenclast(xmm_result, xmm_temp);
2556 
2557     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2558     // no need to store r to memory until we exit
2559     __ addptr(pos, AESBlockSize);
2560     __ subptr(len_reg, AESBlockSize);
2561     __ jcc(Assembler::notEqual, L_loopTop_256);
2562     __ jmp(L_exit);
2563 
2564     return start;
2565   }
2566 
2567 
2568   // CBC AES Decryption.
2569   // In 32-bit stub, because of lack of registers we do not try to parallelize 4 blocks at a time.
2570   //
2571   // Arguments:
2572   //
2573   // Inputs:
2574   //   c_rarg0   - source byte array address
2575   //   c_rarg1   - destination byte array address
2576   //   c_rarg2   - K (key) in little endian int array
2577   //   c_rarg3   - r vector byte array address
2578   //   c_rarg4   - input length
2579   //
2580   // Output:
2581   //   rax       - input length
2582   //
2583 
2584   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
2585     assert(UseAES, "need AES instructions and misaligned SSE support");
2586     __ align(CodeEntryAlignment);
2587     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
2588     address start = __ pc();
2589 
2590     const Register from        = rsi;      // source array address
2591     const Register to          = rdx;      // destination array address
2592     const Register key         = rcx;      // key array address
2593     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2594                                            // and left with the results of the last encryption block
2595     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2596     const Register pos         = rax;
2597 
2598     const int PARALLEL_FACTOR = 4;
2599     const int ROUNDS[3] = { 10, 12, 14 }; //aes rounds for key128, key192, key256
2600 
2601     Label L_exit;
2602     Label L_singleBlock_loopTop[3]; //128, 192, 256
2603     Label L_multiBlock_loopTop[3]; //128, 192, 256
2604 
2605     const XMMRegister xmm_prev_block_cipher = xmm0; // holds cipher of previous block
2606     const XMMRegister xmm_key_shuf_mask = xmm1;
2607 
2608     const XMMRegister xmm_key_tmp0 = xmm2;
2609     const XMMRegister xmm_key_tmp1 = xmm3;
2610 
2611     // registers holding the six results in the parallelized loop
2612     const XMMRegister xmm_result0 = xmm4;
2613     const XMMRegister xmm_result1 = xmm5;
2614     const XMMRegister xmm_result2 = xmm6;
2615     const XMMRegister xmm_result3 = xmm7;
2616 
2617     __ enter(); // required for proper stackwalking of RuntimeStub frame
2618     handleSOERegisters(true /*saving*/);
2619 
2620     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
2621     // context for the registers used, where all instructions below are using 128-bit mode
2622     // On EVEX without VL and BW, these instructions will all be AVX.
2623     if (VM_Version::supports_avx512vlbw()) {
2624       __ movl(rdx, 0xffff);
2625       __ kmovdl(k1, rdx);
2626     }
2627 
2628     // load registers from incoming parameters
2629     const Address  from_param(rbp, 8+0);
2630     const Address  to_param  (rbp, 8+4);
2631     const Address  key_param (rbp, 8+8);
2632     const Address  rvec_param (rbp, 8+12);
2633     const Address  len_param  (rbp, 8+16);
2634 
2635     __ movptr(from , from_param);
2636     __ movptr(to   , to_param);
2637     __ movptr(key  , key_param);
2638     __ movptr(rvec , rvec_param);
2639     __ movptr(len_reg , len_param);
2640 
2641     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2642     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // initialize with initial rvec
2643 
2644     __ xorptr(pos, pos);
2645 
2646     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2647     // rvec is reused
2648     __ movl(rvec, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2649     __ cmpl(rvec, 52);
2650     __ jcc(Assembler::equal, L_multiBlock_loopTop[1]);
2651     __ cmpl(rvec, 60);
2652     __ jcc(Assembler::equal, L_multiBlock_loopTop[2]);
2653 
2654 #define DoFour(opc, src_reg)           \
2655   __ opc(xmm_result0, src_reg);         \
2656   __ opc(xmm_result1, src_reg);         \
2657   __ opc(xmm_result2, src_reg);         \
2658   __ opc(xmm_result3, src_reg);         \
2659 
2660     for (int k = 0; k < 3; ++k) {
2661       __ align(OptoLoopAlignment);
2662       __ BIND(L_multiBlock_loopTop[k]);
2663       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
2664       __ jcc(Assembler::less, L_singleBlock_loopTop[k]);
2665 
2666       __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); // get next 4 blocks into xmmresult registers
2667       __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2668       __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2669       __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2670 
2671       // the java expanded key ordering is rotated one position from what we want
2672       // so we start from 0x10 here and hit 0x00 last
2673       load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2674       DoFour(pxor, xmm_key_tmp0); //xor with first key
2675       // do the aes dec rounds
2676       for (int rnum = 1; rnum <= ROUNDS[k];) {
2677         //load two keys at a time
2678         //k1->0x20, ..., k9->0xa0, k10->0x00
2679         load_key(xmm_key_tmp1, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2680         load_key(xmm_key_tmp0, key, ((rnum + 2) % (ROUNDS[k] + 1)) * 0x10, xmm_key_shuf_mask); // hit 0x00 last!
2681         DoFour(aesdec, xmm_key_tmp1);
2682         rnum++;
2683         if (rnum != ROUNDS[k]) {
2684           DoFour(aesdec, xmm_key_tmp0);
2685         }
2686         else {
2687           DoFour(aesdeclast, xmm_key_tmp0);
2688         }
2689         rnum++;
2690       }
2691 
2692       // for each result, xor with the r vector of previous cipher block
2693       __ pxor(xmm_result0, xmm_prev_block_cipher);
2694       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2695       __ pxor(xmm_result1, xmm_prev_block_cipher);
2696       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2697       __ pxor(xmm_result2, xmm_prev_block_cipher);
2698       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2699       __ pxor(xmm_result3, xmm_prev_block_cipher);
2700       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3 * AESBlockSize)); // this will carry over to next set of blocks
2701 
2702             // store 4 results into the next 64 bytes of output
2703        __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2704        __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2705        __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2706        __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2707 
2708        __ addptr(pos, 4 * AESBlockSize);
2709        __ subptr(len_reg, 4 * AESBlockSize);
2710        __ jmp(L_multiBlock_loopTop[k]);
2711 
2712        //singleBlock starts here
2713        __ align(OptoLoopAlignment);
2714        __ BIND(L_singleBlock_loopTop[k]);
2715        __ cmpptr(len_reg, 0); // any blocks left?
2716        __ jcc(Assembler::equal, L_exit);
2717        __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
2718        __ movdqa(xmm_result1, xmm_result0);
2719 
2720        load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2721        __ pxor(xmm_result0, xmm_key_tmp0);
2722        // do the aes dec rounds
2723        for (int rnum = 1; rnum < ROUNDS[k]; rnum++) {
2724          // the java expanded key ordering is rotated one position from what we want
2725          load_key(xmm_key_tmp0, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2726          __ aesdec(xmm_result0, xmm_key_tmp0);
2727        }
2728        load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
2729        __ aesdeclast(xmm_result0, xmm_key_tmp0);
2730        __ pxor(xmm_result0, xmm_prev_block_cipher); // xor with the current r vector
2731        __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result0); // store into the next 16 bytes of output
2732        // no need to store r to memory until we exit
2733        __ movdqa(xmm_prev_block_cipher, xmm_result1); // set up next r vector with cipher input from this block
2734 
2735        __ addptr(pos, AESBlockSize);
2736        __ subptr(len_reg, AESBlockSize);
2737        __ jmp(L_singleBlock_loopTop[k]);
2738     }//for 128/192/256
2739 
2740     __ BIND(L_exit);
2741     __ movptr(rvec, rvec_param);                        // restore this since reused earlier
2742     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher); // final value of r stored in rvec of CipherBlockChaining object
2743     handleSOERegisters(false /*restoring*/);
2744     __ movptr(rax, len_param);                          // return length
2745     __ leave();                                         // required for proper stackwalking of RuntimeStub frame
2746     __ ret(0);
2747 
2748     return start;
2749   }
2750 
2751   // CTR AES crypt.
2752   // In 32-bit stub, parallelize 4 blocks at a time
2753   // Arguments:
2754   //
2755   // Inputs:
2756   //   c_rarg0   - source byte array address
2757   //   c_rarg1   - destination byte array address
2758   //   c_rarg2   - K (key) in little endian int array
2759   //   c_rarg3   - counter vector byte array address
2760   //   c_rarg4   - input length
2761   //
2762   // Output:
2763   //   rax       - input length
2764   //
2765   address generate_counterMode_AESCrypt_Parallel() {
2766     assert(UseAES, "need AES instructions and misaligned SSE support");
2767     __ align(CodeEntryAlignment);
2768     StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
2769     address start = __ pc();
2770     const Register from        = rsi;      // source array address
2771     const Register to          = rdx;      // destination array address
2772     const Register key         = rcx;      // key array address
2773     const Register counter     = rdi;      // counter byte array initialized from initvector array address
2774                                            // and updated with the incremented counter in the end
2775     const Register len_reg     = rbx;
2776     const Register pos         = rax;
2777 
2778     __ enter(); // required for proper stackwalking of RuntimeStub frame
2779     handleSOERegisters(true /*saving*/); // save rbx, rsi, rdi
2780 
2781     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
2782     // context for the registers used, where all instructions below are using 128-bit mode
2783     // On EVEX without VL and BW, these instructions will all be AVX.
2784     if (VM_Version::supports_avx512vlbw()) {
2785       __ movl(rdx, 0xffff);
2786       __ kmovdl(k1, rdx);
2787     }
2788 
2789     // load registers from incoming parameters
2790     const Address  from_param(rbp, 8+0);
2791     const Address  to_param  (rbp, 8+4);
2792     const Address  key_param (rbp, 8+8);
2793     const Address  rvec_param (rbp, 8+12);
2794     const Address  len_param  (rbp, 8+16);
2795     const Address  saved_counter_param(rbp, 8 + 20);
2796     const Address  used_addr_param(rbp, 8 + 24);
2797 
2798     __ movptr(from , from_param);
2799     __ movptr(to   , to_param);
2800     __ movptr(len_reg , len_param);
2801 
2802     // Use the partially used encrpyted counter from last invocation
2803     Label L_exit_preLoop, L_preLoop_start;
2804 
2805     // Use the registers 'counter' and 'key' here in this preloop
2806     // to hold of last 2 params 'used' and 'saved_encCounter_start'
2807     Register used = counter;
2808     Register saved_encCounter_start = key;
2809     Register used_addr = saved_encCounter_start;
2810 
2811     __ movptr(used_addr, used_addr_param);
2812     __ movptr(used, Address(used_addr, 0));
2813     __ movptr(saved_encCounter_start, saved_counter_param);
2814 
2815     __ BIND(L_preLoop_start);
2816     __ cmpptr(used, 16);
2817     __ jcc(Assembler::aboveEqual, L_exit_preLoop);
2818     __ cmpptr(len_reg, 0);
2819     __ jcc(Assembler::lessEqual, L_exit_preLoop);
2820     __ movb(rax, Address(saved_encCounter_start, used));
2821     __ xorb(rax, Address(from, 0));
2822     __ movb(Address(to, 0), rax);
2823     __ addptr(from, 1);
2824     __ addptr(to, 1);
2825     __ addptr(used, 1);
2826     __ subptr(len_reg, 1);
2827 
2828     __ jmp(L_preLoop_start);
2829 
2830     __ BIND(L_exit_preLoop);
2831     __ movptr(used_addr, used_addr_param);
2832     __ movptr(used_addr, used_addr_param);
2833     __ movl(Address(used_addr, 0), used);
2834 
2835     // load the parameters 'key' and 'counter'
2836     __ movptr(key, key_param);
2837     __ movptr(counter, rvec_param);
2838 
2839     // xmm register assignments for the loops below
2840     const XMMRegister xmm_curr_counter      = xmm0;
2841     const XMMRegister xmm_counter_shuf_mask = xmm1;  // need to be reloaded
2842     const XMMRegister xmm_key_shuf_mask     = xmm2;  // need to be reloaded
2843     const XMMRegister xmm_key               = xmm3;
2844     const XMMRegister xmm_result0           = xmm4;
2845     const XMMRegister xmm_result1           = xmm5;
2846     const XMMRegister xmm_result2           = xmm6;
2847     const XMMRegister xmm_result3           = xmm7;
2848     const XMMRegister xmm_from0             = xmm1;   //reuse XMM register
2849     const XMMRegister xmm_from1             = xmm2;
2850     const XMMRegister xmm_from2             = xmm3;
2851     const XMMRegister xmm_from3             = xmm4;
2852 
2853     //for key_128, key_192, key_256
2854     const int rounds[3] = {10, 12, 14};
2855     Label L_singleBlockLoopTop[3];
2856     Label L_multiBlock_loopTop[3];
2857     Label L_key192_top, L_key256_top;
2858     Label L_incCounter[3][4]; // 3: different key length,  4: 4 blocks at a time
2859     Label L_incCounter_single[3]; //for single block, key128, key192, key256
2860     Label L_processTail_insr[3], L_processTail_4_insr[3], L_processTail_2_insr[3], L_processTail_1_insr[3], L_processTail_exit_insr[3];
2861     Label L_processTail_extr[3], L_processTail_4_extr[3], L_processTail_2_extr[3], L_processTail_1_extr[3], L_processTail_exit_extr[3];
2862 
2863     Label L_exit;
2864     const int PARALLEL_FACTOR = 4;  //because of the limited register number
2865 
2866     // initialize counter with initial counter
2867     __ movdqu(xmm_curr_counter, Address(counter, 0x00));
2868     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2869     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled for increase
2870 
2871     // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
2872     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2873     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2874     __ cmpl(rax, 52);
2875     __ jcc(Assembler::equal, L_key192_top);
2876     __ cmpl(rax, 60);
2877     __ jcc(Assembler::equal, L_key256_top);
2878 
2879     //key128 begins here
2880     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2881 
2882 #define CTR_DoFour(opc, src_reg)               \
2883     __ opc(xmm_result0, src_reg);              \
2884     __ opc(xmm_result1, src_reg);              \
2885     __ opc(xmm_result2, src_reg);              \
2886     __ opc(xmm_result3, src_reg);
2887 
2888     // k == 0 :  generate code for key_128
2889     // k == 1 :  generate code for key_192
2890     // k == 2 :  generate code for key_256
2891     for (int k = 0; k < 3; ++k) {
2892       //multi blocks starts here
2893       __ align(OptoLoopAlignment);
2894       __ BIND(L_multiBlock_loopTop[k]);
2895       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least PARALLEL_FACTOR blocks left
2896       __ jcc(Assembler::less, L_singleBlockLoopTop[k]);
2897 
2898       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2899       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2900 
2901       //load, then increase counters
2902       CTR_DoFour(movdqa, xmm_curr_counter);
2903       __ push(rbx);
2904       inc_counter(rbx, xmm_result1, 0x01, L_incCounter[k][0]);
2905       inc_counter(rbx, xmm_result2, 0x02, L_incCounter[k][1]);
2906       inc_counter(rbx, xmm_result3, 0x03, L_incCounter[k][2]);
2907       inc_counter(rbx, xmm_curr_counter, 0x04, L_incCounter[k][3]);
2908       __ pop (rbx);
2909 
2910       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask); // load Round 0 key. interleaving for better performance
2911 
2912       CTR_DoFour(pshufb, xmm_counter_shuf_mask); // after increased, shuffled counters back for PXOR
2913       CTR_DoFour(pxor, xmm_key);   //PXOR with Round 0 key
2914 
2915       for (int i = 1; i < rounds[k]; ++i) {
2916         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2917         CTR_DoFour(aesenc, xmm_key);
2918       }
2919       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2920       CTR_DoFour(aesenclast, xmm_key);
2921 
2922       // get next PARALLEL_FACTOR blocks into xmm_from registers
2923       __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2924       __ movdqu(xmm_from1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2925       __ movdqu(xmm_from2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2926 
2927       // PXOR with input text
2928       __ pxor(xmm_result0, xmm_from0); //result0 is xmm4
2929       __ pxor(xmm_result1, xmm_from1);
2930       __ pxor(xmm_result2, xmm_from2);
2931 
2932       // store PARALLEL_FACTOR results into the next 64 bytes of output
2933       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2934       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2935       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2936 
2937       // do it here after xmm_result0 is saved, because xmm_from3 reuse the same register of xmm_result0.
2938       __ movdqu(xmm_from3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2939       __ pxor(xmm_result3, xmm_from3);
2940       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2941 
2942       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize); // increase the length of crypt text
2943       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // decrease the remaining length
2944       __ jmp(L_multiBlock_loopTop[k]);
2945 
2946       // singleBlock starts here
2947       __ align(OptoLoopAlignment);
2948       __ BIND(L_singleBlockLoopTop[k]);
2949       __ cmpptr(len_reg, 0);
2950       __ jcc(Assembler::equal, L_exit);
2951       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2952       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2953       __ movdqa(xmm_result0, xmm_curr_counter);
2954       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask);
2955       __ push(rbx);//rbx is used for increasing counter
2956       inc_counter(rbx, xmm_curr_counter, 0x01, L_incCounter_single[k]);
2957       __ pop (rbx);
2958       __ pshufb(xmm_result0, xmm_counter_shuf_mask);
2959       __ pxor(xmm_result0, xmm_key);
2960       for (int i = 1; i < rounds[k]; i++) {
2961         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2962         __ aesenc(xmm_result0, xmm_key);
2963       }
2964       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2965       __ aesenclast(xmm_result0, xmm_key);
2966       __ cmpptr(len_reg, AESBlockSize);
2967       __ jcc(Assembler::less, L_processTail_insr[k]);
2968         __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2969         __ pxor(xmm_result0, xmm_from0);
2970         __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2971         __ addptr(pos, AESBlockSize);
2972         __ subptr(len_reg, AESBlockSize);
2973         __ jmp(L_singleBlockLoopTop[k]);
2974 
2975       __ BIND(L_processTail_insr[k]);                                               // Process the tail part of the input array
2976         __ addptr(pos, len_reg);                                                    // 1. Insert bytes from src array into xmm_from0 register
2977         __ testptr(len_reg, 8);
2978         __ jcc(Assembler::zero, L_processTail_4_insr[k]);
2979           __ subptr(pos,8);
2980           __ pinsrd(xmm_from0, Address(from, pos), 0);
2981           __ pinsrd(xmm_from0, Address(from, pos, Address::times_1, 4), 1);
2982         __ BIND(L_processTail_4_insr[k]);
2983         __ testptr(len_reg, 4);
2984         __ jcc(Assembler::zero, L_processTail_2_insr[k]);
2985           __ subptr(pos,4);
2986           __ pslldq(xmm_from0, 4);
2987           __ pinsrd(xmm_from0, Address(from, pos), 0);
2988         __ BIND(L_processTail_2_insr[k]);
2989         __ testptr(len_reg, 2);
2990         __ jcc(Assembler::zero, L_processTail_1_insr[k]);
2991           __ subptr(pos, 2);
2992           __ pslldq(xmm_from0, 2);
2993           __ pinsrw(xmm_from0, Address(from, pos), 0);
2994         __ BIND(L_processTail_1_insr[k]);
2995         __ testptr(len_reg, 1);
2996         __ jcc(Assembler::zero, L_processTail_exit_insr[k]);
2997           __ subptr(pos, 1);
2998           __ pslldq(xmm_from0, 1);
2999           __ pinsrb(xmm_from0, Address(from, pos), 0);
3000         __ BIND(L_processTail_exit_insr[k]);
3001 
3002         __ movptr(saved_encCounter_start, saved_counter_param);
3003         __ movdqu(Address(saved_encCounter_start, 0), xmm_result0);               // 2. Perform pxor of the encrypted counter and plaintext Bytes.
3004         __ pxor(xmm_result0, xmm_from0);                                          //    Also the encrypted counter is saved for next invocation.
3005 
3006         __ testptr(len_reg, 8);
3007         __ jcc(Assembler::zero, L_processTail_4_extr[k]);                        // 3. Extract bytes from xmm_result0 into the dest. array
3008           __ pextrd(Address(to, pos), xmm_result0, 0);
3009           __ pextrd(Address(to, pos, Address::times_1, 4), xmm_result0, 1);
3010           __ psrldq(xmm_result0, 8);
3011           __ addptr(pos, 8);
3012         __ BIND(L_processTail_4_extr[k]);
3013         __ testptr(len_reg, 4);
3014         __ jcc(Assembler::zero, L_processTail_2_extr[k]);
3015           __ pextrd(Address(to, pos), xmm_result0, 0);
3016           __ psrldq(xmm_result0, 4);
3017           __ addptr(pos, 4);
3018         __ BIND(L_processTail_2_extr[k]);
3019         __ testptr(len_reg, 2);
3020         __ jcc(Assembler::zero, L_processTail_1_extr[k]);
3021           __ pextrb(Address(to, pos), xmm_result0, 0);
3022           __ pextrb(Address(to, pos, Address::times_1, 1), xmm_result0, 1);
3023           __ psrldq(xmm_result0, 2);
3024           __ addptr(pos, 2);
3025         __ BIND(L_processTail_1_extr[k]);
3026         __ testptr(len_reg, 1);
3027         __ jcc(Assembler::zero, L_processTail_exit_extr[k]);
3028           __ pextrb(Address(to, pos), xmm_result0, 0);
3029 
3030         __ BIND(L_processTail_exit_extr[k]);
3031         __ movptr(used_addr, used_addr_param);
3032         __ movl(Address(used_addr, 0), len_reg);
3033         __ jmp(L_exit);
3034     }
3035 
3036     __ BIND(L_exit);
3037     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
3038     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled back.
3039     __ movdqu(Address(counter, 0), xmm_curr_counter); //save counter back
3040     handleSOERegisters(false /*restoring*/);
3041     __ movptr(rax, len_param); // return length
3042     __ leave();                // required for proper stackwalking of RuntimeStub frame
3043     __ ret(0);
3044 
3045     __ BIND (L_key192_top);
3046     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
3047     __ jmp(L_multiBlock_loopTop[1]); //key192
3048 
3049     __ BIND (L_key256_top);
3050     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
3051     __ jmp(L_multiBlock_loopTop[2]); //key192
3052 
3053     return start;
3054   }
3055 
3056   address generate_upper_word_mask() {
3057     __ align(64);
3058     StubCodeMark mark(this, "StubRoutines", "upper_word_mask");
3059     address start = __ pc();
3060     __ emit_data(0x00000000, relocInfo::none, 0);
3061     __ emit_data(0x00000000, relocInfo::none, 0);
3062     __ emit_data(0x00000000, relocInfo::none, 0);
3063     __ emit_data(0xFFFFFFFF, relocInfo::none, 0);
3064     return start;
3065   }
3066 
3067   address generate_shuffle_byte_flip_mask() {
3068     __ align(64);
3069     StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask");
3070     address start = __ pc();
3071     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3072     __ emit_data(0x08090a0b, relocInfo::none, 0);
3073     __ emit_data(0x04050607, relocInfo::none, 0);
3074     __ emit_data(0x00010203, relocInfo::none, 0);
3075     return start;
3076   }
3077 
3078   // ofs and limit are use for multi-block byte array.
3079   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3080   address generate_sha1_implCompress(bool multi_block, const char *name) {
3081     __ align(CodeEntryAlignment);
3082     StubCodeMark mark(this, "StubRoutines", name);
3083     address start = __ pc();
3084 
3085     Register buf   = rax;
3086     Register state = rdx;
3087     Register ofs   = rcx;
3088     Register limit = rdi;
3089 
3090     const Address  buf_param(rbp, 8 + 0);
3091     const Address  state_param(rbp, 8 + 4);
3092     const Address  ofs_param(rbp, 8 + 8);
3093     const Address  limit_param(rbp, 8 + 12);
3094 
3095     const XMMRegister abcd = xmm0;
3096     const XMMRegister e0 = xmm1;
3097     const XMMRegister e1 = xmm2;
3098     const XMMRegister msg0 = xmm3;
3099 
3100     const XMMRegister msg1 = xmm4;
3101     const XMMRegister msg2 = xmm5;
3102     const XMMRegister msg3 = xmm6;
3103     const XMMRegister shuf_mask = xmm7;
3104 
3105     __ enter();
3106     __ subptr(rsp, 8 * wordSize);
3107     if (multi_block) {
3108       __ push(limit);
3109     }
3110     __ movptr(buf, buf_param);
3111     __ movptr(state, state_param);
3112     if (multi_block) {
3113       __ movptr(ofs, ofs_param);
3114       __ movptr(limit, limit_param);
3115     }
3116 
3117     __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask,
3118       buf, state, ofs, limit, rsp, multi_block);
3119 
3120     if (multi_block) {
3121       __ pop(limit);
3122     }
3123     __ addptr(rsp, 8 * wordSize);
3124     __ leave();
3125     __ ret(0);
3126     return start;
3127   }
3128 
3129   address generate_pshuffle_byte_flip_mask() {
3130     __ align(64);
3131     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask");
3132     address start = __ pc();
3133     __ emit_data(0x00010203, relocInfo::none, 0);
3134     __ emit_data(0x04050607, relocInfo::none, 0);
3135     __ emit_data(0x08090a0b, relocInfo::none, 0);
3136     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3137     return start;
3138   }
3139 
3140   // ofs and limit are use for multi-block byte array.
3141   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3142  address generate_sha256_implCompress(bool multi_block, const char *name) {
3143     __ align(CodeEntryAlignment);
3144     StubCodeMark mark(this, "StubRoutines", name);
3145     address start = __ pc();
3146 
3147     Register buf = rbx;
3148     Register state = rsi;
3149     Register ofs = rdx;
3150     Register limit = rcx;
3151 
3152     const Address  buf_param(rbp, 8 + 0);
3153     const Address  state_param(rbp, 8 + 4);
3154     const Address  ofs_param(rbp, 8 + 8);
3155     const Address  limit_param(rbp, 8 + 12);
3156 
3157     const XMMRegister msg = xmm0;
3158     const XMMRegister state0 = xmm1;
3159     const XMMRegister state1 = xmm2;
3160     const XMMRegister msgtmp0 = xmm3;
3161 
3162     const XMMRegister msgtmp1 = xmm4;
3163     const XMMRegister msgtmp2 = xmm5;
3164     const XMMRegister msgtmp3 = xmm6;
3165     const XMMRegister msgtmp4 = xmm7;
3166 
3167     __ enter();
3168     __ subptr(rsp, 8 * wordSize);
3169     handleSOERegisters(true /*saving*/);
3170     __ movptr(buf, buf_param);
3171     __ movptr(state, state_param);
3172     if (multi_block) {
3173      __ movptr(ofs, ofs_param);
3174      __ movptr(limit, limit_param);
3175     }
3176 
3177     __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3178       buf, state, ofs, limit, rsp, multi_block);
3179 
3180     handleSOERegisters(false);
3181     __ addptr(rsp, 8 * wordSize);
3182     __ leave();
3183     __ ret(0);
3184     return start;
3185   }
3186 
3187   // byte swap x86 long
3188   address generate_ghash_long_swap_mask() {
3189     __ align(CodeEntryAlignment);
3190     StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
3191     address start = __ pc();
3192     __ emit_data(0x0b0a0908, relocInfo::none, 0);
3193     __ emit_data(0x0f0e0d0c, relocInfo::none, 0);
3194     __ emit_data(0x03020100, relocInfo::none, 0);
3195     __ emit_data(0x07060504, relocInfo::none, 0);
3196 
3197   return start;
3198   }
3199 
3200   // byte swap x86 byte array
3201   address generate_ghash_byte_swap_mask() {
3202     __ align(CodeEntryAlignment);
3203     StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
3204     address start = __ pc();
3205     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3206     __ emit_data(0x08090a0b, relocInfo::none, 0);
3207     __ emit_data(0x04050607, relocInfo::none, 0);
3208     __ emit_data(0x00010203, relocInfo::none, 0);
3209   return start;
3210   }
3211 
3212   /* Single and multi-block ghash operations */
3213   address generate_ghash_processBlocks() {
3214     assert(UseGHASHIntrinsics, "need GHASH intrinsics and CLMUL support");
3215     __ align(CodeEntryAlignment);
3216     Label L_ghash_loop, L_exit;
3217     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
3218     address start = __ pc();
3219 
3220     const Register state        = rdi;
3221     const Register subkeyH      = rsi;
3222     const Register data         = rdx;
3223     const Register blocks       = rcx;
3224 
3225     const Address  state_param(rbp, 8+0);
3226     const Address  subkeyH_param(rbp, 8+4);
3227     const Address  data_param(rbp, 8+8);
3228     const Address  blocks_param(rbp, 8+12);
3229 
3230     const XMMRegister xmm_temp0 = xmm0;
3231     const XMMRegister xmm_temp1 = xmm1;
3232     const XMMRegister xmm_temp2 = xmm2;
3233     const XMMRegister xmm_temp3 = xmm3;
3234     const XMMRegister xmm_temp4 = xmm4;
3235     const XMMRegister xmm_temp5 = xmm5;
3236     const XMMRegister xmm_temp6 = xmm6;
3237     const XMMRegister xmm_temp7 = xmm7;
3238 
3239     __ enter();
3240     handleSOERegisters(true);  // Save registers
3241 
3242     // For EVEX with VL and BW, provide a standard mask, VL = 128 will guide the merge
3243     // context for the registers used, where all instructions below are using 128-bit mode
3244     // On EVEX without VL and BW, these instructions will all be AVX.
3245     if (VM_Version::supports_avx512vlbw()) {
3246       __ movl(rdx, 0xffff);
3247       __ kmovdl(k1, rdx);
3248     }
3249 
3250     __ movptr(state, state_param);
3251     __ movptr(subkeyH, subkeyH_param);
3252     __ movptr(data, data_param);
3253     __ movptr(blocks, blocks_param);
3254 
3255     __ movdqu(xmm_temp0, Address(state, 0));
3256     __ pshufb(xmm_temp0, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3257 
3258     __ movdqu(xmm_temp1, Address(subkeyH, 0));
3259     __ pshufb(xmm_temp1, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3260 
3261     __ BIND(L_ghash_loop);
3262     __ movdqu(xmm_temp2, Address(data, 0));
3263     __ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
3264 
3265     __ pxor(xmm_temp0, xmm_temp2);
3266 
3267     //
3268     // Multiply with the hash key
3269     //
3270     __ movdqu(xmm_temp3, xmm_temp0);
3271     __ pclmulqdq(xmm_temp3, xmm_temp1, 0);      // xmm3 holds a0*b0
3272     __ movdqu(xmm_temp4, xmm_temp0);
3273     __ pclmulqdq(xmm_temp4, xmm_temp1, 16);     // xmm4 holds a0*b1
3274 
3275     __ movdqu(xmm_temp5, xmm_temp0);
3276     __ pclmulqdq(xmm_temp5, xmm_temp1, 1);      // xmm5 holds a1*b0
3277     __ movdqu(xmm_temp6, xmm_temp0);
3278     __ pclmulqdq(xmm_temp6, xmm_temp1, 17);     // xmm6 holds a1*b1
3279 
3280     __ pxor(xmm_temp4, xmm_temp5);      // xmm4 holds a0*b1 + a1*b0
3281 
3282     __ movdqu(xmm_temp5, xmm_temp4);    // move the contents of xmm4 to xmm5
3283     __ psrldq(xmm_temp4, 8);    // shift by xmm4 64 bits to the right
3284     __ pslldq(xmm_temp5, 8);    // shift by xmm5 64 bits to the left
3285     __ pxor(xmm_temp3, xmm_temp5);
3286     __ pxor(xmm_temp6, xmm_temp4);      // Register pair <xmm6:xmm3> holds the result
3287                                         // of the carry-less multiplication of
3288                                         // xmm0 by xmm1.
3289 
3290     // We shift the result of the multiplication by one bit position
3291     // to the left to cope for the fact that the bits are reversed.
3292     __ movdqu(xmm_temp7, xmm_temp3);
3293     __ movdqu(xmm_temp4, xmm_temp6);
3294     __ pslld (xmm_temp3, 1);
3295     __ pslld(xmm_temp6, 1);
3296     __ psrld(xmm_temp7, 31);
3297     __ psrld(xmm_temp4, 31);
3298     __ movdqu(xmm_temp5, xmm_temp7);
3299     __ pslldq(xmm_temp4, 4);
3300     __ pslldq(xmm_temp7, 4);
3301     __ psrldq(xmm_temp5, 12);
3302     __ por(xmm_temp3, xmm_temp7);
3303     __ por(xmm_temp6, xmm_temp4);
3304     __ por(xmm_temp6, xmm_temp5);
3305 
3306     //
3307     // First phase of the reduction
3308     //
3309     // Move xmm3 into xmm4, xmm5, xmm7 in order to perform the shifts
3310     // independently.
3311     __ movdqu(xmm_temp7, xmm_temp3);
3312     __ movdqu(xmm_temp4, xmm_temp3);
3313     __ movdqu(xmm_temp5, xmm_temp3);
3314     __ pslld(xmm_temp7, 31);    // packed right shift shifting << 31
3315     __ pslld(xmm_temp4, 30);    // packed right shift shifting << 30
3316     __ pslld(xmm_temp5, 25);    // packed right shift shifting << 25
3317     __ pxor(xmm_temp7, xmm_temp4);      // xor the shifted versions
3318     __ pxor(xmm_temp7, xmm_temp5);
3319     __ movdqu(xmm_temp4, xmm_temp7);
3320     __ pslldq(xmm_temp7, 12);
3321     __ psrldq(xmm_temp4, 4);
3322     __ pxor(xmm_temp3, xmm_temp7);      // first phase of the reduction complete
3323 
3324     //
3325     // Second phase of the reduction
3326     //
3327     // Make 3 copies of xmm3 in xmm2, xmm5, xmm7 for doing these
3328     // shift operations.
3329     __ movdqu(xmm_temp2, xmm_temp3);
3330     __ movdqu(xmm_temp7, xmm_temp3);
3331     __ movdqu(xmm_temp5, xmm_temp3);
3332     __ psrld(xmm_temp2, 1);     // packed left shifting >> 1
3333     __ psrld(xmm_temp7, 2);     // packed left shifting >> 2
3334     __ psrld(xmm_temp5, 7);     // packed left shifting >> 7
3335     __ pxor(xmm_temp2, xmm_temp7);      // xor the shifted versions
3336     __ pxor(xmm_temp2, xmm_temp5);
3337     __ pxor(xmm_temp2, xmm_temp4);
3338     __ pxor(xmm_temp3, xmm_temp2);
3339     __ pxor(xmm_temp6, xmm_temp3);      // the result is in xmm6
3340 
3341     __ decrement(blocks);
3342     __ jcc(Assembler::zero, L_exit);
3343     __ movdqu(xmm_temp0, xmm_temp6);
3344     __ addptr(data, 16);
3345     __ jmp(L_ghash_loop);
3346 
3347     __ BIND(L_exit);
3348        // Byte swap 16-byte result
3349     __ pshufb(xmm_temp6, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3350     __ movdqu(Address(state, 0), xmm_temp6);   // store the result
3351 
3352     handleSOERegisters(false);  // restore registers
3353     __ leave();
3354     __ ret(0);
3355     return start;
3356   }
3357 
3358   /**
3359    *  Arguments:
3360    *
3361    * Inputs:
3362    *   rsp(4)   - int crc
3363    *   rsp(8)   - byte* buf
3364    *   rsp(12)  - int length
3365    *
3366    * Ouput:
3367    *       rax   - int crc result
3368    */
3369   address generate_updateBytesCRC32() {
3370     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
3371 
3372     __ align(CodeEntryAlignment);
3373     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
3374 
3375     address start = __ pc();
3376 
3377     const Register crc   = rdx;  // crc
3378     const Register buf   = rsi;  // source java byte array address
3379     const Register len   = rcx;  // length
3380     const Register table = rdi;  // crc_table address (reuse register)
3381     const Register tmp   = rbx;
3382     assert_different_registers(crc, buf, len, table, tmp, rax);
3383 
3384     BLOCK_COMMENT("Entry:");
3385     __ enter(); // required for proper stackwalking of RuntimeStub frame
3386     __ push(rsi);
3387     __ push(rdi);
3388     __ push(rbx);
3389 
3390     Address crc_arg(rbp, 8 + 0);
3391     Address buf_arg(rbp, 8 + 4);
3392     Address len_arg(rbp, 8 + 8);
3393 
3394     // Load up:
3395     __ movl(crc,   crc_arg);
3396     __ movptr(buf, buf_arg);
3397     __ movl(len,   len_arg);
3398 
3399     __ kernel_crc32(crc, buf, len, table, tmp);
3400 
3401     __ movl(rax, crc);
3402     __ pop(rbx);
3403     __ pop(rdi);
3404     __ pop(rsi);
3405     __ leave(); // required for proper stackwalking of RuntimeStub frame
3406     __ ret(0);
3407 
3408     return start;
3409   }
3410 
3411   /**
3412   *  Arguments:
3413   *
3414   * Inputs:
3415   *   rsp(4)   - int crc
3416   *   rsp(8)   - byte* buf
3417   *   rsp(12)  - int length
3418   *   rsp(16)  - table_start - optional (present only when doing a library_calll,
3419   *              not used by x86 algorithm)
3420   *
3421   * Ouput:
3422   *       rax  - int crc result
3423   */
3424   address generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
3425     assert(UseCRC32CIntrinsics, "need SSE4_2");
3426     __ align(CodeEntryAlignment);
3427     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
3428     address start = __ pc();
3429     const Register crc = rax;  // crc
3430     const Register buf = rcx;  // source java byte array address
3431     const Register len = rdx;  // length
3432     const Register d = rbx;
3433     const Register g = rsi;
3434     const Register h = rdi;
3435     const Register empty = 0; // will never be used, in order not
3436                               // to change a signature for crc32c_IPL_Alg2_Alt2
3437                               // between 64/32 I'm just keeping it here
3438     assert_different_registers(crc, buf, len, d, g, h);
3439 
3440     BLOCK_COMMENT("Entry:");
3441     __ enter(); // required for proper stackwalking of RuntimeStub frame
3442     Address crc_arg(rsp, 4 + 4 + 0); // ESP+4 +
3443                                      // we need to add additional 4 because __ enter
3444                                      // have just pushed ebp on a stack
3445     Address buf_arg(rsp, 4 + 4 + 4);
3446     Address len_arg(rsp, 4 + 4 + 8);
3447       // Load up:
3448       __ movl(crc, crc_arg);
3449       __ movl(buf, buf_arg);
3450       __ movl(len, len_arg);
3451       __ push(d);
3452       __ push(g);
3453       __ push(h);
3454       __ crc32c_ipl_alg2_alt2(crc, buf, len,
3455                               d, g, h,
3456                               empty, empty, empty,
3457                               xmm0, xmm1, xmm2,
3458                               is_pclmulqdq_supported);
3459       __ pop(h);
3460       __ pop(g);
3461       __ pop(d);
3462     __ leave(); // required for proper stackwalking of RuntimeStub frame
3463     __ ret(0);
3464 
3465     return start;
3466   }
3467 
3468  address generate_libmExp() {
3469     address start = __ pc();
3470 
3471     const XMMRegister x0  = xmm0;
3472     const XMMRegister x1  = xmm1;
3473     const XMMRegister x2  = xmm2;
3474     const XMMRegister x3  = xmm3;
3475 
3476     const XMMRegister x4  = xmm4;
3477     const XMMRegister x5  = xmm5;
3478     const XMMRegister x6  = xmm6;
3479     const XMMRegister x7  = xmm7;
3480 
3481     const Register tmp   = rbx;
3482 
3483     BLOCK_COMMENT("Entry:");
3484     __ enter(); // required for proper stackwalking of RuntimeStub frame
3485     __ fast_exp(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3486     __ leave(); // required for proper stackwalking of RuntimeStub frame
3487     __ ret(0);
3488 
3489     return start;
3490 
3491   }
3492 
3493  address generate_libmLog() {
3494    address start = __ pc();
3495 
3496    const XMMRegister x0 = xmm0;
3497    const XMMRegister x1 = xmm1;
3498    const XMMRegister x2 = xmm2;
3499    const XMMRegister x3 = xmm3;
3500 
3501    const XMMRegister x4 = xmm4;
3502    const XMMRegister x5 = xmm5;
3503    const XMMRegister x6 = xmm6;
3504    const XMMRegister x7 = xmm7;
3505 
3506    const Register tmp = rbx;
3507 
3508    BLOCK_COMMENT("Entry:");
3509    __ enter(); // required for proper stackwalking of RuntimeStub frame
3510    __ fast_log(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3511    __ leave(); // required for proper stackwalking of RuntimeStub frame
3512    __ ret(0);
3513 
3514    return start;
3515 
3516  }
3517 
3518  address generate_libmLog10() {
3519    address start = __ pc();
3520 
3521    const XMMRegister x0 = xmm0;
3522    const XMMRegister x1 = xmm1;
3523    const XMMRegister x2 = xmm2;
3524    const XMMRegister x3 = xmm3;
3525 
3526    const XMMRegister x4 = xmm4;
3527    const XMMRegister x5 = xmm5;
3528    const XMMRegister x6 = xmm6;
3529    const XMMRegister x7 = xmm7;
3530 
3531    const Register tmp = rbx;
3532 
3533    BLOCK_COMMENT("Entry:");
3534    __ enter(); // required for proper stackwalking of RuntimeStub frame
3535    __ fast_log10(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3536    __ leave(); // required for proper stackwalking of RuntimeStub frame
3537    __ ret(0);
3538 
3539    return start;
3540 
3541  }
3542 
3543  address generate_libmPow() {
3544    address start = __ pc();
3545 
3546    const XMMRegister x0 = xmm0;
3547    const XMMRegister x1 = xmm1;
3548    const XMMRegister x2 = xmm2;
3549    const XMMRegister x3 = xmm3;
3550 
3551    const XMMRegister x4 = xmm4;
3552    const XMMRegister x5 = xmm5;
3553    const XMMRegister x6 = xmm6;
3554    const XMMRegister x7 = xmm7;
3555 
3556    const Register tmp = rbx;
3557 
3558    BLOCK_COMMENT("Entry:");
3559    __ enter(); // required for proper stackwalking of RuntimeStub frame
3560    __ fast_pow(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3561    __ leave(); // required for proper stackwalking of RuntimeStub frame
3562    __ ret(0);
3563 
3564    return start;
3565 
3566  }
3567 
3568  address generate_libm_reduce_pi04l() {
3569    address start = __ pc();
3570 
3571    BLOCK_COMMENT("Entry:");
3572    __ libm_reduce_pi04l(rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3573 
3574    return start;
3575 
3576  }
3577 
3578  address generate_libm_sin_cos_huge() {
3579    address start = __ pc();
3580 
3581    const XMMRegister x0 = xmm0;
3582    const XMMRegister x1 = xmm1;
3583 
3584    BLOCK_COMMENT("Entry:");
3585    __ libm_sincos_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3586 
3587    return start;
3588 
3589  }
3590 
3591  address generate_libmSin() {
3592    address start = __ pc();
3593 
3594    const XMMRegister x0 = xmm0;
3595    const XMMRegister x1 = xmm1;
3596    const XMMRegister x2 = xmm2;
3597    const XMMRegister x3 = xmm3;
3598 
3599    const XMMRegister x4 = xmm4;
3600    const XMMRegister x5 = xmm5;
3601    const XMMRegister x6 = xmm6;
3602    const XMMRegister x7 = xmm7;
3603 
3604    BLOCK_COMMENT("Entry:");
3605    __ enter(); // required for proper stackwalking of RuntimeStub frame
3606    __ fast_sin(x0, x1, x2, x3, x4, x5, x6, x7, rax, rbx, rdx);
3607    __ leave(); // required for proper stackwalking of RuntimeStub frame
3608    __ ret(0);
3609 
3610    return start;
3611 
3612  }
3613 
3614  address generate_libmCos() {
3615    address start = __ pc();
3616 
3617    const XMMRegister x0 = xmm0;
3618    const XMMRegister x1 = xmm1;
3619    const XMMRegister x2 = xmm2;
3620    const XMMRegister x3 = xmm3;
3621 
3622    const XMMRegister x4 = xmm4;
3623    const XMMRegister x5 = xmm5;
3624    const XMMRegister x6 = xmm6;
3625    const XMMRegister x7 = xmm7;
3626 
3627    const Register tmp = rbx;
3628 
3629    BLOCK_COMMENT("Entry:");
3630    __ enter(); // required for proper stackwalking of RuntimeStub frame
3631    __ fast_cos(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3632    __ leave(); // required for proper stackwalking of RuntimeStub frame
3633    __ ret(0);
3634 
3635    return start;
3636 
3637  }
3638 
3639  address generate_libm_tan_cot_huge() {
3640    address start = __ pc();
3641 
3642    const XMMRegister x0 = xmm0;
3643    const XMMRegister x1 = xmm1;
3644 
3645    BLOCK_COMMENT("Entry:");
3646    __ libm_tancot_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3647 
3648    return start;
3649 
3650  }
3651 
3652  address generate_libmTan() {
3653    address start = __ pc();
3654 
3655    const XMMRegister x0 = xmm0;
3656    const XMMRegister x1 = xmm1;
3657    const XMMRegister x2 = xmm2;
3658    const XMMRegister x3 = xmm3;
3659 
3660    const XMMRegister x4 = xmm4;
3661    const XMMRegister x5 = xmm5;
3662    const XMMRegister x6 = xmm6;
3663    const XMMRegister x7 = xmm7;
3664 
3665    const Register tmp = rbx;
3666 
3667    BLOCK_COMMENT("Entry:");
3668    __ enter(); // required for proper stackwalking of RuntimeStub frame
3669    __ fast_tan(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3670    __ leave(); // required for proper stackwalking of RuntimeStub frame
3671    __ ret(0);
3672 
3673    return start;
3674 
3675  }
3676 
3677   // Safefetch stubs.
3678   void generate_safefetch(const char* name, int size, address* entry,
3679                           address* fault_pc, address* continuation_pc) {
3680     // safefetch signatures:
3681     //   int      SafeFetch32(int*      adr, int      errValue);
3682     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3683 
3684     StubCodeMark mark(this, "StubRoutines", name);
3685 
3686     // Entry point, pc or function descriptor.
3687     *entry = __ pc();
3688 
3689     __ movl(rax, Address(rsp, 0x8));
3690     __ movl(rcx, Address(rsp, 0x4));
3691     // Load *adr into eax, may fault.
3692     *fault_pc = __ pc();
3693     switch (size) {
3694       case 4:
3695         // int32_t
3696         __ movl(rax, Address(rcx, 0));
3697         break;
3698       case 8:
3699         // int64_t
3700         Unimplemented();
3701         break;
3702       default:
3703         ShouldNotReachHere();
3704     }
3705 
3706     // Return errValue or *adr.
3707     *continuation_pc = __ pc();
3708     __ ret(0);
3709   }
3710 
3711  public:
3712   // Information about frame layout at time of blocking runtime call.
3713   // Note that we only have to preserve callee-saved registers since
3714   // the compilers are responsible for supplying a continuation point
3715   // if they expect all registers to be preserved.
3716   enum layout {
3717     thread_off,    // last_java_sp
3718     arg1_off,
3719     arg2_off,
3720     rbp_off,       // callee saved register
3721     ret_pc,
3722     framesize
3723   };
3724 
3725  private:
3726 
3727 #undef  __
3728 #define __ masm->
3729 
3730   //------------------------------------------------------------------------------------------------------------------------
3731   // Continuation point for throwing of implicit exceptions that are not handled in
3732   // the current activation. Fabricates an exception oop and initiates normal
3733   // exception dispatching in this frame.
3734   //
3735   // Previously the compiler (c2) allowed for callee save registers on Java calls.
3736   // This is no longer true after adapter frames were removed but could possibly
3737   // be brought back in the future if the interpreter code was reworked and it
3738   // was deemed worthwhile. The comment below was left to describe what must
3739   // happen here if callee saves were resurrected. As it stands now this stub
3740   // could actually be a vanilla BufferBlob and have now oopMap at all.
3741   // Since it doesn't make much difference we've chosen to leave it the
3742   // way it was in the callee save days and keep the comment.
3743 
3744   // If we need to preserve callee-saved values we need a callee-saved oop map and
3745   // therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.
3746   // If the compiler needs all registers to be preserved between the fault
3747   // point and the exception handler then it must assume responsibility for that in
3748   // AbstractCompiler::continuation_for_implicit_null_exception or
3749   // continuation_for_implicit_division_by_zero_exception. All other implicit
3750   // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
3751   // either at call sites or otherwise assume that stack unwinding will be initiated,
3752   // so caller saved registers were assumed volatile in the compiler.
3753   address generate_throw_exception(const char* name, address runtime_entry,
3754                                    Register arg1 = noreg, Register arg2 = noreg) {
3755 
3756     int insts_size = 256;
3757     int locs_size  = 32;
3758 
3759     CodeBuffer code(name, insts_size, locs_size);
3760     OopMapSet* oop_maps  = new OopMapSet();
3761     MacroAssembler* masm = new MacroAssembler(&code);
3762 
3763     address start = __ pc();
3764 
3765     // This is an inlined and slightly modified version of call_VM
3766     // which has the ability to fetch the return PC out of
3767     // thread-local storage and also sets up last_Java_sp slightly
3768     // differently than the real call_VM
3769     Register java_thread = rbx;
3770     __ get_thread(java_thread);
3771 
3772     __ enter(); // required for proper stackwalking of RuntimeStub frame
3773 
3774     // pc and rbp, already pushed
3775     __ subptr(rsp, (framesize-2) * wordSize); // prolog
3776 
3777     // Frame is now completed as far as size and linkage.
3778 
3779     int frame_complete = __ pc() - start;
3780 
3781     // push java thread (becomes first argument of C function)
3782     __ movptr(Address(rsp, thread_off * wordSize), java_thread);
3783     if (arg1 != noreg) {
3784       __ movptr(Address(rsp, arg1_off * wordSize), arg1);
3785     }
3786     if (arg2 != noreg) {
3787       assert(arg1 != noreg, "missing reg arg");
3788       __ movptr(Address(rsp, arg2_off * wordSize), arg2);
3789     }
3790 
3791     // Set up last_Java_sp and last_Java_fp
3792     __ set_last_Java_frame(java_thread, rsp, rbp, NULL);
3793 
3794     // Call runtime
3795     BLOCK_COMMENT("call runtime_entry");
3796     __ call(RuntimeAddress(runtime_entry));
3797     // Generate oop map
3798     OopMap* map =  new OopMap(framesize, 0);
3799     oop_maps->add_gc_map(__ pc() - start, map);
3800 
3801     // restore the thread (cannot use the pushed argument since arguments
3802     // may be overwritten by C code generated by an optimizing compiler);
3803     // however can use the register value directly if it is callee saved.
3804     __ get_thread(java_thread);
3805 
3806     __ reset_last_Java_frame(java_thread, true, false);
3807 
3808     __ leave(); // required for proper stackwalking of RuntimeStub frame
3809 
3810     // check for pending exceptions
3811 #ifdef ASSERT
3812     Label L;
3813     __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3814     __ jcc(Assembler::notEqual, L);
3815     __ should_not_reach_here();
3816     __ bind(L);
3817 #endif /* ASSERT */
3818     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3819 
3820 
3821     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);
3822     return stub->entry_point();
3823   }
3824 
3825 
3826   void create_control_words() {
3827     // Round to nearest, 53-bit mode, exceptions masked
3828     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
3829     // Round to zero, 53-bit mode, exception mased
3830     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
3831     // Round to nearest, 24-bit mode, exceptions masked
3832     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
3833     // Round to nearest, 64-bit mode, exceptions masked
3834     StubRoutines::_fpu_cntrl_wrd_64    = 0x037F;
3835     // Round to nearest, 64-bit mode, exceptions masked
3836     StubRoutines::_mxcsr_std           = 0x1F80;
3837     // Note: the following two constants are 80-bit values
3838     //       layout is critical for correct loading by FPU.
3839     // Bias for strict fp multiply/divide
3840     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
3841     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
3842     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
3843     // Un-Bias for strict fp multiply/divide
3844     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
3845     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
3846     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
3847   }
3848 
3849   //---------------------------------------------------------------------------
3850   // Initialization
3851 
3852   void generate_initial() {
3853     // Generates all stubs and initializes the entry points
3854 
3855     //------------------------------------------------------------------------------------------------------------------------
3856     // entry points that exist in all platforms
3857     // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
3858     //       the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
3859     StubRoutines::_forward_exception_entry      = generate_forward_exception();
3860 
3861     StubRoutines::_call_stub_entry              =
3862       generate_call_stub(StubRoutines::_call_stub_return_address);
3863     // is referenced by megamorphic call
3864     StubRoutines::_catch_exception_entry        = generate_catch_exception();
3865 
3866     // These are currently used by Solaris/Intel
3867     StubRoutines::_atomic_xchg_entry            = generate_atomic_xchg();
3868 
3869     StubRoutines::_handler_for_unsafe_access_entry =
3870       generate_handler_for_unsafe_access();
3871 
3872     // platform dependent
3873     create_control_words();
3874 
3875     StubRoutines::x86::_verify_mxcsr_entry                 = generate_verify_mxcsr();
3876     StubRoutines::x86::_verify_fpu_cntrl_wrd_entry         = generate_verify_fpu_cntrl_wrd();
3877     StubRoutines::_d2i_wrapper                              = generate_d2i_wrapper(T_INT,
3878                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
3879     StubRoutines::_d2l_wrapper                              = generate_d2i_wrapper(T_LONG,
3880                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
3881 
3882     // Build this early so it's available for the interpreter
3883     StubRoutines::_throw_StackOverflowError_entry          = generate_throw_exception("StackOverflowError throw_exception",
3884                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
3885     StubRoutines::_throw_delayed_StackOverflowError_entry  = generate_throw_exception("delayed StackOverflowError throw_exception",
3886                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_delayed_StackOverflowError));
3887 
3888     if (UseCRC32Intrinsics) {
3889       // set table address before stub generation which use it
3890       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
3891       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
3892     }
3893 
3894     if (UseCRC32CIntrinsics) {
3895       bool supports_clmul = VM_Version::supports_clmul();
3896       StubRoutines::x86::generate_CRC32C_table(supports_clmul);
3897       StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table;
3898       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
3899     }
3900     if (VM_Version::supports_sse2() && UseLibmIntrinsic) {
3901       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3902           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3903           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3904         StubRoutines::x86::_L_2il0floatpacket_0_adr = (address)StubRoutines::x86::_L_2il0floatpacket_0;
3905         StubRoutines::x86::_Pi4Inv_adr = (address)StubRoutines::x86::_Pi4Inv;
3906         StubRoutines::x86::_Pi4x3_adr = (address)StubRoutines::x86::_Pi4x3;
3907         StubRoutines::x86::_Pi4x4_adr = (address)StubRoutines::x86::_Pi4x4;
3908         StubRoutines::x86::_ones_adr = (address)StubRoutines::x86::_ones;
3909       }
3910       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) {
3911         StubRoutines::_dexp = generate_libmExp();
3912       }
3913       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) {
3914         StubRoutines::_dlog = generate_libmLog();
3915       }
3916       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) {
3917         StubRoutines::_dlog10 = generate_libmLog10();
3918       }
3919       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) {
3920         StubRoutines::_dpow = generate_libmPow();
3921       }
3922       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3923         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3924         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3925         StubRoutines::_dlibm_reduce_pi04l = generate_libm_reduce_pi04l();
3926       }
3927       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3928         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3929         StubRoutines::_dlibm_sin_cos_huge = generate_libm_sin_cos_huge();
3930       }
3931       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
3932         StubRoutines::_dsin = generate_libmSin();
3933       }
3934       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3935         StubRoutines::_dcos = generate_libmCos();
3936       }
3937       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3938         StubRoutines::_dlibm_tan_cot_huge = generate_libm_tan_cot_huge();
3939         StubRoutines::_dtan = generate_libmTan();
3940       }
3941     }
3942   }
3943 
3944   void generate_all() {
3945     // Generates all stubs and initializes the entry points
3946 
3947     // These entry points require SharedInfo::stack0 to be set up in non-core builds
3948     // and need to be relocatable, so they each fabricate a RuntimeStub internally.
3949     StubRoutines::_throw_AbstractMethodError_entry         = generate_throw_exception("AbstractMethodError throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
3950     StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
3951     StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
3952 
3953     //------------------------------------------------------------------------------------------------------------------------
3954     // entry points that are platform specific
3955 
3956     // support for verify_oop (must happen after universe_init)
3957     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
3958 
3959     // arraycopy stubs used by compilers
3960     generate_arraycopy_stubs();
3961 
3962     // don't bother generating these AES intrinsic stubs unless global flag is set
3963     if (UseAESIntrinsics) {
3964       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // might be needed by the others
3965 
3966       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
3967       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
3968       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
3969       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
3970     }
3971 
3972     if (UseAESCTRIntrinsics) {
3973       StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
3974       StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
3975     }
3976 
3977     if (UseSHA1Intrinsics) {
3978       StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask();
3979       StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask();
3980       StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
3981       StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
3982     }
3983     if (UseSHA256Intrinsics) {
3984       StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
3985       StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
3986       StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
3987       StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
3988     }
3989 
3990     // Generate GHASH intrinsics code
3991     if (UseGHASHIntrinsics) {
3992       StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
3993       StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
3994       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
3995     }
3996 
3997     // Safefetch stubs.
3998     generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
3999                                                    &StubRoutines::_safefetch32_fault_pc,
4000                                                    &StubRoutines::_safefetch32_continuation_pc);
4001     StubRoutines::_safefetchN_entry           = StubRoutines::_safefetch32_entry;
4002     StubRoutines::_safefetchN_fault_pc        = StubRoutines::_safefetch32_fault_pc;
4003     StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc;
4004   }
4005 
4006 
4007  public:
4008   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
4009     if (all) {
4010       generate_all();
4011     } else {
4012       generate_initial();
4013     }
4014   }
4015 }; // end class declaration
4016 
4017 
4018 void StubGenerator_generate(CodeBuffer* code, bool all) {
4019   StubGenerator g(code, all);
4020 }