1 /*
   2  * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/barrierSetAssembler.hpp"
  30 #include "gc/shared/barrierSetNMethod.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "memory/universe.hpp"
  33 #include "nativeInst_x86.hpp"
  34 #include "oops/instanceOop.hpp"
  35 #include "oops/method.hpp"
  36 #include "oops/objArrayKlass.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "prims/methodHandles.hpp"
  39 #include "runtime/frame.inline.hpp"
  40 #include "runtime/handles.inline.hpp"
  41 #include "runtime/sharedRuntime.hpp"
  42 #include "runtime/stubCodeGenerator.hpp"
  43 #include "runtime/stubRoutines.hpp"
  44 #include "runtime/thread.inline.hpp"
  45 #ifdef COMPILER2
  46 #include "opto/runtime.hpp"
  47 #endif
  48 
  49 // Declaration and definition of StubGenerator (no .hpp file).
  50 // For a more detailed description of the stub routine structure
  51 // see the comment in stubRoutines.hpp
  52 
  53 #define __ _masm->
  54 #define a__ ((Assembler*)_masm)->
  55 
  56 #ifdef PRODUCT
  57 #define BLOCK_COMMENT(str) /* nothing */
  58 #else
  59 #define BLOCK_COMMENT(str) __ block_comment(str)
  60 #endif
  61 
  62 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  63 
  64 const int MXCSR_MASK  = 0xFFC0;  // Mask out any pending exceptions
  65 const int FPU_CNTRL_WRD_MASK = 0xFFFF;
  66 
  67 // -------------------------------------------------------------------------------------------------------------------------
  68 // Stub Code definitions
  69 
  70 class StubGenerator: public StubCodeGenerator {
  71  private:
  72 
  73 #ifdef PRODUCT
  74 #define inc_counter_np(counter) ((void)0)
  75 #else
  76   void inc_counter_np_(int& counter) {
  77     __ incrementl(ExternalAddress((address)&counter));
  78   }
  79 #define inc_counter_np(counter) \
  80   BLOCK_COMMENT("inc_counter " #counter); \
  81   inc_counter_np_(counter);
  82 #endif //PRODUCT
  83 
  84   void inc_copy_counter_np(BasicType t) {
  85 #ifndef PRODUCT
  86     switch (t) {
  87     case T_BYTE:    inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;
  88     case T_SHORT:   inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;
  89     case T_INT:     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;
  90     case T_LONG:    inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;
  91     case T_OBJECT:  inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;
  92     default:        ShouldNotReachHere();
  93     }
  94 #endif //PRODUCT
  95   }
  96 
  97   //------------------------------------------------------------------------------------------------------------------------
  98   // Call stubs are used to call Java from C
  99   //
 100   //    [ return_from_Java     ] <--- rsp
 101   //    [ argument word n      ]
 102   //      ...
 103   // -N [ argument word 1      ]
 104   // -7 [ Possible padding for stack alignment ]
 105   // -6 [ Possible padding for stack alignment ]
 106   // -5 [ Possible padding for stack alignment ]
 107   // -4 [ mxcsr save           ] <--- rsp_after_call
 108   // -3 [ saved rbx,            ]
 109   // -2 [ saved rsi            ]
 110   // -1 [ saved rdi            ]
 111   //  0 [ saved rbp,            ] <--- rbp,
 112   //  1 [ return address       ]
 113   //  2 [ ptr. to call wrapper ]
 114   //  3 [ result               ]
 115   //  4 [ result_type          ]
 116   //  5 [ method               ]
 117   //  6 [ entry_point          ]
 118   //  7 [ parameters           ]
 119   //  8 [ parameter_size       ]
 120   //  9 [ thread               ]
 121 
 122 
 123   address generate_call_stub(address& return_address) {
 124     StubCodeMark mark(this, "StubRoutines", "call_stub");
 125     address start = __ pc();
 126 
 127     // stub code parameters / addresses
 128     assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");
 129     bool  sse_save = false;
 130     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!
 131     const int     locals_count_in_bytes  (4*wordSize);
 132     const Address mxcsr_save    (rbp, -4 * wordSize);
 133     const Address saved_rbx     (rbp, -3 * wordSize);
 134     const Address saved_rsi     (rbp, -2 * wordSize);
 135     const Address saved_rdi     (rbp, -1 * wordSize);
 136     const Address result        (rbp,  3 * wordSize);
 137     const Address result_type   (rbp,  4 * wordSize);
 138     const Address method        (rbp,  5 * wordSize);
 139     const Address entry_point   (rbp,  6 * wordSize);
 140     const Address parameters    (rbp,  7 * wordSize);
 141     const Address parameter_size(rbp,  8 * wordSize);
 142     const Address thread        (rbp,  9 * wordSize); // same as in generate_catch_exception()!
 143     sse_save =  UseSSE > 0;
 144 
 145     // stub code
 146     __ enter();
 147     __ movptr(rcx, parameter_size);              // parameter counter
 148     __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes
 149     __ addptr(rcx, locals_count_in_bytes);       // reserve space for register saves
 150     __ subptr(rsp, rcx);
 151     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
 152 
 153     // save rdi, rsi, & rbx, according to C calling conventions
 154     __ movptr(saved_rdi, rdi);
 155     __ movptr(saved_rsi, rsi);
 156     __ movptr(saved_rbx, rbx);
 157 
 158     // save and initialize %mxcsr
 159     if (sse_save) {
 160       Label skip_ldmx;
 161       __ stmxcsr(mxcsr_save);
 162       __ movl(rax, mxcsr_save);
 163       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 164       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 165       __ cmp32(rax, mxcsr_std);
 166       __ jcc(Assembler::equal, skip_ldmx);
 167       __ ldmxcsr(mxcsr_std);
 168       __ bind(skip_ldmx);
 169     }
 170 
 171     // make sure the control word is correct.
 172     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 173 
 174 #ifdef ASSERT
 175     // make sure we have no pending exceptions
 176     { Label L;
 177       __ movptr(rcx, thread);
 178       __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 179       __ jcc(Assembler::equal, L);
 180       __ stop("StubRoutines::call_stub: entered with pending exception");
 181       __ bind(L);
 182     }
 183 #endif
 184 
 185     // pass parameters if any
 186     BLOCK_COMMENT("pass parameters if any");
 187     Label parameters_done;
 188     __ movl(rcx, parameter_size);  // parameter counter
 189     __ testl(rcx, rcx);
 190     __ jcc(Assembler::zero, parameters_done);
 191 
 192     // parameter passing loop
 193 
 194     Label loop;
 195     // Copy Java parameters in reverse order (receiver last)
 196     // Note that the argument order is inverted in the process
 197     // source is rdx[rcx: N-1..0]
 198     // dest   is rsp[rbx: 0..N-1]
 199 
 200     __ movptr(rdx, parameters);          // parameter pointer
 201     __ xorptr(rbx, rbx);
 202 
 203     __ BIND(loop);
 204 
 205     // get parameter
 206     __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
 207     __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
 208                     Interpreter::expr_offset_in_bytes(0)), rax);          // store parameter
 209     __ increment(rbx);
 210     __ decrement(rcx);
 211     __ jcc(Assembler::notZero, loop);
 212 
 213     // call Java function
 214     __ BIND(parameters_done);
 215     __ movptr(rbx, method);           // get Method*
 216     __ movptr(rax, entry_point);      // get entry_point
 217     __ mov(rsi, rsp);                 // set sender sp
 218     BLOCK_COMMENT("call Java function");
 219     __ call(rax);
 220 
 221     BLOCK_COMMENT("call_stub_return_address:");
 222     return_address = __ pc();
 223 
 224 #ifdef COMPILER2
 225     {
 226       Label L_skip;
 227       if (UseSSE >= 2) {
 228         __ verify_FPU(0, "call_stub_return");
 229       } else {
 230         for (int i = 1; i < 8; i++) {
 231           __ ffree(i);
 232         }
 233 
 234         // UseSSE <= 1 so double result should be left on TOS
 235         __ movl(rsi, result_type);
 236         __ cmpl(rsi, T_DOUBLE);
 237         __ jcc(Assembler::equal, L_skip);
 238         if (UseSSE == 0) {
 239           // UseSSE == 0 so float result should be left on TOS
 240           __ cmpl(rsi, T_FLOAT);
 241           __ jcc(Assembler::equal, L_skip);
 242         }
 243         __ ffree(0);
 244       }
 245       __ BIND(L_skip);
 246     }
 247 #endif // COMPILER2
 248 
 249     // store result depending on type
 250     // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 251     __ movptr(rdi, result);
 252     Label is_long, is_float, is_double, exit;
 253     __ movl(rsi, result_type);
 254     __ cmpl(rsi, T_LONG);
 255     __ jcc(Assembler::equal, is_long);
 256     __ cmpl(rsi, T_FLOAT);
 257     __ jcc(Assembler::equal, is_float);
 258     __ cmpl(rsi, T_DOUBLE);
 259     __ jcc(Assembler::equal, is_double);
 260 
 261     // handle T_INT case
 262     __ movl(Address(rdi, 0), rax);
 263     __ BIND(exit);
 264 
 265     // check that FPU stack is empty
 266     __ verify_FPU(0, "generate_call_stub");
 267 
 268     // pop parameters
 269     __ lea(rsp, rsp_after_call);
 270 
 271     // restore %mxcsr
 272     if (sse_save) {
 273       __ ldmxcsr(mxcsr_save);
 274     }
 275 
 276     // restore rdi, rsi and rbx,
 277     __ movptr(rbx, saved_rbx);
 278     __ movptr(rsi, saved_rsi);
 279     __ movptr(rdi, saved_rdi);
 280     __ addptr(rsp, 4*wordSize);
 281 
 282     // return
 283     __ pop(rbp);
 284     __ ret(0);
 285 
 286     // handle return types different from T_INT
 287     __ BIND(is_long);
 288     __ movl(Address(rdi, 0 * wordSize), rax);
 289     __ movl(Address(rdi, 1 * wordSize), rdx);
 290     __ jmp(exit);
 291 
 292     __ BIND(is_float);
 293     // interpreter uses xmm0 for return values
 294     if (UseSSE >= 1) {
 295       __ movflt(Address(rdi, 0), xmm0);
 296     } else {
 297       __ fstp_s(Address(rdi, 0));
 298     }
 299     __ jmp(exit);
 300 
 301     __ BIND(is_double);
 302     // interpreter uses xmm0 for return values
 303     if (UseSSE >= 2) {
 304       __ movdbl(Address(rdi, 0), xmm0);
 305     } else {
 306       __ fstp_d(Address(rdi, 0));
 307     }
 308     __ jmp(exit);
 309 
 310     return start;
 311   }
 312 
 313 
 314   //------------------------------------------------------------------------------------------------------------------------
 315   // Return point for a Java call if there's an exception thrown in Java code.
 316   // The exception is caught and transformed into a pending exception stored in
 317   // JavaThread that can be tested from within the VM.
 318   //
 319   // Note: Usually the parameters are removed by the callee. In case of an exception
 320   //       crossing an activation frame boundary, that is not the case if the callee
 321   //       is compiled code => need to setup the rsp.
 322   //
 323   // rax,: exception oop
 324 
 325   address generate_catch_exception() {
 326     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 327     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!
 328     const Address thread        (rbp,  9 * wordSize); // same as in generate_call_stub()!
 329     address start = __ pc();
 330 
 331     // get thread directly
 332     __ movptr(rcx, thread);
 333 #ifdef ASSERT
 334     // verify that threads correspond
 335     { Label L;
 336       __ get_thread(rbx);
 337       __ cmpptr(rbx, rcx);
 338       __ jcc(Assembler::equal, L);
 339       __ stop("StubRoutines::catch_exception: threads must correspond");
 340       __ bind(L);
 341     }
 342 #endif
 343     // set pending exception
 344     __ verify_oop(rax);
 345     __ movptr(Address(rcx, Thread::pending_exception_offset()), rax          );
 346     __ lea(Address(rcx, Thread::exception_file_offset   ()),
 347            ExternalAddress((address)__FILE__));
 348     __ movl(Address(rcx, Thread::exception_line_offset   ()), __LINE__ );
 349     // complete return to VM
 350     assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
 351     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 352 
 353     return start;
 354   }
 355 
 356 
 357   //------------------------------------------------------------------------------------------------------------------------
 358   // Continuation point for runtime calls returning with a pending exception.
 359   // The pending exception check happened in the runtime or native call stub.
 360   // The pending exception in Thread is converted into a Java-level exception.
 361   //
 362   // Contract with Java-level exception handlers:
 363   // rax: exception
 364   // rdx: throwing pc
 365   //
 366   // NOTE: At entry of this stub, exception-pc must be on stack !!
 367 
 368   address generate_forward_exception() {
 369     StubCodeMark mark(this, "StubRoutines", "forward exception");
 370     address start = __ pc();
 371     const Register thread = rcx;
 372 
 373     // other registers used in this stub
 374     const Register exception_oop = rax;
 375     const Register handler_addr  = rbx;
 376     const Register exception_pc  = rdx;
 377 
 378     // Upon entry, the sp points to the return address returning into Java
 379     // (interpreted or compiled) code; i.e., the return address becomes the
 380     // throwing pc.
 381     //
 382     // Arguments pushed before the runtime call are still on the stack but
 383     // the exception handler will reset the stack pointer -> ignore them.
 384     // A potential result in registers can be ignored as well.
 385 
 386 #ifdef ASSERT
 387     // make sure this code is only executed if there is a pending exception
 388     { Label L;
 389       __ get_thread(thread);
 390       __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 391       __ jcc(Assembler::notEqual, L);
 392       __ stop("StubRoutines::forward exception: no pending exception (1)");
 393       __ bind(L);
 394     }
 395 #endif
 396 
 397     // compute exception handler into rbx,
 398     __ get_thread(thread);
 399     __ movptr(exception_pc, Address(rsp, 0));
 400     BLOCK_COMMENT("call exception_handler_for_return_address");
 401     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
 402     __ mov(handler_addr, rax);
 403 
 404     // setup rax & rdx, remove return address & clear pending exception
 405     __ get_thread(thread);
 406     __ pop(exception_pc);
 407     __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
 408     __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
 409 
 410 #ifdef ASSERT
 411     // make sure exception is set
 412     { Label L;
 413       __ testptr(exception_oop, exception_oop);
 414       __ jcc(Assembler::notEqual, L);
 415       __ stop("StubRoutines::forward exception: no pending exception (2)");
 416       __ bind(L);
 417     }
 418 #endif
 419 
 420     // Verify that there is really a valid exception in RAX.
 421     __ verify_oop(exception_oop);
 422 
 423     // continue at exception handler (return address removed)
 424     // rax: exception
 425     // rbx: exception handler
 426     // rdx: throwing pc
 427     __ jmp(handler_addr);
 428 
 429     return start;
 430   }
 431 
 432 
 433   //----------------------------------------------------------------------------------------------------
 434   // Support for int32_t Atomic::xchg(int32_t exchange_value, volatile int32_t* dest)
 435   //
 436   // xchg exists as far back as 8086, lock needed for MP only
 437   // Stack layout immediately after call:
 438   //
 439   // 0 [ret addr ] <--- rsp
 440   // 1 [  ex     ]
 441   // 2 [  dest   ]
 442   //
 443   // Result:   *dest <- ex, return (old *dest)
 444   //
 445   // Note: win32 does not currently use this code
 446 
 447   address generate_atomic_xchg() {
 448     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 449     address start = __ pc();
 450 
 451     __ push(rdx);
 452     Address exchange(rsp, 2 * wordSize);
 453     Address dest_addr(rsp, 3 * wordSize);
 454     __ movl(rax, exchange);
 455     __ movptr(rdx, dest_addr);
 456     __ xchgl(rax, Address(rdx, 0));
 457     __ pop(rdx);
 458     __ ret(0);
 459 
 460     return start;
 461   }
 462 
 463   //----------------------------------------------------------------------------------------------------
 464   // Support for void verify_mxcsr()
 465   //
 466   // This routine is used with -Xcheck:jni to verify that native
 467   // JNI code does not return to Java code without restoring the
 468   // MXCSR register to our expected state.
 469 
 470 
 471   address generate_verify_mxcsr() {
 472     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 473     address start = __ pc();
 474 
 475     const Address mxcsr_save(rsp, 0);
 476 
 477     if (CheckJNICalls && UseSSE > 0 ) {
 478       Label ok_ret;
 479       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 480       __ push(rax);
 481       __ subptr(rsp, wordSize);      // allocate a temp location
 482       __ stmxcsr(mxcsr_save);
 483       __ movl(rax, mxcsr_save);
 484       __ andl(rax, MXCSR_MASK);
 485       __ cmp32(rax, mxcsr_std);
 486       __ jcc(Assembler::equal, ok_ret);
 487 
 488       __ warn("MXCSR changed by native JNI code.");
 489 
 490       __ ldmxcsr(mxcsr_std);
 491 
 492       __ bind(ok_ret);
 493       __ addptr(rsp, wordSize);
 494       __ pop(rax);
 495     }
 496 
 497     __ ret(0);
 498 
 499     return start;
 500   }
 501 
 502 
 503   //---------------------------------------------------------------------------
 504   // Support for void verify_fpu_cntrl_wrd()
 505   //
 506   // This routine is used with -Xcheck:jni to verify that native
 507   // JNI code does not return to Java code without restoring the
 508   // FP control word to our expected state.
 509 
 510   address generate_verify_fpu_cntrl_wrd() {
 511     StubCodeMark mark(this, "StubRoutines", "verify_spcw");
 512     address start = __ pc();
 513 
 514     const Address fpu_cntrl_wrd_save(rsp, 0);
 515 
 516     if (CheckJNICalls) {
 517       Label ok_ret;
 518       __ push(rax);
 519       __ subptr(rsp, wordSize);      // allocate a temp location
 520       __ fnstcw(fpu_cntrl_wrd_save);
 521       __ movl(rax, fpu_cntrl_wrd_save);
 522       __ andl(rax, FPU_CNTRL_WRD_MASK);
 523       ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());
 524       __ cmp32(rax, fpu_std);
 525       __ jcc(Assembler::equal, ok_ret);
 526 
 527       __ warn("Floating point control word changed by native JNI code.");
 528 
 529       __ fldcw(fpu_std);
 530 
 531       __ bind(ok_ret);
 532       __ addptr(rsp, wordSize);
 533       __ pop(rax);
 534     }
 535 
 536     __ ret(0);
 537 
 538     return start;
 539   }
 540 
 541   //---------------------------------------------------------------------------
 542   // Wrapper for slow-case handling of double-to-integer conversion
 543   // d2i or f2i fast case failed either because it is nan or because
 544   // of under/overflow.
 545   // Input:  FPU TOS: float value
 546   // Output: rax, (rdx): integer (long) result
 547 
 548   address generate_d2i_wrapper(BasicType t, address fcn) {
 549     StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");
 550     address start = __ pc();
 551 
 552   // Capture info about frame layout
 553   enum layout { FPUState_off         = 0,
 554                 rbp_off              = FPUStateSizeInWords,
 555                 rdi_off,
 556                 rsi_off,
 557                 rcx_off,
 558                 rbx_off,
 559                 saved_argument_off,
 560                 saved_argument_off2, // 2nd half of double
 561                 framesize
 562   };
 563 
 564   assert(FPUStateSizeInWords == 27, "update stack layout");
 565 
 566     // Save outgoing argument to stack across push_FPU_state()
 567     __ subptr(rsp, wordSize * 2);
 568     __ fstp_d(Address(rsp, 0));
 569 
 570     // Save CPU & FPU state
 571     __ push(rbx);
 572     __ push(rcx);
 573     __ push(rsi);
 574     __ push(rdi);
 575     __ push(rbp);
 576     __ push_FPU_state();
 577 
 578     // push_FPU_state() resets the FP top of stack
 579     // Load original double into FP top of stack
 580     __ fld_d(Address(rsp, saved_argument_off * wordSize));
 581     // Store double into stack as outgoing argument
 582     __ subptr(rsp, wordSize*2);
 583     __ fst_d(Address(rsp, 0));
 584 
 585     // Prepare FPU for doing math in C-land
 586     __ empty_FPU_stack();
 587     // Call the C code to massage the double.  Result in EAX
 588     if (t == T_INT)
 589       { BLOCK_COMMENT("SharedRuntime::d2i"); }
 590     else if (t == T_LONG)
 591       { BLOCK_COMMENT("SharedRuntime::d2l"); }
 592     __ call_VM_leaf( fcn, 2 );
 593 
 594     // Restore CPU & FPU state
 595     __ pop_FPU_state();
 596     __ pop(rbp);
 597     __ pop(rdi);
 598     __ pop(rsi);
 599     __ pop(rcx);
 600     __ pop(rbx);
 601     __ addptr(rsp, wordSize * 2);
 602 
 603     __ ret(0);
 604 
 605     return start;
 606   }
 607   //---------------------------------------------------------------------------------------------------
 608 
 609   address generate_vector_mask(const char *stub_name, int32_t mask) {
 610     __ align(CodeEntryAlignment);
 611     StubCodeMark mark(this, "StubRoutines", stub_name);
 612     address start = __ pc();
 613 
 614     for (int i = 0; i < 16; i++) {
 615       __ emit_data(mask, relocInfo::none, 0);
 616     }
 617 
 618     return start;
 619   }
 620 
 621   address generate_vector_mask_long_double(const char *stub_name, int32_t maskhi, int32_t masklo) {
 622     __ align(CodeEntryAlignment);
 623     StubCodeMark mark(this, "StubRoutines", stub_name);
 624     address start = __ pc();
 625 
 626     for (int i = 0; i < 8; i++) {
 627       __ emit_data(masklo, relocInfo::none, 0);
 628       __ emit_data(maskhi, relocInfo::none, 0);
 629     }
 630 
 631     return start;
 632   }
 633 
 634   //----------------------------------------------------------------------------------------------------
 635 
 636   address generate_vector_byte_perm_mask(const char *stub_name) {
 637     __ align(CodeEntryAlignment);
 638     StubCodeMark mark(this, "StubRoutines", stub_name);
 639     address start = __ pc();
 640 
 641     __ emit_data(0x00000001, relocInfo::none, 0);
 642     __ emit_data(0x00000000, relocInfo::none, 0);
 643     __ emit_data(0x00000003, relocInfo::none, 0);
 644     __ emit_data(0x00000000, relocInfo::none, 0);
 645     __ emit_data(0x00000005, relocInfo::none, 0);
 646     __ emit_data(0x00000000, relocInfo::none, 0);
 647     __ emit_data(0x00000007, relocInfo::none, 0);
 648     __ emit_data(0x00000000, relocInfo::none, 0);
 649     __ emit_data(0x00000000, relocInfo::none, 0);
 650     __ emit_data(0x00000000, relocInfo::none, 0);
 651     __ emit_data(0x00000002, relocInfo::none, 0);
 652     __ emit_data(0x00000000, relocInfo::none, 0);
 653     __ emit_data(0x00000004, relocInfo::none, 0);
 654     __ emit_data(0x00000000, relocInfo::none, 0);
 655     __ emit_data(0x00000006, relocInfo::none, 0);
 656     __ emit_data(0x00000000, relocInfo::none, 0);
 657 
 658     return start;
 659   }
 660 
 661   //----------------------------------------------------------------------------------------------------
 662   // Non-destructive plausibility checks for oops
 663 
 664   address generate_verify_oop() {
 665     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 666     address start = __ pc();
 667 
 668     // Incoming arguments on stack after saving rax,:
 669     //
 670     // [tos    ]: saved rdx
 671     // [tos + 1]: saved EFLAGS
 672     // [tos + 2]: return address
 673     // [tos + 3]: char* error message
 674     // [tos + 4]: oop   object to verify
 675     // [tos + 5]: saved rax, - saved by caller and bashed
 676 
 677     Label exit, error;
 678     __ pushf();
 679     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 680     __ push(rdx);                                // save rdx
 681     // make sure object is 'reasonable'
 682     __ movptr(rax, Address(rsp, 4 * wordSize));    // get object
 683     __ testptr(rax, rax);
 684     __ jcc(Assembler::zero, exit);               // if obj is NULL it is ok
 685 
 686     // Check if the oop is in the right area of memory
 687     const int oop_mask = Universe::verify_oop_mask();
 688     const int oop_bits = Universe::verify_oop_bits();
 689     __ mov(rdx, rax);
 690     __ andptr(rdx, oop_mask);
 691     __ cmpptr(rdx, oop_bits);
 692     __ jcc(Assembler::notZero, error);
 693 
 694     // make sure klass is 'reasonable', which is not zero.
 695     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
 696     __ testptr(rax, rax);
 697     __ jcc(Assembler::zero, error);              // if klass is NULL it is broken
 698 
 699     // return if everything seems ok
 700     __ bind(exit);
 701     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 702     __ pop(rdx);                                 // restore rdx
 703     __ popf();                                   // restore EFLAGS
 704     __ ret(3 * wordSize);                        // pop arguments
 705 
 706     // handle errors
 707     __ bind(error);
 708     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 709     __ pop(rdx);                                 // get saved rdx back
 710     __ popf();                                   // get saved EFLAGS off stack -- will be ignored
 711     __ pusha();                                  // push registers (eip = return address & msg are already pushed)
 712     BLOCK_COMMENT("call MacroAssembler::debug");
 713     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 714     __ hlt();
 715     return start;
 716   }
 717 
 718 
 719   // Copy 64 bytes chunks
 720   //
 721   // Inputs:
 722   //   from        - source array address
 723   //   to_from     - destination array address - from
 724   //   qword_count - 8-bytes element count, negative
 725   //
 726   void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
 727     assert( UseSSE >= 2, "supported cpu only" );
 728     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 729 
 730     // Copy 64-byte chunks
 731     __ jmpb(L_copy_64_bytes);
 732     __ align(OptoLoopAlignment);
 733   __ BIND(L_copy_64_bytes_loop);
 734 
 735     if (UseUnalignedLoadStores) {
 736       if (UseAVX > 2) {
 737         __ evmovdqul(xmm0, Address(from, 0), Assembler::AVX_512bit);
 738         __ evmovdqul(Address(from, to_from, Address::times_1, 0), xmm0, Assembler::AVX_512bit);
 739       } else if (UseAVX == 2) {
 740         __ vmovdqu(xmm0, Address(from,  0));
 741         __ vmovdqu(Address(from, to_from, Address::times_1,  0), xmm0);
 742         __ vmovdqu(xmm1, Address(from, 32));
 743         __ vmovdqu(Address(from, to_from, Address::times_1, 32), xmm1);
 744       } else {
 745         __ movdqu(xmm0, Address(from, 0));
 746         __ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
 747         __ movdqu(xmm1, Address(from, 16));
 748         __ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
 749         __ movdqu(xmm2, Address(from, 32));
 750         __ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
 751         __ movdqu(xmm3, Address(from, 48));
 752         __ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
 753       }
 754     } else {
 755       __ movq(xmm0, Address(from, 0));
 756       __ movq(Address(from, to_from, Address::times_1, 0), xmm0);
 757       __ movq(xmm1, Address(from, 8));
 758       __ movq(Address(from, to_from, Address::times_1, 8), xmm1);
 759       __ movq(xmm2, Address(from, 16));
 760       __ movq(Address(from, to_from, Address::times_1, 16), xmm2);
 761       __ movq(xmm3, Address(from, 24));
 762       __ movq(Address(from, to_from, Address::times_1, 24), xmm3);
 763       __ movq(xmm4, Address(from, 32));
 764       __ movq(Address(from, to_from, Address::times_1, 32), xmm4);
 765       __ movq(xmm5, Address(from, 40));
 766       __ movq(Address(from, to_from, Address::times_1, 40), xmm5);
 767       __ movq(xmm6, Address(from, 48));
 768       __ movq(Address(from, to_from, Address::times_1, 48), xmm6);
 769       __ movq(xmm7, Address(from, 56));
 770       __ movq(Address(from, to_from, Address::times_1, 56), xmm7);
 771     }
 772 
 773     __ addl(from, 64);
 774   __ BIND(L_copy_64_bytes);
 775     __ subl(qword_count, 8);
 776     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 777 
 778     if (UseUnalignedLoadStores && (UseAVX == 2)) {
 779       // clean upper bits of YMM registers
 780       __ vpxor(xmm0, xmm0);
 781       __ vpxor(xmm1, xmm1);
 782     }
 783     __ addl(qword_count, 8);
 784     __ jccb(Assembler::zero, L_exit);
 785     //
 786     // length is too short, just copy qwords
 787     //
 788   __ BIND(L_copy_8_bytes);
 789     __ movq(xmm0, Address(from, 0));
 790     __ movq(Address(from, to_from, Address::times_1), xmm0);
 791     __ addl(from, 8);
 792     __ decrement(qword_count);
 793     __ jcc(Assembler::greater, L_copy_8_bytes);
 794   __ BIND(L_exit);
 795   }
 796 
 797   // Copy 64 bytes chunks
 798   //
 799   // Inputs:
 800   //   from        - source array address
 801   //   to_from     - destination array address - from
 802   //   qword_count - 8-bytes element count, negative
 803   //
 804   void mmx_copy_forward(Register from, Register to_from, Register qword_count) {
 805     assert( VM_Version::supports_mmx(), "supported cpu only" );
 806     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 807     // Copy 64-byte chunks
 808     __ jmpb(L_copy_64_bytes);
 809     __ align(OptoLoopAlignment);
 810   __ BIND(L_copy_64_bytes_loop);
 811     __ movq(mmx0, Address(from, 0));
 812     __ movq(mmx1, Address(from, 8));
 813     __ movq(mmx2, Address(from, 16));
 814     __ movq(Address(from, to_from, Address::times_1, 0), mmx0);
 815     __ movq(mmx3, Address(from, 24));
 816     __ movq(Address(from, to_from, Address::times_1, 8), mmx1);
 817     __ movq(mmx4, Address(from, 32));
 818     __ movq(Address(from, to_from, Address::times_1, 16), mmx2);
 819     __ movq(mmx5, Address(from, 40));
 820     __ movq(Address(from, to_from, Address::times_1, 24), mmx3);
 821     __ movq(mmx6, Address(from, 48));
 822     __ movq(Address(from, to_from, Address::times_1, 32), mmx4);
 823     __ movq(mmx7, Address(from, 56));
 824     __ movq(Address(from, to_from, Address::times_1, 40), mmx5);
 825     __ movq(Address(from, to_from, Address::times_1, 48), mmx6);
 826     __ movq(Address(from, to_from, Address::times_1, 56), mmx7);
 827     __ addptr(from, 64);
 828   __ BIND(L_copy_64_bytes);
 829     __ subl(qword_count, 8);
 830     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 831     __ addl(qword_count, 8);
 832     __ jccb(Assembler::zero, L_exit);
 833     //
 834     // length is too short, just copy qwords
 835     //
 836   __ BIND(L_copy_8_bytes);
 837     __ movq(mmx0, Address(from, 0));
 838     __ movq(Address(from, to_from, Address::times_1), mmx0);
 839     __ addptr(from, 8);
 840     __ decrement(qword_count);
 841     __ jcc(Assembler::greater, L_copy_8_bytes);
 842   __ BIND(L_exit);
 843     __ emms();
 844   }
 845 
 846   address generate_disjoint_copy(BasicType t, bool aligned,
 847                                  Address::ScaleFactor sf,
 848                                  address* entry, const char *name,
 849                                  bool dest_uninitialized = false) {
 850     __ align(CodeEntryAlignment);
 851     StubCodeMark mark(this, "StubRoutines", name);
 852     address start = __ pc();
 853 
 854     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
 855     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;
 856 
 857     int shift = Address::times_ptr - sf;
 858 
 859     const Register from     = rsi;  // source array address
 860     const Register to       = rdi;  // destination array address
 861     const Register count    = rcx;  // elements count
 862     const Register to_from  = to;   // (to - from)
 863     const Register saved_to = rdx;  // saved destination array address
 864 
 865     __ enter(); // required for proper stackwalking of RuntimeStub frame
 866     __ push(rsi);
 867     __ push(rdi);
 868     __ movptr(from , Address(rsp, 12+ 4));
 869     __ movptr(to   , Address(rsp, 12+ 8));
 870     __ movl(count, Address(rsp, 12+ 12));
 871 
 872     if (entry != NULL) {
 873       *entry = __ pc(); // Entry point from conjoint arraycopy stub.
 874       BLOCK_COMMENT("Entry:");
 875     }
 876 
 877     if (t == T_OBJECT) {
 878       __ testl(count, count);
 879       __ jcc(Assembler::zero, L_0_count);
 880     }
 881 
 882     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
 883     if (dest_uninitialized) {
 884       decorators |= IS_DEST_UNINITIALIZED;
 885     }
 886     if (aligned) {
 887       decorators |= ARRAYCOPY_ALIGNED;
 888     }
 889 
 890     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 891     bs->arraycopy_prologue(_masm, decorators, t, from, to, count);
 892     {
 893       bool add_entry = (t != T_OBJECT && (!aligned || t == T_INT));
 894       // UnsafeCopyMemory page error: continue after ucm
 895       UnsafeCopyMemoryMark ucmm(this, add_entry, true);
 896       __ subptr(to, from); // to --> to_from
 897       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
 898       __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
 899       if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 900         // align source address at 4 bytes address boundary
 901         if (t == T_BYTE) {
 902           // One byte misalignment happens only for byte arrays
 903           __ testl(from, 1);
 904           __ jccb(Assembler::zero, L_skip_align1);
 905           __ movb(rax, Address(from, 0));
 906           __ movb(Address(from, to_from, Address::times_1, 0), rax);
 907           __ increment(from);
 908           __ decrement(count);
 909         __ BIND(L_skip_align1);
 910         }
 911         // Two bytes misalignment happens only for byte and short (char) arrays
 912         __ testl(from, 2);
 913         __ jccb(Assembler::zero, L_skip_align2);
 914         __ movw(rax, Address(from, 0));
 915         __ movw(Address(from, to_from, Address::times_1, 0), rax);
 916         __ addptr(from, 2);
 917         __ subl(count, 1<<(shift-1));
 918       __ BIND(L_skip_align2);
 919       }
 920       if (!VM_Version::supports_mmx()) {
 921         __ mov(rax, count);      // save 'count'
 922         __ shrl(count, shift); // bytes count
 923         __ addptr(to_from, from);// restore 'to'
 924         __ rep_mov();
 925         __ subptr(to_from, from);// restore 'to_from'
 926         __ mov(count, rax);      // restore 'count'
 927         __ jmpb(L_copy_2_bytes); // all dwords were copied
 928       } else {
 929         if (!UseUnalignedLoadStores) {
 930           // align to 8 bytes, we know we are 4 byte aligned to start
 931           __ testptr(from, 4);
 932           __ jccb(Assembler::zero, L_copy_64_bytes);
 933           __ movl(rax, Address(from, 0));
 934           __ movl(Address(from, to_from, Address::times_1, 0), rax);
 935           __ addptr(from, 4);
 936           __ subl(count, 1<<shift);
 937          }
 938       __ BIND(L_copy_64_bytes);
 939         __ mov(rax, count);
 940         __ shrl(rax, shift+1);  // 8 bytes chunk count
 941         //
 942         // Copy 8-byte chunks through MMX registers, 8 per iteration of the loop
 943         //
 944         if (UseXMMForArrayCopy) {
 945           xmm_copy_forward(from, to_from, rax);
 946         } else {
 947           mmx_copy_forward(from, to_from, rax);
 948         }
 949       }
 950       // copy tailing dword
 951     __ BIND(L_copy_4_bytes);
 952       __ testl(count, 1<<shift);
 953       __ jccb(Assembler::zero, L_copy_2_bytes);
 954       __ movl(rax, Address(from, 0));
 955       __ movl(Address(from, to_from, Address::times_1, 0), rax);
 956       if (t == T_BYTE || t == T_SHORT) {
 957         __ addptr(from, 4);
 958       __ BIND(L_copy_2_bytes);
 959         // copy tailing word
 960         __ testl(count, 1<<(shift-1));
 961         __ jccb(Assembler::zero, L_copy_byte);
 962         __ movw(rax, Address(from, 0));
 963         __ movw(Address(from, to_from, Address::times_1, 0), rax);
 964         if (t == T_BYTE) {
 965           __ addptr(from, 2);
 966         __ BIND(L_copy_byte);
 967           // copy tailing byte
 968           __ testl(count, 1);
 969           __ jccb(Assembler::zero, L_exit);
 970           __ movb(rax, Address(from, 0));
 971           __ movb(Address(from, to_from, Address::times_1, 0), rax);
 972         __ BIND(L_exit);
 973         } else {
 974         __ BIND(L_copy_byte);
 975         }
 976       } else {
 977       __ BIND(L_copy_2_bytes);
 978       }
 979     }
 980 
 981     if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
 982       __ emms();
 983     }
 984     __ movl(count, Address(rsp, 12+12)); // reread 'count'
 985     bs->arraycopy_epilogue(_masm, decorators, t, from, to, count);
 986 
 987     if (t == T_OBJECT) {
 988     __ BIND(L_0_count);
 989     }
 990     inc_copy_counter_np(t);
 991     __ pop(rdi);
 992     __ pop(rsi);
 993     __ leave(); // required for proper stackwalking of RuntimeStub frame
 994     __ vzeroupper();
 995     __ xorptr(rax, rax); // return 0
 996     __ ret(0);
 997     return start;
 998   }
 999 
1000 
1001   address generate_fill(BasicType t, bool aligned, const char *name) {
1002     __ align(CodeEntryAlignment);
1003     StubCodeMark mark(this, "StubRoutines", name);
1004     address start = __ pc();
1005 
1006     BLOCK_COMMENT("Entry:");
1007 
1008     const Register to       = rdi;  // source array address
1009     const Register value    = rdx;  // value
1010     const Register count    = rsi;  // elements count
1011 
1012     __ enter(); // required for proper stackwalking of RuntimeStub frame
1013     __ push(rsi);
1014     __ push(rdi);
1015     __ movptr(to   , Address(rsp, 12+ 4));
1016     __ movl(value, Address(rsp, 12+ 8));
1017     __ movl(count, Address(rsp, 12+ 12));
1018 
1019     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1020 
1021     __ pop(rdi);
1022     __ pop(rsi);
1023     __ leave(); // required for proper stackwalking of RuntimeStub frame
1024     __ ret(0);
1025     return start;
1026   }
1027 
1028   address generate_conjoint_copy(BasicType t, bool aligned,
1029                                  Address::ScaleFactor sf,
1030                                  address nooverlap_target,
1031                                  address* entry, const char *name,
1032                                  bool dest_uninitialized = false) {
1033     __ align(CodeEntryAlignment);
1034     StubCodeMark mark(this, "StubRoutines", name);
1035     address start = __ pc();
1036 
1037     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
1038     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;
1039 
1040     int shift = Address::times_ptr - sf;
1041 
1042     const Register src   = rax;  // source array address
1043     const Register dst   = rdx;  // destination array address
1044     const Register from  = rsi;  // source array address
1045     const Register to    = rdi;  // destination array address
1046     const Register count = rcx;  // elements count
1047     const Register end   = rax;  // array end address
1048 
1049     __ enter(); // required for proper stackwalking of RuntimeStub frame
1050     __ push(rsi);
1051     __ push(rdi);
1052     __ movptr(src  , Address(rsp, 12+ 4));   // from
1053     __ movptr(dst  , Address(rsp, 12+ 8));   // to
1054     __ movl2ptr(count, Address(rsp, 12+12)); // count
1055 
1056     if (entry != NULL) {
1057       *entry = __ pc(); // Entry point from generic arraycopy stub.
1058       BLOCK_COMMENT("Entry:");
1059     }
1060 
1061     // nooverlap_target expects arguments in rsi and rdi.
1062     __ mov(from, src);
1063     __ mov(to  , dst);
1064 
1065     // arrays overlap test: dispatch to disjoint stub if necessary.
1066     RuntimeAddress nooverlap(nooverlap_target);
1067     __ cmpptr(dst, src);
1068     __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
1069     __ jump_cc(Assembler::belowEqual, nooverlap);
1070     __ cmpptr(dst, end);
1071     __ jump_cc(Assembler::aboveEqual, nooverlap);
1072 
1073     if (t == T_OBJECT) {
1074       __ testl(count, count);
1075       __ jcc(Assembler::zero, L_0_count);
1076     }
1077 
1078     DecoratorSet decorators = IN_HEAP | IS_ARRAY;
1079     if (dest_uninitialized) {
1080       decorators |= IS_DEST_UNINITIALIZED;
1081     }
1082     if (aligned) {
1083       decorators |= ARRAYCOPY_ALIGNED;
1084     }
1085 
1086     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1087     bs->arraycopy_prologue(_masm, decorators, t, from, to, count);
1088 
1089     {
1090       bool add_entry = (t != T_OBJECT && (!aligned || t == T_INT));
1091       // UnsafeCopyMemory page error: continue after ucm
1092       UnsafeCopyMemoryMark ucmm(this, add_entry, true);
1093       // copy from high to low
1094       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1095       __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
1096       if (t == T_BYTE || t == T_SHORT) {
1097         // Align the end of destination array at 4 bytes address boundary
1098         __ lea(end, Address(dst, count, sf, 0));
1099         if (t == T_BYTE) {
1100           // One byte misalignment happens only for byte arrays
1101           __ testl(end, 1);
1102           __ jccb(Assembler::zero, L_skip_align1);
1103           __ decrement(count);
1104           __ movb(rdx, Address(from, count, sf, 0));
1105           __ movb(Address(to, count, sf, 0), rdx);
1106         __ BIND(L_skip_align1);
1107         }
1108         // Two bytes misalignment happens only for byte and short (char) arrays
1109         __ testl(end, 2);
1110         __ jccb(Assembler::zero, L_skip_align2);
1111         __ subptr(count, 1<<(shift-1));
1112         __ movw(rdx, Address(from, count, sf, 0));
1113         __ movw(Address(to, count, sf, 0), rdx);
1114       __ BIND(L_skip_align2);
1115         __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1116         __ jcc(Assembler::below, L_copy_4_bytes);
1117       }
1118 
1119       if (!VM_Version::supports_mmx()) {
1120         __ std();
1121         __ mov(rax, count); // Save 'count'
1122         __ mov(rdx, to);    // Save 'to'
1123         __ lea(rsi, Address(from, count, sf, -4));
1124         __ lea(rdi, Address(to  , count, sf, -4));
1125         __ shrptr(count, shift); // bytes count
1126         __ rep_mov();
1127         __ cld();
1128         __ mov(count, rax); // restore 'count'
1129         __ andl(count, (1<<shift)-1);      // mask the number of rest elements
1130         __ movptr(from, Address(rsp, 12+4)); // reread 'from'
1131         __ mov(to, rdx);   // restore 'to'
1132         __ jmpb(L_copy_2_bytes); // all dword were copied
1133       } else {
1134         // Align to 8 bytes the end of array. It is aligned to 4 bytes already.
1135         __ testptr(end, 4);
1136         __ jccb(Assembler::zero, L_copy_8_bytes);
1137         __ subl(count, 1<<shift);
1138         __ movl(rdx, Address(from, count, sf, 0));
1139         __ movl(Address(to, count, sf, 0), rdx);
1140         __ jmpb(L_copy_8_bytes);
1141 
1142         __ align(OptoLoopAlignment);
1143         // Move 8 bytes
1144       __ BIND(L_copy_8_bytes_loop);
1145         if (UseXMMForArrayCopy) {
1146           __ movq(xmm0, Address(from, count, sf, 0));
1147           __ movq(Address(to, count, sf, 0), xmm0);
1148         } else {
1149           __ movq(mmx0, Address(from, count, sf, 0));
1150           __ movq(Address(to, count, sf, 0), mmx0);
1151         }
1152       __ BIND(L_copy_8_bytes);
1153         __ subl(count, 2<<shift);
1154         __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1155         __ addl(count, 2<<shift);
1156         if (!UseXMMForArrayCopy) {
1157           __ emms();
1158         }
1159       }
1160     __ BIND(L_copy_4_bytes);
1161       // copy prefix qword
1162       __ testl(count, 1<<shift);
1163       __ jccb(Assembler::zero, L_copy_2_bytes);
1164       __ movl(rdx, Address(from, count, sf, -4));
1165       __ movl(Address(to, count, sf, -4), rdx);
1166 
1167       if (t == T_BYTE || t == T_SHORT) {
1168           __ subl(count, (1<<shift));
1169         __ BIND(L_copy_2_bytes);
1170           // copy prefix dword
1171           __ testl(count, 1<<(shift-1));
1172           __ jccb(Assembler::zero, L_copy_byte);
1173           __ movw(rdx, Address(from, count, sf, -2));
1174           __ movw(Address(to, count, sf, -2), rdx);
1175           if (t == T_BYTE) {
1176             __ subl(count, 1<<(shift-1));
1177           __ BIND(L_copy_byte);
1178             // copy prefix byte
1179             __ testl(count, 1);
1180             __ jccb(Assembler::zero, L_exit);
1181             __ movb(rdx, Address(from, 0));
1182             __ movb(Address(to, 0), rdx);
1183           __ BIND(L_exit);
1184           } else {
1185           __ BIND(L_copy_byte);
1186           }
1187       } else {
1188       __ BIND(L_copy_2_bytes);
1189       }
1190     }
1191 
1192     if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
1193       __ emms();
1194     }
1195     __ movl2ptr(count, Address(rsp, 12+12)); // reread count
1196     bs->arraycopy_epilogue(_masm, decorators, t, from, to, count);
1197 
1198     if (t == T_OBJECT) {
1199     __ BIND(L_0_count);
1200     }
1201     inc_copy_counter_np(t);
1202     __ pop(rdi);
1203     __ pop(rsi);
1204     __ leave(); // required for proper stackwalking of RuntimeStub frame
1205     __ xorptr(rax, rax); // return 0
1206     __ ret(0);
1207     return start;
1208   }
1209 
1210 
1211   address generate_disjoint_long_copy(address* entry, const char *name) {
1212     __ align(CodeEntryAlignment);
1213     StubCodeMark mark(this, "StubRoutines", name);
1214     address start = __ pc();
1215 
1216     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1217     const Register from       = rax;  // source array address
1218     const Register to         = rdx;  // destination array address
1219     const Register count      = rcx;  // elements count
1220     const Register to_from    = rdx;  // (to - from)
1221 
1222     __ enter(); // required for proper stackwalking of RuntimeStub frame
1223     __ movptr(from , Address(rsp, 8+0));       // from
1224     __ movptr(to   , Address(rsp, 8+4));       // to
1225     __ movl2ptr(count, Address(rsp, 8+8));     // count
1226 
1227     *entry = __ pc(); // Entry point from conjoint arraycopy stub.
1228     BLOCK_COMMENT("Entry:");
1229 
1230     {
1231       // UnsafeCopyMemory page error: continue after ucm
1232       UnsafeCopyMemoryMark ucmm(this, true, true);
1233       __ subptr(to, from); // to --> to_from
1234       if (VM_Version::supports_mmx()) {
1235         if (UseXMMForArrayCopy) {
1236           xmm_copy_forward(from, to_from, count);
1237         } else {
1238           mmx_copy_forward(from, to_from, count);
1239         }
1240       } else {
1241         __ jmpb(L_copy_8_bytes);
1242         __ align(OptoLoopAlignment);
1243       __ BIND(L_copy_8_bytes_loop);
1244         __ fild_d(Address(from, 0));
1245         __ fistp_d(Address(from, to_from, Address::times_1));
1246         __ addptr(from, 8);
1247       __ BIND(L_copy_8_bytes);
1248         __ decrement(count);
1249         __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1250       }
1251     }
1252     if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
1253       __ emms();
1254     }
1255     inc_copy_counter_np(T_LONG);
1256     __ leave(); // required for proper stackwalking of RuntimeStub frame
1257     __ vzeroupper();
1258     __ xorptr(rax, rax); // return 0
1259     __ ret(0);
1260     return start;
1261   }
1262 
1263   address generate_conjoint_long_copy(address nooverlap_target,
1264                                       address* entry, const char *name) {
1265     __ align(CodeEntryAlignment);
1266     StubCodeMark mark(this, "StubRoutines", name);
1267     address start = __ pc();
1268 
1269     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1270     const Register from       = rax;  // source array address
1271     const Register to         = rdx;  // destination array address
1272     const Register count      = rcx;  // elements count
1273     const Register end_from   = rax;  // source array end address
1274 
1275     __ enter(); // required for proper stackwalking of RuntimeStub frame
1276     __ movptr(from , Address(rsp, 8+0));       // from
1277     __ movptr(to   , Address(rsp, 8+4));       // to
1278     __ movl2ptr(count, Address(rsp, 8+8));     // count
1279 
1280     *entry = __ pc(); // Entry point from generic arraycopy stub.
1281     BLOCK_COMMENT("Entry:");
1282 
1283     // arrays overlap test
1284     __ cmpptr(to, from);
1285     RuntimeAddress nooverlap(nooverlap_target);
1286     __ jump_cc(Assembler::belowEqual, nooverlap);
1287     __ lea(end_from, Address(from, count, Address::times_8, 0));
1288     __ cmpptr(to, end_from);
1289     __ movptr(from, Address(rsp, 8));  // from
1290     __ jump_cc(Assembler::aboveEqual, nooverlap);
1291 
1292     {
1293       // UnsafeCopyMemory page error: continue after ucm
1294       UnsafeCopyMemoryMark ucmm(this, true, true);
1295 
1296       __ jmpb(L_copy_8_bytes);
1297 
1298       __ align(OptoLoopAlignment);
1299     __ BIND(L_copy_8_bytes_loop);
1300       if (VM_Version::supports_mmx()) {
1301         if (UseXMMForArrayCopy) {
1302           __ movq(xmm0, Address(from, count, Address::times_8));
1303           __ movq(Address(to, count, Address::times_8), xmm0);
1304         } else {
1305           __ movq(mmx0, Address(from, count, Address::times_8));
1306           __ movq(Address(to, count, Address::times_8), mmx0);
1307         }
1308       } else {
1309         __ fild_d(Address(from, count, Address::times_8));
1310         __ fistp_d(Address(to, count, Address::times_8));
1311       }
1312     __ BIND(L_copy_8_bytes);
1313       __ decrement(count);
1314       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1315 
1316     }
1317     if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
1318       __ emms();
1319     }
1320     inc_copy_counter_np(T_LONG);
1321     __ leave(); // required for proper stackwalking of RuntimeStub frame
1322     __ xorptr(rax, rax); // return 0
1323     __ ret(0);
1324     return start;
1325   }
1326 
1327 
1328   // Helper for generating a dynamic type check.
1329   // The sub_klass must be one of {rbx, rdx, rsi}.
1330   // The temp is killed.
1331   void generate_type_check(Register sub_klass,
1332                            Address& super_check_offset_addr,
1333                            Address& super_klass_addr,
1334                            Register temp,
1335                            Label* L_success, Label* L_failure) {
1336     BLOCK_COMMENT("type_check:");
1337 
1338     Label L_fallthrough;
1339 #define LOCAL_JCC(assembler_con, label_ptr)                             \
1340     if (label_ptr != NULL)  __ jcc(assembler_con, *(label_ptr));        \
1341     else                    __ jcc(assembler_con, L_fallthrough) /*omit semi*/
1342 
1343     // The following is a strange variation of the fast path which requires
1344     // one less register, because needed values are on the argument stack.
1345     // __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
1346     //                                  L_success, L_failure, NULL);
1347     assert_different_registers(sub_klass, temp);
1348 
1349     int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1350 
1351     // if the pointers are equal, we are done (e.g., String[] elements)
1352     __ cmpptr(sub_klass, super_klass_addr);
1353     LOCAL_JCC(Assembler::equal, L_success);
1354 
1355     // check the supertype display:
1356     __ movl2ptr(temp, super_check_offset_addr);
1357     Address super_check_addr(sub_klass, temp, Address::times_1, 0);
1358     __ movptr(temp, super_check_addr); // load displayed supertype
1359     __ cmpptr(temp, super_klass_addr); // test the super type
1360     LOCAL_JCC(Assembler::equal, L_success);
1361 
1362     // if it was a primary super, we can just fail immediately
1363     __ cmpl(super_check_offset_addr, sc_offset);
1364     LOCAL_JCC(Assembler::notEqual, L_failure);
1365 
1366     // The repne_scan instruction uses fixed registers, which will get spilled.
1367     // We happen to know this works best when super_klass is in rax.
1368     Register super_klass = temp;
1369     __ movptr(super_klass, super_klass_addr);
1370     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
1371                                      L_success, L_failure);
1372 
1373     __ bind(L_fallthrough);
1374 
1375     if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }
1376     if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }
1377 
1378 #undef LOCAL_JCC
1379   }
1380 
1381   //
1382   //  Generate checkcasting array copy stub
1383   //
1384   //  Input:
1385   //    4(rsp)   - source array address
1386   //    8(rsp)   - destination array address
1387   //   12(rsp)   - element count, can be zero
1388   //   16(rsp)   - size_t ckoff (super_check_offset)
1389   //   20(rsp)   - oop ckval (super_klass)
1390   //
1391   //  Output:
1392   //    rax, ==  0  -  success
1393   //    rax, == -1^K - failure, where K is partial transfer count
1394   //
1395   address generate_checkcast_copy(const char *name, address* entry, bool dest_uninitialized = false) {
1396     __ align(CodeEntryAlignment);
1397     StubCodeMark mark(this, "StubRoutines", name);
1398     address start = __ pc();
1399 
1400     Label L_load_element, L_store_element, L_do_card_marks, L_done;
1401 
1402     // register use:
1403     //  rax, rdx, rcx -- loop control (end_from, end_to, count)
1404     //  rdi, rsi      -- element access (oop, klass)
1405     //  rbx,           -- temp
1406     const Register from       = rax;    // source array address
1407     const Register to         = rdx;    // destination array address
1408     const Register length     = rcx;    // elements count
1409     const Register elem       = rdi;    // each oop copied
1410     const Register elem_klass = rsi;    // each elem._klass (sub_klass)
1411     const Register temp       = rbx;    // lone remaining temp
1412 
1413     __ enter(); // required for proper stackwalking of RuntimeStub frame
1414 
1415     __ push(rsi);
1416     __ push(rdi);
1417     __ push(rbx);
1418 
1419     Address   from_arg(rsp, 16+ 4);     // from
1420     Address     to_arg(rsp, 16+ 8);     // to
1421     Address length_arg(rsp, 16+12);     // elements count
1422     Address  ckoff_arg(rsp, 16+16);     // super_check_offset
1423     Address  ckval_arg(rsp, 16+20);     // super_klass
1424 
1425     // Load up:
1426     __ movptr(from,     from_arg);
1427     __ movptr(to,         to_arg);
1428     __ movl2ptr(length, length_arg);
1429 
1430     if (entry != NULL) {
1431       *entry = __ pc(); // Entry point from generic arraycopy stub.
1432       BLOCK_COMMENT("Entry:");
1433     }
1434 
1435     //---------------------------------------------------------------
1436     // Assembler stub will be used for this call to arraycopy
1437     // if the two arrays are subtypes of Object[] but the
1438     // destination array type is not equal to or a supertype
1439     // of the source type.  Each element must be separately
1440     // checked.
1441 
1442     // Loop-invariant addresses.  They are exclusive end pointers.
1443     Address end_from_addr(from, length, Address::times_ptr, 0);
1444     Address   end_to_addr(to,   length, Address::times_ptr, 0);
1445 
1446     Register end_from = from;           // re-use
1447     Register end_to   = to;             // re-use
1448     Register count    = length;         // re-use
1449 
1450     // Loop-variant addresses.  They assume post-incremented count < 0.
1451     Address from_element_addr(end_from, count, Address::times_ptr, 0);
1452     Address   to_element_addr(end_to,   count, Address::times_ptr, 0);
1453     Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
1454 
1455     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST;
1456     if (dest_uninitialized) {
1457       decorators |= IS_DEST_UNINITIALIZED;
1458     }
1459 
1460     BasicType type = T_OBJECT;
1461     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1462     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1463 
1464     // Copy from low to high addresses, indexed from the end of each array.
1465     __ lea(end_from, end_from_addr);
1466     __ lea(end_to,   end_to_addr);
1467     assert(length == count, "");        // else fix next line:
1468     __ negptr(count);                   // negate and test the length
1469     __ jccb(Assembler::notZero, L_load_element);
1470 
1471     // Empty array:  Nothing to do.
1472     __ xorptr(rax, rax);                  // return 0 on (trivial) success
1473     __ jmp(L_done);
1474 
1475     // ======== begin loop ========
1476     // (Loop is rotated; its entry is L_load_element.)
1477     // Loop control:
1478     //   for (count = -count; count != 0; count++)
1479     // Base pointers src, dst are biased by 8*count,to last element.
1480     __ align(OptoLoopAlignment);
1481 
1482     __ BIND(L_store_element);
1483     __ movptr(to_element_addr, elem);     // store the oop
1484     __ increment(count);                // increment the count toward zero
1485     __ jccb(Assembler::zero, L_do_card_marks);
1486 
1487     // ======== loop entry is here ========
1488     __ BIND(L_load_element);
1489     __ movptr(elem, from_element_addr);   // load the oop
1490     __ testptr(elem, elem);
1491     __ jccb(Assembler::zero, L_store_element);
1492 
1493     // (Could do a trick here:  Remember last successful non-null
1494     // element stored and make a quick oop equality check on it.)
1495 
1496     __ movptr(elem_klass, elem_klass_addr); // query the object klass
1497     generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,
1498                         &L_store_element, NULL);
1499     // (On fall-through, we have failed the element type check.)
1500     // ======== end loop ========
1501 
1502     // It was a real error; we must depend on the caller to finish the job.
1503     // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
1504     // Emit GC store barriers for the oops we have copied (length_arg + count),
1505     // and report their number to the caller.
1506     assert_different_registers(to, count, rax);
1507     Label L_post_barrier;
1508     __ addl(count, length_arg);         // transfers = (length - remaining)
1509     __ movl2ptr(rax, count);            // save the value
1510     __ notptr(rax);                     // report (-1^K) to caller (does not affect flags)
1511     __ jccb(Assembler::notZero, L_post_barrier);
1512     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
1513 
1514     // Come here on success only.
1515     __ BIND(L_do_card_marks);
1516     __ xorptr(rax, rax);                // return 0 on success
1517     __ movl2ptr(count, length_arg);
1518 
1519     __ BIND(L_post_barrier);
1520     __ movptr(to, to_arg);              // reload
1521     bs->arraycopy_epilogue(_masm, decorators, type, from, to, count);
1522 
1523     // Common exit point (success or failure).
1524     __ BIND(L_done);
1525     __ pop(rbx);
1526     __ pop(rdi);
1527     __ pop(rsi);
1528     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1529     __ leave(); // required for proper stackwalking of RuntimeStub frame
1530     __ ret(0);
1531 
1532     return start;
1533   }
1534 
1535   //
1536   //  Generate 'unsafe' array copy stub
1537   //  Though just as safe as the other stubs, it takes an unscaled
1538   //  size_t argument instead of an element count.
1539   //
1540   //  Input:
1541   //    4(rsp)   - source array address
1542   //    8(rsp)   - destination array address
1543   //   12(rsp)   - byte count, can be zero
1544   //
1545   //  Output:
1546   //    rax, ==  0  -  success
1547   //    rax, == -1  -  need to call System.arraycopy
1548   //
1549   // Examines the alignment of the operands and dispatches
1550   // to a long, int, short, or byte copy loop.
1551   //
1552   address generate_unsafe_copy(const char *name,
1553                                address byte_copy_entry,
1554                                address short_copy_entry,
1555                                address int_copy_entry,
1556                                address long_copy_entry) {
1557 
1558     Label L_long_aligned, L_int_aligned, L_short_aligned;
1559 
1560     __ align(CodeEntryAlignment);
1561     StubCodeMark mark(this, "StubRoutines", name);
1562     address start = __ pc();
1563 
1564     const Register from       = rax;  // source array address
1565     const Register to         = rdx;  // destination array address
1566     const Register count      = rcx;  // elements count
1567 
1568     __ enter(); // required for proper stackwalking of RuntimeStub frame
1569     __ push(rsi);
1570     __ push(rdi);
1571     Address  from_arg(rsp, 12+ 4);      // from
1572     Address    to_arg(rsp, 12+ 8);      // to
1573     Address count_arg(rsp, 12+12);      // byte count
1574 
1575     // Load up:
1576     __ movptr(from ,  from_arg);
1577     __ movptr(to   ,    to_arg);
1578     __ movl2ptr(count, count_arg);
1579 
1580     // bump this on entry, not on exit:
1581     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
1582 
1583     const Register bits = rsi;
1584     __ mov(bits, from);
1585     __ orptr(bits, to);
1586     __ orptr(bits, count);
1587 
1588     __ testl(bits, BytesPerLong-1);
1589     __ jccb(Assembler::zero, L_long_aligned);
1590 
1591     __ testl(bits, BytesPerInt-1);
1592     __ jccb(Assembler::zero, L_int_aligned);
1593 
1594     __ testl(bits, BytesPerShort-1);
1595     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
1596 
1597     __ BIND(L_short_aligned);
1598     __ shrptr(count, LogBytesPerShort); // size => short_count
1599     __ movl(count_arg, count);          // update 'count'
1600     __ jump(RuntimeAddress(short_copy_entry));
1601 
1602     __ BIND(L_int_aligned);
1603     __ shrptr(count, LogBytesPerInt); // size => int_count
1604     __ movl(count_arg, count);          // update 'count'
1605     __ jump(RuntimeAddress(int_copy_entry));
1606 
1607     __ BIND(L_long_aligned);
1608     __ shrptr(count, LogBytesPerLong); // size => qword_count
1609     __ movl(count_arg, count);          // update 'count'
1610     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1611     __ pop(rsi);
1612     __ jump(RuntimeAddress(long_copy_entry));
1613 
1614     return start;
1615   }
1616 
1617 
1618   // Perform range checks on the proposed arraycopy.
1619   // Smashes src_pos and dst_pos.  (Uses them up for temps.)
1620   void arraycopy_range_checks(Register src,
1621                               Register src_pos,
1622                               Register dst,
1623                               Register dst_pos,
1624                               Address& length,
1625                               Label& L_failed) {
1626     BLOCK_COMMENT("arraycopy_range_checks:");
1627     const Register src_end = src_pos;   // source array end position
1628     const Register dst_end = dst_pos;   // destination array end position
1629     __ addl(src_end, length); // src_pos + length
1630     __ addl(dst_end, length); // dst_pos + length
1631 
1632     //  if (src_pos + length > arrayOop(src)->length() ) FAIL;
1633     __ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
1634     __ jcc(Assembler::above, L_failed);
1635 
1636     //  if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
1637     __ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
1638     __ jcc(Assembler::above, L_failed);
1639 
1640     BLOCK_COMMENT("arraycopy_range_checks done");
1641   }
1642 
1643 
1644   //
1645   //  Generate generic array copy stubs
1646   //
1647   //  Input:
1648   //     4(rsp)    -  src oop
1649   //     8(rsp)    -  src_pos
1650   //    12(rsp)    -  dst oop
1651   //    16(rsp)    -  dst_pos
1652   //    20(rsp)    -  element count
1653   //
1654   //  Output:
1655   //    rax, ==  0  -  success
1656   //    rax, == -1^K - failure, where K is partial transfer count
1657   //
1658   address generate_generic_copy(const char *name,
1659                                 address entry_jbyte_arraycopy,
1660                                 address entry_jshort_arraycopy,
1661                                 address entry_jint_arraycopy,
1662                                 address entry_oop_arraycopy,
1663                                 address entry_jlong_arraycopy,
1664                                 address entry_checkcast_arraycopy) {
1665     Label L_failed, L_failed_0, L_objArray;
1666 
1667     { int modulus = CodeEntryAlignment;
1668       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
1669       int advance = target - (__ offset() % modulus);
1670       if (advance < 0)  advance += modulus;
1671       if (advance > 0)  __ nop(advance);
1672     }
1673     StubCodeMark mark(this, "StubRoutines", name);
1674 
1675     // Short-hop target to L_failed.  Makes for denser prologue code.
1676     __ BIND(L_failed_0);
1677     __ jmp(L_failed);
1678     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
1679 
1680     __ align(CodeEntryAlignment);
1681     address start = __ pc();
1682 
1683     __ enter(); // required for proper stackwalking of RuntimeStub frame
1684     __ push(rsi);
1685     __ push(rdi);
1686 
1687     // bump this on entry, not on exit:
1688     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
1689 
1690     // Input values
1691     Address SRC     (rsp, 12+ 4);
1692     Address SRC_POS (rsp, 12+ 8);
1693     Address DST     (rsp, 12+12);
1694     Address DST_POS (rsp, 12+16);
1695     Address LENGTH  (rsp, 12+20);
1696 
1697     //-----------------------------------------------------------------------
1698     // Assembler stub will be used for this call to arraycopy
1699     // if the following conditions are met:
1700     //
1701     // (1) src and dst must not be null.
1702     // (2) src_pos must not be negative.
1703     // (3) dst_pos must not be negative.
1704     // (4) length  must not be negative.
1705     // (5) src klass and dst klass should be the same and not NULL.
1706     // (6) src and dst should be arrays.
1707     // (7) src_pos + length must not exceed length of src.
1708     // (8) dst_pos + length must not exceed length of dst.
1709     //
1710 
1711     const Register src     = rax;       // source array oop
1712     const Register src_pos = rsi;
1713     const Register dst     = rdx;       // destination array oop
1714     const Register dst_pos = rdi;
1715     const Register length  = rcx;       // transfer count
1716 
1717     //  if (src == NULL) return -1;
1718     __ movptr(src, SRC);      // src oop
1719     __ testptr(src, src);
1720     __ jccb(Assembler::zero, L_failed_0);
1721 
1722     //  if (src_pos < 0) return -1;
1723     __ movl2ptr(src_pos, SRC_POS);  // src_pos
1724     __ testl(src_pos, src_pos);
1725     __ jccb(Assembler::negative, L_failed_0);
1726 
1727     //  if (dst == NULL) return -1;
1728     __ movptr(dst, DST);      // dst oop
1729     __ testptr(dst, dst);
1730     __ jccb(Assembler::zero, L_failed_0);
1731 
1732     //  if (dst_pos < 0) return -1;
1733     __ movl2ptr(dst_pos, DST_POS);  // dst_pos
1734     __ testl(dst_pos, dst_pos);
1735     __ jccb(Assembler::negative, L_failed_0);
1736 
1737     //  if (length < 0) return -1;
1738     __ movl2ptr(length, LENGTH);   // length
1739     __ testl(length, length);
1740     __ jccb(Assembler::negative, L_failed_0);
1741 
1742     //  if (src->klass() == NULL) return -1;
1743     Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());
1744     Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());
1745     const Register rcx_src_klass = rcx;    // array klass
1746     __ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
1747 
1748 #ifdef ASSERT
1749     //  assert(src->klass() != NULL);
1750     BLOCK_COMMENT("assert klasses not null");
1751     { Label L1, L2;
1752       __ testptr(rcx_src_klass, rcx_src_klass);
1753       __ jccb(Assembler::notZero, L2);   // it is broken if klass is NULL
1754       __ bind(L1);
1755       __ stop("broken null klass");
1756       __ bind(L2);
1757       __ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
1758       __ jccb(Assembler::equal, L1);      // this would be broken also
1759       BLOCK_COMMENT("assert done");
1760     }
1761 #endif //ASSERT
1762 
1763     // Load layout helper (32-bits)
1764     //
1765     //  |array_tag|     | header_size | element_type |     |log2_element_size|
1766     // 32        30    24            16              8     2                 0
1767     //
1768     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
1769     //
1770 
1771     int lh_offset = in_bytes(Klass::layout_helper_offset());
1772     Address src_klass_lh_addr(rcx_src_klass, lh_offset);
1773 
1774     // Handle objArrays completely differently...
1775     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
1776     __ cmpl(src_klass_lh_addr, objArray_lh);
1777     __ jcc(Assembler::equal, L_objArray);
1778 
1779     //  if (src->klass() != dst->klass()) return -1;
1780     __ cmpptr(rcx_src_klass, dst_klass_addr);
1781     __ jccb(Assembler::notEqual, L_failed_0);
1782 
1783     const Register rcx_lh = rcx;  // layout helper
1784     assert(rcx_lh == rcx_src_klass, "known alias");
1785     __ movl(rcx_lh, src_klass_lh_addr);
1786 
1787     //  if (!src->is_Array()) return -1;
1788     __ cmpl(rcx_lh, Klass::_lh_neutral_value);
1789     __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
1790 
1791     // At this point, it is known to be a typeArray (array_tag 0x3).
1792 #ifdef ASSERT
1793     { Label L;
1794       __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
1795       __ jcc(Assembler::greaterEqual, L); // signed cmp
1796       __ stop("must be a primitive array");
1797       __ bind(L);
1798     }
1799 #endif
1800 
1801     assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);
1802     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1803 
1804     // TypeArrayKlass
1805     //
1806     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
1807     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
1808     //
1809     const Register rsi_offset = rsi; // array offset
1810     const Register src_array  = src; // src array offset
1811     const Register dst_array  = dst; // dst array offset
1812     const Register rdi_elsize = rdi; // log2 element size
1813 
1814     __ mov(rsi_offset, rcx_lh);
1815     __ shrptr(rsi_offset, Klass::_lh_header_size_shift);
1816     __ andptr(rsi_offset, Klass::_lh_header_size_mask);   // array_offset
1817     __ addptr(src_array, rsi_offset);  // src array offset
1818     __ addptr(dst_array, rsi_offset);  // dst array offset
1819     __ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
1820 
1821     // next registers should be set before the jump to corresponding stub
1822     const Register from       = src; // source array address
1823     const Register to         = dst; // destination array address
1824     const Register count      = rcx; // elements count
1825     // some of them should be duplicated on stack
1826 #define FROM   Address(rsp, 12+ 4)
1827 #define TO     Address(rsp, 12+ 8)   // Not used now
1828 #define COUNT  Address(rsp, 12+12)   // Only for oop arraycopy
1829 
1830     BLOCK_COMMENT("scale indexes to element size");
1831     __ movl2ptr(rsi, SRC_POS);  // src_pos
1832     __ shlptr(rsi);             // src_pos << rcx (log2 elsize)
1833     assert(src_array == from, "");
1834     __ addptr(from, rsi);       // from = src_array + SRC_POS << log2 elsize
1835     __ movl2ptr(rdi, DST_POS);  // dst_pos
1836     __ shlptr(rdi);             // dst_pos << rcx (log2 elsize)
1837     assert(dst_array == to, "");
1838     __ addptr(to,  rdi);        // to   = dst_array + DST_POS << log2 elsize
1839     __ movptr(FROM, from);      // src_addr
1840     __ mov(rdi_elsize, rcx_lh); // log2 elsize
1841     __ movl2ptr(count, LENGTH); // elements count
1842 
1843     BLOCK_COMMENT("choose copy loop based on element size");
1844     __ cmpl(rdi_elsize, 0);
1845 
1846     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
1847     __ cmpl(rdi_elsize, LogBytesPerShort);
1848     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
1849     __ cmpl(rdi_elsize, LogBytesPerInt);
1850     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
1851 #ifdef ASSERT
1852     __ cmpl(rdi_elsize, LogBytesPerLong);
1853     __ jccb(Assembler::notEqual, L_failed);
1854 #endif
1855     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1856     __ pop(rsi);
1857     __ jump(RuntimeAddress(entry_jlong_arraycopy));
1858 
1859   __ BIND(L_failed);
1860     __ xorptr(rax, rax);
1861     __ notptr(rax); // return -1
1862     __ pop(rdi);
1863     __ pop(rsi);
1864     __ leave(); // required for proper stackwalking of RuntimeStub frame
1865     __ ret(0);
1866 
1867     // ObjArrayKlass
1868   __ BIND(L_objArray);
1869     // live at this point:  rcx_src_klass, src[_pos], dst[_pos]
1870 
1871     Label L_plain_copy, L_checkcast_copy;
1872     //  test array classes for subtyping
1873     __ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
1874     __ jccb(Assembler::notEqual, L_checkcast_copy);
1875 
1876     // Identically typed arrays can be copied without element-wise checks.
1877     assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);
1878     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1879 
1880   __ BIND(L_plain_copy);
1881     __ movl2ptr(count, LENGTH); // elements count
1882     __ movl2ptr(src_pos, SRC_POS);  // reload src_pos
1883     __ lea(from, Address(src, src_pos, Address::times_ptr,
1884                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
1885     __ movl2ptr(dst_pos, DST_POS);  // reload dst_pos
1886     __ lea(to,   Address(dst, dst_pos, Address::times_ptr,
1887                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
1888     __ movptr(FROM,  from);   // src_addr
1889     __ movptr(TO,    to);     // dst_addr
1890     __ movl(COUNT, count);  // count
1891     __ jump(RuntimeAddress(entry_oop_arraycopy));
1892 
1893   __ BIND(L_checkcast_copy);
1894     // live at this point:  rcx_src_klass, dst[_pos], src[_pos]
1895     {
1896       // Handy offsets:
1897       int  ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
1898       int sco_offset = in_bytes(Klass::super_check_offset_offset());
1899 
1900       Register rsi_dst_klass = rsi;
1901       Register rdi_temp      = rdi;
1902       assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");
1903       assert(rdi_temp      == dst_pos, "expected alias w/ dst_pos");
1904       Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);
1905 
1906       // Before looking at dst.length, make sure dst is also an objArray.
1907       __ movptr(rsi_dst_klass, dst_klass_addr);
1908       __ cmpl(dst_klass_lh_addr, objArray_lh);
1909       __ jccb(Assembler::notEqual, L_failed);
1910 
1911       // It is safe to examine both src.length and dst.length.
1912       __ movl2ptr(src_pos, SRC_POS);        // reload rsi
1913       arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1914       // (Now src_pos and dst_pos are killed, but not src and dst.)
1915 
1916       // We'll need this temp (don't forget to pop it after the type check).
1917       __ push(rbx);
1918       Register rbx_src_klass = rbx;
1919 
1920       __ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
1921       __ movptr(rsi_dst_klass, dst_klass_addr);
1922       Address super_check_offset_addr(rsi_dst_klass, sco_offset);
1923       Label L_fail_array_check;
1924       generate_type_check(rbx_src_klass,
1925                           super_check_offset_addr, dst_klass_addr,
1926                           rdi_temp, NULL, &L_fail_array_check);
1927       // (On fall-through, we have passed the array type check.)
1928       __ pop(rbx);
1929       __ jmp(L_plain_copy);
1930 
1931       __ BIND(L_fail_array_check);
1932       // Reshuffle arguments so we can call checkcast_arraycopy:
1933 
1934       // match initial saves for checkcast_arraycopy
1935       // push(rsi);    // already done; see above
1936       // push(rdi);    // already done; see above
1937       // push(rbx);    // already done; see above
1938 
1939       // Marshal outgoing arguments now, freeing registers.
1940       Address   from_arg(rsp, 16+ 4);   // from
1941       Address     to_arg(rsp, 16+ 8);   // to
1942       Address length_arg(rsp, 16+12);   // elements count
1943       Address  ckoff_arg(rsp, 16+16);   // super_check_offset
1944       Address  ckval_arg(rsp, 16+20);   // super_klass
1945 
1946       Address SRC_POS_arg(rsp, 16+ 8);
1947       Address DST_POS_arg(rsp, 16+16);
1948       Address  LENGTH_arg(rsp, 16+20);
1949       // push rbx, changed the incoming offsets (why not just use rbp,??)
1950       // assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");
1951 
1952       __ movptr(rbx, Address(rsi_dst_klass, ek_offset));
1953       __ movl2ptr(length, LENGTH_arg);    // reload elements count
1954       __ movl2ptr(src_pos, SRC_POS_arg);  // reload src_pos
1955       __ movl2ptr(dst_pos, DST_POS_arg);  // reload dst_pos
1956 
1957       __ movptr(ckval_arg, rbx);          // destination element type
1958       __ movl(rbx, Address(rbx, sco_offset));
1959       __ movl(ckoff_arg, rbx);          // corresponding class check offset
1960 
1961       __ movl(length_arg, length);      // outgoing length argument
1962 
1963       __ lea(from, Address(src, src_pos, Address::times_ptr,
1964                             arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1965       __ movptr(from_arg, from);
1966 
1967       __ lea(to, Address(dst, dst_pos, Address::times_ptr,
1968                           arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1969       __ movptr(to_arg, to);
1970       __ jump(RuntimeAddress(entry_checkcast_arraycopy));
1971     }
1972 
1973     return start;
1974   }
1975 
1976   void generate_arraycopy_stubs() {
1977     address entry;
1978     address entry_jbyte_arraycopy;
1979     address entry_jshort_arraycopy;
1980     address entry_jint_arraycopy;
1981     address entry_oop_arraycopy;
1982     address entry_jlong_arraycopy;
1983     address entry_checkcast_arraycopy;
1984 
1985     StubRoutines::_arrayof_jbyte_disjoint_arraycopy =
1986         generate_disjoint_copy(T_BYTE,  true, Address::times_1, &entry,
1987                                "arrayof_jbyte_disjoint_arraycopy");
1988     StubRoutines::_arrayof_jbyte_arraycopy =
1989         generate_conjoint_copy(T_BYTE,  true, Address::times_1,  entry,
1990                                NULL, "arrayof_jbyte_arraycopy");
1991     StubRoutines::_jbyte_disjoint_arraycopy =
1992         generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,
1993                                "jbyte_disjoint_arraycopy");
1994     StubRoutines::_jbyte_arraycopy =
1995         generate_conjoint_copy(T_BYTE, false, Address::times_1,  entry,
1996                                &entry_jbyte_arraycopy, "jbyte_arraycopy");
1997 
1998     StubRoutines::_arrayof_jshort_disjoint_arraycopy =
1999         generate_disjoint_copy(T_SHORT,  true, Address::times_2, &entry,
2000                                "arrayof_jshort_disjoint_arraycopy");
2001     StubRoutines::_arrayof_jshort_arraycopy =
2002         generate_conjoint_copy(T_SHORT,  true, Address::times_2,  entry,
2003                                NULL, "arrayof_jshort_arraycopy");
2004     StubRoutines::_jshort_disjoint_arraycopy =
2005         generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,
2006                                "jshort_disjoint_arraycopy");
2007     StubRoutines::_jshort_arraycopy =
2008         generate_conjoint_copy(T_SHORT, false, Address::times_2,  entry,
2009                                &entry_jshort_arraycopy, "jshort_arraycopy");
2010 
2011     // Next arrays are always aligned on 4 bytes at least.
2012     StubRoutines::_jint_disjoint_arraycopy =
2013         generate_disjoint_copy(T_INT, true, Address::times_4, &entry,
2014                                "jint_disjoint_arraycopy");
2015     StubRoutines::_jint_arraycopy =
2016         generate_conjoint_copy(T_INT, true, Address::times_4,  entry,
2017                                &entry_jint_arraycopy, "jint_arraycopy");
2018 
2019     StubRoutines::_oop_disjoint_arraycopy =
2020         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
2021                                "oop_disjoint_arraycopy");
2022     StubRoutines::_oop_arraycopy =
2023         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
2024                                &entry_oop_arraycopy, "oop_arraycopy");
2025 
2026     StubRoutines::_oop_disjoint_arraycopy_uninit =
2027         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
2028                                "oop_disjoint_arraycopy_uninit",
2029                                /*dest_uninitialized*/true);
2030     StubRoutines::_oop_arraycopy_uninit =
2031         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
2032                                NULL, "oop_arraycopy_uninit",
2033                                /*dest_uninitialized*/true);
2034 
2035     StubRoutines::_jlong_disjoint_arraycopy =
2036         generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");
2037     StubRoutines::_jlong_arraycopy =
2038         generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,
2039                                     "jlong_arraycopy");
2040 
2041     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2042     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2043     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2044     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2045     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2046     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2047 
2048     StubRoutines::_arrayof_jint_disjoint_arraycopy       = StubRoutines::_jint_disjoint_arraycopy;
2049     StubRoutines::_arrayof_oop_disjoint_arraycopy        = StubRoutines::_oop_disjoint_arraycopy;
2050     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit;
2051     StubRoutines::_arrayof_jlong_disjoint_arraycopy      = StubRoutines::_jlong_disjoint_arraycopy;
2052 
2053     StubRoutines::_arrayof_jint_arraycopy       = StubRoutines::_jint_arraycopy;
2054     StubRoutines::_arrayof_oop_arraycopy        = StubRoutines::_oop_arraycopy;
2055     StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit;
2056     StubRoutines::_arrayof_jlong_arraycopy      = StubRoutines::_jlong_arraycopy;
2057 
2058     StubRoutines::_checkcast_arraycopy =
2059         generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2060     StubRoutines::_checkcast_arraycopy_uninit =
2061         generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, /*dest_uninitialized*/true);
2062 
2063     StubRoutines::_unsafe_arraycopy =
2064         generate_unsafe_copy("unsafe_arraycopy",
2065                                entry_jbyte_arraycopy,
2066                                entry_jshort_arraycopy,
2067                                entry_jint_arraycopy,
2068                                entry_jlong_arraycopy);
2069 
2070     StubRoutines::_generic_arraycopy =
2071         generate_generic_copy("generic_arraycopy",
2072                                entry_jbyte_arraycopy,
2073                                entry_jshort_arraycopy,
2074                                entry_jint_arraycopy,
2075                                entry_oop_arraycopy,
2076                                entry_jlong_arraycopy,
2077                                entry_checkcast_arraycopy);
2078   }
2079 
2080   // AES intrinsic stubs
2081   enum {AESBlockSize = 16};
2082 
2083   address generate_key_shuffle_mask() {
2084     __ align(16);
2085     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
2086     address start = __ pc();
2087     __ emit_data(0x00010203, relocInfo::none, 0 );
2088     __ emit_data(0x04050607, relocInfo::none, 0 );
2089     __ emit_data(0x08090a0b, relocInfo::none, 0 );
2090     __ emit_data(0x0c0d0e0f, relocInfo::none, 0 );
2091     return start;
2092   }
2093 
2094   address generate_counter_shuffle_mask() {
2095     __ align(16);
2096     StubCodeMark mark(this, "StubRoutines", "counter_shuffle_mask");
2097     address start = __ pc();
2098     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
2099     __ emit_data(0x08090a0b, relocInfo::none, 0);
2100     __ emit_data(0x04050607, relocInfo::none, 0);
2101     __ emit_data(0x00010203, relocInfo::none, 0);
2102     return start;
2103   }
2104 
2105   // Utility routine for loading a 128-bit key word in little endian format
2106   // can optionally specify that the shuffle mask is already in an xmmregister
2107   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2108     __ movdqu(xmmdst, Address(key, offset));
2109     if (xmm_shuf_mask != NULL) {
2110       __ pshufb(xmmdst, xmm_shuf_mask);
2111     } else {
2112       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2113     }
2114   }
2115 
2116   // aesenc using specified key+offset
2117   // can optionally specify that the shuffle mask is already in an xmmregister
2118   void aes_enc_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2119     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2120     __ aesenc(xmmdst, xmmtmp);
2121   }
2122 
2123   // aesdec using specified key+offset
2124   // can optionally specify that the shuffle mask is already in an xmmregister
2125   void aes_dec_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2126     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2127     __ aesdec(xmmdst, xmmtmp);
2128   }
2129 
2130   // Utility routine for increase 128bit counter (iv in CTR mode)
2131   //  XMM_128bit,  D3, D2, D1, D0
2132   void inc_counter(Register reg, XMMRegister xmmdst, int inc_delta, Label& next_block) {
2133     __ pextrd(reg, xmmdst, 0x0);
2134     __ addl(reg, inc_delta);
2135     __ pinsrd(xmmdst, reg, 0x0);
2136     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2137 
2138     __ pextrd(reg, xmmdst, 0x01); // Carry-> D1
2139     __ addl(reg, 0x01);
2140     __ pinsrd(xmmdst, reg, 0x01);
2141     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2142 
2143     __ pextrd(reg, xmmdst, 0x02); // Carry-> D2
2144     __ addl(reg, 0x01);
2145     __ pinsrd(xmmdst, reg, 0x02);
2146     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2147 
2148     __ pextrd(reg, xmmdst, 0x03); // Carry -> D3
2149     __ addl(reg, 0x01);
2150     __ pinsrd(xmmdst, reg, 0x03);
2151 
2152     __ BIND(next_block);          // next instruction
2153   }
2154 
2155 
2156   // Arguments:
2157   //
2158   // Inputs:
2159   //   c_rarg0   - source byte array address
2160   //   c_rarg1   - destination byte array address
2161   //   c_rarg2   - K (key) in little endian int array
2162   //
2163   address generate_aescrypt_encryptBlock() {
2164     assert(UseAES, "need AES instructions and misaligned SSE support");
2165     __ align(CodeEntryAlignment);
2166     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
2167     Label L_doLast;
2168     address start = __ pc();
2169 
2170     const Register from        = rdx;      // source array address
2171     const Register to          = rdx;      // destination array address
2172     const Register key         = rcx;      // key array address
2173     const Register keylen      = rax;
2174     const Address  from_param(rbp, 8+0);
2175     const Address  to_param  (rbp, 8+4);
2176     const Address  key_param (rbp, 8+8);
2177 
2178     const XMMRegister xmm_result = xmm0;
2179     const XMMRegister xmm_key_shuf_mask = xmm1;
2180     const XMMRegister xmm_temp1  = xmm2;
2181     const XMMRegister xmm_temp2  = xmm3;
2182     const XMMRegister xmm_temp3  = xmm4;
2183     const XMMRegister xmm_temp4  = xmm5;
2184 
2185     __ enter();   // required for proper stackwalking of RuntimeStub frame
2186 
2187     __ movptr(from, from_param);
2188     __ movptr(key, key_param);
2189 
2190     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2191     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2192 
2193     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2194     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
2195     __ movptr(to, to_param);
2196 
2197     // For encryption, the java expanded key ordering is just what we need
2198 
2199     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
2200     __ pxor(xmm_result, xmm_temp1);
2201 
2202     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2203     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2204     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2205     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2206 
2207     __ aesenc(xmm_result, xmm_temp1);
2208     __ aesenc(xmm_result, xmm_temp2);
2209     __ aesenc(xmm_result, xmm_temp3);
2210     __ aesenc(xmm_result, xmm_temp4);
2211 
2212     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2213     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2214     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2215     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2216 
2217     __ aesenc(xmm_result, xmm_temp1);
2218     __ aesenc(xmm_result, xmm_temp2);
2219     __ aesenc(xmm_result, xmm_temp3);
2220     __ aesenc(xmm_result, xmm_temp4);
2221 
2222     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2223     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2224 
2225     __ cmpl(keylen, 44);
2226     __ jccb(Assembler::equal, L_doLast);
2227 
2228     __ aesenc(xmm_result, xmm_temp1);
2229     __ aesenc(xmm_result, xmm_temp2);
2230 
2231     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2232     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2233 
2234     __ cmpl(keylen, 52);
2235     __ jccb(Assembler::equal, L_doLast);
2236 
2237     __ aesenc(xmm_result, xmm_temp1);
2238     __ aesenc(xmm_result, xmm_temp2);
2239 
2240     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2241     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2242 
2243     __ BIND(L_doLast);
2244     __ aesenc(xmm_result, xmm_temp1);
2245     __ aesenclast(xmm_result, xmm_temp2);
2246     __ movdqu(Address(to, 0), xmm_result);        // store the result
2247     __ xorptr(rax, rax); // return 0
2248     __ leave(); // required for proper stackwalking of RuntimeStub frame
2249     __ ret(0);
2250 
2251     return start;
2252   }
2253 
2254 
2255   // Arguments:
2256   //
2257   // Inputs:
2258   //   c_rarg0   - source byte array address
2259   //   c_rarg1   - destination byte array address
2260   //   c_rarg2   - K (key) in little endian int array
2261   //
2262   address generate_aescrypt_decryptBlock() {
2263     assert(UseAES, "need AES instructions and misaligned SSE support");
2264     __ align(CodeEntryAlignment);
2265     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
2266     Label L_doLast;
2267     address start = __ pc();
2268 
2269     const Register from        = rdx;      // source array address
2270     const Register to          = rdx;      // destination array address
2271     const Register key         = rcx;      // key array address
2272     const Register keylen      = rax;
2273     const Address  from_param(rbp, 8+0);
2274     const Address  to_param  (rbp, 8+4);
2275     const Address  key_param (rbp, 8+8);
2276 
2277     const XMMRegister xmm_result = xmm0;
2278     const XMMRegister xmm_key_shuf_mask = xmm1;
2279     const XMMRegister xmm_temp1  = xmm2;
2280     const XMMRegister xmm_temp2  = xmm3;
2281     const XMMRegister xmm_temp3  = xmm4;
2282     const XMMRegister xmm_temp4  = xmm5;
2283 
2284     __ enter(); // required for proper stackwalking of RuntimeStub frame
2285 
2286     __ movptr(from, from_param);
2287     __ movptr(key, key_param);
2288 
2289     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2290     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2291 
2292     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2293     __ movdqu(xmm_result, Address(from, 0));
2294     __ movptr(to, to_param);
2295 
2296     // for decryption java expanded key ordering is rotated one position from what we want
2297     // so we start from 0x10 here and hit 0x00 last
2298     // we don't know if the key is aligned, hence not using load-execute form
2299     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2300     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2301     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2302     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2303 
2304     __ pxor  (xmm_result, xmm_temp1);
2305     __ aesdec(xmm_result, xmm_temp2);
2306     __ aesdec(xmm_result, xmm_temp3);
2307     __ aesdec(xmm_result, xmm_temp4);
2308 
2309     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2310     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2311     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2312     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2313 
2314     __ aesdec(xmm_result, xmm_temp1);
2315     __ aesdec(xmm_result, xmm_temp2);
2316     __ aesdec(xmm_result, xmm_temp3);
2317     __ aesdec(xmm_result, xmm_temp4);
2318 
2319     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2320     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2321     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
2322 
2323     __ cmpl(keylen, 44);
2324     __ jccb(Assembler::equal, L_doLast);
2325 
2326     __ aesdec(xmm_result, xmm_temp1);
2327     __ aesdec(xmm_result, xmm_temp2);
2328 
2329     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2330     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2331 
2332     __ cmpl(keylen, 52);
2333     __ jccb(Assembler::equal, L_doLast);
2334 
2335     __ aesdec(xmm_result, xmm_temp1);
2336     __ aesdec(xmm_result, xmm_temp2);
2337 
2338     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2339     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2340 
2341     __ BIND(L_doLast);
2342     __ aesdec(xmm_result, xmm_temp1);
2343     __ aesdec(xmm_result, xmm_temp2);
2344 
2345     // for decryption the aesdeclast operation is always on key+0x00
2346     __ aesdeclast(xmm_result, xmm_temp3);
2347     __ movdqu(Address(to, 0), xmm_result);  // store the result
2348     __ xorptr(rax, rax); // return 0
2349     __ leave(); // required for proper stackwalking of RuntimeStub frame
2350     __ ret(0);
2351 
2352     return start;
2353   }
2354 
2355   void handleSOERegisters(bool saving) {
2356     const int saveFrameSizeInBytes = 4 * wordSize;
2357     const Address saved_rbx     (rbp, -3 * wordSize);
2358     const Address saved_rsi     (rbp, -2 * wordSize);
2359     const Address saved_rdi     (rbp, -1 * wordSize);
2360 
2361     if (saving) {
2362       __ subptr(rsp, saveFrameSizeInBytes);
2363       __ movptr(saved_rsi, rsi);
2364       __ movptr(saved_rdi, rdi);
2365       __ movptr(saved_rbx, rbx);
2366     } else {
2367       // restoring
2368       __ movptr(rsi, saved_rsi);
2369       __ movptr(rdi, saved_rdi);
2370       __ movptr(rbx, saved_rbx);
2371     }
2372   }
2373 
2374   // Arguments:
2375   //
2376   // Inputs:
2377   //   c_rarg0   - source byte array address
2378   //   c_rarg1   - destination byte array address
2379   //   c_rarg2   - K (key) in little endian int array
2380   //   c_rarg3   - r vector byte array address
2381   //   c_rarg4   - input length
2382   //
2383   // Output:
2384   //   rax       - input length
2385   //
2386   address generate_cipherBlockChaining_encryptAESCrypt() {
2387     assert(UseAES, "need AES instructions and misaligned SSE support");
2388     __ align(CodeEntryAlignment);
2389     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
2390     address start = __ pc();
2391 
2392     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
2393     const Register from        = rsi;      // source array address
2394     const Register to          = rdx;      // destination array address
2395     const Register key         = rcx;      // key array address
2396     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2397                                            // and left with the results of the last encryption block
2398     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2399     const Register pos         = rax;
2400 
2401     // xmm register assignments for the loops below
2402     const XMMRegister xmm_result = xmm0;
2403     const XMMRegister xmm_temp   = xmm1;
2404     // first 6 keys preloaded into xmm2-xmm7
2405     const int XMM_REG_NUM_KEY_FIRST = 2;
2406     const int XMM_REG_NUM_KEY_LAST  = 7;
2407     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
2408 
2409     __ enter(); // required for proper stackwalking of RuntimeStub frame
2410     handleSOERegisters(true /*saving*/);
2411 
2412     // load registers from incoming parameters
2413     const Address  from_param(rbp, 8+0);
2414     const Address  to_param  (rbp, 8+4);
2415     const Address  key_param (rbp, 8+8);
2416     const Address  rvec_param (rbp, 8+12);
2417     const Address  len_param  (rbp, 8+16);
2418     __ movptr(from , from_param);
2419     __ movptr(to   , to_param);
2420     __ movptr(key  , key_param);
2421     __ movptr(rvec , rvec_param);
2422     __ movptr(len_reg , len_param);
2423 
2424     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
2425     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2426     // load up xmm regs 2 thru 7 with keys 0-5
2427     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2428       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
2429       offset += 0x10;
2430     }
2431 
2432     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
2433 
2434     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2435     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2436     __ cmpl(rax, 44);
2437     __ jcc(Assembler::notEqual, L_key_192_256);
2438 
2439     // 128 bit code follows here
2440     __ movl(pos, 0);
2441     __ align(OptoLoopAlignment);
2442     __ BIND(L_loopTop_128);
2443     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2444     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2445 
2446     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2447     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2448       __ aesenc(xmm_result, as_XMMRegister(rnum));
2449     }
2450     for (int key_offset = 0x60; key_offset <= 0x90; key_offset += 0x10) {
2451       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2452     }
2453     load_key(xmm_temp, key, 0xa0);
2454     __ aesenclast(xmm_result, xmm_temp);
2455 
2456     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
2457     // no need to store r to memory until we exit
2458     __ addptr(pos, AESBlockSize);
2459     __ subptr(len_reg, AESBlockSize);
2460     __ jcc(Assembler::notEqual, L_loopTop_128);
2461 
2462     __ BIND(L_exit);
2463     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
2464 
2465     handleSOERegisters(false /*restoring*/);
2466     __ movptr(rax, len_param); // return length
2467     __ leave();                                  // required for proper stackwalking of RuntimeStub frame
2468     __ ret(0);
2469 
2470     __ BIND(L_key_192_256);
2471     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
2472     __ cmpl(rax, 52);
2473     __ jcc(Assembler::notEqual, L_key_256);
2474 
2475     // 192-bit code follows here (could be changed to use more xmm registers)
2476     __ movl(pos, 0);
2477     __ align(OptoLoopAlignment);
2478     __ BIND(L_loopTop_192);
2479     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2480     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2481 
2482     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2483     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2484       __ aesenc(xmm_result, as_XMMRegister(rnum));
2485     }
2486     for (int key_offset = 0x60; key_offset <= 0xb0; key_offset += 0x10) {
2487       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2488     }
2489     load_key(xmm_temp, key, 0xc0);
2490     __ aesenclast(xmm_result, xmm_temp);
2491 
2492     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2493     // no need to store r to memory until we exit
2494     __ addptr(pos, AESBlockSize);
2495     __ subptr(len_reg, AESBlockSize);
2496     __ jcc(Assembler::notEqual, L_loopTop_192);
2497     __ jmp(L_exit);
2498 
2499     __ BIND(L_key_256);
2500     // 256-bit code follows here (could be changed to use more xmm registers)
2501     __ movl(pos, 0);
2502     __ align(OptoLoopAlignment);
2503     __ BIND(L_loopTop_256);
2504     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2505     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2506 
2507     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2508     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2509       __ aesenc(xmm_result, as_XMMRegister(rnum));
2510     }
2511     for (int key_offset = 0x60; key_offset <= 0xd0; key_offset += 0x10) {
2512       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2513     }
2514     load_key(xmm_temp, key, 0xe0);
2515     __ aesenclast(xmm_result, xmm_temp);
2516 
2517     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2518     // no need to store r to memory until we exit
2519     __ addptr(pos, AESBlockSize);
2520     __ subptr(len_reg, AESBlockSize);
2521     __ jcc(Assembler::notEqual, L_loopTop_256);
2522     __ jmp(L_exit);
2523 
2524     return start;
2525   }
2526 
2527 
2528   // CBC AES Decryption.
2529   // In 32-bit stub, because of lack of registers we do not try to parallelize 4 blocks at a time.
2530   //
2531   // Arguments:
2532   //
2533   // Inputs:
2534   //   c_rarg0   - source byte array address
2535   //   c_rarg1   - destination byte array address
2536   //   c_rarg2   - K (key) in little endian int array
2537   //   c_rarg3   - r vector byte array address
2538   //   c_rarg4   - input length
2539   //
2540   // Output:
2541   //   rax       - input length
2542   //
2543 
2544   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
2545     assert(UseAES, "need AES instructions and misaligned SSE support");
2546     __ align(CodeEntryAlignment);
2547     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
2548     address start = __ pc();
2549 
2550     const Register from        = rsi;      // source array address
2551     const Register to          = rdx;      // destination array address
2552     const Register key         = rcx;      // key array address
2553     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2554                                            // and left with the results of the last encryption block
2555     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2556     const Register pos         = rax;
2557 
2558     const int PARALLEL_FACTOR = 4;
2559     const int ROUNDS[3] = { 10, 12, 14 }; //aes rounds for key128, key192, key256
2560 
2561     Label L_exit;
2562     Label L_singleBlock_loopTop[3]; //128, 192, 256
2563     Label L_multiBlock_loopTop[3]; //128, 192, 256
2564 
2565     const XMMRegister xmm_prev_block_cipher = xmm0; // holds cipher of previous block
2566     const XMMRegister xmm_key_shuf_mask = xmm1;
2567 
2568     const XMMRegister xmm_key_tmp0 = xmm2;
2569     const XMMRegister xmm_key_tmp1 = xmm3;
2570 
2571     // registers holding the six results in the parallelized loop
2572     const XMMRegister xmm_result0 = xmm4;
2573     const XMMRegister xmm_result1 = xmm5;
2574     const XMMRegister xmm_result2 = xmm6;
2575     const XMMRegister xmm_result3 = xmm7;
2576 
2577     __ enter(); // required for proper stackwalking of RuntimeStub frame
2578     handleSOERegisters(true /*saving*/);
2579 
2580     // load registers from incoming parameters
2581     const Address  from_param(rbp, 8+0);
2582     const Address  to_param  (rbp, 8+4);
2583     const Address  key_param (rbp, 8+8);
2584     const Address  rvec_param (rbp, 8+12);
2585     const Address  len_param  (rbp, 8+16);
2586 
2587     __ movptr(from , from_param);
2588     __ movptr(to   , to_param);
2589     __ movptr(key  , key_param);
2590     __ movptr(rvec , rvec_param);
2591     __ movptr(len_reg , len_param);
2592 
2593     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2594     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // initialize with initial rvec
2595 
2596     __ xorptr(pos, pos);
2597 
2598     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2599     // rvec is reused
2600     __ movl(rvec, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2601     __ cmpl(rvec, 52);
2602     __ jcc(Assembler::equal, L_multiBlock_loopTop[1]);
2603     __ cmpl(rvec, 60);
2604     __ jcc(Assembler::equal, L_multiBlock_loopTop[2]);
2605 
2606 #define DoFour(opc, src_reg)           \
2607   __ opc(xmm_result0, src_reg);         \
2608   __ opc(xmm_result1, src_reg);         \
2609   __ opc(xmm_result2, src_reg);         \
2610   __ opc(xmm_result3, src_reg);         \
2611 
2612     for (int k = 0; k < 3; ++k) {
2613       __ align(OptoLoopAlignment);
2614       __ BIND(L_multiBlock_loopTop[k]);
2615       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
2616       __ jcc(Assembler::less, L_singleBlock_loopTop[k]);
2617 
2618       __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); // get next 4 blocks into xmmresult registers
2619       __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2620       __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2621       __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2622 
2623       // the java expanded key ordering is rotated one position from what we want
2624       // so we start from 0x10 here and hit 0x00 last
2625       load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2626       DoFour(pxor, xmm_key_tmp0); //xor with first key
2627       // do the aes dec rounds
2628       for (int rnum = 1; rnum <= ROUNDS[k];) {
2629         //load two keys at a time
2630         //k1->0x20, ..., k9->0xa0, k10->0x00
2631         load_key(xmm_key_tmp1, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2632         load_key(xmm_key_tmp0, key, ((rnum + 2) % (ROUNDS[k] + 1)) * 0x10, xmm_key_shuf_mask); // hit 0x00 last!
2633         DoFour(aesdec, xmm_key_tmp1);
2634         rnum++;
2635         if (rnum != ROUNDS[k]) {
2636           DoFour(aesdec, xmm_key_tmp0);
2637         }
2638         else {
2639           DoFour(aesdeclast, xmm_key_tmp0);
2640         }
2641         rnum++;
2642       }
2643 
2644       // for each result, xor with the r vector of previous cipher block
2645       __ pxor(xmm_result0, xmm_prev_block_cipher);
2646       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2647       __ pxor(xmm_result1, xmm_prev_block_cipher);
2648       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2649       __ pxor(xmm_result2, xmm_prev_block_cipher);
2650       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2651       __ pxor(xmm_result3, xmm_prev_block_cipher);
2652       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3 * AESBlockSize)); // this will carry over to next set of blocks
2653 
2654             // store 4 results into the next 64 bytes of output
2655        __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2656        __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2657        __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2658        __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2659 
2660        __ addptr(pos, 4 * AESBlockSize);
2661        __ subptr(len_reg, 4 * AESBlockSize);
2662        __ jmp(L_multiBlock_loopTop[k]);
2663 
2664        //singleBlock starts here
2665        __ align(OptoLoopAlignment);
2666        __ BIND(L_singleBlock_loopTop[k]);
2667        __ cmpptr(len_reg, 0); // any blocks left?
2668        __ jcc(Assembler::equal, L_exit);
2669        __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
2670        __ movdqa(xmm_result1, xmm_result0);
2671 
2672        load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2673        __ pxor(xmm_result0, xmm_key_tmp0);
2674        // do the aes dec rounds
2675        for (int rnum = 1; rnum < ROUNDS[k]; rnum++) {
2676          // the java expanded key ordering is rotated one position from what we want
2677          load_key(xmm_key_tmp0, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2678          __ aesdec(xmm_result0, xmm_key_tmp0);
2679        }
2680        load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
2681        __ aesdeclast(xmm_result0, xmm_key_tmp0);
2682        __ pxor(xmm_result0, xmm_prev_block_cipher); // xor with the current r vector
2683        __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result0); // store into the next 16 bytes of output
2684        // no need to store r to memory until we exit
2685        __ movdqa(xmm_prev_block_cipher, xmm_result1); // set up next r vector with cipher input from this block
2686 
2687        __ addptr(pos, AESBlockSize);
2688        __ subptr(len_reg, AESBlockSize);
2689        __ jmp(L_singleBlock_loopTop[k]);
2690     }//for 128/192/256
2691 
2692     __ BIND(L_exit);
2693     __ movptr(rvec, rvec_param);                        // restore this since reused earlier
2694     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher); // final value of r stored in rvec of CipherBlockChaining object
2695     handleSOERegisters(false /*restoring*/);
2696     __ movptr(rax, len_param);                          // return length
2697     __ leave();                                         // required for proper stackwalking of RuntimeStub frame
2698     __ ret(0);
2699 
2700     return start;
2701   }
2702 
2703   // CTR AES crypt.
2704   // In 32-bit stub, parallelize 4 blocks at a time
2705   // Arguments:
2706   //
2707   // Inputs:
2708   //   c_rarg0   - source byte array address
2709   //   c_rarg1   - destination byte array address
2710   //   c_rarg2   - K (key) in little endian int array
2711   //   c_rarg3   - counter vector byte array address
2712   //   c_rarg4   - input length
2713   //
2714   // Output:
2715   //   rax       - input length
2716   //
2717   address generate_counterMode_AESCrypt_Parallel() {
2718     assert(UseAES, "need AES instructions and misaligned SSE support");
2719     __ align(CodeEntryAlignment);
2720     StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
2721     address start = __ pc();
2722     const Register from        = rsi;      // source array address
2723     const Register to          = rdx;      // destination array address
2724     const Register key         = rcx;      // key array address
2725     const Register counter     = rdi;      // counter byte array initialized from initvector array address
2726                                            // and updated with the incremented counter in the end
2727     const Register len_reg     = rbx;
2728     const Register pos         = rax;
2729 
2730     __ enter(); // required for proper stackwalking of RuntimeStub frame
2731     handleSOERegisters(true /*saving*/); // save rbx, rsi, rdi
2732 
2733     // load registers from incoming parameters
2734     const Address  from_param(rbp, 8+0);
2735     const Address  to_param  (rbp, 8+4);
2736     const Address  key_param (rbp, 8+8);
2737     const Address  rvec_param (rbp, 8+12);
2738     const Address  len_param  (rbp, 8+16);
2739     const Address  saved_counter_param(rbp, 8 + 20);
2740     const Address  used_addr_param(rbp, 8 + 24);
2741 
2742     __ movptr(from , from_param);
2743     __ movptr(to   , to_param);
2744     __ movptr(len_reg , len_param);
2745 
2746     // Use the partially used encrpyted counter from last invocation
2747     Label L_exit_preLoop, L_preLoop_start;
2748 
2749     // Use the registers 'counter' and 'key' here in this preloop
2750     // to hold of last 2 params 'used' and 'saved_encCounter_start'
2751     Register used = counter;
2752     Register saved_encCounter_start = key;
2753     Register used_addr = saved_encCounter_start;
2754 
2755     __ movptr(used_addr, used_addr_param);
2756     __ movptr(used, Address(used_addr, 0));
2757     __ movptr(saved_encCounter_start, saved_counter_param);
2758 
2759     __ BIND(L_preLoop_start);
2760     __ cmpptr(used, 16);
2761     __ jcc(Assembler::aboveEqual, L_exit_preLoop);
2762     __ cmpptr(len_reg, 0);
2763     __ jcc(Assembler::lessEqual, L_exit_preLoop);
2764     __ movb(rax, Address(saved_encCounter_start, used));
2765     __ xorb(rax, Address(from, 0));
2766     __ movb(Address(to, 0), rax);
2767     __ addptr(from, 1);
2768     __ addptr(to, 1);
2769     __ addptr(used, 1);
2770     __ subptr(len_reg, 1);
2771 
2772     __ jmp(L_preLoop_start);
2773 
2774     __ BIND(L_exit_preLoop);
2775     __ movptr(used_addr, used_addr_param);
2776     __ movptr(used_addr, used_addr_param);
2777     __ movl(Address(used_addr, 0), used);
2778 
2779     // load the parameters 'key' and 'counter'
2780     __ movptr(key, key_param);
2781     __ movptr(counter, rvec_param);
2782 
2783     // xmm register assignments for the loops below
2784     const XMMRegister xmm_curr_counter      = xmm0;
2785     const XMMRegister xmm_counter_shuf_mask = xmm1;  // need to be reloaded
2786     const XMMRegister xmm_key_shuf_mask     = xmm2;  // need to be reloaded
2787     const XMMRegister xmm_key               = xmm3;
2788     const XMMRegister xmm_result0           = xmm4;
2789     const XMMRegister xmm_result1           = xmm5;
2790     const XMMRegister xmm_result2           = xmm6;
2791     const XMMRegister xmm_result3           = xmm7;
2792     const XMMRegister xmm_from0             = xmm1;   //reuse XMM register
2793     const XMMRegister xmm_from1             = xmm2;
2794     const XMMRegister xmm_from2             = xmm3;
2795     const XMMRegister xmm_from3             = xmm4;
2796 
2797     //for key_128, key_192, key_256
2798     const int rounds[3] = {10, 12, 14};
2799     Label L_singleBlockLoopTop[3];
2800     Label L_multiBlock_loopTop[3];
2801     Label L_key192_top, L_key256_top;
2802     Label L_incCounter[3][4]; // 3: different key length,  4: 4 blocks at a time
2803     Label L_incCounter_single[3]; //for single block, key128, key192, key256
2804     Label L_processTail_insr[3], L_processTail_4_insr[3], L_processTail_2_insr[3], L_processTail_1_insr[3], L_processTail_exit_insr[3];
2805     Label L_processTail_extr[3], L_processTail_4_extr[3], L_processTail_2_extr[3], L_processTail_1_extr[3], L_processTail_exit_extr[3];
2806 
2807     Label L_exit;
2808     const int PARALLEL_FACTOR = 4;  //because of the limited register number
2809 
2810     // initialize counter with initial counter
2811     __ movdqu(xmm_curr_counter, Address(counter, 0x00));
2812     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2813     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled for increase
2814 
2815     // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
2816     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2817     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2818     __ cmpl(rax, 52);
2819     __ jcc(Assembler::equal, L_key192_top);
2820     __ cmpl(rax, 60);
2821     __ jcc(Assembler::equal, L_key256_top);
2822 
2823     //key128 begins here
2824     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2825 
2826 #define CTR_DoFour(opc, src_reg)               \
2827     __ opc(xmm_result0, src_reg);              \
2828     __ opc(xmm_result1, src_reg);              \
2829     __ opc(xmm_result2, src_reg);              \
2830     __ opc(xmm_result3, src_reg);
2831 
2832     // k == 0 :  generate code for key_128
2833     // k == 1 :  generate code for key_192
2834     // k == 2 :  generate code for key_256
2835     for (int k = 0; k < 3; ++k) {
2836       //multi blocks starts here
2837       __ align(OptoLoopAlignment);
2838       __ BIND(L_multiBlock_loopTop[k]);
2839       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least PARALLEL_FACTOR blocks left
2840       __ jcc(Assembler::less, L_singleBlockLoopTop[k]);
2841 
2842       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2843       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2844 
2845       //load, then increase counters
2846       CTR_DoFour(movdqa, xmm_curr_counter);
2847       __ push(rbx);
2848       inc_counter(rbx, xmm_result1, 0x01, L_incCounter[k][0]);
2849       inc_counter(rbx, xmm_result2, 0x02, L_incCounter[k][1]);
2850       inc_counter(rbx, xmm_result3, 0x03, L_incCounter[k][2]);
2851       inc_counter(rbx, xmm_curr_counter, 0x04, L_incCounter[k][3]);
2852       __ pop (rbx);
2853 
2854       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask); // load Round 0 key. interleaving for better performance
2855 
2856       CTR_DoFour(pshufb, xmm_counter_shuf_mask); // after increased, shuffled counters back for PXOR
2857       CTR_DoFour(pxor, xmm_key);   //PXOR with Round 0 key
2858 
2859       for (int i = 1; i < rounds[k]; ++i) {
2860         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2861         CTR_DoFour(aesenc, xmm_key);
2862       }
2863       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2864       CTR_DoFour(aesenclast, xmm_key);
2865 
2866       // get next PARALLEL_FACTOR blocks into xmm_from registers
2867       __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2868       __ movdqu(xmm_from1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2869       __ movdqu(xmm_from2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2870 
2871       // PXOR with input text
2872       __ pxor(xmm_result0, xmm_from0); //result0 is xmm4
2873       __ pxor(xmm_result1, xmm_from1);
2874       __ pxor(xmm_result2, xmm_from2);
2875 
2876       // store PARALLEL_FACTOR results into the next 64 bytes of output
2877       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2878       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2879       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2880 
2881       // do it here after xmm_result0 is saved, because xmm_from3 reuse the same register of xmm_result0.
2882       __ movdqu(xmm_from3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2883       __ pxor(xmm_result3, xmm_from3);
2884       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2885 
2886       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize); // increase the length of crypt text
2887       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // decrease the remaining length
2888       __ jmp(L_multiBlock_loopTop[k]);
2889 
2890       // singleBlock starts here
2891       __ align(OptoLoopAlignment);
2892       __ BIND(L_singleBlockLoopTop[k]);
2893       __ cmpptr(len_reg, 0);
2894       __ jcc(Assembler::equal, L_exit);
2895       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2896       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2897       __ movdqa(xmm_result0, xmm_curr_counter);
2898       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask);
2899       __ push(rbx);//rbx is used for increasing counter
2900       inc_counter(rbx, xmm_curr_counter, 0x01, L_incCounter_single[k]);
2901       __ pop (rbx);
2902       __ pshufb(xmm_result0, xmm_counter_shuf_mask);
2903       __ pxor(xmm_result0, xmm_key);
2904       for (int i = 1; i < rounds[k]; i++) {
2905         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2906         __ aesenc(xmm_result0, xmm_key);
2907       }
2908       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2909       __ aesenclast(xmm_result0, xmm_key);
2910       __ cmpptr(len_reg, AESBlockSize);
2911       __ jcc(Assembler::less, L_processTail_insr[k]);
2912         __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2913         __ pxor(xmm_result0, xmm_from0);
2914         __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2915         __ addptr(pos, AESBlockSize);
2916         __ subptr(len_reg, AESBlockSize);
2917         __ jmp(L_singleBlockLoopTop[k]);
2918 
2919       __ BIND(L_processTail_insr[k]);                                               // Process the tail part of the input array
2920         __ addptr(pos, len_reg);                                                    // 1. Insert bytes from src array into xmm_from0 register
2921         __ testptr(len_reg, 8);
2922         __ jcc(Assembler::zero, L_processTail_4_insr[k]);
2923           __ subptr(pos,8);
2924           __ pinsrd(xmm_from0, Address(from, pos), 0);
2925           __ pinsrd(xmm_from0, Address(from, pos, Address::times_1, 4), 1);
2926         __ BIND(L_processTail_4_insr[k]);
2927         __ testptr(len_reg, 4);
2928         __ jcc(Assembler::zero, L_processTail_2_insr[k]);
2929           __ subptr(pos,4);
2930           __ pslldq(xmm_from0, 4);
2931           __ pinsrd(xmm_from0, Address(from, pos), 0);
2932         __ BIND(L_processTail_2_insr[k]);
2933         __ testptr(len_reg, 2);
2934         __ jcc(Assembler::zero, L_processTail_1_insr[k]);
2935           __ subptr(pos, 2);
2936           __ pslldq(xmm_from0, 2);
2937           __ pinsrw(xmm_from0, Address(from, pos), 0);
2938         __ BIND(L_processTail_1_insr[k]);
2939         __ testptr(len_reg, 1);
2940         __ jcc(Assembler::zero, L_processTail_exit_insr[k]);
2941           __ subptr(pos, 1);
2942           __ pslldq(xmm_from0, 1);
2943           __ pinsrb(xmm_from0, Address(from, pos), 0);
2944         __ BIND(L_processTail_exit_insr[k]);
2945 
2946         __ movptr(saved_encCounter_start, saved_counter_param);
2947         __ movdqu(Address(saved_encCounter_start, 0), xmm_result0);               // 2. Perform pxor of the encrypted counter and plaintext Bytes.
2948         __ pxor(xmm_result0, xmm_from0);                                          //    Also the encrypted counter is saved for next invocation.
2949 
2950         __ testptr(len_reg, 8);
2951         __ jcc(Assembler::zero, L_processTail_4_extr[k]);                        // 3. Extract bytes from xmm_result0 into the dest. array
2952           __ pextrd(Address(to, pos), xmm_result0, 0);
2953           __ pextrd(Address(to, pos, Address::times_1, 4), xmm_result0, 1);
2954           __ psrldq(xmm_result0, 8);
2955           __ addptr(pos, 8);
2956         __ BIND(L_processTail_4_extr[k]);
2957         __ testptr(len_reg, 4);
2958         __ jcc(Assembler::zero, L_processTail_2_extr[k]);
2959           __ pextrd(Address(to, pos), xmm_result0, 0);
2960           __ psrldq(xmm_result0, 4);
2961           __ addptr(pos, 4);
2962         __ BIND(L_processTail_2_extr[k]);
2963         __ testptr(len_reg, 2);
2964         __ jcc(Assembler::zero, L_processTail_1_extr[k]);
2965           __ pextrb(Address(to, pos), xmm_result0, 0);
2966           __ pextrb(Address(to, pos, Address::times_1, 1), xmm_result0, 1);
2967           __ psrldq(xmm_result0, 2);
2968           __ addptr(pos, 2);
2969         __ BIND(L_processTail_1_extr[k]);
2970         __ testptr(len_reg, 1);
2971         __ jcc(Assembler::zero, L_processTail_exit_extr[k]);
2972           __ pextrb(Address(to, pos), xmm_result0, 0);
2973 
2974         __ BIND(L_processTail_exit_extr[k]);
2975         __ movptr(used_addr, used_addr_param);
2976         __ movl(Address(used_addr, 0), len_reg);
2977         __ jmp(L_exit);
2978     }
2979 
2980     __ BIND(L_exit);
2981     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2982     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled back.
2983     __ movdqu(Address(counter, 0), xmm_curr_counter); //save counter back
2984     handleSOERegisters(false /*restoring*/);
2985     __ movptr(rax, len_param); // return length
2986     __ leave();                // required for proper stackwalking of RuntimeStub frame
2987     __ ret(0);
2988 
2989     __ BIND (L_key192_top);
2990     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2991     __ jmp(L_multiBlock_loopTop[1]); //key192
2992 
2993     __ BIND (L_key256_top);
2994     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2995     __ jmp(L_multiBlock_loopTop[2]); //key192
2996 
2997     return start;
2998   }
2999 
3000   address generate_upper_word_mask() {
3001     __ align(64);
3002     StubCodeMark mark(this, "StubRoutines", "upper_word_mask");
3003     address start = __ pc();
3004     __ emit_data(0x00000000, relocInfo::none, 0);
3005     __ emit_data(0x00000000, relocInfo::none, 0);
3006     __ emit_data(0x00000000, relocInfo::none, 0);
3007     __ emit_data(0xFFFFFFFF, relocInfo::none, 0);
3008     return start;
3009   }
3010 
3011   address generate_shuffle_byte_flip_mask() {
3012     __ align(64);
3013     StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask");
3014     address start = __ pc();
3015     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3016     __ emit_data(0x08090a0b, relocInfo::none, 0);
3017     __ emit_data(0x04050607, relocInfo::none, 0);
3018     __ emit_data(0x00010203, relocInfo::none, 0);
3019     return start;
3020   }
3021 
3022   // ofs and limit are use for multi-block byte array.
3023   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3024   address generate_sha1_implCompress(bool multi_block, const char *name) {
3025     __ align(CodeEntryAlignment);
3026     StubCodeMark mark(this, "StubRoutines", name);
3027     address start = __ pc();
3028 
3029     Register buf   = rax;
3030     Register state = rdx;
3031     Register ofs   = rcx;
3032     Register limit = rdi;
3033 
3034     const Address  buf_param(rbp, 8 + 0);
3035     const Address  state_param(rbp, 8 + 4);
3036     const Address  ofs_param(rbp, 8 + 8);
3037     const Address  limit_param(rbp, 8 + 12);
3038 
3039     const XMMRegister abcd = xmm0;
3040     const XMMRegister e0 = xmm1;
3041     const XMMRegister e1 = xmm2;
3042     const XMMRegister msg0 = xmm3;
3043 
3044     const XMMRegister msg1 = xmm4;
3045     const XMMRegister msg2 = xmm5;
3046     const XMMRegister msg3 = xmm6;
3047     const XMMRegister shuf_mask = xmm7;
3048 
3049     __ enter();
3050     __ subptr(rsp, 8 * wordSize);
3051     handleSOERegisters(true /*saving*/);
3052 
3053     __ movptr(buf, buf_param);
3054     __ movptr(state, state_param);
3055     if (multi_block) {
3056       __ movptr(ofs, ofs_param);
3057       __ movptr(limit, limit_param);
3058     }
3059 
3060     __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask,
3061       buf, state, ofs, limit, rsp, multi_block);
3062 
3063     handleSOERegisters(false /*restoring*/);
3064     __ addptr(rsp, 8 * wordSize);
3065     __ leave();
3066     __ ret(0);
3067     return start;
3068   }
3069 
3070   address generate_pshuffle_byte_flip_mask() {
3071     __ align(64);
3072     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask");
3073     address start = __ pc();
3074     __ emit_data(0x00010203, relocInfo::none, 0);
3075     __ emit_data(0x04050607, relocInfo::none, 0);
3076     __ emit_data(0x08090a0b, relocInfo::none, 0);
3077     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3078     return start;
3079   }
3080 
3081   // ofs and limit are use for multi-block byte array.
3082   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
3083  address generate_sha256_implCompress(bool multi_block, const char *name) {
3084     __ align(CodeEntryAlignment);
3085     StubCodeMark mark(this, "StubRoutines", name);
3086     address start = __ pc();
3087 
3088     Register buf = rbx;
3089     Register state = rsi;
3090     Register ofs = rdx;
3091     Register limit = rcx;
3092 
3093     const Address  buf_param(rbp, 8 + 0);
3094     const Address  state_param(rbp, 8 + 4);
3095     const Address  ofs_param(rbp, 8 + 8);
3096     const Address  limit_param(rbp, 8 + 12);
3097 
3098     const XMMRegister msg = xmm0;
3099     const XMMRegister state0 = xmm1;
3100     const XMMRegister state1 = xmm2;
3101     const XMMRegister msgtmp0 = xmm3;
3102 
3103     const XMMRegister msgtmp1 = xmm4;
3104     const XMMRegister msgtmp2 = xmm5;
3105     const XMMRegister msgtmp3 = xmm6;
3106     const XMMRegister msgtmp4 = xmm7;
3107 
3108     __ enter();
3109     __ subptr(rsp, 8 * wordSize);
3110     handleSOERegisters(true /*saving*/);
3111     __ movptr(buf, buf_param);
3112     __ movptr(state, state_param);
3113     if (multi_block) {
3114      __ movptr(ofs, ofs_param);
3115      __ movptr(limit, limit_param);
3116     }
3117 
3118     __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3119       buf, state, ofs, limit, rsp, multi_block);
3120 
3121     handleSOERegisters(false);
3122     __ addptr(rsp, 8 * wordSize);
3123     __ leave();
3124     __ ret(0);
3125     return start;
3126   }
3127 
3128   // byte swap x86 long
3129   address generate_ghash_long_swap_mask() {
3130     __ align(CodeEntryAlignment);
3131     StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
3132     address start = __ pc();
3133     __ emit_data(0x0b0a0908, relocInfo::none, 0);
3134     __ emit_data(0x0f0e0d0c, relocInfo::none, 0);
3135     __ emit_data(0x03020100, relocInfo::none, 0);
3136     __ emit_data(0x07060504, relocInfo::none, 0);
3137 
3138   return start;
3139   }
3140 
3141   // byte swap x86 byte array
3142   address generate_ghash_byte_swap_mask() {
3143     __ align(CodeEntryAlignment);
3144     StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
3145     address start = __ pc();
3146     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3147     __ emit_data(0x08090a0b, relocInfo::none, 0);
3148     __ emit_data(0x04050607, relocInfo::none, 0);
3149     __ emit_data(0x00010203, relocInfo::none, 0);
3150   return start;
3151   }
3152 
3153   /* Single and multi-block ghash operations */
3154   address generate_ghash_processBlocks() {
3155     assert(UseGHASHIntrinsics, "need GHASH intrinsics and CLMUL support");
3156     __ align(CodeEntryAlignment);
3157     Label L_ghash_loop, L_exit;
3158     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
3159     address start = __ pc();
3160 
3161     const Register state        = rdi;
3162     const Register subkeyH      = rsi;
3163     const Register data         = rdx;
3164     const Register blocks       = rcx;
3165 
3166     const Address  state_param(rbp, 8+0);
3167     const Address  subkeyH_param(rbp, 8+4);
3168     const Address  data_param(rbp, 8+8);
3169     const Address  blocks_param(rbp, 8+12);
3170 
3171     const XMMRegister xmm_temp0 = xmm0;
3172     const XMMRegister xmm_temp1 = xmm1;
3173     const XMMRegister xmm_temp2 = xmm2;
3174     const XMMRegister xmm_temp3 = xmm3;
3175     const XMMRegister xmm_temp4 = xmm4;
3176     const XMMRegister xmm_temp5 = xmm5;
3177     const XMMRegister xmm_temp6 = xmm6;
3178     const XMMRegister xmm_temp7 = xmm7;
3179 
3180     __ enter();
3181     handleSOERegisters(true);  // Save registers
3182 
3183     __ movptr(state, state_param);
3184     __ movptr(subkeyH, subkeyH_param);
3185     __ movptr(data, data_param);
3186     __ movptr(blocks, blocks_param);
3187 
3188     __ movdqu(xmm_temp0, Address(state, 0));
3189     __ pshufb(xmm_temp0, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3190 
3191     __ movdqu(xmm_temp1, Address(subkeyH, 0));
3192     __ pshufb(xmm_temp1, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3193 
3194     __ BIND(L_ghash_loop);
3195     __ movdqu(xmm_temp2, Address(data, 0));
3196     __ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
3197 
3198     __ pxor(xmm_temp0, xmm_temp2);
3199 
3200     //
3201     // Multiply with the hash key
3202     //
3203     __ movdqu(xmm_temp3, xmm_temp0);
3204     __ pclmulqdq(xmm_temp3, xmm_temp1, 0);      // xmm3 holds a0*b0
3205     __ movdqu(xmm_temp4, xmm_temp0);
3206     __ pclmulqdq(xmm_temp4, xmm_temp1, 16);     // xmm4 holds a0*b1
3207 
3208     __ movdqu(xmm_temp5, xmm_temp0);
3209     __ pclmulqdq(xmm_temp5, xmm_temp1, 1);      // xmm5 holds a1*b0
3210     __ movdqu(xmm_temp6, xmm_temp0);
3211     __ pclmulqdq(xmm_temp6, xmm_temp1, 17);     // xmm6 holds a1*b1
3212 
3213     __ pxor(xmm_temp4, xmm_temp5);      // xmm4 holds a0*b1 + a1*b0
3214 
3215     __ movdqu(xmm_temp5, xmm_temp4);    // move the contents of xmm4 to xmm5
3216     __ psrldq(xmm_temp4, 8);    // shift by xmm4 64 bits to the right
3217     __ pslldq(xmm_temp5, 8);    // shift by xmm5 64 bits to the left
3218     __ pxor(xmm_temp3, xmm_temp5);
3219     __ pxor(xmm_temp6, xmm_temp4);      // Register pair <xmm6:xmm3> holds the result
3220                                         // of the carry-less multiplication of
3221                                         // xmm0 by xmm1.
3222 
3223     // We shift the result of the multiplication by one bit position
3224     // to the left to cope for the fact that the bits are reversed.
3225     __ movdqu(xmm_temp7, xmm_temp3);
3226     __ movdqu(xmm_temp4, xmm_temp6);
3227     __ pslld (xmm_temp3, 1);
3228     __ pslld(xmm_temp6, 1);
3229     __ psrld(xmm_temp7, 31);
3230     __ psrld(xmm_temp4, 31);
3231     __ movdqu(xmm_temp5, xmm_temp7);
3232     __ pslldq(xmm_temp4, 4);
3233     __ pslldq(xmm_temp7, 4);
3234     __ psrldq(xmm_temp5, 12);
3235     __ por(xmm_temp3, xmm_temp7);
3236     __ por(xmm_temp6, xmm_temp4);
3237     __ por(xmm_temp6, xmm_temp5);
3238 
3239     //
3240     // First phase of the reduction
3241     //
3242     // Move xmm3 into xmm4, xmm5, xmm7 in order to perform the shifts
3243     // independently.
3244     __ movdqu(xmm_temp7, xmm_temp3);
3245     __ movdqu(xmm_temp4, xmm_temp3);
3246     __ movdqu(xmm_temp5, xmm_temp3);
3247     __ pslld(xmm_temp7, 31);    // packed right shift shifting << 31
3248     __ pslld(xmm_temp4, 30);    // packed right shift shifting << 30
3249     __ pslld(xmm_temp5, 25);    // packed right shift shifting << 25
3250     __ pxor(xmm_temp7, xmm_temp4);      // xor the shifted versions
3251     __ pxor(xmm_temp7, xmm_temp5);
3252     __ movdqu(xmm_temp4, xmm_temp7);
3253     __ pslldq(xmm_temp7, 12);
3254     __ psrldq(xmm_temp4, 4);
3255     __ pxor(xmm_temp3, xmm_temp7);      // first phase of the reduction complete
3256 
3257     //
3258     // Second phase of the reduction
3259     //
3260     // Make 3 copies of xmm3 in xmm2, xmm5, xmm7 for doing these
3261     // shift operations.
3262     __ movdqu(xmm_temp2, xmm_temp3);
3263     __ movdqu(xmm_temp7, xmm_temp3);
3264     __ movdqu(xmm_temp5, xmm_temp3);
3265     __ psrld(xmm_temp2, 1);     // packed left shifting >> 1
3266     __ psrld(xmm_temp7, 2);     // packed left shifting >> 2
3267     __ psrld(xmm_temp5, 7);     // packed left shifting >> 7
3268     __ pxor(xmm_temp2, xmm_temp7);      // xor the shifted versions
3269     __ pxor(xmm_temp2, xmm_temp5);
3270     __ pxor(xmm_temp2, xmm_temp4);
3271     __ pxor(xmm_temp3, xmm_temp2);
3272     __ pxor(xmm_temp6, xmm_temp3);      // the result is in xmm6
3273 
3274     __ decrement(blocks);
3275     __ jcc(Assembler::zero, L_exit);
3276     __ movdqu(xmm_temp0, xmm_temp6);
3277     __ addptr(data, 16);
3278     __ jmp(L_ghash_loop);
3279 
3280     __ BIND(L_exit);
3281        // Byte swap 16-byte result
3282     __ pshufb(xmm_temp6, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3283     __ movdqu(Address(state, 0), xmm_temp6);   // store the result
3284 
3285     handleSOERegisters(false);  // restore registers
3286     __ leave();
3287     __ ret(0);
3288     return start;
3289   }
3290 
3291   /**
3292    *  Arguments:
3293    *
3294    * Inputs:
3295    *   rsp(4)   - int crc
3296    *   rsp(8)   - byte* buf
3297    *   rsp(12)  - int length
3298    *
3299    * Ouput:
3300    *       rax   - int crc result
3301    */
3302   address generate_updateBytesCRC32() {
3303     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
3304 
3305     __ align(CodeEntryAlignment);
3306     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
3307 
3308     address start = __ pc();
3309 
3310     const Register crc   = rdx;  // crc
3311     const Register buf   = rsi;  // source java byte array address
3312     const Register len   = rcx;  // length
3313     const Register table = rdi;  // crc_table address (reuse register)
3314     const Register tmp   = rbx;
3315     assert_different_registers(crc, buf, len, table, tmp, rax);
3316 
3317     BLOCK_COMMENT("Entry:");
3318     __ enter(); // required for proper stackwalking of RuntimeStub frame
3319     __ push(rsi);
3320     __ push(rdi);
3321     __ push(rbx);
3322 
3323     Address crc_arg(rbp, 8 + 0);
3324     Address buf_arg(rbp, 8 + 4);
3325     Address len_arg(rbp, 8 + 8);
3326 
3327     // Load up:
3328     __ movl(crc,   crc_arg);
3329     __ movptr(buf, buf_arg);
3330     __ movl(len,   len_arg);
3331 
3332     __ kernel_crc32(crc, buf, len, table, tmp);
3333 
3334     __ movl(rax, crc);
3335     __ pop(rbx);
3336     __ pop(rdi);
3337     __ pop(rsi);
3338     __ vzeroupper();
3339     __ leave(); // required for proper stackwalking of RuntimeStub frame
3340     __ ret(0);
3341 
3342     return start;
3343   }
3344 
3345   /**
3346   *  Arguments:
3347   *
3348   * Inputs:
3349   *   rsp(4)   - int crc
3350   *   rsp(8)   - byte* buf
3351   *   rsp(12)  - int length
3352   *   rsp(16)  - table_start - optional (present only when doing a library_calll,
3353   *              not used by x86 algorithm)
3354   *
3355   * Ouput:
3356   *       rax  - int crc result
3357   */
3358   address generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
3359     assert(UseCRC32CIntrinsics, "need SSE4_2");
3360     __ align(CodeEntryAlignment);
3361     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
3362     address start = __ pc();
3363     const Register crc = rax;  // crc
3364     const Register buf = rcx;  // source java byte array address
3365     const Register len = rdx;  // length
3366     const Register d = rbx;
3367     const Register g = rsi;
3368     const Register h = rdi;
3369     const Register empty = 0; // will never be used, in order not
3370                               // to change a signature for crc32c_IPL_Alg2_Alt2
3371                               // between 64/32 I'm just keeping it here
3372     assert_different_registers(crc, buf, len, d, g, h);
3373 
3374     BLOCK_COMMENT("Entry:");
3375     __ enter(); // required for proper stackwalking of RuntimeStub frame
3376     Address crc_arg(rsp, 4 + 4 + 0); // ESP+4 +
3377                                      // we need to add additional 4 because __ enter
3378                                      // have just pushed ebp on a stack
3379     Address buf_arg(rsp, 4 + 4 + 4);
3380     Address len_arg(rsp, 4 + 4 + 8);
3381       // Load up:
3382       __ movl(crc, crc_arg);
3383       __ movl(buf, buf_arg);
3384       __ movl(len, len_arg);
3385       __ push(d);
3386       __ push(g);
3387       __ push(h);
3388       __ crc32c_ipl_alg2_alt2(crc, buf, len,
3389                               d, g, h,
3390                               empty, empty, empty,
3391                               xmm0, xmm1, xmm2,
3392                               is_pclmulqdq_supported);
3393       __ pop(h);
3394       __ pop(g);
3395       __ pop(d);
3396     __ vzeroupper();
3397     __ leave(); // required for proper stackwalking of RuntimeStub frame
3398     __ ret(0);
3399 
3400     return start;
3401   }
3402 
3403  address generate_libmExp() {
3404     StubCodeMark mark(this, "StubRoutines", "libmExp");
3405 
3406     address start = __ pc();
3407 
3408     const XMMRegister x0  = xmm0;
3409     const XMMRegister x1  = xmm1;
3410     const XMMRegister x2  = xmm2;
3411     const XMMRegister x3  = xmm3;
3412 
3413     const XMMRegister x4  = xmm4;
3414     const XMMRegister x5  = xmm5;
3415     const XMMRegister x6  = xmm6;
3416     const XMMRegister x7  = xmm7;
3417 
3418     const Register tmp   = rbx;
3419 
3420     BLOCK_COMMENT("Entry:");
3421     __ enter(); // required for proper stackwalking of RuntimeStub frame
3422     __ fast_exp(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3423     __ leave(); // required for proper stackwalking of RuntimeStub frame
3424     __ ret(0);
3425 
3426     return start;
3427 
3428   }
3429 
3430  address generate_libmLog() {
3431    StubCodeMark mark(this, "StubRoutines", "libmLog");
3432 
3433    address start = __ pc();
3434 
3435    const XMMRegister x0 = xmm0;
3436    const XMMRegister x1 = xmm1;
3437    const XMMRegister x2 = xmm2;
3438    const XMMRegister x3 = xmm3;
3439 
3440    const XMMRegister x4 = xmm4;
3441    const XMMRegister x5 = xmm5;
3442    const XMMRegister x6 = xmm6;
3443    const XMMRegister x7 = xmm7;
3444 
3445    const Register tmp = rbx;
3446 
3447    BLOCK_COMMENT("Entry:");
3448    __ enter(); // required for proper stackwalking of RuntimeStub frame
3449    __ fast_log(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3450    __ leave(); // required for proper stackwalking of RuntimeStub frame
3451    __ ret(0);
3452 
3453    return start;
3454 
3455  }
3456 
3457  address generate_libmLog10() {
3458    StubCodeMark mark(this, "StubRoutines", "libmLog10");
3459 
3460    address start = __ pc();
3461 
3462    const XMMRegister x0 = xmm0;
3463    const XMMRegister x1 = xmm1;
3464    const XMMRegister x2 = xmm2;
3465    const XMMRegister x3 = xmm3;
3466 
3467    const XMMRegister x4 = xmm4;
3468    const XMMRegister x5 = xmm5;
3469    const XMMRegister x6 = xmm6;
3470    const XMMRegister x7 = xmm7;
3471 
3472    const Register tmp = rbx;
3473 
3474    BLOCK_COMMENT("Entry:");
3475    __ enter(); // required for proper stackwalking of RuntimeStub frame
3476    __ fast_log10(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3477    __ leave(); // required for proper stackwalking of RuntimeStub frame
3478    __ ret(0);
3479 
3480    return start;
3481 
3482  }
3483 
3484  address generate_libmPow() {
3485    StubCodeMark mark(this, "StubRoutines", "libmPow");
3486 
3487    address start = __ pc();
3488 
3489    const XMMRegister x0 = xmm0;
3490    const XMMRegister x1 = xmm1;
3491    const XMMRegister x2 = xmm2;
3492    const XMMRegister x3 = xmm3;
3493 
3494    const XMMRegister x4 = xmm4;
3495    const XMMRegister x5 = xmm5;
3496    const XMMRegister x6 = xmm6;
3497    const XMMRegister x7 = xmm7;
3498 
3499    const Register tmp = rbx;
3500 
3501    BLOCK_COMMENT("Entry:");
3502    __ enter(); // required for proper stackwalking of RuntimeStub frame
3503    __ fast_pow(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3504    __ leave(); // required for proper stackwalking of RuntimeStub frame
3505    __ ret(0);
3506 
3507    return start;
3508 
3509  }
3510 
3511  address generate_libm_reduce_pi04l() {
3512    StubCodeMark mark(this, "StubRoutines", "libm_reduce_pi04l");
3513 
3514    address start = __ pc();
3515 
3516    BLOCK_COMMENT("Entry:");
3517    __ libm_reduce_pi04l(rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3518 
3519    return start;
3520 
3521  }
3522 
3523  address generate_libm_sin_cos_huge() {
3524    StubCodeMark mark(this, "StubRoutines", "libm_sin_cos_huge");
3525 
3526    address start = __ pc();
3527 
3528    const XMMRegister x0 = xmm0;
3529    const XMMRegister x1 = xmm1;
3530 
3531    BLOCK_COMMENT("Entry:");
3532    __ libm_sincos_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3533 
3534    return start;
3535 
3536  }
3537 
3538  address generate_libmSin() {
3539    StubCodeMark mark(this, "StubRoutines", "libmSin");
3540 
3541    address start = __ pc();
3542 
3543    const XMMRegister x0 = xmm0;
3544    const XMMRegister x1 = xmm1;
3545    const XMMRegister x2 = xmm2;
3546    const XMMRegister x3 = xmm3;
3547 
3548    const XMMRegister x4 = xmm4;
3549    const XMMRegister x5 = xmm5;
3550    const XMMRegister x6 = xmm6;
3551    const XMMRegister x7 = xmm7;
3552 
3553    BLOCK_COMMENT("Entry:");
3554    __ enter(); // required for proper stackwalking of RuntimeStub frame
3555    __ fast_sin(x0, x1, x2, x3, x4, x5, x6, x7, rax, rbx, rdx);
3556    __ leave(); // required for proper stackwalking of RuntimeStub frame
3557    __ ret(0);
3558 
3559    return start;
3560 
3561  }
3562 
3563  address generate_libmCos() {
3564    StubCodeMark mark(this, "StubRoutines", "libmCos");
3565 
3566    address start = __ pc();
3567 
3568    const XMMRegister x0 = xmm0;
3569    const XMMRegister x1 = xmm1;
3570    const XMMRegister x2 = xmm2;
3571    const XMMRegister x3 = xmm3;
3572 
3573    const XMMRegister x4 = xmm4;
3574    const XMMRegister x5 = xmm5;
3575    const XMMRegister x6 = xmm6;
3576    const XMMRegister x7 = xmm7;
3577 
3578    const Register tmp = rbx;
3579 
3580    BLOCK_COMMENT("Entry:");
3581    __ enter(); // required for proper stackwalking of RuntimeStub frame
3582    __ fast_cos(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3583    __ leave(); // required for proper stackwalking of RuntimeStub frame
3584    __ ret(0);
3585 
3586    return start;
3587 
3588  }
3589 
3590  address generate_libm_tan_cot_huge() {
3591    StubCodeMark mark(this, "StubRoutines", "libm_tan_cot_huge");
3592 
3593    address start = __ pc();
3594 
3595    const XMMRegister x0 = xmm0;
3596    const XMMRegister x1 = xmm1;
3597 
3598    BLOCK_COMMENT("Entry:");
3599    __ libm_tancot_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3600 
3601    return start;
3602 
3603  }
3604 
3605  address generate_libmTan() {
3606    StubCodeMark mark(this, "StubRoutines", "libmTan");
3607 
3608    address start = __ pc();
3609 
3610    const XMMRegister x0 = xmm0;
3611    const XMMRegister x1 = xmm1;
3612    const XMMRegister x2 = xmm2;
3613    const XMMRegister x3 = xmm3;
3614 
3615    const XMMRegister x4 = xmm4;
3616    const XMMRegister x5 = xmm5;
3617    const XMMRegister x6 = xmm6;
3618    const XMMRegister x7 = xmm7;
3619 
3620    const Register tmp = rbx;
3621 
3622    BLOCK_COMMENT("Entry:");
3623    __ enter(); // required for proper stackwalking of RuntimeStub frame
3624    __ fast_tan(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3625    __ leave(); // required for proper stackwalking of RuntimeStub frame
3626    __ ret(0);
3627 
3628    return start;
3629 
3630  }
3631 
3632   // Safefetch stubs.
3633   void generate_safefetch(const char* name, int size, address* entry,
3634                           address* fault_pc, address* continuation_pc) {
3635     // safefetch signatures:
3636     //   int      SafeFetch32(int*      adr, int      errValue);
3637     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3638 
3639     StubCodeMark mark(this, "StubRoutines", name);
3640 
3641     // Entry point, pc or function descriptor.
3642     *entry = __ pc();
3643 
3644     __ movl(rax, Address(rsp, 0x8));
3645     __ movl(rcx, Address(rsp, 0x4));
3646     // Load *adr into eax, may fault.
3647     *fault_pc = __ pc();
3648     switch (size) {
3649       case 4:
3650         // int32_t
3651         __ movl(rax, Address(rcx, 0));
3652         break;
3653       case 8:
3654         // int64_t
3655         Unimplemented();
3656         break;
3657       default:
3658         ShouldNotReachHere();
3659     }
3660 
3661     // Return errValue or *adr.
3662     *continuation_pc = __ pc();
3663     __ ret(0);
3664   }
3665 
3666   address generate_method_entry_barrier() {
3667     __ align(CodeEntryAlignment);
3668     StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier");
3669 
3670     Label deoptimize_label;
3671 
3672     address start = __ pc();
3673 
3674     __ push(-1); // cookie, this is used for writing the new rsp when deoptimizing
3675 
3676     BLOCK_COMMENT("Entry:");
3677     __ enter(); // save rbp
3678 
3679     // save rbx, because we want to use that value.
3680     // We could do without it but then we depend on the number of slots used by pusha
3681     __ push(rbx);
3682 
3683     __ lea(rbx, Address(rsp, wordSize * 3)); // 1 for cookie, 1 for rbp, 1 for rbx - this should be the return address
3684 
3685     __ pusha();
3686 
3687     // xmm0 and xmm1 may be used for passing float/double arguments
3688     const int xmm_size = wordSize * 2;
3689     const int xmm_spill_size = xmm_size * 2;
3690     __ subptr(rsp, xmm_spill_size);
3691     __ movdqu(Address(rsp, xmm_size * 1), xmm1);
3692     __ movdqu(Address(rsp, xmm_size * 0), xmm0);
3693 
3694     __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(address*)>(BarrierSetNMethod::nmethod_stub_entry_barrier)), rbx);
3695 
3696     __ movdqu(xmm0, Address(rsp, xmm_size * 0));
3697     __ movdqu(xmm1, Address(rsp, xmm_size * 1));
3698     __ addptr(rsp, xmm_spill_size);
3699 
3700     __ cmpl(rax, 1); // 1 means deoptimize
3701     __ jcc(Assembler::equal, deoptimize_label);
3702 
3703     __ popa();
3704     __ pop(rbx);
3705 
3706     __ leave();
3707 
3708     __ addptr(rsp, 1 * wordSize); // cookie
3709     __ ret(0);
3710 
3711     __ BIND(deoptimize_label);
3712 
3713     __ popa();
3714     __ pop(rbx);
3715 
3716     __ leave();
3717 
3718     // this can be taken out, but is good for verification purposes. getting a SIGSEGV
3719     // here while still having a correct stack is valuable
3720     __ testptr(rsp, Address(rsp, 0));
3721 
3722     __ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier
3723     __ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point
3724 
3725     return start;
3726   }
3727 
3728  public:
3729   // Information about frame layout at time of blocking runtime call.
3730   // Note that we only have to preserve callee-saved registers since
3731   // the compilers are responsible for supplying a continuation point
3732   // if they expect all registers to be preserved.
3733   enum layout {
3734     thread_off,    // last_java_sp
3735     arg1_off,
3736     arg2_off,
3737     rbp_off,       // callee saved register
3738     ret_pc,
3739     framesize
3740   };
3741 
3742  private:
3743 
3744 #undef  __
3745 #define __ masm->
3746 
3747   //------------------------------------------------------------------------------------------------------------------------
3748   // Continuation point for throwing of implicit exceptions that are not handled in
3749   // the current activation. Fabricates an exception oop and initiates normal
3750   // exception dispatching in this frame.
3751   //
3752   // Previously the compiler (c2) allowed for callee save registers on Java calls.
3753   // This is no longer true after adapter frames were removed but could possibly
3754   // be brought back in the future if the interpreter code was reworked and it
3755   // was deemed worthwhile. The comment below was left to describe what must
3756   // happen here if callee saves were resurrected. As it stands now this stub
3757   // could actually be a vanilla BufferBlob and have now oopMap at all.
3758   // Since it doesn't make much difference we've chosen to leave it the
3759   // way it was in the callee save days and keep the comment.
3760 
3761   // If we need to preserve callee-saved values we need a callee-saved oop map and
3762   // therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.
3763   // If the compiler needs all registers to be preserved between the fault
3764   // point and the exception handler then it must assume responsibility for that in
3765   // AbstractCompiler::continuation_for_implicit_null_exception or
3766   // continuation_for_implicit_division_by_zero_exception. All other implicit
3767   // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
3768   // either at call sites or otherwise assume that stack unwinding will be initiated,
3769   // so caller saved registers were assumed volatile in the compiler.
3770   address generate_throw_exception(const char* name, address runtime_entry,
3771                                    Register arg1 = noreg, Register arg2 = noreg) {
3772 
3773     int insts_size = 256;
3774     int locs_size  = 32;
3775 
3776     CodeBuffer code(name, insts_size, locs_size);
3777     OopMapSet* oop_maps  = new OopMapSet();
3778     MacroAssembler* masm = new MacroAssembler(&code);
3779 
3780     address start = __ pc();
3781 
3782     // This is an inlined and slightly modified version of call_VM
3783     // which has the ability to fetch the return PC out of
3784     // thread-local storage and also sets up last_Java_sp slightly
3785     // differently than the real call_VM
3786     Register java_thread = rbx;
3787     __ get_thread(java_thread);
3788 
3789     __ enter(); // required for proper stackwalking of RuntimeStub frame
3790 
3791     // pc and rbp, already pushed
3792     __ subptr(rsp, (framesize-2) * wordSize); // prolog
3793 
3794     // Frame is now completed as far as size and linkage.
3795 
3796     int frame_complete = __ pc() - start;
3797 
3798     // push java thread (becomes first argument of C function)
3799     __ movptr(Address(rsp, thread_off * wordSize), java_thread);
3800     if (arg1 != noreg) {
3801       __ movptr(Address(rsp, arg1_off * wordSize), arg1);
3802     }
3803     if (arg2 != noreg) {
3804       assert(arg1 != noreg, "missing reg arg");
3805       __ movptr(Address(rsp, arg2_off * wordSize), arg2);
3806     }
3807 
3808     // Set up last_Java_sp and last_Java_fp
3809     __ set_last_Java_frame(java_thread, rsp, rbp, NULL);
3810 
3811     // Call runtime
3812     BLOCK_COMMENT("call runtime_entry");
3813     __ call(RuntimeAddress(runtime_entry));
3814     // Generate oop map
3815     OopMap* map =  new OopMap(framesize, 0);
3816     oop_maps->add_gc_map(__ pc() - start, map);
3817 
3818     // restore the thread (cannot use the pushed argument since arguments
3819     // may be overwritten by C code generated by an optimizing compiler);
3820     // however can use the register value directly if it is callee saved.
3821     __ get_thread(java_thread);
3822 
3823     __ reset_last_Java_frame(java_thread, true);
3824 
3825     __ leave(); // required for proper stackwalking of RuntimeStub frame
3826 
3827     // check for pending exceptions
3828 #ifdef ASSERT
3829     Label L;
3830     __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3831     __ jcc(Assembler::notEqual, L);
3832     __ should_not_reach_here();
3833     __ bind(L);
3834 #endif /* ASSERT */
3835     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3836 
3837 
3838     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);
3839     return stub->entry_point();
3840   }
3841 
3842 
3843   void create_control_words() {
3844     // Round to nearest, 53-bit mode, exceptions masked
3845     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
3846     // Round to zero, 53-bit mode, exception mased
3847     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
3848     // Round to nearest, 24-bit mode, exceptions masked
3849     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
3850     // Round to nearest, 64-bit mode, exceptions masked
3851     StubRoutines::_mxcsr_std           = 0x1F80;
3852     // Note: the following two constants are 80-bit values
3853     //       layout is critical for correct loading by FPU.
3854     // Bias for strict fp multiply/divide
3855     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
3856     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
3857     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
3858     // Un-Bias for strict fp multiply/divide
3859     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
3860     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
3861     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
3862   }
3863 
3864   //---------------------------------------------------------------------------
3865   // Initialization
3866 
3867   void generate_initial() {
3868     // Generates all stubs and initializes the entry points
3869 
3870     //------------------------------------------------------------------------------------------------------------------------
3871     // entry points that exist in all platforms
3872     // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
3873     //       the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
3874     StubRoutines::_forward_exception_entry      = generate_forward_exception();
3875 
3876     StubRoutines::_call_stub_entry              =
3877       generate_call_stub(StubRoutines::_call_stub_return_address);
3878     // is referenced by megamorphic call
3879     StubRoutines::_catch_exception_entry        = generate_catch_exception();
3880 
3881     // These are currently used by Solaris/Intel
3882     StubRoutines::_atomic_xchg_entry            = generate_atomic_xchg();
3883 
3884     // platform dependent
3885     create_control_words();
3886 
3887     StubRoutines::x86::_verify_mxcsr_entry                 = generate_verify_mxcsr();
3888     StubRoutines::x86::_verify_fpu_cntrl_wrd_entry         = generate_verify_fpu_cntrl_wrd();
3889     StubRoutines::_d2i_wrapper                              = generate_d2i_wrapper(T_INT,
3890                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
3891     StubRoutines::_d2l_wrapper                              = generate_d2i_wrapper(T_LONG,
3892                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
3893 
3894     // Build this early so it's available for the interpreter
3895     StubRoutines::_throw_StackOverflowError_entry          = generate_throw_exception("StackOverflowError throw_exception",
3896                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
3897     StubRoutines::_throw_delayed_StackOverflowError_entry  = generate_throw_exception("delayed StackOverflowError throw_exception",
3898                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_delayed_StackOverflowError));
3899 
3900     if (UseCRC32Intrinsics) {
3901       // set table address before stub generation which use it
3902       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
3903       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
3904     }
3905 
3906     if (UseCRC32CIntrinsics) {
3907       bool supports_clmul = VM_Version::supports_clmul();
3908       StubRoutines::x86::generate_CRC32C_table(supports_clmul);
3909       StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table;
3910       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
3911     }
3912     if (VM_Version::supports_sse2() && UseLibmIntrinsic && InlineIntrinsics) {
3913       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3914           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3915           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3916         StubRoutines::x86::_L_2il0floatpacket_0_adr = (address)StubRoutines::x86::_L_2il0floatpacket_0;
3917         StubRoutines::x86::_Pi4Inv_adr = (address)StubRoutines::x86::_Pi4Inv;
3918         StubRoutines::x86::_Pi4x3_adr = (address)StubRoutines::x86::_Pi4x3;
3919         StubRoutines::x86::_Pi4x4_adr = (address)StubRoutines::x86::_Pi4x4;
3920         StubRoutines::x86::_ones_adr = (address)StubRoutines::x86::_ones;
3921       }
3922       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) {
3923         StubRoutines::_dexp = generate_libmExp();
3924       }
3925       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) {
3926         StubRoutines::_dlog = generate_libmLog();
3927       }
3928       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) {
3929         StubRoutines::_dlog10 = generate_libmLog10();
3930       }
3931       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) {
3932         StubRoutines::_dpow = generate_libmPow();
3933       }
3934       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3935         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3936         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3937         StubRoutines::_dlibm_reduce_pi04l = generate_libm_reduce_pi04l();
3938       }
3939       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3940         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3941         StubRoutines::_dlibm_sin_cos_huge = generate_libm_sin_cos_huge();
3942       }
3943       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
3944         StubRoutines::_dsin = generate_libmSin();
3945       }
3946       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3947         StubRoutines::_dcos = generate_libmCos();
3948       }
3949       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3950         StubRoutines::_dlibm_tan_cot_huge = generate_libm_tan_cot_huge();
3951         StubRoutines::_dtan = generate_libmTan();
3952       }
3953     }
3954   }
3955 
3956   void generate_all() {
3957     // Generates all stubs and initializes the entry points
3958 
3959     // These entry points require SharedInfo::stack0 to be set up in non-core builds
3960     // and need to be relocatable, so they each fabricate a RuntimeStub internally.
3961     StubRoutines::_throw_AbstractMethodError_entry         = generate_throw_exception("AbstractMethodError throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
3962     StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
3963     StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
3964 
3965     //------------------------------------------------------------------------------------------------------------------------
3966     // entry points that are platform specific
3967 
3968     StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF);
3969     StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x80000000);
3970     StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask_long_double("vector_double_sign_mask", 0x7FFFFFFF, 0xFFFFFFFF);
3971     StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask_long_double("vector_double_sign_flip", 0x80000000, 0x00000000);
3972     StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff);
3973     StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
3974     StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask_long_double("vector_long_sign_mask", 0x80000000, 0x00000000);
3975 
3976     // support for verify_oop (must happen after universe_init)
3977     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
3978 
3979     // arraycopy stubs used by compilers
3980     generate_arraycopy_stubs();
3981 
3982     // don't bother generating these AES intrinsic stubs unless global flag is set
3983     if (UseAESIntrinsics) {
3984       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // might be needed by the others
3985 
3986       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
3987       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
3988       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
3989       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
3990     }
3991 
3992     if (UseAESCTRIntrinsics) {
3993       StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
3994       StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
3995     }
3996 
3997     if (UseSHA1Intrinsics) {
3998       StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask();
3999       StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask();
4000       StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
4001       StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
4002     }
4003     if (UseSHA256Intrinsics) {
4004       StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
4005       StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
4006       StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
4007       StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
4008     }
4009 
4010     // Generate GHASH intrinsics code
4011     if (UseGHASHIntrinsics) {
4012       StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
4013       StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
4014       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
4015     }
4016 
4017     // Safefetch stubs.
4018     generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
4019                                                    &StubRoutines::_safefetch32_fault_pc,
4020                                                    &StubRoutines::_safefetch32_continuation_pc);
4021     StubRoutines::_safefetchN_entry           = StubRoutines::_safefetch32_entry;
4022     StubRoutines::_safefetchN_fault_pc        = StubRoutines::_safefetch32_fault_pc;
4023     StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc;
4024 
4025     BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
4026     if (bs_nm != NULL) {
4027       StubRoutines::x86::_method_entry_barrier = generate_method_entry_barrier();
4028     }
4029   }
4030 
4031 
4032  public:
4033   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
4034     if (all) {
4035       generate_all();
4036     } else {
4037       generate_initial();
4038     }
4039   }
4040 }; // end class declaration
4041 
4042 #define UCM_TABLE_MAX_ENTRIES 8
4043 void StubGenerator_generate(CodeBuffer* code, bool all) {
4044   if (UnsafeCopyMemory::_table == NULL) {
4045     UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
4046   }
4047   StubGenerator g(code, all);
4048 }