1 /*
   2  * Copyright (c) 1999, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "gc/shared/barrierSet.hpp"
  29 #include "gc/shared/barrierSetAssembler.hpp"
  30 #include "gc/shared/barrierSetNMethod.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "memory/universe.hpp"
  33 #include "nativeInst_x86.hpp"
  34 #include "oops/instanceOop.hpp"
  35 #include "oops/method.hpp"
  36 #include "oops/objArrayKlass.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "prims/methodHandles.hpp"
  39 #include "runtime/frame.inline.hpp"
  40 #include "runtime/handles.inline.hpp"
  41 #include "runtime/sharedRuntime.hpp"
  42 #include "runtime/stubCodeGenerator.hpp"
  43 #include "runtime/stubRoutines.hpp"
  44 #include "runtime/thread.inline.hpp"
  45 #ifdef COMPILER2
  46 #include "opto/runtime.hpp"
  47 #endif
  48 
  49 // Declaration and definition of StubGenerator (no .hpp file).
  50 // For a more detailed description of the stub routine structure
  51 // see the comment in stubRoutines.hpp
  52 
  53 #define __ _masm->
  54 #define a__ ((Assembler*)_masm)->
  55 
  56 #ifdef PRODUCT
  57 #define BLOCK_COMMENT(str) /* nothing */
  58 #else
  59 #define BLOCK_COMMENT(str) __ block_comment(str)
  60 #endif
  61 
  62 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  63 
  64 const int MXCSR_MASK  = 0xFFC0;  // Mask out any pending exceptions
  65 const int FPU_CNTRL_WRD_MASK = 0xFFFF;
  66 
  67 // -------------------------------------------------------------------------------------------------------------------------
  68 // Stub Code definitions
  69 
  70 class StubGenerator: public StubCodeGenerator {
  71  private:
  72 
  73 #ifdef PRODUCT
  74 #define inc_counter_np(counter) ((void)0)
  75 #else
  76   void inc_counter_np_(int& counter) {
  77     __ incrementl(ExternalAddress((address)&counter));
  78   }
  79 #define inc_counter_np(counter) \
  80   BLOCK_COMMENT("inc_counter " #counter); \
  81   inc_counter_np_(counter);
  82 #endif //PRODUCT
  83 
  84   void inc_copy_counter_np(BasicType t) {
  85 #ifndef PRODUCT
  86     switch (t) {
  87     case T_BYTE:    inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;
  88     case T_SHORT:   inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;
  89     case T_INT:     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;
  90     case T_LONG:    inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;
  91     case T_OBJECT:  inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;
  92     default:        ShouldNotReachHere();
  93     }
  94 #endif //PRODUCT
  95   }
  96 
  97   //------------------------------------------------------------------------------------------------------------------------
  98   // Call stubs are used to call Java from C
  99   //
 100   //    [ return_from_Java     ] <--- rsp
 101   //    [ argument word n      ]
 102   //      ...
 103   // -N [ argument word 1      ]
 104   // -7 [ Possible padding for stack alignment ]
 105   // -6 [ Possible padding for stack alignment ]
 106   // -5 [ Possible padding for stack alignment ]
 107   // -4 [ mxcsr save           ] <--- rsp_after_call
 108   // -3 [ saved rbx,            ]
 109   // -2 [ saved rsi            ]
 110   // -1 [ saved rdi            ]
 111   //  0 [ saved rbp,            ] <--- rbp,
 112   //  1 [ return address       ]
 113   //  2 [ ptr. to call wrapper ]
 114   //  3 [ result               ]
 115   //  4 [ result_type          ]
 116   //  5 [ method               ]
 117   //  6 [ entry_point          ]
 118   //  7 [ parameters           ]
 119   //  8 [ parameter_size       ]
 120   //  9 [ thread               ]
 121 
 122 
 123   address generate_call_stub(address& return_address) {
 124     StubCodeMark mark(this, "StubRoutines", "call_stub");
 125     address start = __ pc();
 126 
 127     // stub code parameters / addresses
 128     assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");
 129     bool  sse_save = false;
 130     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!
 131     const int     locals_count_in_bytes  (4*wordSize);
 132     const Address mxcsr_save    (rbp, -4 * wordSize);
 133     const Address saved_rbx     (rbp, -3 * wordSize);
 134     const Address saved_rsi     (rbp, -2 * wordSize);
 135     const Address saved_rdi     (rbp, -1 * wordSize);
 136     const Address result        (rbp,  3 * wordSize);
 137     const Address result_type   (rbp,  4 * wordSize);
 138     const Address method        (rbp,  5 * wordSize);
 139     const Address entry_point   (rbp,  6 * wordSize);
 140     const Address parameters    (rbp,  7 * wordSize);
 141     const Address parameter_size(rbp,  8 * wordSize);
 142     const Address thread        (rbp,  9 * wordSize); // same as in generate_catch_exception()!
 143     sse_save =  UseSSE > 0;
 144 
 145     // stub code
 146     __ enter();
 147     __ movptr(rcx, parameter_size);              // parameter counter
 148     __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes
 149     __ addptr(rcx, locals_count_in_bytes);       // reserve space for register saves
 150     __ subptr(rsp, rcx);
 151     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
 152 
 153     // save rdi, rsi, & rbx, according to C calling conventions
 154     __ movptr(saved_rdi, rdi);
 155     __ movptr(saved_rsi, rsi);
 156     __ movptr(saved_rbx, rbx);
 157 
 158     // save and initialize %mxcsr
 159     if (sse_save) {
 160       Label skip_ldmx;
 161       __ stmxcsr(mxcsr_save);
 162       __ movl(rax, mxcsr_save);
 163       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 164       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 165       __ cmp32(rax, mxcsr_std);
 166       __ jcc(Assembler::equal, skip_ldmx);
 167       __ ldmxcsr(mxcsr_std);
 168       __ bind(skip_ldmx);
 169     }
 170 
 171     // make sure the control word is correct.
 172     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 173 
 174 #ifdef ASSERT
 175     // make sure we have no pending exceptions
 176     { Label L;
 177       __ movptr(rcx, thread);
 178       __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 179       __ jcc(Assembler::equal, L);
 180       __ stop("StubRoutines::call_stub: entered with pending exception");
 181       __ bind(L);
 182     }
 183 #endif
 184 
 185     // pass parameters if any
 186     BLOCK_COMMENT("pass parameters if any");
 187     Label parameters_done;
 188     __ movl(rcx, parameter_size);  // parameter counter
 189     __ testl(rcx, rcx);
 190     __ jcc(Assembler::zero, parameters_done);
 191 
 192     // parameter passing loop
 193 
 194     Label loop;
 195     // Copy Java parameters in reverse order (receiver last)
 196     // Note that the argument order is inverted in the process
 197     // source is rdx[rcx: N-1..0]
 198     // dest   is rsp[rbx: 0..N-1]
 199 
 200     __ movptr(rdx, parameters);          // parameter pointer
 201     __ xorptr(rbx, rbx);
 202 
 203     __ BIND(loop);
 204 
 205     // get parameter
 206     __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
 207     __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
 208                     Interpreter::expr_offset_in_bytes(0)), rax);          // store parameter
 209     __ increment(rbx);
 210     __ decrement(rcx);
 211     __ jcc(Assembler::notZero, loop);
 212 
 213     // call Java function
 214     __ BIND(parameters_done);
 215     __ movptr(rbx, method);           // get Method*
 216     __ movptr(rax, entry_point);      // get entry_point
 217     __ mov(rsi, rsp);                 // set sender sp
 218     BLOCK_COMMENT("call Java function");
 219     __ call(rax);
 220 
 221     BLOCK_COMMENT("call_stub_return_address:");
 222     return_address = __ pc();
 223 
 224 #ifdef COMPILER2
 225     {
 226       Label L_skip;
 227       if (UseSSE >= 2) {
 228         __ verify_FPU(0, "call_stub_return");
 229       } else {
 230         for (int i = 1; i < 8; i++) {
 231           __ ffree(i);
 232         }
 233 
 234         // UseSSE <= 1 so double result should be left on TOS
 235         __ movl(rsi, result_type);
 236         __ cmpl(rsi, T_DOUBLE);
 237         __ jcc(Assembler::equal, L_skip);
 238         if (UseSSE == 0) {
 239           // UseSSE == 0 so float result should be left on TOS
 240           __ cmpl(rsi, T_FLOAT);
 241           __ jcc(Assembler::equal, L_skip);
 242         }
 243         __ ffree(0);
 244       }
 245       __ BIND(L_skip);
 246     }
 247 #endif // COMPILER2
 248 
 249     // store result depending on type
 250     // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 251     __ movptr(rdi, result);
 252     Label is_long, is_float, is_double, exit;
 253     __ movl(rsi, result_type);
 254     __ cmpl(rsi, T_LONG);
 255     __ jcc(Assembler::equal, is_long);
 256     __ cmpl(rsi, T_FLOAT);
 257     __ jcc(Assembler::equal, is_float);
 258     __ cmpl(rsi, T_DOUBLE);
 259     __ jcc(Assembler::equal, is_double);
 260 
 261     // handle T_INT case
 262     __ movl(Address(rdi, 0), rax);
 263     __ BIND(exit);
 264 
 265     // check that FPU stack is empty
 266     __ verify_FPU(0, "generate_call_stub");
 267 
 268     // pop parameters
 269     __ lea(rsp, rsp_after_call);
 270 
 271     // restore %mxcsr
 272     if (sse_save) {
 273       __ ldmxcsr(mxcsr_save);
 274     }
 275 
 276     // restore rdi, rsi and rbx,
 277     __ movptr(rbx, saved_rbx);
 278     __ movptr(rsi, saved_rsi);
 279     __ movptr(rdi, saved_rdi);
 280     __ addptr(rsp, 4*wordSize);
 281 
 282     // return
 283     __ pop(rbp);
 284     __ ret(0);
 285 
 286     // handle return types different from T_INT
 287     __ BIND(is_long);
 288     __ movl(Address(rdi, 0 * wordSize), rax);
 289     __ movl(Address(rdi, 1 * wordSize), rdx);
 290     __ jmp(exit);
 291 
 292     __ BIND(is_float);
 293     // interpreter uses xmm0 for return values
 294     if (UseSSE >= 1) {
 295       __ movflt(Address(rdi, 0), xmm0);
 296     } else {
 297       __ fstp_s(Address(rdi, 0));
 298     }
 299     __ jmp(exit);
 300 
 301     __ BIND(is_double);
 302     // interpreter uses xmm0 for return values
 303     if (UseSSE >= 2) {
 304       __ movdbl(Address(rdi, 0), xmm0);
 305     } else {
 306       __ fstp_d(Address(rdi, 0));
 307     }
 308     __ jmp(exit);
 309 
 310     return start;
 311   }
 312 
 313 
 314   //------------------------------------------------------------------------------------------------------------------------
 315   // Return point for a Java call if there's an exception thrown in Java code.
 316   // The exception is caught and transformed into a pending exception stored in
 317   // JavaThread that can be tested from within the VM.
 318   //
 319   // Note: Usually the parameters are removed by the callee. In case of an exception
 320   //       crossing an activation frame boundary, that is not the case if the callee
 321   //       is compiled code => need to setup the rsp.
 322   //
 323   // rax,: exception oop
 324 
 325   address generate_catch_exception() {
 326     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 327     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!
 328     const Address thread        (rbp,  9 * wordSize); // same as in generate_call_stub()!
 329     address start = __ pc();
 330 
 331     // get thread directly
 332     __ movptr(rcx, thread);
 333 #ifdef ASSERT
 334     // verify that threads correspond
 335     { Label L;
 336       __ get_thread(rbx);
 337       __ cmpptr(rbx, rcx);
 338       __ jcc(Assembler::equal, L);
 339       __ stop("StubRoutines::catch_exception: threads must correspond");
 340       __ bind(L);
 341     }
 342 #endif
 343     // set pending exception
 344     __ verify_oop(rax);
 345     __ movptr(Address(rcx, Thread::pending_exception_offset()), rax          );
 346     __ lea(Address(rcx, Thread::exception_file_offset   ()),
 347            ExternalAddress((address)__FILE__));
 348     __ movl(Address(rcx, Thread::exception_line_offset   ()), __LINE__ );
 349     // complete return to VM
 350     assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
 351     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 352 
 353     return start;
 354   }
 355 
 356 
 357   //------------------------------------------------------------------------------------------------------------------------
 358   // Continuation point for runtime calls returning with a pending exception.
 359   // The pending exception check happened in the runtime or native call stub.
 360   // The pending exception in Thread is converted into a Java-level exception.
 361   //
 362   // Contract with Java-level exception handlers:
 363   // rax: exception
 364   // rdx: throwing pc
 365   //
 366   // NOTE: At entry of this stub, exception-pc must be on stack !!
 367 
 368   address generate_forward_exception() {
 369     StubCodeMark mark(this, "StubRoutines", "forward exception");
 370     address start = __ pc();
 371     const Register thread = rcx;
 372 
 373     // other registers used in this stub
 374     const Register exception_oop = rax;
 375     const Register handler_addr  = rbx;
 376     const Register exception_pc  = rdx;
 377 
 378     // Upon entry, the sp points to the return address returning into Java
 379     // (interpreted or compiled) code; i.e., the return address becomes the
 380     // throwing pc.
 381     //
 382     // Arguments pushed before the runtime call are still on the stack but
 383     // the exception handler will reset the stack pointer -> ignore them.
 384     // A potential result in registers can be ignored as well.
 385 
 386 #ifdef ASSERT
 387     // make sure this code is only executed if there is a pending exception
 388     { Label L;
 389       __ get_thread(thread);
 390       __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 391       __ jcc(Assembler::notEqual, L);
 392       __ stop("StubRoutines::forward exception: no pending exception (1)");
 393       __ bind(L);
 394     }
 395 #endif
 396 
 397     // compute exception handler into rbx,
 398     __ get_thread(thread);
 399     __ movptr(exception_pc, Address(rsp, 0));
 400     BLOCK_COMMENT("call exception_handler_for_return_address");
 401     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
 402     __ mov(handler_addr, rax);
 403 
 404     // setup rax & rdx, remove return address & clear pending exception
 405     __ get_thread(thread);
 406     __ pop(exception_pc);
 407     __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
 408     __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
 409 
 410 #ifdef ASSERT
 411     // make sure exception is set
 412     { Label L;
 413       __ testptr(exception_oop, exception_oop);
 414       __ jcc(Assembler::notEqual, L);
 415       __ stop("StubRoutines::forward exception: no pending exception (2)");
 416       __ bind(L);
 417     }
 418 #endif
 419 
 420     // Verify that there is really a valid exception in RAX.
 421     __ verify_oop(exception_oop);
 422 
 423     // continue at exception handler (return address removed)
 424     // rax: exception
 425     // rbx: exception handler
 426     // rdx: throwing pc
 427     __ jmp(handler_addr);
 428 
 429     return start;
 430   }
 431 
































 432   //----------------------------------------------------------------------------------------------------
 433   // Support for void verify_mxcsr()
 434   //
 435   // This routine is used with -Xcheck:jni to verify that native
 436   // JNI code does not return to Java code without restoring the
 437   // MXCSR register to our expected state.
 438 
 439 
 440   address generate_verify_mxcsr() {
 441     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 442     address start = __ pc();
 443 
 444     const Address mxcsr_save(rsp, 0);
 445 
 446     if (CheckJNICalls && UseSSE > 0 ) {
 447       Label ok_ret;
 448       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 449       __ push(rax);
 450       __ subptr(rsp, wordSize);      // allocate a temp location
 451       __ stmxcsr(mxcsr_save);
 452       __ movl(rax, mxcsr_save);
 453       __ andl(rax, MXCSR_MASK);
 454       __ cmp32(rax, mxcsr_std);
 455       __ jcc(Assembler::equal, ok_ret);
 456 
 457       __ warn("MXCSR changed by native JNI code.");
 458 
 459       __ ldmxcsr(mxcsr_std);
 460 
 461       __ bind(ok_ret);
 462       __ addptr(rsp, wordSize);
 463       __ pop(rax);
 464     }
 465 
 466     __ ret(0);
 467 
 468     return start;
 469   }
 470 
 471 
 472   //---------------------------------------------------------------------------
 473   // Support for void verify_fpu_cntrl_wrd()
 474   //
 475   // This routine is used with -Xcheck:jni to verify that native
 476   // JNI code does not return to Java code without restoring the
 477   // FP control word to our expected state.
 478 
 479   address generate_verify_fpu_cntrl_wrd() {
 480     StubCodeMark mark(this, "StubRoutines", "verify_spcw");
 481     address start = __ pc();
 482 
 483     const Address fpu_cntrl_wrd_save(rsp, 0);
 484 
 485     if (CheckJNICalls) {
 486       Label ok_ret;
 487       __ push(rax);
 488       __ subptr(rsp, wordSize);      // allocate a temp location
 489       __ fnstcw(fpu_cntrl_wrd_save);
 490       __ movl(rax, fpu_cntrl_wrd_save);
 491       __ andl(rax, FPU_CNTRL_WRD_MASK);
 492       ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());
 493       __ cmp32(rax, fpu_std);
 494       __ jcc(Assembler::equal, ok_ret);
 495 
 496       __ warn("Floating point control word changed by native JNI code.");
 497 
 498       __ fldcw(fpu_std);
 499 
 500       __ bind(ok_ret);
 501       __ addptr(rsp, wordSize);
 502       __ pop(rax);
 503     }
 504 
 505     __ ret(0);
 506 
 507     return start;
 508   }
 509 
 510   //---------------------------------------------------------------------------
 511   // Wrapper for slow-case handling of double-to-integer conversion
 512   // d2i or f2i fast case failed either because it is nan or because
 513   // of under/overflow.
 514   // Input:  FPU TOS: float value
 515   // Output: rax, (rdx): integer (long) result
 516 
 517   address generate_d2i_wrapper(BasicType t, address fcn) {
 518     StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");
 519     address start = __ pc();
 520 
 521   // Capture info about frame layout
 522   enum layout { FPUState_off         = 0,
 523                 rbp_off              = FPUStateSizeInWords,
 524                 rdi_off,
 525                 rsi_off,
 526                 rcx_off,
 527                 rbx_off,
 528                 saved_argument_off,
 529                 saved_argument_off2, // 2nd half of double
 530                 framesize
 531   };
 532 
 533   assert(FPUStateSizeInWords == 27, "update stack layout");
 534 
 535     // Save outgoing argument to stack across push_FPU_state()
 536     __ subptr(rsp, wordSize * 2);
 537     __ fstp_d(Address(rsp, 0));
 538 
 539     // Save CPU & FPU state
 540     __ push(rbx);
 541     __ push(rcx);
 542     __ push(rsi);
 543     __ push(rdi);
 544     __ push(rbp);
 545     __ push_FPU_state();
 546 
 547     // push_FPU_state() resets the FP top of stack
 548     // Load original double into FP top of stack
 549     __ fld_d(Address(rsp, saved_argument_off * wordSize));
 550     // Store double into stack as outgoing argument
 551     __ subptr(rsp, wordSize*2);
 552     __ fst_d(Address(rsp, 0));
 553 
 554     // Prepare FPU for doing math in C-land
 555     __ empty_FPU_stack();
 556     // Call the C code to massage the double.  Result in EAX
 557     if (t == T_INT)
 558       { BLOCK_COMMENT("SharedRuntime::d2i"); }
 559     else if (t == T_LONG)
 560       { BLOCK_COMMENT("SharedRuntime::d2l"); }
 561     __ call_VM_leaf( fcn, 2 );
 562 
 563     // Restore CPU & FPU state
 564     __ pop_FPU_state();
 565     __ pop(rbp);
 566     __ pop(rdi);
 567     __ pop(rsi);
 568     __ pop(rcx);
 569     __ pop(rbx);
 570     __ addptr(rsp, wordSize * 2);
 571 
 572     __ ret(0);
 573 
 574     return start;
 575   }
 576   //---------------------------------------------------------------------------------------------------
 577 
 578   address generate_vector_mask(const char *stub_name, int32_t mask) {
 579     __ align(CodeEntryAlignment);
 580     StubCodeMark mark(this, "StubRoutines", stub_name);
 581     address start = __ pc();
 582 
 583     for (int i = 0; i < 16; i++) {
 584       __ emit_data(mask, relocInfo::none, 0);
 585     }
 586 
 587     return start;
 588   }
 589 
 590   address generate_vector_mask_long_double(const char *stub_name, int32_t maskhi, int32_t masklo) {
 591     __ align(CodeEntryAlignment);
 592     StubCodeMark mark(this, "StubRoutines", stub_name);
 593     address start = __ pc();
 594 
 595     for (int i = 0; i < 8; i++) {
 596       __ emit_data(masklo, relocInfo::none, 0);
 597       __ emit_data(maskhi, relocInfo::none, 0);
 598     }
 599 
 600     return start;
 601   }
 602 
 603   //----------------------------------------------------------------------------------------------------
 604 
 605   address generate_vector_byte_perm_mask(const char *stub_name) {
 606     __ align(CodeEntryAlignment);
 607     StubCodeMark mark(this, "StubRoutines", stub_name);
 608     address start = __ pc();
 609 
 610     __ emit_data(0x00000001, relocInfo::none, 0);
 611     __ emit_data(0x00000000, relocInfo::none, 0);
 612     __ emit_data(0x00000003, relocInfo::none, 0);
 613     __ emit_data(0x00000000, relocInfo::none, 0);
 614     __ emit_data(0x00000005, relocInfo::none, 0);
 615     __ emit_data(0x00000000, relocInfo::none, 0);
 616     __ emit_data(0x00000007, relocInfo::none, 0);
 617     __ emit_data(0x00000000, relocInfo::none, 0);
 618     __ emit_data(0x00000000, relocInfo::none, 0);
 619     __ emit_data(0x00000000, relocInfo::none, 0);
 620     __ emit_data(0x00000002, relocInfo::none, 0);
 621     __ emit_data(0x00000000, relocInfo::none, 0);
 622     __ emit_data(0x00000004, relocInfo::none, 0);
 623     __ emit_data(0x00000000, relocInfo::none, 0);
 624     __ emit_data(0x00000006, relocInfo::none, 0);
 625     __ emit_data(0x00000000, relocInfo::none, 0);
 626 
 627     return start;
 628   }
 629 
 630   //----------------------------------------------------------------------------------------------------
 631   // Non-destructive plausibility checks for oops
 632 
 633   address generate_verify_oop() {
 634     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 635     address start = __ pc();
 636 
 637     // Incoming arguments on stack after saving rax,:
 638     //
 639     // [tos    ]: saved rdx
 640     // [tos + 1]: saved EFLAGS
 641     // [tos + 2]: return address
 642     // [tos + 3]: char* error message
 643     // [tos + 4]: oop   object to verify
 644     // [tos + 5]: saved rax, - saved by caller and bashed
 645 
 646     Label exit, error;
 647     __ pushf();
 648     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 649     __ push(rdx);                                // save rdx
 650     // make sure object is 'reasonable'
 651     __ movptr(rax, Address(rsp, 4 * wordSize));    // get object
 652     __ testptr(rax, rax);
 653     __ jcc(Assembler::zero, exit);               // if obj is NULL it is ok
 654 
 655     // Check if the oop is in the right area of memory
 656     const int oop_mask = Universe::verify_oop_mask();
 657     const int oop_bits = Universe::verify_oop_bits();
 658     __ mov(rdx, rax);
 659     __ andptr(rdx, oop_mask);
 660     __ cmpptr(rdx, oop_bits);
 661     __ jcc(Assembler::notZero, error);
 662 
 663     // make sure klass is 'reasonable', which is not zero.
 664     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
 665     __ testptr(rax, rax);
 666     __ jcc(Assembler::zero, error);              // if klass is NULL it is broken
 667 
 668     // return if everything seems ok
 669     __ bind(exit);
 670     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 671     __ pop(rdx);                                 // restore rdx
 672     __ popf();                                   // restore EFLAGS
 673     __ ret(3 * wordSize);                        // pop arguments
 674 
 675     // handle errors
 676     __ bind(error);
 677     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 678     __ pop(rdx);                                 // get saved rdx back
 679     __ popf();                                   // get saved EFLAGS off stack -- will be ignored
 680     __ pusha();                                  // push registers (eip = return address & msg are already pushed)
 681     BLOCK_COMMENT("call MacroAssembler::debug");
 682     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 683     __ hlt();
 684     return start;
 685   }
 686 
 687 
 688   // Copy 64 bytes chunks
 689   //
 690   // Inputs:
 691   //   from        - source array address
 692   //   to_from     - destination array address - from
 693   //   qword_count - 8-bytes element count, negative
 694   //
 695   void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
 696     assert( UseSSE >= 2, "supported cpu only" );
 697     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 698 
 699     // Copy 64-byte chunks
 700     __ jmpb(L_copy_64_bytes);
 701     __ align(OptoLoopAlignment);
 702   __ BIND(L_copy_64_bytes_loop);
 703 
 704     if (UseUnalignedLoadStores) {
 705       if (UseAVX > 2) {
 706         __ evmovdqul(xmm0, Address(from, 0), Assembler::AVX_512bit);
 707         __ evmovdqul(Address(from, to_from, Address::times_1, 0), xmm0, Assembler::AVX_512bit);
 708       } else if (UseAVX == 2) {
 709         __ vmovdqu(xmm0, Address(from,  0));
 710         __ vmovdqu(Address(from, to_from, Address::times_1,  0), xmm0);
 711         __ vmovdqu(xmm1, Address(from, 32));
 712         __ vmovdqu(Address(from, to_from, Address::times_1, 32), xmm1);
 713       } else {
 714         __ movdqu(xmm0, Address(from, 0));
 715         __ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
 716         __ movdqu(xmm1, Address(from, 16));
 717         __ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
 718         __ movdqu(xmm2, Address(from, 32));
 719         __ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
 720         __ movdqu(xmm3, Address(from, 48));
 721         __ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
 722       }
 723     } else {
 724       __ movq(xmm0, Address(from, 0));
 725       __ movq(Address(from, to_from, Address::times_1, 0), xmm0);
 726       __ movq(xmm1, Address(from, 8));
 727       __ movq(Address(from, to_from, Address::times_1, 8), xmm1);
 728       __ movq(xmm2, Address(from, 16));
 729       __ movq(Address(from, to_from, Address::times_1, 16), xmm2);
 730       __ movq(xmm3, Address(from, 24));
 731       __ movq(Address(from, to_from, Address::times_1, 24), xmm3);
 732       __ movq(xmm4, Address(from, 32));
 733       __ movq(Address(from, to_from, Address::times_1, 32), xmm4);
 734       __ movq(xmm5, Address(from, 40));
 735       __ movq(Address(from, to_from, Address::times_1, 40), xmm5);
 736       __ movq(xmm6, Address(from, 48));
 737       __ movq(Address(from, to_from, Address::times_1, 48), xmm6);
 738       __ movq(xmm7, Address(from, 56));
 739       __ movq(Address(from, to_from, Address::times_1, 56), xmm7);
 740     }
 741 
 742     __ addl(from, 64);
 743   __ BIND(L_copy_64_bytes);
 744     __ subl(qword_count, 8);
 745     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 746 
 747     if (UseUnalignedLoadStores && (UseAVX == 2)) {
 748       // clean upper bits of YMM registers
 749       __ vpxor(xmm0, xmm0);
 750       __ vpxor(xmm1, xmm1);
 751     }
 752     __ addl(qword_count, 8);
 753     __ jccb(Assembler::zero, L_exit);
 754     //
 755     // length is too short, just copy qwords
 756     //
 757   __ BIND(L_copy_8_bytes);
 758     __ movq(xmm0, Address(from, 0));
 759     __ movq(Address(from, to_from, Address::times_1), xmm0);
 760     __ addl(from, 8);
 761     __ decrement(qword_count);
 762     __ jcc(Assembler::greater, L_copy_8_bytes);
 763   __ BIND(L_exit);
 764   }
 765 
 766   address generate_disjoint_copy(BasicType t, bool aligned,
 767                                  Address::ScaleFactor sf,
 768                                  address* entry, const char *name,
 769                                  bool dest_uninitialized = false) {
 770     __ align(CodeEntryAlignment);
 771     StubCodeMark mark(this, "StubRoutines", name);
 772     address start = __ pc();
 773 
 774     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
 775     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;
 776 
 777     int shift = Address::times_ptr - sf;
 778 
 779     const Register from     = rsi;  // source array address
 780     const Register to       = rdi;  // destination array address
 781     const Register count    = rcx;  // elements count
 782     const Register to_from  = to;   // (to - from)
 783     const Register saved_to = rdx;  // saved destination array address
 784 
 785     __ enter(); // required for proper stackwalking of RuntimeStub frame
 786     __ push(rsi);
 787     __ push(rdi);
 788     __ movptr(from , Address(rsp, 12+ 4));
 789     __ movptr(to   , Address(rsp, 12+ 8));
 790     __ movl(count, Address(rsp, 12+ 12));
 791 
 792     if (entry != NULL) {
 793       *entry = __ pc(); // Entry point from conjoint arraycopy stub.
 794       BLOCK_COMMENT("Entry:");
 795     }
 796 
 797     if (t == T_OBJECT) {
 798       __ testl(count, count);
 799       __ jcc(Assembler::zero, L_0_count);
 800     }
 801 
 802     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_DISJOINT;
 803     if (dest_uninitialized) {
 804       decorators |= IS_DEST_UNINITIALIZED;
 805     }
 806     if (aligned) {
 807       decorators |= ARRAYCOPY_ALIGNED;
 808     }
 809 
 810     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
 811     bs->arraycopy_prologue(_masm, decorators, t, from, to, count);
 812     {
 813       bool add_entry = (t != T_OBJECT && (!aligned || t == T_INT));
 814       // UnsafeCopyMemory page error: continue after ucm
 815       UnsafeCopyMemoryMark ucmm(this, add_entry, true);
 816       __ subptr(to, from); // to --> to_from
 817       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
 818       __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
 819       if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 820         // align source address at 4 bytes address boundary
 821         if (t == T_BYTE) {
 822           // One byte misalignment happens only for byte arrays
 823           __ testl(from, 1);
 824           __ jccb(Assembler::zero, L_skip_align1);
 825           __ movb(rax, Address(from, 0));
 826           __ movb(Address(from, to_from, Address::times_1, 0), rax);
 827           __ increment(from);
 828           __ decrement(count);
 829         __ BIND(L_skip_align1);
 830         }
 831         // Two bytes misalignment happens only for byte and short (char) arrays
 832         __ testl(from, 2);
 833         __ jccb(Assembler::zero, L_skip_align2);
 834         __ movw(rax, Address(from, 0));
 835         __ movw(Address(from, to_from, Address::times_1, 0), rax);
 836         __ addptr(from, 2);
 837         __ subl(count, 1<<(shift-1));
 838       __ BIND(L_skip_align2);
 839       }
 840       if (!UseXMMForArrayCopy) {
 841         __ mov(rax, count);      // save 'count'
 842         __ shrl(count, shift); // bytes count
 843         __ addptr(to_from, from);// restore 'to'
 844         __ rep_mov();
 845         __ subptr(to_from, from);// restore 'to_from'
 846         __ mov(count, rax);      // restore 'count'
 847         __ jmpb(L_copy_2_bytes); // all dwords were copied
 848       } else {
 849         if (!UseUnalignedLoadStores) {
 850           // align to 8 bytes, we know we are 4 byte aligned to start
 851           __ testptr(from, 4);
 852           __ jccb(Assembler::zero, L_copy_64_bytes);
 853           __ movl(rax, Address(from, 0));
 854           __ movl(Address(from, to_from, Address::times_1, 0), rax);
 855           __ addptr(from, 4);
 856           __ subl(count, 1<<shift);
 857         }
 858       __ BIND(L_copy_64_bytes);
 859         __ mov(rax, count);
 860         __ shrl(rax, shift+1);  // 8 bytes chunk count
 861         //
 862         // Copy 8-byte chunks through XMM registers, 8 per iteration of the loop
 863         //
 864         xmm_copy_forward(from, to_from, rax);
 865       }
 866       // copy tailing dword
 867     __ BIND(L_copy_4_bytes);
 868       __ testl(count, 1<<shift);
 869       __ jccb(Assembler::zero, L_copy_2_bytes);
 870       __ movl(rax, Address(from, 0));
 871       __ movl(Address(from, to_from, Address::times_1, 0), rax);
 872       if (t == T_BYTE || t == T_SHORT) {
 873         __ addptr(from, 4);
 874       __ BIND(L_copy_2_bytes);
 875         // copy tailing word
 876         __ testl(count, 1<<(shift-1));
 877         __ jccb(Assembler::zero, L_copy_byte);
 878         __ movw(rax, Address(from, 0));
 879         __ movw(Address(from, to_from, Address::times_1, 0), rax);
 880         if (t == T_BYTE) {
 881           __ addptr(from, 2);
 882         __ BIND(L_copy_byte);
 883           // copy tailing byte
 884           __ testl(count, 1);
 885           __ jccb(Assembler::zero, L_exit);
 886           __ movb(rax, Address(from, 0));
 887           __ movb(Address(from, to_from, Address::times_1, 0), rax);
 888         __ BIND(L_exit);
 889         } else {
 890         __ BIND(L_copy_byte);
 891         }
 892       } else {
 893       __ BIND(L_copy_2_bytes);
 894       }
 895     }
 896 
 897     __ movl(count, Address(rsp, 12+12)); // reread 'count'
 898     bs->arraycopy_epilogue(_masm, decorators, t, from, to, count);
 899 
 900     if (t == T_OBJECT) {
 901     __ BIND(L_0_count);
 902     }
 903     inc_copy_counter_np(t);
 904     __ pop(rdi);
 905     __ pop(rsi);
 906     __ leave(); // required for proper stackwalking of RuntimeStub frame
 907     __ vzeroupper();
 908     __ xorptr(rax, rax); // return 0
 909     __ ret(0);
 910     return start;
 911   }
 912 
 913 
 914   address generate_fill(BasicType t, bool aligned, const char *name) {
 915     __ align(CodeEntryAlignment);
 916     StubCodeMark mark(this, "StubRoutines", name);
 917     address start = __ pc();
 918 
 919     BLOCK_COMMENT("Entry:");
 920 
 921     const Register to       = rdi;  // source array address
 922     const Register value    = rdx;  // value
 923     const Register count    = rsi;  // elements count
 924 
 925     __ enter(); // required for proper stackwalking of RuntimeStub frame
 926     __ push(rsi);
 927     __ push(rdi);
 928     __ movptr(to   , Address(rsp, 12+ 4));
 929     __ movl(value, Address(rsp, 12+ 8));
 930     __ movl(count, Address(rsp, 12+ 12));
 931 
 932     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
 933 
 934     __ pop(rdi);
 935     __ pop(rsi);
 936     __ leave(); // required for proper stackwalking of RuntimeStub frame
 937     __ ret(0);
 938     return start;
 939   }
 940 
 941   address generate_conjoint_copy(BasicType t, bool aligned,
 942                                  Address::ScaleFactor sf,
 943                                  address nooverlap_target,
 944                                  address* entry, const char *name,
 945                                  bool dest_uninitialized = false) {
 946     __ align(CodeEntryAlignment);
 947     StubCodeMark mark(this, "StubRoutines", name);
 948     address start = __ pc();
 949 
 950     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
 951     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;
 952 
 953     int shift = Address::times_ptr - sf;
 954 
 955     const Register src   = rax;  // source array address
 956     const Register dst   = rdx;  // destination array address
 957     const Register from  = rsi;  // source array address
 958     const Register to    = rdi;  // destination array address
 959     const Register count = rcx;  // elements count
 960     const Register end   = rax;  // array end address
 961 
 962     __ enter(); // required for proper stackwalking of RuntimeStub frame
 963     __ push(rsi);
 964     __ push(rdi);
 965     __ movptr(src  , Address(rsp, 12+ 4));   // from
 966     __ movptr(dst  , Address(rsp, 12+ 8));   // to
 967     __ movl2ptr(count, Address(rsp, 12+12)); // count
 968 
 969     if (entry != NULL) {
 970       *entry = __ pc(); // Entry point from generic arraycopy stub.
 971       BLOCK_COMMENT("Entry:");
 972     }
 973 
 974     // nooverlap_target expects arguments in rsi and rdi.
 975     __ mov(from, src);
 976     __ mov(to  , dst);
 977 
 978     // arrays overlap test: dispatch to disjoint stub if necessary.
 979     RuntimeAddress nooverlap(nooverlap_target);
 980     __ cmpptr(dst, src);
 981     __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
 982     __ jump_cc(Assembler::belowEqual, nooverlap);
 983     __ cmpptr(dst, end);
 984     __ jump_cc(Assembler::aboveEqual, nooverlap);
 985 
 986     if (t == T_OBJECT) {
 987       __ testl(count, count);
 988       __ jcc(Assembler::zero, L_0_count);
 989     }
 990 
 991     DecoratorSet decorators = IN_HEAP | IS_ARRAY;
 992     if (dest_uninitialized) {
 993       decorators |= IS_DEST_UNINITIALIZED;
 994     }
 995     if (aligned) {
 996       decorators |= ARRAYCOPY_ALIGNED;
 997     }
 998 
 999     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1000     bs->arraycopy_prologue(_masm, decorators, t, from, to, count);
1001 
1002     {
1003       bool add_entry = (t != T_OBJECT && (!aligned || t == T_INT));
1004       // UnsafeCopyMemory page error: continue after ucm
1005       UnsafeCopyMemoryMark ucmm(this, add_entry, true);
1006       // copy from high to low
1007       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1008       __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
1009       if (t == T_BYTE || t == T_SHORT) {
1010         // Align the end of destination array at 4 bytes address boundary
1011         __ lea(end, Address(dst, count, sf, 0));
1012         if (t == T_BYTE) {
1013           // One byte misalignment happens only for byte arrays
1014           __ testl(end, 1);
1015           __ jccb(Assembler::zero, L_skip_align1);
1016           __ decrement(count);
1017           __ movb(rdx, Address(from, count, sf, 0));
1018           __ movb(Address(to, count, sf, 0), rdx);
1019         __ BIND(L_skip_align1);
1020         }
1021         // Two bytes misalignment happens only for byte and short (char) arrays
1022         __ testl(end, 2);
1023         __ jccb(Assembler::zero, L_skip_align2);
1024         __ subptr(count, 1<<(shift-1));
1025         __ movw(rdx, Address(from, count, sf, 0));
1026         __ movw(Address(to, count, sf, 0), rdx);
1027       __ BIND(L_skip_align2);
1028         __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1029         __ jcc(Assembler::below, L_copy_4_bytes);
1030       }
1031 
1032       if (!UseXMMForArrayCopy) {
1033         __ std();
1034         __ mov(rax, count); // Save 'count'
1035         __ mov(rdx, to);    // Save 'to'
1036         __ lea(rsi, Address(from, count, sf, -4));
1037         __ lea(rdi, Address(to  , count, sf, -4));
1038         __ shrptr(count, shift); // bytes count
1039         __ rep_mov();
1040         __ cld();
1041         __ mov(count, rax); // restore 'count'
1042         __ andl(count, (1<<shift)-1);      // mask the number of rest elements
1043         __ movptr(from, Address(rsp, 12+4)); // reread 'from'
1044         __ mov(to, rdx);   // restore 'to'
1045         __ jmpb(L_copy_2_bytes); // all dword were copied
1046       } else {
1047         // Align to 8 bytes the end of array. It is aligned to 4 bytes already.
1048         __ testptr(end, 4);
1049         __ jccb(Assembler::zero, L_copy_8_bytes);
1050         __ subl(count, 1<<shift);
1051         __ movl(rdx, Address(from, count, sf, 0));
1052         __ movl(Address(to, count, sf, 0), rdx);
1053         __ jmpb(L_copy_8_bytes);
1054 
1055         __ align(OptoLoopAlignment);
1056         // Move 8 bytes
1057       __ BIND(L_copy_8_bytes_loop);
1058         __ movq(xmm0, Address(from, count, sf, 0));
1059         __ movq(Address(to, count, sf, 0), xmm0);
1060       __ BIND(L_copy_8_bytes);
1061         __ subl(count, 2<<shift);
1062         __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1063         __ addl(count, 2<<shift);
1064       }
1065     __ BIND(L_copy_4_bytes);
1066       // copy prefix qword
1067       __ testl(count, 1<<shift);
1068       __ jccb(Assembler::zero, L_copy_2_bytes);
1069       __ movl(rdx, Address(from, count, sf, -4));
1070       __ movl(Address(to, count, sf, -4), rdx);
1071 
1072       if (t == T_BYTE || t == T_SHORT) {
1073           __ subl(count, (1<<shift));
1074         __ BIND(L_copy_2_bytes);
1075           // copy prefix dword
1076           __ testl(count, 1<<(shift-1));
1077           __ jccb(Assembler::zero, L_copy_byte);
1078           __ movw(rdx, Address(from, count, sf, -2));
1079           __ movw(Address(to, count, sf, -2), rdx);
1080           if (t == T_BYTE) {
1081             __ subl(count, 1<<(shift-1));
1082           __ BIND(L_copy_byte);
1083             // copy prefix byte
1084             __ testl(count, 1);
1085             __ jccb(Assembler::zero, L_exit);
1086             __ movb(rdx, Address(from, 0));
1087             __ movb(Address(to, 0), rdx);
1088           __ BIND(L_exit);
1089           } else {
1090           __ BIND(L_copy_byte);
1091           }
1092       } else {
1093       __ BIND(L_copy_2_bytes);
1094       }
1095     }
1096 
1097     __ movl2ptr(count, Address(rsp, 12+12)); // reread count
1098     bs->arraycopy_epilogue(_masm, decorators, t, from, to, count);
1099 
1100     if (t == T_OBJECT) {
1101     __ BIND(L_0_count);
1102     }
1103     inc_copy_counter_np(t);
1104     __ pop(rdi);
1105     __ pop(rsi);
1106     __ leave(); // required for proper stackwalking of RuntimeStub frame
1107     __ xorptr(rax, rax); // return 0
1108     __ ret(0);
1109     return start;
1110   }
1111 
1112 
1113   address generate_disjoint_long_copy(address* entry, const char *name) {
1114     __ align(CodeEntryAlignment);
1115     StubCodeMark mark(this, "StubRoutines", name);
1116     address start = __ pc();
1117 
1118     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1119     const Register from       = rax;  // source array address
1120     const Register to         = rdx;  // destination array address
1121     const Register count      = rcx;  // elements count
1122     const Register to_from    = rdx;  // (to - from)
1123 
1124     __ enter(); // required for proper stackwalking of RuntimeStub frame
1125     __ movptr(from , Address(rsp, 8+0));       // from
1126     __ movptr(to   , Address(rsp, 8+4));       // to
1127     __ movl2ptr(count, Address(rsp, 8+8));     // count
1128 
1129     *entry = __ pc(); // Entry point from conjoint arraycopy stub.
1130     BLOCK_COMMENT("Entry:");
1131 
1132     {
1133       // UnsafeCopyMemory page error: continue after ucm
1134       UnsafeCopyMemoryMark ucmm(this, true, true);
1135       __ subptr(to, from); // to --> to_from
1136       if (UseXMMForArrayCopy) {
1137         xmm_copy_forward(from, to_from, count);
1138       } else {
1139         __ jmpb(L_copy_8_bytes);
1140         __ align(OptoLoopAlignment);
1141       __ BIND(L_copy_8_bytes_loop);
1142         __ fild_d(Address(from, 0));
1143         __ fistp_d(Address(from, to_from, Address::times_1));
1144         __ addptr(from, 8);
1145       __ BIND(L_copy_8_bytes);
1146         __ decrement(count);
1147         __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1148       }
1149     }
1150     inc_copy_counter_np(T_LONG);
1151     __ leave(); // required for proper stackwalking of RuntimeStub frame
1152     __ vzeroupper();
1153     __ xorptr(rax, rax); // return 0
1154     __ ret(0);
1155     return start;
1156   }
1157 
1158   address generate_conjoint_long_copy(address nooverlap_target,
1159                                       address* entry, const char *name) {
1160     __ align(CodeEntryAlignment);
1161     StubCodeMark mark(this, "StubRoutines", name);
1162     address start = __ pc();
1163 
1164     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1165     const Register from       = rax;  // source array address
1166     const Register to         = rdx;  // destination array address
1167     const Register count      = rcx;  // elements count
1168     const Register end_from   = rax;  // source array end address
1169 
1170     __ enter(); // required for proper stackwalking of RuntimeStub frame
1171     __ movptr(from , Address(rsp, 8+0));       // from
1172     __ movptr(to   , Address(rsp, 8+4));       // to
1173     __ movl2ptr(count, Address(rsp, 8+8));     // count
1174 
1175     *entry = __ pc(); // Entry point from generic arraycopy stub.
1176     BLOCK_COMMENT("Entry:");
1177 
1178     // arrays overlap test
1179     __ cmpptr(to, from);
1180     RuntimeAddress nooverlap(nooverlap_target);
1181     __ jump_cc(Assembler::belowEqual, nooverlap);
1182     __ lea(end_from, Address(from, count, Address::times_8, 0));
1183     __ cmpptr(to, end_from);
1184     __ movptr(from, Address(rsp, 8));  // from
1185     __ jump_cc(Assembler::aboveEqual, nooverlap);
1186 
1187     {
1188       // UnsafeCopyMemory page error: continue after ucm
1189       UnsafeCopyMemoryMark ucmm(this, true, true);
1190 
1191       __ jmpb(L_copy_8_bytes);
1192 
1193       __ align(OptoLoopAlignment);
1194     __ BIND(L_copy_8_bytes_loop);
1195       if (UseXMMForArrayCopy) {
1196         __ movq(xmm0, Address(from, count, Address::times_8));
1197         __ movq(Address(to, count, Address::times_8), xmm0);
1198       } else {
1199         __ fild_d(Address(from, count, Address::times_8));
1200         __ fistp_d(Address(to, count, Address::times_8));
1201       }
1202     __ BIND(L_copy_8_bytes);
1203       __ decrement(count);
1204       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1205 
1206     }
1207     inc_copy_counter_np(T_LONG);
1208     __ leave(); // required for proper stackwalking of RuntimeStub frame
1209     __ xorptr(rax, rax); // return 0
1210     __ ret(0);
1211     return start;
1212   }
1213 
1214 
1215   // Helper for generating a dynamic type check.
1216   // The sub_klass must be one of {rbx, rdx, rsi}.
1217   // The temp is killed.
1218   void generate_type_check(Register sub_klass,
1219                            Address& super_check_offset_addr,
1220                            Address& super_klass_addr,
1221                            Register temp,
1222                            Label* L_success, Label* L_failure) {
1223     BLOCK_COMMENT("type_check:");
1224 
1225     Label L_fallthrough;
1226 #define LOCAL_JCC(assembler_con, label_ptr)                             \
1227     if (label_ptr != NULL)  __ jcc(assembler_con, *(label_ptr));        \
1228     else                    __ jcc(assembler_con, L_fallthrough) /*omit semi*/
1229 
1230     // The following is a strange variation of the fast path which requires
1231     // one less register, because needed values are on the argument stack.
1232     // __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
1233     //                                  L_success, L_failure, NULL);
1234     assert_different_registers(sub_klass, temp);
1235 
1236     int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
1237 
1238     // if the pointers are equal, we are done (e.g., String[] elements)
1239     __ cmpptr(sub_klass, super_klass_addr);
1240     LOCAL_JCC(Assembler::equal, L_success);
1241 
1242     // check the supertype display:
1243     __ movl2ptr(temp, super_check_offset_addr);
1244     Address super_check_addr(sub_klass, temp, Address::times_1, 0);
1245     __ movptr(temp, super_check_addr); // load displayed supertype
1246     __ cmpptr(temp, super_klass_addr); // test the super type
1247     LOCAL_JCC(Assembler::equal, L_success);
1248 
1249     // if it was a primary super, we can just fail immediately
1250     __ cmpl(super_check_offset_addr, sc_offset);
1251     LOCAL_JCC(Assembler::notEqual, L_failure);
1252 
1253     // The repne_scan instruction uses fixed registers, which will get spilled.
1254     // We happen to know this works best when super_klass is in rax.
1255     Register super_klass = temp;
1256     __ movptr(super_klass, super_klass_addr);
1257     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
1258                                      L_success, L_failure);
1259 
1260     __ bind(L_fallthrough);
1261 
1262     if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }
1263     if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }
1264 
1265 #undef LOCAL_JCC
1266   }
1267 
1268   //
1269   //  Generate checkcasting array copy stub
1270   //
1271   //  Input:
1272   //    4(rsp)   - source array address
1273   //    8(rsp)   - destination array address
1274   //   12(rsp)   - element count, can be zero
1275   //   16(rsp)   - size_t ckoff (super_check_offset)
1276   //   20(rsp)   - oop ckval (super_klass)
1277   //
1278   //  Output:
1279   //    rax, ==  0  -  success
1280   //    rax, == -1^K - failure, where K is partial transfer count
1281   //
1282   address generate_checkcast_copy(const char *name, address* entry, bool dest_uninitialized = false) {
1283     __ align(CodeEntryAlignment);
1284     StubCodeMark mark(this, "StubRoutines", name);
1285     address start = __ pc();
1286 
1287     Label L_load_element, L_store_element, L_do_card_marks, L_done;
1288 
1289     // register use:
1290     //  rax, rdx, rcx -- loop control (end_from, end_to, count)
1291     //  rdi, rsi      -- element access (oop, klass)
1292     //  rbx,           -- temp
1293     const Register from       = rax;    // source array address
1294     const Register to         = rdx;    // destination array address
1295     const Register length     = rcx;    // elements count
1296     const Register elem       = rdi;    // each oop copied
1297     const Register elem_klass = rsi;    // each elem._klass (sub_klass)
1298     const Register temp       = rbx;    // lone remaining temp
1299 
1300     __ enter(); // required for proper stackwalking of RuntimeStub frame
1301 
1302     __ push(rsi);
1303     __ push(rdi);
1304     __ push(rbx);
1305 
1306     Address   from_arg(rsp, 16+ 4);     // from
1307     Address     to_arg(rsp, 16+ 8);     // to
1308     Address length_arg(rsp, 16+12);     // elements count
1309     Address  ckoff_arg(rsp, 16+16);     // super_check_offset
1310     Address  ckval_arg(rsp, 16+20);     // super_klass
1311 
1312     // Load up:
1313     __ movptr(from,     from_arg);
1314     __ movptr(to,         to_arg);
1315     __ movl2ptr(length, length_arg);
1316 
1317     if (entry != NULL) {
1318       *entry = __ pc(); // Entry point from generic arraycopy stub.
1319       BLOCK_COMMENT("Entry:");
1320     }
1321 
1322     //---------------------------------------------------------------
1323     // Assembler stub will be used for this call to arraycopy
1324     // if the two arrays are subtypes of Object[] but the
1325     // destination array type is not equal to or a supertype
1326     // of the source type.  Each element must be separately
1327     // checked.
1328 
1329     // Loop-invariant addresses.  They are exclusive end pointers.
1330     Address end_from_addr(from, length, Address::times_ptr, 0);
1331     Address   end_to_addr(to,   length, Address::times_ptr, 0);
1332 
1333     Register end_from = from;           // re-use
1334     Register end_to   = to;             // re-use
1335     Register count    = length;         // re-use
1336 
1337     // Loop-variant addresses.  They assume post-incremented count < 0.
1338     Address from_element_addr(end_from, count, Address::times_ptr, 0);
1339     Address   to_element_addr(end_to,   count, Address::times_ptr, 0);
1340     Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
1341 
1342     DecoratorSet decorators = IN_HEAP | IS_ARRAY | ARRAYCOPY_CHECKCAST;
1343     if (dest_uninitialized) {
1344       decorators |= IS_DEST_UNINITIALIZED;
1345     }
1346 
1347     BasicType type = T_OBJECT;
1348     BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
1349     bs->arraycopy_prologue(_masm, decorators, type, from, to, count);
1350 
1351     // Copy from low to high addresses, indexed from the end of each array.
1352     __ lea(end_from, end_from_addr);
1353     __ lea(end_to,   end_to_addr);
1354     assert(length == count, "");        // else fix next line:
1355     __ negptr(count);                   // negate and test the length
1356     __ jccb(Assembler::notZero, L_load_element);
1357 
1358     // Empty array:  Nothing to do.
1359     __ xorptr(rax, rax);                  // return 0 on (trivial) success
1360     __ jmp(L_done);
1361 
1362     // ======== begin loop ========
1363     // (Loop is rotated; its entry is L_load_element.)
1364     // Loop control:
1365     //   for (count = -count; count != 0; count++)
1366     // Base pointers src, dst are biased by 8*count,to last element.
1367     __ align(OptoLoopAlignment);
1368 
1369     __ BIND(L_store_element);
1370     __ movptr(to_element_addr, elem);     // store the oop
1371     __ increment(count);                // increment the count toward zero
1372     __ jccb(Assembler::zero, L_do_card_marks);
1373 
1374     // ======== loop entry is here ========
1375     __ BIND(L_load_element);
1376     __ movptr(elem, from_element_addr);   // load the oop
1377     __ testptr(elem, elem);
1378     __ jccb(Assembler::zero, L_store_element);
1379 
1380     // (Could do a trick here:  Remember last successful non-null
1381     // element stored and make a quick oop equality check on it.)
1382 
1383     __ movptr(elem_klass, elem_klass_addr); // query the object klass
1384     generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,
1385                         &L_store_element, NULL);
1386     // (On fall-through, we have failed the element type check.)
1387     // ======== end loop ========
1388 
1389     // It was a real error; we must depend on the caller to finish the job.
1390     // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
1391     // Emit GC store barriers for the oops we have copied (length_arg + count),
1392     // and report their number to the caller.
1393     assert_different_registers(to, count, rax);
1394     Label L_post_barrier;
1395     __ addl(count, length_arg);         // transfers = (length - remaining)
1396     __ movl2ptr(rax, count);            // save the value
1397     __ notptr(rax);                     // report (-1^K) to caller (does not affect flags)
1398     __ jccb(Assembler::notZero, L_post_barrier);
1399     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
1400 
1401     // Come here on success only.
1402     __ BIND(L_do_card_marks);
1403     __ xorptr(rax, rax);                // return 0 on success
1404     __ movl2ptr(count, length_arg);
1405 
1406     __ BIND(L_post_barrier);
1407     __ movptr(to, to_arg);              // reload
1408     bs->arraycopy_epilogue(_masm, decorators, type, from, to, count);
1409 
1410     // Common exit point (success or failure).
1411     __ BIND(L_done);
1412     __ pop(rbx);
1413     __ pop(rdi);
1414     __ pop(rsi);
1415     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1416     __ leave(); // required for proper stackwalking of RuntimeStub frame
1417     __ ret(0);
1418 
1419     return start;
1420   }
1421 
1422   //
1423   //  Generate 'unsafe' array copy stub
1424   //  Though just as safe as the other stubs, it takes an unscaled
1425   //  size_t argument instead of an element count.
1426   //
1427   //  Input:
1428   //    4(rsp)   - source array address
1429   //    8(rsp)   - destination array address
1430   //   12(rsp)   - byte count, can be zero
1431   //
1432   //  Output:
1433   //    rax, ==  0  -  success
1434   //    rax, == -1  -  need to call System.arraycopy
1435   //
1436   // Examines the alignment of the operands and dispatches
1437   // to a long, int, short, or byte copy loop.
1438   //
1439   address generate_unsafe_copy(const char *name,
1440                                address byte_copy_entry,
1441                                address short_copy_entry,
1442                                address int_copy_entry,
1443                                address long_copy_entry) {
1444 
1445     Label L_long_aligned, L_int_aligned, L_short_aligned;
1446 
1447     __ align(CodeEntryAlignment);
1448     StubCodeMark mark(this, "StubRoutines", name);
1449     address start = __ pc();
1450 
1451     const Register from       = rax;  // source array address
1452     const Register to         = rdx;  // destination array address
1453     const Register count      = rcx;  // elements count
1454 
1455     __ enter(); // required for proper stackwalking of RuntimeStub frame
1456     __ push(rsi);
1457     __ push(rdi);
1458     Address  from_arg(rsp, 12+ 4);      // from
1459     Address    to_arg(rsp, 12+ 8);      // to
1460     Address count_arg(rsp, 12+12);      // byte count
1461 
1462     // Load up:
1463     __ movptr(from ,  from_arg);
1464     __ movptr(to   ,    to_arg);
1465     __ movl2ptr(count, count_arg);
1466 
1467     // bump this on entry, not on exit:
1468     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
1469 
1470     const Register bits = rsi;
1471     __ mov(bits, from);
1472     __ orptr(bits, to);
1473     __ orptr(bits, count);
1474 
1475     __ testl(bits, BytesPerLong-1);
1476     __ jccb(Assembler::zero, L_long_aligned);
1477 
1478     __ testl(bits, BytesPerInt-1);
1479     __ jccb(Assembler::zero, L_int_aligned);
1480 
1481     __ testl(bits, BytesPerShort-1);
1482     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
1483 
1484     __ BIND(L_short_aligned);
1485     __ shrptr(count, LogBytesPerShort); // size => short_count
1486     __ movl(count_arg, count);          // update 'count'
1487     __ jump(RuntimeAddress(short_copy_entry));
1488 
1489     __ BIND(L_int_aligned);
1490     __ shrptr(count, LogBytesPerInt); // size => int_count
1491     __ movl(count_arg, count);          // update 'count'
1492     __ jump(RuntimeAddress(int_copy_entry));
1493 
1494     __ BIND(L_long_aligned);
1495     __ shrptr(count, LogBytesPerLong); // size => qword_count
1496     __ movl(count_arg, count);          // update 'count'
1497     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1498     __ pop(rsi);
1499     __ jump(RuntimeAddress(long_copy_entry));
1500 
1501     return start;
1502   }
1503 
1504 
1505   // Perform range checks on the proposed arraycopy.
1506   // Smashes src_pos and dst_pos.  (Uses them up for temps.)
1507   void arraycopy_range_checks(Register src,
1508                               Register src_pos,
1509                               Register dst,
1510                               Register dst_pos,
1511                               Address& length,
1512                               Label& L_failed) {
1513     BLOCK_COMMENT("arraycopy_range_checks:");
1514     const Register src_end = src_pos;   // source array end position
1515     const Register dst_end = dst_pos;   // destination array end position
1516     __ addl(src_end, length); // src_pos + length
1517     __ addl(dst_end, length); // dst_pos + length
1518 
1519     //  if (src_pos + length > arrayOop(src)->length() ) FAIL;
1520     __ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
1521     __ jcc(Assembler::above, L_failed);
1522 
1523     //  if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
1524     __ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
1525     __ jcc(Assembler::above, L_failed);
1526 
1527     BLOCK_COMMENT("arraycopy_range_checks done");
1528   }
1529 
1530 
1531   //
1532   //  Generate generic array copy stubs
1533   //
1534   //  Input:
1535   //     4(rsp)    -  src oop
1536   //     8(rsp)    -  src_pos
1537   //    12(rsp)    -  dst oop
1538   //    16(rsp)    -  dst_pos
1539   //    20(rsp)    -  element count
1540   //
1541   //  Output:
1542   //    rax, ==  0  -  success
1543   //    rax, == -1^K - failure, where K is partial transfer count
1544   //
1545   address generate_generic_copy(const char *name,
1546                                 address entry_jbyte_arraycopy,
1547                                 address entry_jshort_arraycopy,
1548                                 address entry_jint_arraycopy,
1549                                 address entry_oop_arraycopy,
1550                                 address entry_jlong_arraycopy,
1551                                 address entry_checkcast_arraycopy) {
1552     Label L_failed, L_failed_0, L_objArray;
1553 
1554     { int modulus = CodeEntryAlignment;
1555       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
1556       int advance = target - (__ offset() % modulus);
1557       if (advance < 0)  advance += modulus;
1558       if (advance > 0)  __ nop(advance);
1559     }
1560     StubCodeMark mark(this, "StubRoutines", name);
1561 
1562     // Short-hop target to L_failed.  Makes for denser prologue code.
1563     __ BIND(L_failed_0);
1564     __ jmp(L_failed);
1565     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
1566 
1567     __ align(CodeEntryAlignment);
1568     address start = __ pc();
1569 
1570     __ enter(); // required for proper stackwalking of RuntimeStub frame
1571     __ push(rsi);
1572     __ push(rdi);
1573 
1574     // bump this on entry, not on exit:
1575     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
1576 
1577     // Input values
1578     Address SRC     (rsp, 12+ 4);
1579     Address SRC_POS (rsp, 12+ 8);
1580     Address DST     (rsp, 12+12);
1581     Address DST_POS (rsp, 12+16);
1582     Address LENGTH  (rsp, 12+20);
1583 
1584     //-----------------------------------------------------------------------
1585     // Assembler stub will be used for this call to arraycopy
1586     // if the following conditions are met:
1587     //
1588     // (1) src and dst must not be null.
1589     // (2) src_pos must not be negative.
1590     // (3) dst_pos must not be negative.
1591     // (4) length  must not be negative.
1592     // (5) src klass and dst klass should be the same and not NULL.
1593     // (6) src and dst should be arrays.
1594     // (7) src_pos + length must not exceed length of src.
1595     // (8) dst_pos + length must not exceed length of dst.
1596     //
1597 
1598     const Register src     = rax;       // source array oop
1599     const Register src_pos = rsi;
1600     const Register dst     = rdx;       // destination array oop
1601     const Register dst_pos = rdi;
1602     const Register length  = rcx;       // transfer count
1603 
1604     //  if (src == NULL) return -1;
1605     __ movptr(src, SRC);      // src oop
1606     __ testptr(src, src);
1607     __ jccb(Assembler::zero, L_failed_0);
1608 
1609     //  if (src_pos < 0) return -1;
1610     __ movl2ptr(src_pos, SRC_POS);  // src_pos
1611     __ testl(src_pos, src_pos);
1612     __ jccb(Assembler::negative, L_failed_0);
1613 
1614     //  if (dst == NULL) return -1;
1615     __ movptr(dst, DST);      // dst oop
1616     __ testptr(dst, dst);
1617     __ jccb(Assembler::zero, L_failed_0);
1618 
1619     //  if (dst_pos < 0) return -1;
1620     __ movl2ptr(dst_pos, DST_POS);  // dst_pos
1621     __ testl(dst_pos, dst_pos);
1622     __ jccb(Assembler::negative, L_failed_0);
1623 
1624     //  if (length < 0) return -1;
1625     __ movl2ptr(length, LENGTH);   // length
1626     __ testl(length, length);
1627     __ jccb(Assembler::negative, L_failed_0);
1628 
1629     //  if (src->klass() == NULL) return -1;
1630     Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());
1631     Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());
1632     const Register rcx_src_klass = rcx;    // array klass
1633     __ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
1634 
1635 #ifdef ASSERT
1636     //  assert(src->klass() != NULL);
1637     BLOCK_COMMENT("assert klasses not null");
1638     { Label L1, L2;
1639       __ testptr(rcx_src_klass, rcx_src_klass);
1640       __ jccb(Assembler::notZero, L2);   // it is broken if klass is NULL
1641       __ bind(L1);
1642       __ stop("broken null klass");
1643       __ bind(L2);
1644       __ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
1645       __ jccb(Assembler::equal, L1);      // this would be broken also
1646       BLOCK_COMMENT("assert done");
1647     }
1648 #endif //ASSERT
1649 
1650     // Load layout helper (32-bits)
1651     //
1652     //  |array_tag|     | header_size | element_type |     |log2_element_size|
1653     // 32        30    24            16              8     2                 0
1654     //
1655     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
1656     //
1657 
1658     int lh_offset = in_bytes(Klass::layout_helper_offset());
1659     Address src_klass_lh_addr(rcx_src_klass, lh_offset);
1660 
1661     // Handle objArrays completely differently...
1662     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
1663     __ cmpl(src_klass_lh_addr, objArray_lh);
1664     __ jcc(Assembler::equal, L_objArray);
1665 
1666     //  if (src->klass() != dst->klass()) return -1;
1667     __ cmpptr(rcx_src_klass, dst_klass_addr);
1668     __ jccb(Assembler::notEqual, L_failed_0);
1669 
1670     const Register rcx_lh = rcx;  // layout helper
1671     assert(rcx_lh == rcx_src_klass, "known alias");
1672     __ movl(rcx_lh, src_klass_lh_addr);
1673 
1674     //  if (!src->is_Array()) return -1;
1675     __ cmpl(rcx_lh, Klass::_lh_neutral_value);
1676     __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
1677 
1678     // At this point, it is known to be a typeArray (array_tag 0x3).
1679 #ifdef ASSERT
1680     { Label L;
1681       __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
1682       __ jcc(Assembler::greaterEqual, L); // signed cmp
1683       __ stop("must be a primitive array");
1684       __ bind(L);
1685     }
1686 #endif
1687 
1688     assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);
1689     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1690 
1691     // TypeArrayKlass
1692     //
1693     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
1694     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
1695     //
1696     const Register rsi_offset = rsi; // array offset
1697     const Register src_array  = src; // src array offset
1698     const Register dst_array  = dst; // dst array offset
1699     const Register rdi_elsize = rdi; // log2 element size
1700 
1701     __ mov(rsi_offset, rcx_lh);
1702     __ shrptr(rsi_offset, Klass::_lh_header_size_shift);
1703     __ andptr(rsi_offset, Klass::_lh_header_size_mask);   // array_offset
1704     __ addptr(src_array, rsi_offset);  // src array offset
1705     __ addptr(dst_array, rsi_offset);  // dst array offset
1706     __ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
1707 
1708     // next registers should be set before the jump to corresponding stub
1709     const Register from       = src; // source array address
1710     const Register to         = dst; // destination array address
1711     const Register count      = rcx; // elements count
1712     // some of them should be duplicated on stack
1713 #define FROM   Address(rsp, 12+ 4)
1714 #define TO     Address(rsp, 12+ 8)   // Not used now
1715 #define COUNT  Address(rsp, 12+12)   // Only for oop arraycopy
1716 
1717     BLOCK_COMMENT("scale indexes to element size");
1718     __ movl2ptr(rsi, SRC_POS);  // src_pos
1719     __ shlptr(rsi);             // src_pos << rcx (log2 elsize)
1720     assert(src_array == from, "");
1721     __ addptr(from, rsi);       // from = src_array + SRC_POS << log2 elsize
1722     __ movl2ptr(rdi, DST_POS);  // dst_pos
1723     __ shlptr(rdi);             // dst_pos << rcx (log2 elsize)
1724     assert(dst_array == to, "");
1725     __ addptr(to,  rdi);        // to   = dst_array + DST_POS << log2 elsize
1726     __ movptr(FROM, from);      // src_addr
1727     __ mov(rdi_elsize, rcx_lh); // log2 elsize
1728     __ movl2ptr(count, LENGTH); // elements count
1729 
1730     BLOCK_COMMENT("choose copy loop based on element size");
1731     __ cmpl(rdi_elsize, 0);
1732 
1733     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
1734     __ cmpl(rdi_elsize, LogBytesPerShort);
1735     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
1736     __ cmpl(rdi_elsize, LogBytesPerInt);
1737     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
1738 #ifdef ASSERT
1739     __ cmpl(rdi_elsize, LogBytesPerLong);
1740     __ jccb(Assembler::notEqual, L_failed);
1741 #endif
1742     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1743     __ pop(rsi);
1744     __ jump(RuntimeAddress(entry_jlong_arraycopy));
1745 
1746   __ BIND(L_failed);
1747     __ xorptr(rax, rax);
1748     __ notptr(rax); // return -1
1749     __ pop(rdi);
1750     __ pop(rsi);
1751     __ leave(); // required for proper stackwalking of RuntimeStub frame
1752     __ ret(0);
1753 
1754     // ObjArrayKlass
1755   __ BIND(L_objArray);
1756     // live at this point:  rcx_src_klass, src[_pos], dst[_pos]
1757 
1758     Label L_plain_copy, L_checkcast_copy;
1759     //  test array classes for subtyping
1760     __ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
1761     __ jccb(Assembler::notEqual, L_checkcast_copy);
1762 
1763     // Identically typed arrays can be copied without element-wise checks.
1764     assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);
1765     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1766 
1767   __ BIND(L_plain_copy);
1768     __ movl2ptr(count, LENGTH); // elements count
1769     __ movl2ptr(src_pos, SRC_POS);  // reload src_pos
1770     __ lea(from, Address(src, src_pos, Address::times_ptr,
1771                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
1772     __ movl2ptr(dst_pos, DST_POS);  // reload dst_pos
1773     __ lea(to,   Address(dst, dst_pos, Address::times_ptr,
1774                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
1775     __ movptr(FROM,  from);   // src_addr
1776     __ movptr(TO,    to);     // dst_addr
1777     __ movl(COUNT, count);  // count
1778     __ jump(RuntimeAddress(entry_oop_arraycopy));
1779 
1780   __ BIND(L_checkcast_copy);
1781     // live at this point:  rcx_src_klass, dst[_pos], src[_pos]
1782     {
1783       // Handy offsets:
1784       int  ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
1785       int sco_offset = in_bytes(Klass::super_check_offset_offset());
1786 
1787       Register rsi_dst_klass = rsi;
1788       Register rdi_temp      = rdi;
1789       assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");
1790       assert(rdi_temp      == dst_pos, "expected alias w/ dst_pos");
1791       Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);
1792 
1793       // Before looking at dst.length, make sure dst is also an objArray.
1794       __ movptr(rsi_dst_klass, dst_klass_addr);
1795       __ cmpl(dst_klass_lh_addr, objArray_lh);
1796       __ jccb(Assembler::notEqual, L_failed);
1797 
1798       // It is safe to examine both src.length and dst.length.
1799       __ movl2ptr(src_pos, SRC_POS);        // reload rsi
1800       arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1801       // (Now src_pos and dst_pos are killed, but not src and dst.)
1802 
1803       // We'll need this temp (don't forget to pop it after the type check).
1804       __ push(rbx);
1805       Register rbx_src_klass = rbx;
1806 
1807       __ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
1808       __ movptr(rsi_dst_klass, dst_klass_addr);
1809       Address super_check_offset_addr(rsi_dst_klass, sco_offset);
1810       Label L_fail_array_check;
1811       generate_type_check(rbx_src_klass,
1812                           super_check_offset_addr, dst_klass_addr,
1813                           rdi_temp, NULL, &L_fail_array_check);
1814       // (On fall-through, we have passed the array type check.)
1815       __ pop(rbx);
1816       __ jmp(L_plain_copy);
1817 
1818       __ BIND(L_fail_array_check);
1819       // Reshuffle arguments so we can call checkcast_arraycopy:
1820 
1821       // match initial saves for checkcast_arraycopy
1822       // push(rsi);    // already done; see above
1823       // push(rdi);    // already done; see above
1824       // push(rbx);    // already done; see above
1825 
1826       // Marshal outgoing arguments now, freeing registers.
1827       Address   from_arg(rsp, 16+ 4);   // from
1828       Address     to_arg(rsp, 16+ 8);   // to
1829       Address length_arg(rsp, 16+12);   // elements count
1830       Address  ckoff_arg(rsp, 16+16);   // super_check_offset
1831       Address  ckval_arg(rsp, 16+20);   // super_klass
1832 
1833       Address SRC_POS_arg(rsp, 16+ 8);
1834       Address DST_POS_arg(rsp, 16+16);
1835       Address  LENGTH_arg(rsp, 16+20);
1836       // push rbx, changed the incoming offsets (why not just use rbp,??)
1837       // assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");
1838 
1839       __ movptr(rbx, Address(rsi_dst_klass, ek_offset));
1840       __ movl2ptr(length, LENGTH_arg);    // reload elements count
1841       __ movl2ptr(src_pos, SRC_POS_arg);  // reload src_pos
1842       __ movl2ptr(dst_pos, DST_POS_arg);  // reload dst_pos
1843 
1844       __ movptr(ckval_arg, rbx);          // destination element type
1845       __ movl(rbx, Address(rbx, sco_offset));
1846       __ movl(ckoff_arg, rbx);          // corresponding class check offset
1847 
1848       __ movl(length_arg, length);      // outgoing length argument
1849 
1850       __ lea(from, Address(src, src_pos, Address::times_ptr,
1851                             arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1852       __ movptr(from_arg, from);
1853 
1854       __ lea(to, Address(dst, dst_pos, Address::times_ptr,
1855                           arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1856       __ movptr(to_arg, to);
1857       __ jump(RuntimeAddress(entry_checkcast_arraycopy));
1858     }
1859 
1860     return start;
1861   }
1862 
1863   void generate_arraycopy_stubs() {
1864     address entry;
1865     address entry_jbyte_arraycopy;
1866     address entry_jshort_arraycopy;
1867     address entry_jint_arraycopy;
1868     address entry_oop_arraycopy;
1869     address entry_jlong_arraycopy;
1870     address entry_checkcast_arraycopy;
1871 
1872     StubRoutines::_arrayof_jbyte_disjoint_arraycopy =
1873         generate_disjoint_copy(T_BYTE,  true, Address::times_1, &entry,
1874                                "arrayof_jbyte_disjoint_arraycopy");
1875     StubRoutines::_arrayof_jbyte_arraycopy =
1876         generate_conjoint_copy(T_BYTE,  true, Address::times_1,  entry,
1877                                NULL, "arrayof_jbyte_arraycopy");
1878     StubRoutines::_jbyte_disjoint_arraycopy =
1879         generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,
1880                                "jbyte_disjoint_arraycopy");
1881     StubRoutines::_jbyte_arraycopy =
1882         generate_conjoint_copy(T_BYTE, false, Address::times_1,  entry,
1883                                &entry_jbyte_arraycopy, "jbyte_arraycopy");
1884 
1885     StubRoutines::_arrayof_jshort_disjoint_arraycopy =
1886         generate_disjoint_copy(T_SHORT,  true, Address::times_2, &entry,
1887                                "arrayof_jshort_disjoint_arraycopy");
1888     StubRoutines::_arrayof_jshort_arraycopy =
1889         generate_conjoint_copy(T_SHORT,  true, Address::times_2,  entry,
1890                                NULL, "arrayof_jshort_arraycopy");
1891     StubRoutines::_jshort_disjoint_arraycopy =
1892         generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,
1893                                "jshort_disjoint_arraycopy");
1894     StubRoutines::_jshort_arraycopy =
1895         generate_conjoint_copy(T_SHORT, false, Address::times_2,  entry,
1896                                &entry_jshort_arraycopy, "jshort_arraycopy");
1897 
1898     // Next arrays are always aligned on 4 bytes at least.
1899     StubRoutines::_jint_disjoint_arraycopy =
1900         generate_disjoint_copy(T_INT, true, Address::times_4, &entry,
1901                                "jint_disjoint_arraycopy");
1902     StubRoutines::_jint_arraycopy =
1903         generate_conjoint_copy(T_INT, true, Address::times_4,  entry,
1904                                &entry_jint_arraycopy, "jint_arraycopy");
1905 
1906     StubRoutines::_oop_disjoint_arraycopy =
1907         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
1908                                "oop_disjoint_arraycopy");
1909     StubRoutines::_oop_arraycopy =
1910         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
1911                                &entry_oop_arraycopy, "oop_arraycopy");
1912 
1913     StubRoutines::_oop_disjoint_arraycopy_uninit =
1914         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
1915                                "oop_disjoint_arraycopy_uninit",
1916                                /*dest_uninitialized*/true);
1917     StubRoutines::_oop_arraycopy_uninit =
1918         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
1919                                NULL, "oop_arraycopy_uninit",
1920                                /*dest_uninitialized*/true);
1921 
1922     StubRoutines::_jlong_disjoint_arraycopy =
1923         generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");
1924     StubRoutines::_jlong_arraycopy =
1925         generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,
1926                                     "jlong_arraycopy");
1927 
1928     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
1929     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
1930     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
1931     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
1932     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
1933     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
1934 
1935     StubRoutines::_arrayof_jint_disjoint_arraycopy       = StubRoutines::_jint_disjoint_arraycopy;
1936     StubRoutines::_arrayof_oop_disjoint_arraycopy        = StubRoutines::_oop_disjoint_arraycopy;
1937     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit;
1938     StubRoutines::_arrayof_jlong_disjoint_arraycopy      = StubRoutines::_jlong_disjoint_arraycopy;
1939 
1940     StubRoutines::_arrayof_jint_arraycopy       = StubRoutines::_jint_arraycopy;
1941     StubRoutines::_arrayof_oop_arraycopy        = StubRoutines::_oop_arraycopy;
1942     StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit;
1943     StubRoutines::_arrayof_jlong_arraycopy      = StubRoutines::_jlong_arraycopy;
1944 
1945     StubRoutines::_checkcast_arraycopy =
1946         generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
1947     StubRoutines::_checkcast_arraycopy_uninit =
1948         generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, /*dest_uninitialized*/true);
1949 
1950     StubRoutines::_unsafe_arraycopy =
1951         generate_unsafe_copy("unsafe_arraycopy",
1952                                entry_jbyte_arraycopy,
1953                                entry_jshort_arraycopy,
1954                                entry_jint_arraycopy,
1955                                entry_jlong_arraycopy);
1956 
1957     StubRoutines::_generic_arraycopy =
1958         generate_generic_copy("generic_arraycopy",
1959                                entry_jbyte_arraycopy,
1960                                entry_jshort_arraycopy,
1961                                entry_jint_arraycopy,
1962                                entry_oop_arraycopy,
1963                                entry_jlong_arraycopy,
1964                                entry_checkcast_arraycopy);
1965   }
1966 
1967   // AES intrinsic stubs
1968   enum {AESBlockSize = 16};
1969 
1970   address generate_key_shuffle_mask() {
1971     __ align(16);
1972     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
1973     address start = __ pc();
1974     __ emit_data(0x00010203, relocInfo::none, 0 );
1975     __ emit_data(0x04050607, relocInfo::none, 0 );
1976     __ emit_data(0x08090a0b, relocInfo::none, 0 );
1977     __ emit_data(0x0c0d0e0f, relocInfo::none, 0 );
1978     return start;
1979   }
1980 
1981   address generate_counter_shuffle_mask() {
1982     __ align(16);
1983     StubCodeMark mark(this, "StubRoutines", "counter_shuffle_mask");
1984     address start = __ pc();
1985     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
1986     __ emit_data(0x08090a0b, relocInfo::none, 0);
1987     __ emit_data(0x04050607, relocInfo::none, 0);
1988     __ emit_data(0x00010203, relocInfo::none, 0);
1989     return start;
1990   }
1991 
1992   // Utility routine for loading a 128-bit key word in little endian format
1993   // can optionally specify that the shuffle mask is already in an xmmregister
1994   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
1995     __ movdqu(xmmdst, Address(key, offset));
1996     if (xmm_shuf_mask != NULL) {
1997       __ pshufb(xmmdst, xmm_shuf_mask);
1998     } else {
1999       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2000     }
2001   }
2002 
2003   // aesenc using specified key+offset
2004   // can optionally specify that the shuffle mask is already in an xmmregister
2005   void aes_enc_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2006     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2007     __ aesenc(xmmdst, xmmtmp);
2008   }
2009 
2010   // aesdec using specified key+offset
2011   // can optionally specify that the shuffle mask is already in an xmmregister
2012   void aes_dec_key(XMMRegister xmmdst, XMMRegister xmmtmp, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
2013     load_key(xmmtmp, key, offset, xmm_shuf_mask);
2014     __ aesdec(xmmdst, xmmtmp);
2015   }
2016 
2017   // Utility routine for increase 128bit counter (iv in CTR mode)
2018   //  XMM_128bit,  D3, D2, D1, D0
2019   void inc_counter(Register reg, XMMRegister xmmdst, int inc_delta, Label& next_block) {
2020     __ pextrd(reg, xmmdst, 0x0);
2021     __ addl(reg, inc_delta);
2022     __ pinsrd(xmmdst, reg, 0x0);
2023     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2024 
2025     __ pextrd(reg, xmmdst, 0x01); // Carry-> D1
2026     __ addl(reg, 0x01);
2027     __ pinsrd(xmmdst, reg, 0x01);
2028     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2029 
2030     __ pextrd(reg, xmmdst, 0x02); // Carry-> D2
2031     __ addl(reg, 0x01);
2032     __ pinsrd(xmmdst, reg, 0x02);
2033     __ jcc(Assembler::carryClear, next_block); // jump if no carry
2034 
2035     __ pextrd(reg, xmmdst, 0x03); // Carry -> D3
2036     __ addl(reg, 0x01);
2037     __ pinsrd(xmmdst, reg, 0x03);
2038 
2039     __ BIND(next_block);          // next instruction
2040   }
2041 
2042 
2043   // Arguments:
2044   //
2045   // Inputs:
2046   //   c_rarg0   - source byte array address
2047   //   c_rarg1   - destination byte array address
2048   //   c_rarg2   - K (key) in little endian int array
2049   //
2050   address generate_aescrypt_encryptBlock() {
2051     assert(UseAES, "need AES instructions and misaligned SSE support");
2052     __ align(CodeEntryAlignment);
2053     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
2054     Label L_doLast;
2055     address start = __ pc();
2056 
2057     const Register from        = rdx;      // source array address
2058     const Register to          = rdx;      // destination array address
2059     const Register key         = rcx;      // key array address
2060     const Register keylen      = rax;
2061     const Address  from_param(rbp, 8+0);
2062     const Address  to_param  (rbp, 8+4);
2063     const Address  key_param (rbp, 8+8);
2064 
2065     const XMMRegister xmm_result = xmm0;
2066     const XMMRegister xmm_key_shuf_mask = xmm1;
2067     const XMMRegister xmm_temp1  = xmm2;
2068     const XMMRegister xmm_temp2  = xmm3;
2069     const XMMRegister xmm_temp3  = xmm4;
2070     const XMMRegister xmm_temp4  = xmm5;
2071 
2072     __ enter();   // required for proper stackwalking of RuntimeStub frame
2073 
2074     __ movptr(from, from_param);
2075     __ movptr(key, key_param);
2076 
2077     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2078     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2079 
2080     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2081     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
2082     __ movptr(to, to_param);
2083 
2084     // For encryption, the java expanded key ordering is just what we need
2085 
2086     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
2087     __ pxor(xmm_result, xmm_temp1);
2088 
2089     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2090     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2091     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2092     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2093 
2094     __ aesenc(xmm_result, xmm_temp1);
2095     __ aesenc(xmm_result, xmm_temp2);
2096     __ aesenc(xmm_result, xmm_temp3);
2097     __ aesenc(xmm_result, xmm_temp4);
2098 
2099     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2100     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2101     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2102     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2103 
2104     __ aesenc(xmm_result, xmm_temp1);
2105     __ aesenc(xmm_result, xmm_temp2);
2106     __ aesenc(xmm_result, xmm_temp3);
2107     __ aesenc(xmm_result, xmm_temp4);
2108 
2109     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2110     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2111 
2112     __ cmpl(keylen, 44);
2113     __ jccb(Assembler::equal, L_doLast);
2114 
2115     __ aesenc(xmm_result, xmm_temp1);
2116     __ aesenc(xmm_result, xmm_temp2);
2117 
2118     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2119     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2120 
2121     __ cmpl(keylen, 52);
2122     __ jccb(Assembler::equal, L_doLast);
2123 
2124     __ aesenc(xmm_result, xmm_temp1);
2125     __ aesenc(xmm_result, xmm_temp2);
2126 
2127     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2128     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2129 
2130     __ BIND(L_doLast);
2131     __ aesenc(xmm_result, xmm_temp1);
2132     __ aesenclast(xmm_result, xmm_temp2);
2133     __ movdqu(Address(to, 0), xmm_result);        // store the result
2134     __ xorptr(rax, rax); // return 0
2135     __ leave(); // required for proper stackwalking of RuntimeStub frame
2136     __ ret(0);
2137 
2138     return start;
2139   }
2140 
2141 
2142   // Arguments:
2143   //
2144   // Inputs:
2145   //   c_rarg0   - source byte array address
2146   //   c_rarg1   - destination byte array address
2147   //   c_rarg2   - K (key) in little endian int array
2148   //
2149   address generate_aescrypt_decryptBlock() {
2150     assert(UseAES, "need AES instructions and misaligned SSE support");
2151     __ align(CodeEntryAlignment);
2152     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
2153     Label L_doLast;
2154     address start = __ pc();
2155 
2156     const Register from        = rdx;      // source array address
2157     const Register to          = rdx;      // destination array address
2158     const Register key         = rcx;      // key array address
2159     const Register keylen      = rax;
2160     const Address  from_param(rbp, 8+0);
2161     const Address  to_param  (rbp, 8+4);
2162     const Address  key_param (rbp, 8+8);
2163 
2164     const XMMRegister xmm_result = xmm0;
2165     const XMMRegister xmm_key_shuf_mask = xmm1;
2166     const XMMRegister xmm_temp1  = xmm2;
2167     const XMMRegister xmm_temp2  = xmm3;
2168     const XMMRegister xmm_temp3  = xmm4;
2169     const XMMRegister xmm_temp4  = xmm5;
2170 
2171     __ enter(); // required for proper stackwalking of RuntimeStub frame
2172 
2173     __ movptr(from, from_param);
2174     __ movptr(key, key_param);
2175 
2176     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
2177     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2178 
2179     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2180     __ movdqu(xmm_result, Address(from, 0));
2181     __ movptr(to, to_param);
2182 
2183     // for decryption java expanded key ordering is rotated one position from what we want
2184     // so we start from 0x10 here and hit 0x00 last
2185     // we don't know if the key is aligned, hence not using load-execute form
2186     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
2187     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
2188     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
2189     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
2190 
2191     __ pxor  (xmm_result, xmm_temp1);
2192     __ aesdec(xmm_result, xmm_temp2);
2193     __ aesdec(xmm_result, xmm_temp3);
2194     __ aesdec(xmm_result, xmm_temp4);
2195 
2196     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
2197     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
2198     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
2199     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
2200 
2201     __ aesdec(xmm_result, xmm_temp1);
2202     __ aesdec(xmm_result, xmm_temp2);
2203     __ aesdec(xmm_result, xmm_temp3);
2204     __ aesdec(xmm_result, xmm_temp4);
2205 
2206     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
2207     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
2208     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
2209 
2210     __ cmpl(keylen, 44);
2211     __ jccb(Assembler::equal, L_doLast);
2212 
2213     __ aesdec(xmm_result, xmm_temp1);
2214     __ aesdec(xmm_result, xmm_temp2);
2215 
2216     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
2217     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
2218 
2219     __ cmpl(keylen, 52);
2220     __ jccb(Assembler::equal, L_doLast);
2221 
2222     __ aesdec(xmm_result, xmm_temp1);
2223     __ aesdec(xmm_result, xmm_temp2);
2224 
2225     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
2226     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
2227 
2228     __ BIND(L_doLast);
2229     __ aesdec(xmm_result, xmm_temp1);
2230     __ aesdec(xmm_result, xmm_temp2);
2231 
2232     // for decryption the aesdeclast operation is always on key+0x00
2233     __ aesdeclast(xmm_result, xmm_temp3);
2234     __ movdqu(Address(to, 0), xmm_result);  // store the result
2235     __ xorptr(rax, rax); // return 0
2236     __ leave(); // required for proper stackwalking of RuntimeStub frame
2237     __ ret(0);
2238 
2239     return start;
2240   }
2241 
2242   void handleSOERegisters(bool saving) {
2243     const int saveFrameSizeInBytes = 4 * wordSize;
2244     const Address saved_rbx     (rbp, -3 * wordSize);
2245     const Address saved_rsi     (rbp, -2 * wordSize);
2246     const Address saved_rdi     (rbp, -1 * wordSize);
2247 
2248     if (saving) {
2249       __ subptr(rsp, saveFrameSizeInBytes);
2250       __ movptr(saved_rsi, rsi);
2251       __ movptr(saved_rdi, rdi);
2252       __ movptr(saved_rbx, rbx);
2253     } else {
2254       // restoring
2255       __ movptr(rsi, saved_rsi);
2256       __ movptr(rdi, saved_rdi);
2257       __ movptr(rbx, saved_rbx);
2258     }
2259   }
2260 
2261   // Arguments:
2262   //
2263   // Inputs:
2264   //   c_rarg0   - source byte array address
2265   //   c_rarg1   - destination byte array address
2266   //   c_rarg2   - K (key) in little endian int array
2267   //   c_rarg3   - r vector byte array address
2268   //   c_rarg4   - input length
2269   //
2270   // Output:
2271   //   rax       - input length
2272   //
2273   address generate_cipherBlockChaining_encryptAESCrypt() {
2274     assert(UseAES, "need AES instructions and misaligned SSE support");
2275     __ align(CodeEntryAlignment);
2276     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
2277     address start = __ pc();
2278 
2279     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
2280     const Register from        = rsi;      // source array address
2281     const Register to          = rdx;      // destination array address
2282     const Register key         = rcx;      // key array address
2283     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2284                                            // and left with the results of the last encryption block
2285     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2286     const Register pos         = rax;
2287 
2288     // xmm register assignments for the loops below
2289     const XMMRegister xmm_result = xmm0;
2290     const XMMRegister xmm_temp   = xmm1;
2291     // first 6 keys preloaded into xmm2-xmm7
2292     const int XMM_REG_NUM_KEY_FIRST = 2;
2293     const int XMM_REG_NUM_KEY_LAST  = 7;
2294     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
2295 
2296     __ enter(); // required for proper stackwalking of RuntimeStub frame
2297     handleSOERegisters(true /*saving*/);
2298 
2299     // load registers from incoming parameters
2300     const Address  from_param(rbp, 8+0);
2301     const Address  to_param  (rbp, 8+4);
2302     const Address  key_param (rbp, 8+8);
2303     const Address  rvec_param (rbp, 8+12);
2304     const Address  len_param  (rbp, 8+16);
2305     __ movptr(from , from_param);
2306     __ movptr(to   , to_param);
2307     __ movptr(key  , key_param);
2308     __ movptr(rvec , rvec_param);
2309     __ movptr(len_reg , len_param);
2310 
2311     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
2312     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2313     // load up xmm regs 2 thru 7 with keys 0-5
2314     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2315       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
2316       offset += 0x10;
2317     }
2318 
2319     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
2320 
2321     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2322     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2323     __ cmpl(rax, 44);
2324     __ jcc(Assembler::notEqual, L_key_192_256);
2325 
2326     // 128 bit code follows here
2327     __ movl(pos, 0);
2328     __ align(OptoLoopAlignment);
2329     __ BIND(L_loopTop_128);
2330     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2331     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2332 
2333     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2334     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2335       __ aesenc(xmm_result, as_XMMRegister(rnum));
2336     }
2337     for (int key_offset = 0x60; key_offset <= 0x90; key_offset += 0x10) {
2338       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2339     }
2340     load_key(xmm_temp, key, 0xa0);
2341     __ aesenclast(xmm_result, xmm_temp);
2342 
2343     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
2344     // no need to store r to memory until we exit
2345     __ addptr(pos, AESBlockSize);
2346     __ subptr(len_reg, AESBlockSize);
2347     __ jcc(Assembler::notEqual, L_loopTop_128);
2348 
2349     __ BIND(L_exit);
2350     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
2351 
2352     handleSOERegisters(false /*restoring*/);
2353     __ movptr(rax, len_param); // return length
2354     __ leave();                                  // required for proper stackwalking of RuntimeStub frame
2355     __ ret(0);
2356 
2357     __ BIND(L_key_192_256);
2358     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
2359     __ cmpl(rax, 52);
2360     __ jcc(Assembler::notEqual, L_key_256);
2361 
2362     // 192-bit code follows here (could be changed to use more xmm registers)
2363     __ movl(pos, 0);
2364     __ align(OptoLoopAlignment);
2365     __ BIND(L_loopTop_192);
2366     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2367     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2368 
2369     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2370     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2371       __ aesenc(xmm_result, as_XMMRegister(rnum));
2372     }
2373     for (int key_offset = 0x60; key_offset <= 0xb0; key_offset += 0x10) {
2374       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2375     }
2376     load_key(xmm_temp, key, 0xc0);
2377     __ aesenclast(xmm_result, xmm_temp);
2378 
2379     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2380     // no need to store r to memory until we exit
2381     __ addptr(pos, AESBlockSize);
2382     __ subptr(len_reg, AESBlockSize);
2383     __ jcc(Assembler::notEqual, L_loopTop_192);
2384     __ jmp(L_exit);
2385 
2386     __ BIND(L_key_256);
2387     // 256-bit code follows here (could be changed to use more xmm registers)
2388     __ movl(pos, 0);
2389     __ align(OptoLoopAlignment);
2390     __ BIND(L_loopTop_256);
2391     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
2392     __ pxor  (xmm_result, xmm_temp);                                // xor with the current r vector
2393 
2394     __ pxor  (xmm_result, xmm_key0);                                // do the aes rounds
2395     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST; rnum++) {
2396       __ aesenc(xmm_result, as_XMMRegister(rnum));
2397     }
2398     for (int key_offset = 0x60; key_offset <= 0xd0; key_offset += 0x10) {
2399       aes_enc_key(xmm_result, xmm_temp, key, key_offset);
2400     }
2401     load_key(xmm_temp, key, 0xe0);
2402     __ aesenclast(xmm_result, xmm_temp);
2403 
2404     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);   // store into the next 16 bytes of output
2405     // no need to store r to memory until we exit
2406     __ addptr(pos, AESBlockSize);
2407     __ subptr(len_reg, AESBlockSize);
2408     __ jcc(Assembler::notEqual, L_loopTop_256);
2409     __ jmp(L_exit);
2410 
2411     return start;
2412   }
2413 
2414 
2415   // CBC AES Decryption.
2416   // In 32-bit stub, because of lack of registers we do not try to parallelize 4 blocks at a time.
2417   //
2418   // Arguments:
2419   //
2420   // Inputs:
2421   //   c_rarg0   - source byte array address
2422   //   c_rarg1   - destination byte array address
2423   //   c_rarg2   - K (key) in little endian int array
2424   //   c_rarg3   - r vector byte array address
2425   //   c_rarg4   - input length
2426   //
2427   // Output:
2428   //   rax       - input length
2429   //
2430 
2431   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
2432     assert(UseAES, "need AES instructions and misaligned SSE support");
2433     __ align(CodeEntryAlignment);
2434     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
2435     address start = __ pc();
2436 
2437     const Register from        = rsi;      // source array address
2438     const Register to          = rdx;      // destination array address
2439     const Register key         = rcx;      // key array address
2440     const Register rvec        = rdi;      // r byte array initialized from initvector array address
2441                                            // and left with the results of the last encryption block
2442     const Register len_reg     = rbx;      // src len (must be multiple of blocksize 16)
2443     const Register pos         = rax;
2444 
2445     const int PARALLEL_FACTOR = 4;
2446     const int ROUNDS[3] = { 10, 12, 14 }; //aes rounds for key128, key192, key256
2447 
2448     Label L_exit;
2449     Label L_singleBlock_loopTop[3]; //128, 192, 256
2450     Label L_multiBlock_loopTop[3]; //128, 192, 256
2451 
2452     const XMMRegister xmm_prev_block_cipher = xmm0; // holds cipher of previous block
2453     const XMMRegister xmm_key_shuf_mask = xmm1;
2454 
2455     const XMMRegister xmm_key_tmp0 = xmm2;
2456     const XMMRegister xmm_key_tmp1 = xmm3;
2457 
2458     // registers holding the six results in the parallelized loop
2459     const XMMRegister xmm_result0 = xmm4;
2460     const XMMRegister xmm_result1 = xmm5;
2461     const XMMRegister xmm_result2 = xmm6;
2462     const XMMRegister xmm_result3 = xmm7;
2463 
2464     __ enter(); // required for proper stackwalking of RuntimeStub frame
2465     handleSOERegisters(true /*saving*/);
2466 
2467     // load registers from incoming parameters
2468     const Address  from_param(rbp, 8+0);
2469     const Address  to_param  (rbp, 8+4);
2470     const Address  key_param (rbp, 8+8);
2471     const Address  rvec_param (rbp, 8+12);
2472     const Address  len_param  (rbp, 8+16);
2473 
2474     __ movptr(from , from_param);
2475     __ movptr(to   , to_param);
2476     __ movptr(key  , key_param);
2477     __ movptr(rvec , rvec_param);
2478     __ movptr(len_reg , len_param);
2479 
2480     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2481     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00)); // initialize with initial rvec
2482 
2483     __ xorptr(pos, pos);
2484 
2485     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
2486     // rvec is reused
2487     __ movl(rvec, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2488     __ cmpl(rvec, 52);
2489     __ jcc(Assembler::equal, L_multiBlock_loopTop[1]);
2490     __ cmpl(rvec, 60);
2491     __ jcc(Assembler::equal, L_multiBlock_loopTop[2]);
2492 
2493 #define DoFour(opc, src_reg)           \
2494   __ opc(xmm_result0, src_reg);         \
2495   __ opc(xmm_result1, src_reg);         \
2496   __ opc(xmm_result2, src_reg);         \
2497   __ opc(xmm_result3, src_reg);         \
2498 
2499     for (int k = 0; k < 3; ++k) {
2500       __ align(OptoLoopAlignment);
2501       __ BIND(L_multiBlock_loopTop[k]);
2502       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least 4 blocks left
2503       __ jcc(Assembler::less, L_singleBlock_loopTop[k]);
2504 
2505       __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0 * AESBlockSize)); // get next 4 blocks into xmmresult registers
2506       __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2507       __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2508       __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2509 
2510       // the java expanded key ordering is rotated one position from what we want
2511       // so we start from 0x10 here and hit 0x00 last
2512       load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2513       DoFour(pxor, xmm_key_tmp0); //xor with first key
2514       // do the aes dec rounds
2515       for (int rnum = 1; rnum <= ROUNDS[k];) {
2516         //load two keys at a time
2517         //k1->0x20, ..., k9->0xa0, k10->0x00
2518         load_key(xmm_key_tmp1, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2519         load_key(xmm_key_tmp0, key, ((rnum + 2) % (ROUNDS[k] + 1)) * 0x10, xmm_key_shuf_mask); // hit 0x00 last!
2520         DoFour(aesdec, xmm_key_tmp1);
2521         rnum++;
2522         if (rnum != ROUNDS[k]) {
2523           DoFour(aesdec, xmm_key_tmp0);
2524         }
2525         else {
2526           DoFour(aesdeclast, xmm_key_tmp0);
2527         }
2528         rnum++;
2529       }
2530 
2531       // for each result, xor with the r vector of previous cipher block
2532       __ pxor(xmm_result0, xmm_prev_block_cipher);
2533       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2534       __ pxor(xmm_result1, xmm_prev_block_cipher);
2535       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2536       __ pxor(xmm_result2, xmm_prev_block_cipher);
2537       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2538       __ pxor(xmm_result3, xmm_prev_block_cipher);
2539       __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3 * AESBlockSize)); // this will carry over to next set of blocks
2540 
2541             // store 4 results into the next 64 bytes of output
2542        __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2543        __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2544        __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2545        __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2546 
2547        __ addptr(pos, 4 * AESBlockSize);
2548        __ subptr(len_reg, 4 * AESBlockSize);
2549        __ jmp(L_multiBlock_loopTop[k]);
2550 
2551        //singleBlock starts here
2552        __ align(OptoLoopAlignment);
2553        __ BIND(L_singleBlock_loopTop[k]);
2554        __ cmpptr(len_reg, 0); // any blocks left?
2555        __ jcc(Assembler::equal, L_exit);
2556        __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
2557        __ movdqa(xmm_result1, xmm_result0);
2558 
2559        load_key(xmm_key_tmp0, key, 0x10, xmm_key_shuf_mask);
2560        __ pxor(xmm_result0, xmm_key_tmp0);
2561        // do the aes dec rounds
2562        for (int rnum = 1; rnum < ROUNDS[k]; rnum++) {
2563          // the java expanded key ordering is rotated one position from what we want
2564          load_key(xmm_key_tmp0, key, (rnum + 1) * 0x10, xmm_key_shuf_mask);
2565          __ aesdec(xmm_result0, xmm_key_tmp0);
2566        }
2567        load_key(xmm_key_tmp0, key, 0x00, xmm_key_shuf_mask);
2568        __ aesdeclast(xmm_result0, xmm_key_tmp0);
2569        __ pxor(xmm_result0, xmm_prev_block_cipher); // xor with the current r vector
2570        __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result0); // store into the next 16 bytes of output
2571        // no need to store r to memory until we exit
2572        __ movdqa(xmm_prev_block_cipher, xmm_result1); // set up next r vector with cipher input from this block
2573 
2574        __ addptr(pos, AESBlockSize);
2575        __ subptr(len_reg, AESBlockSize);
2576        __ jmp(L_singleBlock_loopTop[k]);
2577     }//for 128/192/256
2578 
2579     __ BIND(L_exit);
2580     __ movptr(rvec, rvec_param);                        // restore this since reused earlier
2581     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher); // final value of r stored in rvec of CipherBlockChaining object
2582     handleSOERegisters(false /*restoring*/);
2583     __ movptr(rax, len_param);                          // return length
2584     __ leave();                                         // required for proper stackwalking of RuntimeStub frame
2585     __ ret(0);
2586 
2587     return start;
2588   }
2589 
2590   // CTR AES crypt.
2591   // In 32-bit stub, parallelize 4 blocks at a time
2592   // Arguments:
2593   //
2594   // Inputs:
2595   //   c_rarg0   - source byte array address
2596   //   c_rarg1   - destination byte array address
2597   //   c_rarg2   - K (key) in little endian int array
2598   //   c_rarg3   - counter vector byte array address
2599   //   c_rarg4   - input length
2600   //
2601   // Output:
2602   //   rax       - input length
2603   //
2604   address generate_counterMode_AESCrypt_Parallel() {
2605     assert(UseAES, "need AES instructions and misaligned SSE support");
2606     __ align(CodeEntryAlignment);
2607     StubCodeMark mark(this, "StubRoutines", "counterMode_AESCrypt");
2608     address start = __ pc();
2609     const Register from        = rsi;      // source array address
2610     const Register to          = rdx;      // destination array address
2611     const Register key         = rcx;      // key array address
2612     const Register counter     = rdi;      // counter byte array initialized from initvector array address
2613                                            // and updated with the incremented counter in the end
2614     const Register len_reg     = rbx;
2615     const Register pos         = rax;
2616 
2617     __ enter(); // required for proper stackwalking of RuntimeStub frame
2618     handleSOERegisters(true /*saving*/); // save rbx, rsi, rdi
2619 
2620     // load registers from incoming parameters
2621     const Address  from_param(rbp, 8+0);
2622     const Address  to_param  (rbp, 8+4);
2623     const Address  key_param (rbp, 8+8);
2624     const Address  rvec_param (rbp, 8+12);
2625     const Address  len_param  (rbp, 8+16);
2626     const Address  saved_counter_param(rbp, 8 + 20);
2627     const Address  used_addr_param(rbp, 8 + 24);
2628 
2629     __ movptr(from , from_param);
2630     __ movptr(to   , to_param);
2631     __ movptr(len_reg , len_param);
2632 
2633     // Use the partially used encrpyted counter from last invocation
2634     Label L_exit_preLoop, L_preLoop_start;
2635 
2636     // Use the registers 'counter' and 'key' here in this preloop
2637     // to hold of last 2 params 'used' and 'saved_encCounter_start'
2638     Register used = counter;
2639     Register saved_encCounter_start = key;
2640     Register used_addr = saved_encCounter_start;
2641 
2642     __ movptr(used_addr, used_addr_param);
2643     __ movptr(used, Address(used_addr, 0));
2644     __ movptr(saved_encCounter_start, saved_counter_param);
2645 
2646     __ BIND(L_preLoop_start);
2647     __ cmpptr(used, 16);
2648     __ jcc(Assembler::aboveEqual, L_exit_preLoop);
2649     __ cmpptr(len_reg, 0);
2650     __ jcc(Assembler::lessEqual, L_exit_preLoop);
2651     __ movb(rax, Address(saved_encCounter_start, used));
2652     __ xorb(rax, Address(from, 0));
2653     __ movb(Address(to, 0), rax);
2654     __ addptr(from, 1);
2655     __ addptr(to, 1);
2656     __ addptr(used, 1);
2657     __ subptr(len_reg, 1);
2658 
2659     __ jmp(L_preLoop_start);
2660 
2661     __ BIND(L_exit_preLoop);
2662     __ movptr(used_addr, used_addr_param);
2663     __ movptr(used_addr, used_addr_param);
2664     __ movl(Address(used_addr, 0), used);
2665 
2666     // load the parameters 'key' and 'counter'
2667     __ movptr(key, key_param);
2668     __ movptr(counter, rvec_param);
2669 
2670     // xmm register assignments for the loops below
2671     const XMMRegister xmm_curr_counter      = xmm0;
2672     const XMMRegister xmm_counter_shuf_mask = xmm1;  // need to be reloaded
2673     const XMMRegister xmm_key_shuf_mask     = xmm2;  // need to be reloaded
2674     const XMMRegister xmm_key               = xmm3;
2675     const XMMRegister xmm_result0           = xmm4;
2676     const XMMRegister xmm_result1           = xmm5;
2677     const XMMRegister xmm_result2           = xmm6;
2678     const XMMRegister xmm_result3           = xmm7;
2679     const XMMRegister xmm_from0             = xmm1;   //reuse XMM register
2680     const XMMRegister xmm_from1             = xmm2;
2681     const XMMRegister xmm_from2             = xmm3;
2682     const XMMRegister xmm_from3             = xmm4;
2683 
2684     //for key_128, key_192, key_256
2685     const int rounds[3] = {10, 12, 14};
2686     Label L_singleBlockLoopTop[3];
2687     Label L_multiBlock_loopTop[3];
2688     Label L_key192_top, L_key256_top;
2689     Label L_incCounter[3][4]; // 3: different key length,  4: 4 blocks at a time
2690     Label L_incCounter_single[3]; //for single block, key128, key192, key256
2691     Label L_processTail_insr[3], L_processTail_4_insr[3], L_processTail_2_insr[3], L_processTail_1_insr[3], L_processTail_exit_insr[3];
2692     Label L_processTail_extr[3], L_processTail_4_extr[3], L_processTail_2_extr[3], L_processTail_1_extr[3], L_processTail_exit_extr[3];
2693 
2694     Label L_exit;
2695     const int PARALLEL_FACTOR = 4;  //because of the limited register number
2696 
2697     // initialize counter with initial counter
2698     __ movdqu(xmm_curr_counter, Address(counter, 0x00));
2699     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2700     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled for increase
2701 
2702     // key length could be only {11, 13, 15} * 4 = {44, 52, 60}
2703     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2704     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
2705     __ cmpl(rax, 52);
2706     __ jcc(Assembler::equal, L_key192_top);
2707     __ cmpl(rax, 60);
2708     __ jcc(Assembler::equal, L_key256_top);
2709 
2710     //key128 begins here
2711     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2712 
2713 #define CTR_DoFour(opc, src_reg)               \
2714     __ opc(xmm_result0, src_reg);              \
2715     __ opc(xmm_result1, src_reg);              \
2716     __ opc(xmm_result2, src_reg);              \
2717     __ opc(xmm_result3, src_reg);
2718 
2719     // k == 0 :  generate code for key_128
2720     // k == 1 :  generate code for key_192
2721     // k == 2 :  generate code for key_256
2722     for (int k = 0; k < 3; ++k) {
2723       //multi blocks starts here
2724       __ align(OptoLoopAlignment);
2725       __ BIND(L_multiBlock_loopTop[k]);
2726       __ cmpptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // see if at least PARALLEL_FACTOR blocks left
2727       __ jcc(Assembler::less, L_singleBlockLoopTop[k]);
2728 
2729       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2730       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2731 
2732       //load, then increase counters
2733       CTR_DoFour(movdqa, xmm_curr_counter);
2734       __ push(rbx);
2735       inc_counter(rbx, xmm_result1, 0x01, L_incCounter[k][0]);
2736       inc_counter(rbx, xmm_result2, 0x02, L_incCounter[k][1]);
2737       inc_counter(rbx, xmm_result3, 0x03, L_incCounter[k][2]);
2738       inc_counter(rbx, xmm_curr_counter, 0x04, L_incCounter[k][3]);
2739       __ pop (rbx);
2740 
2741       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask); // load Round 0 key. interleaving for better performance
2742 
2743       CTR_DoFour(pshufb, xmm_counter_shuf_mask); // after increased, shuffled counters back for PXOR
2744       CTR_DoFour(pxor, xmm_key);   //PXOR with Round 0 key
2745 
2746       for (int i = 1; i < rounds[k]; ++i) {
2747         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2748         CTR_DoFour(aesenc, xmm_key);
2749       }
2750       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2751       CTR_DoFour(aesenclast, xmm_key);
2752 
2753       // get next PARALLEL_FACTOR blocks into xmm_from registers
2754       __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2755       __ movdqu(xmm_from1, Address(from, pos, Address::times_1, 1 * AESBlockSize));
2756       __ movdqu(xmm_from2, Address(from, pos, Address::times_1, 2 * AESBlockSize));
2757 
2758       // PXOR with input text
2759       __ pxor(xmm_result0, xmm_from0); //result0 is xmm4
2760       __ pxor(xmm_result1, xmm_from1);
2761       __ pxor(xmm_result2, xmm_from2);
2762 
2763       // store PARALLEL_FACTOR results into the next 64 bytes of output
2764       __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2765       __ movdqu(Address(to, pos, Address::times_1, 1 * AESBlockSize), xmm_result1);
2766       __ movdqu(Address(to, pos, Address::times_1, 2 * AESBlockSize), xmm_result2);
2767 
2768       // do it here after xmm_result0 is saved, because xmm_from3 reuse the same register of xmm_result0.
2769       __ movdqu(xmm_from3, Address(from, pos, Address::times_1, 3 * AESBlockSize));
2770       __ pxor(xmm_result3, xmm_from3);
2771       __ movdqu(Address(to, pos, Address::times_1, 3 * AESBlockSize), xmm_result3);
2772 
2773       __ addptr(pos, PARALLEL_FACTOR * AESBlockSize); // increase the length of crypt text
2774       __ subptr(len_reg, PARALLEL_FACTOR * AESBlockSize); // decrease the remaining length
2775       __ jmp(L_multiBlock_loopTop[k]);
2776 
2777       // singleBlock starts here
2778       __ align(OptoLoopAlignment);
2779       __ BIND(L_singleBlockLoopTop[k]);
2780       __ cmpptr(len_reg, 0);
2781       __ jcc(Assembler::equal, L_exit);
2782       __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
2783       __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2784       __ movdqa(xmm_result0, xmm_curr_counter);
2785       load_key(xmm_key, key, 0x00, xmm_key_shuf_mask);
2786       __ push(rbx);//rbx is used for increasing counter
2787       inc_counter(rbx, xmm_curr_counter, 0x01, L_incCounter_single[k]);
2788       __ pop (rbx);
2789       __ pshufb(xmm_result0, xmm_counter_shuf_mask);
2790       __ pxor(xmm_result0, xmm_key);
2791       for (int i = 1; i < rounds[k]; i++) {
2792         load_key(xmm_key, key, (0x10 * i), xmm_key_shuf_mask);
2793         __ aesenc(xmm_result0, xmm_key);
2794       }
2795       load_key(xmm_key, key, (0x10 * rounds[k]), xmm_key_shuf_mask);
2796       __ aesenclast(xmm_result0, xmm_key);
2797       __ cmpptr(len_reg, AESBlockSize);
2798       __ jcc(Assembler::less, L_processTail_insr[k]);
2799         __ movdqu(xmm_from0, Address(from, pos, Address::times_1, 0 * AESBlockSize));
2800         __ pxor(xmm_result0, xmm_from0);
2801         __ movdqu(Address(to, pos, Address::times_1, 0 * AESBlockSize), xmm_result0);
2802         __ addptr(pos, AESBlockSize);
2803         __ subptr(len_reg, AESBlockSize);
2804         __ jmp(L_singleBlockLoopTop[k]);
2805 
2806       __ BIND(L_processTail_insr[k]);                                               // Process the tail part of the input array
2807         __ addptr(pos, len_reg);                                                    // 1. Insert bytes from src array into xmm_from0 register
2808         __ testptr(len_reg, 8);
2809         __ jcc(Assembler::zero, L_processTail_4_insr[k]);
2810           __ subptr(pos,8);
2811           __ pinsrd(xmm_from0, Address(from, pos), 0);
2812           __ pinsrd(xmm_from0, Address(from, pos, Address::times_1, 4), 1);
2813         __ BIND(L_processTail_4_insr[k]);
2814         __ testptr(len_reg, 4);
2815         __ jcc(Assembler::zero, L_processTail_2_insr[k]);
2816           __ subptr(pos,4);
2817           __ pslldq(xmm_from0, 4);
2818           __ pinsrd(xmm_from0, Address(from, pos), 0);
2819         __ BIND(L_processTail_2_insr[k]);
2820         __ testptr(len_reg, 2);
2821         __ jcc(Assembler::zero, L_processTail_1_insr[k]);
2822           __ subptr(pos, 2);
2823           __ pslldq(xmm_from0, 2);
2824           __ pinsrw(xmm_from0, Address(from, pos), 0);
2825         __ BIND(L_processTail_1_insr[k]);
2826         __ testptr(len_reg, 1);
2827         __ jcc(Assembler::zero, L_processTail_exit_insr[k]);
2828           __ subptr(pos, 1);
2829           __ pslldq(xmm_from0, 1);
2830           __ pinsrb(xmm_from0, Address(from, pos), 0);
2831         __ BIND(L_processTail_exit_insr[k]);
2832 
2833         __ movptr(saved_encCounter_start, saved_counter_param);
2834         __ movdqu(Address(saved_encCounter_start, 0), xmm_result0);               // 2. Perform pxor of the encrypted counter and plaintext Bytes.
2835         __ pxor(xmm_result0, xmm_from0);                                          //    Also the encrypted counter is saved for next invocation.
2836 
2837         __ testptr(len_reg, 8);
2838         __ jcc(Assembler::zero, L_processTail_4_extr[k]);                        // 3. Extract bytes from xmm_result0 into the dest. array
2839           __ pextrd(Address(to, pos), xmm_result0, 0);
2840           __ pextrd(Address(to, pos, Address::times_1, 4), xmm_result0, 1);
2841           __ psrldq(xmm_result0, 8);
2842           __ addptr(pos, 8);
2843         __ BIND(L_processTail_4_extr[k]);
2844         __ testptr(len_reg, 4);
2845         __ jcc(Assembler::zero, L_processTail_2_extr[k]);
2846           __ pextrd(Address(to, pos), xmm_result0, 0);
2847           __ psrldq(xmm_result0, 4);
2848           __ addptr(pos, 4);
2849         __ BIND(L_processTail_2_extr[k]);
2850         __ testptr(len_reg, 2);
2851         __ jcc(Assembler::zero, L_processTail_1_extr[k]);
2852           __ pextrb(Address(to, pos), xmm_result0, 0);
2853           __ pextrb(Address(to, pos, Address::times_1, 1), xmm_result0, 1);
2854           __ psrldq(xmm_result0, 2);
2855           __ addptr(pos, 2);
2856         __ BIND(L_processTail_1_extr[k]);
2857         __ testptr(len_reg, 1);
2858         __ jcc(Assembler::zero, L_processTail_exit_extr[k]);
2859           __ pextrb(Address(to, pos), xmm_result0, 0);
2860 
2861         __ BIND(L_processTail_exit_extr[k]);
2862         __ movptr(used_addr, used_addr_param);
2863         __ movl(Address(used_addr, 0), len_reg);
2864         __ jmp(L_exit);
2865     }
2866 
2867     __ BIND(L_exit);
2868     __ movdqu(xmm_counter_shuf_mask, ExternalAddress(StubRoutines::x86::counter_shuffle_mask_addr()));
2869     __ pshufb(xmm_curr_counter, xmm_counter_shuf_mask); //counter is shuffled back.
2870     __ movdqu(Address(counter, 0), xmm_curr_counter); //save counter back
2871     handleSOERegisters(false /*restoring*/);
2872     __ movptr(rax, len_param); // return length
2873     __ leave();                // required for proper stackwalking of RuntimeStub frame
2874     __ ret(0);
2875 
2876     __ BIND (L_key192_top);
2877     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2878     __ jmp(L_multiBlock_loopTop[1]); //key192
2879 
2880     __ BIND (L_key256_top);
2881     __ movptr(pos, 0); // init pos before L_multiBlock_loopTop
2882     __ jmp(L_multiBlock_loopTop[2]); //key192
2883 
2884     return start;
2885   }
2886 
2887   address generate_upper_word_mask() {
2888     __ align(64);
2889     StubCodeMark mark(this, "StubRoutines", "upper_word_mask");
2890     address start = __ pc();
2891     __ emit_data(0x00000000, relocInfo::none, 0);
2892     __ emit_data(0x00000000, relocInfo::none, 0);
2893     __ emit_data(0x00000000, relocInfo::none, 0);
2894     __ emit_data(0xFFFFFFFF, relocInfo::none, 0);
2895     return start;
2896   }
2897 
2898   address generate_shuffle_byte_flip_mask() {
2899     __ align(64);
2900     StubCodeMark mark(this, "StubRoutines", "shuffle_byte_flip_mask");
2901     address start = __ pc();
2902     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
2903     __ emit_data(0x08090a0b, relocInfo::none, 0);
2904     __ emit_data(0x04050607, relocInfo::none, 0);
2905     __ emit_data(0x00010203, relocInfo::none, 0);
2906     return start;
2907   }
2908 
2909   // ofs and limit are use for multi-block byte array.
2910   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
2911   address generate_sha1_implCompress(bool multi_block, const char *name) {
2912     __ align(CodeEntryAlignment);
2913     StubCodeMark mark(this, "StubRoutines", name);
2914     address start = __ pc();
2915 
2916     Register buf   = rax;
2917     Register state = rdx;
2918     Register ofs   = rcx;
2919     Register limit = rdi;
2920 
2921     const Address  buf_param(rbp, 8 + 0);
2922     const Address  state_param(rbp, 8 + 4);
2923     const Address  ofs_param(rbp, 8 + 8);
2924     const Address  limit_param(rbp, 8 + 12);
2925 
2926     const XMMRegister abcd = xmm0;
2927     const XMMRegister e0 = xmm1;
2928     const XMMRegister e1 = xmm2;
2929     const XMMRegister msg0 = xmm3;
2930 
2931     const XMMRegister msg1 = xmm4;
2932     const XMMRegister msg2 = xmm5;
2933     const XMMRegister msg3 = xmm6;
2934     const XMMRegister shuf_mask = xmm7;
2935 
2936     __ enter();
2937     __ subptr(rsp, 8 * wordSize);
2938     handleSOERegisters(true /*saving*/);
2939 
2940     __ movptr(buf, buf_param);
2941     __ movptr(state, state_param);
2942     if (multi_block) {
2943       __ movptr(ofs, ofs_param);
2944       __ movptr(limit, limit_param);
2945     }
2946 
2947     __ fast_sha1(abcd, e0, e1, msg0, msg1, msg2, msg3, shuf_mask,
2948       buf, state, ofs, limit, rsp, multi_block);
2949 
2950     handleSOERegisters(false /*restoring*/);
2951     __ addptr(rsp, 8 * wordSize);
2952     __ leave();
2953     __ ret(0);
2954     return start;
2955   }
2956 
2957   address generate_pshuffle_byte_flip_mask() {
2958     __ align(64);
2959     StubCodeMark mark(this, "StubRoutines", "pshuffle_byte_flip_mask");
2960     address start = __ pc();
2961     __ emit_data(0x00010203, relocInfo::none, 0);
2962     __ emit_data(0x04050607, relocInfo::none, 0);
2963     __ emit_data(0x08090a0b, relocInfo::none, 0);
2964     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
2965     return start;
2966   }
2967 
2968   // ofs and limit are use for multi-block byte array.
2969   // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit)
2970  address generate_sha256_implCompress(bool multi_block, const char *name) {
2971     __ align(CodeEntryAlignment);
2972     StubCodeMark mark(this, "StubRoutines", name);
2973     address start = __ pc();
2974 
2975     Register buf = rbx;
2976     Register state = rsi;
2977     Register ofs = rdx;
2978     Register limit = rcx;
2979 
2980     const Address  buf_param(rbp, 8 + 0);
2981     const Address  state_param(rbp, 8 + 4);
2982     const Address  ofs_param(rbp, 8 + 8);
2983     const Address  limit_param(rbp, 8 + 12);
2984 
2985     const XMMRegister msg = xmm0;
2986     const XMMRegister state0 = xmm1;
2987     const XMMRegister state1 = xmm2;
2988     const XMMRegister msgtmp0 = xmm3;
2989 
2990     const XMMRegister msgtmp1 = xmm4;
2991     const XMMRegister msgtmp2 = xmm5;
2992     const XMMRegister msgtmp3 = xmm6;
2993     const XMMRegister msgtmp4 = xmm7;
2994 
2995     __ enter();
2996     __ subptr(rsp, 8 * wordSize);
2997     handleSOERegisters(true /*saving*/);
2998     __ movptr(buf, buf_param);
2999     __ movptr(state, state_param);
3000     if (multi_block) {
3001      __ movptr(ofs, ofs_param);
3002      __ movptr(limit, limit_param);
3003     }
3004 
3005     __ fast_sha256(msg, state0, state1, msgtmp0, msgtmp1, msgtmp2, msgtmp3, msgtmp4,
3006       buf, state, ofs, limit, rsp, multi_block);
3007 
3008     handleSOERegisters(false);
3009     __ addptr(rsp, 8 * wordSize);
3010     __ leave();
3011     __ ret(0);
3012     return start;
3013   }
3014 
3015   // byte swap x86 long
3016   address generate_ghash_long_swap_mask() {
3017     __ align(CodeEntryAlignment);
3018     StubCodeMark mark(this, "StubRoutines", "ghash_long_swap_mask");
3019     address start = __ pc();
3020     __ emit_data(0x0b0a0908, relocInfo::none, 0);
3021     __ emit_data(0x0f0e0d0c, relocInfo::none, 0);
3022     __ emit_data(0x03020100, relocInfo::none, 0);
3023     __ emit_data(0x07060504, relocInfo::none, 0);
3024 
3025   return start;
3026   }
3027 
3028   // byte swap x86 byte array
3029   address generate_ghash_byte_swap_mask() {
3030     __ align(CodeEntryAlignment);
3031     StubCodeMark mark(this, "StubRoutines", "ghash_byte_swap_mask");
3032     address start = __ pc();
3033     __ emit_data(0x0c0d0e0f, relocInfo::none, 0);
3034     __ emit_data(0x08090a0b, relocInfo::none, 0);
3035     __ emit_data(0x04050607, relocInfo::none, 0);
3036     __ emit_data(0x00010203, relocInfo::none, 0);
3037   return start;
3038   }
3039 
3040   /* Single and multi-block ghash operations */
3041   address generate_ghash_processBlocks() {
3042     assert(UseGHASHIntrinsics, "need GHASH intrinsics and CLMUL support");
3043     __ align(CodeEntryAlignment);
3044     Label L_ghash_loop, L_exit;
3045     StubCodeMark mark(this, "StubRoutines", "ghash_processBlocks");
3046     address start = __ pc();
3047 
3048     const Register state        = rdi;
3049     const Register subkeyH      = rsi;
3050     const Register data         = rdx;
3051     const Register blocks       = rcx;
3052 
3053     const Address  state_param(rbp, 8+0);
3054     const Address  subkeyH_param(rbp, 8+4);
3055     const Address  data_param(rbp, 8+8);
3056     const Address  blocks_param(rbp, 8+12);
3057 
3058     const XMMRegister xmm_temp0 = xmm0;
3059     const XMMRegister xmm_temp1 = xmm1;
3060     const XMMRegister xmm_temp2 = xmm2;
3061     const XMMRegister xmm_temp3 = xmm3;
3062     const XMMRegister xmm_temp4 = xmm4;
3063     const XMMRegister xmm_temp5 = xmm5;
3064     const XMMRegister xmm_temp6 = xmm6;
3065     const XMMRegister xmm_temp7 = xmm7;
3066 
3067     __ enter();
3068     handleSOERegisters(true);  // Save registers
3069 
3070     __ movptr(state, state_param);
3071     __ movptr(subkeyH, subkeyH_param);
3072     __ movptr(data, data_param);
3073     __ movptr(blocks, blocks_param);
3074 
3075     __ movdqu(xmm_temp0, Address(state, 0));
3076     __ pshufb(xmm_temp0, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3077 
3078     __ movdqu(xmm_temp1, Address(subkeyH, 0));
3079     __ pshufb(xmm_temp1, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3080 
3081     __ BIND(L_ghash_loop);
3082     __ movdqu(xmm_temp2, Address(data, 0));
3083     __ pshufb(xmm_temp2, ExternalAddress(StubRoutines::x86::ghash_byte_swap_mask_addr()));
3084 
3085     __ pxor(xmm_temp0, xmm_temp2);
3086 
3087     //
3088     // Multiply with the hash key
3089     //
3090     __ movdqu(xmm_temp3, xmm_temp0);
3091     __ pclmulqdq(xmm_temp3, xmm_temp1, 0);      // xmm3 holds a0*b0
3092     __ movdqu(xmm_temp4, xmm_temp0);
3093     __ pclmulqdq(xmm_temp4, xmm_temp1, 16);     // xmm4 holds a0*b1
3094 
3095     __ movdqu(xmm_temp5, xmm_temp0);
3096     __ pclmulqdq(xmm_temp5, xmm_temp1, 1);      // xmm5 holds a1*b0
3097     __ movdqu(xmm_temp6, xmm_temp0);
3098     __ pclmulqdq(xmm_temp6, xmm_temp1, 17);     // xmm6 holds a1*b1
3099 
3100     __ pxor(xmm_temp4, xmm_temp5);      // xmm4 holds a0*b1 + a1*b0
3101 
3102     __ movdqu(xmm_temp5, xmm_temp4);    // move the contents of xmm4 to xmm5
3103     __ psrldq(xmm_temp4, 8);    // shift by xmm4 64 bits to the right
3104     __ pslldq(xmm_temp5, 8);    // shift by xmm5 64 bits to the left
3105     __ pxor(xmm_temp3, xmm_temp5);
3106     __ pxor(xmm_temp6, xmm_temp4);      // Register pair <xmm6:xmm3> holds the result
3107                                         // of the carry-less multiplication of
3108                                         // xmm0 by xmm1.
3109 
3110     // We shift the result of the multiplication by one bit position
3111     // to the left to cope for the fact that the bits are reversed.
3112     __ movdqu(xmm_temp7, xmm_temp3);
3113     __ movdqu(xmm_temp4, xmm_temp6);
3114     __ pslld (xmm_temp3, 1);
3115     __ pslld(xmm_temp6, 1);
3116     __ psrld(xmm_temp7, 31);
3117     __ psrld(xmm_temp4, 31);
3118     __ movdqu(xmm_temp5, xmm_temp7);
3119     __ pslldq(xmm_temp4, 4);
3120     __ pslldq(xmm_temp7, 4);
3121     __ psrldq(xmm_temp5, 12);
3122     __ por(xmm_temp3, xmm_temp7);
3123     __ por(xmm_temp6, xmm_temp4);
3124     __ por(xmm_temp6, xmm_temp5);
3125 
3126     //
3127     // First phase of the reduction
3128     //
3129     // Move xmm3 into xmm4, xmm5, xmm7 in order to perform the shifts
3130     // independently.
3131     __ movdqu(xmm_temp7, xmm_temp3);
3132     __ movdqu(xmm_temp4, xmm_temp3);
3133     __ movdqu(xmm_temp5, xmm_temp3);
3134     __ pslld(xmm_temp7, 31);    // packed right shift shifting << 31
3135     __ pslld(xmm_temp4, 30);    // packed right shift shifting << 30
3136     __ pslld(xmm_temp5, 25);    // packed right shift shifting << 25
3137     __ pxor(xmm_temp7, xmm_temp4);      // xor the shifted versions
3138     __ pxor(xmm_temp7, xmm_temp5);
3139     __ movdqu(xmm_temp4, xmm_temp7);
3140     __ pslldq(xmm_temp7, 12);
3141     __ psrldq(xmm_temp4, 4);
3142     __ pxor(xmm_temp3, xmm_temp7);      // first phase of the reduction complete
3143 
3144     //
3145     // Second phase of the reduction
3146     //
3147     // Make 3 copies of xmm3 in xmm2, xmm5, xmm7 for doing these
3148     // shift operations.
3149     __ movdqu(xmm_temp2, xmm_temp3);
3150     __ movdqu(xmm_temp7, xmm_temp3);
3151     __ movdqu(xmm_temp5, xmm_temp3);
3152     __ psrld(xmm_temp2, 1);     // packed left shifting >> 1
3153     __ psrld(xmm_temp7, 2);     // packed left shifting >> 2
3154     __ psrld(xmm_temp5, 7);     // packed left shifting >> 7
3155     __ pxor(xmm_temp2, xmm_temp7);      // xor the shifted versions
3156     __ pxor(xmm_temp2, xmm_temp5);
3157     __ pxor(xmm_temp2, xmm_temp4);
3158     __ pxor(xmm_temp3, xmm_temp2);
3159     __ pxor(xmm_temp6, xmm_temp3);      // the result is in xmm6
3160 
3161     __ decrement(blocks);
3162     __ jcc(Assembler::zero, L_exit);
3163     __ movdqu(xmm_temp0, xmm_temp6);
3164     __ addptr(data, 16);
3165     __ jmp(L_ghash_loop);
3166 
3167     __ BIND(L_exit);
3168        // Byte swap 16-byte result
3169     __ pshufb(xmm_temp6, ExternalAddress(StubRoutines::x86::ghash_long_swap_mask_addr()));
3170     __ movdqu(Address(state, 0), xmm_temp6);   // store the result
3171 
3172     handleSOERegisters(false);  // restore registers
3173     __ leave();
3174     __ ret(0);
3175     return start;
3176   }
3177 
3178   /**
3179    *  Arguments:
3180    *
3181    * Inputs:
3182    *   rsp(4)   - int crc
3183    *   rsp(8)   - byte* buf
3184    *   rsp(12)  - int length
3185    *
3186    * Ouput:
3187    *       rax   - int crc result
3188    */
3189   address generate_updateBytesCRC32() {
3190     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
3191 
3192     __ align(CodeEntryAlignment);
3193     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
3194 
3195     address start = __ pc();
3196 
3197     const Register crc   = rdx;  // crc
3198     const Register buf   = rsi;  // source java byte array address
3199     const Register len   = rcx;  // length
3200     const Register table = rdi;  // crc_table address (reuse register)
3201     const Register tmp   = rbx;
3202     assert_different_registers(crc, buf, len, table, tmp, rax);
3203 
3204     BLOCK_COMMENT("Entry:");
3205     __ enter(); // required for proper stackwalking of RuntimeStub frame
3206     __ push(rsi);
3207     __ push(rdi);
3208     __ push(rbx);
3209 
3210     Address crc_arg(rbp, 8 + 0);
3211     Address buf_arg(rbp, 8 + 4);
3212     Address len_arg(rbp, 8 + 8);
3213 
3214     // Load up:
3215     __ movl(crc,   crc_arg);
3216     __ movptr(buf, buf_arg);
3217     __ movl(len,   len_arg);
3218 
3219     __ kernel_crc32(crc, buf, len, table, tmp);
3220 
3221     __ movl(rax, crc);
3222     __ pop(rbx);
3223     __ pop(rdi);
3224     __ pop(rsi);
3225     __ vzeroupper();
3226     __ leave(); // required for proper stackwalking of RuntimeStub frame
3227     __ ret(0);
3228 
3229     return start;
3230   }
3231 
3232   /**
3233   *  Arguments:
3234   *
3235   * Inputs:
3236   *   rsp(4)   - int crc
3237   *   rsp(8)   - byte* buf
3238   *   rsp(12)  - int length
3239   *   rsp(16)  - table_start - optional (present only when doing a library_calll,
3240   *              not used by x86 algorithm)
3241   *
3242   * Ouput:
3243   *       rax  - int crc result
3244   */
3245   address generate_updateBytesCRC32C(bool is_pclmulqdq_supported) {
3246     assert(UseCRC32CIntrinsics, "need SSE4_2");
3247     __ align(CodeEntryAlignment);
3248     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32C");
3249     address start = __ pc();
3250     const Register crc = rax;  // crc
3251     const Register buf = rcx;  // source java byte array address
3252     const Register len = rdx;  // length
3253     const Register d = rbx;
3254     const Register g = rsi;
3255     const Register h = rdi;
3256     const Register empty = 0; // will never be used, in order not
3257                               // to change a signature for crc32c_IPL_Alg2_Alt2
3258                               // between 64/32 I'm just keeping it here
3259     assert_different_registers(crc, buf, len, d, g, h);
3260 
3261     BLOCK_COMMENT("Entry:");
3262     __ enter(); // required for proper stackwalking of RuntimeStub frame
3263     Address crc_arg(rsp, 4 + 4 + 0); // ESP+4 +
3264                                      // we need to add additional 4 because __ enter
3265                                      // have just pushed ebp on a stack
3266     Address buf_arg(rsp, 4 + 4 + 4);
3267     Address len_arg(rsp, 4 + 4 + 8);
3268       // Load up:
3269       __ movl(crc, crc_arg);
3270       __ movl(buf, buf_arg);
3271       __ movl(len, len_arg);
3272       __ push(d);
3273       __ push(g);
3274       __ push(h);
3275       __ crc32c_ipl_alg2_alt2(crc, buf, len,
3276                               d, g, h,
3277                               empty, empty, empty,
3278                               xmm0, xmm1, xmm2,
3279                               is_pclmulqdq_supported);
3280       __ pop(h);
3281       __ pop(g);
3282       __ pop(d);
3283     __ vzeroupper();
3284     __ leave(); // required for proper stackwalking of RuntimeStub frame
3285     __ ret(0);
3286 
3287     return start;
3288   }
3289 
3290  address generate_libmExp() {
3291     StubCodeMark mark(this, "StubRoutines", "libmExp");
3292 
3293     address start = __ pc();
3294 
3295     const XMMRegister x0  = xmm0;
3296     const XMMRegister x1  = xmm1;
3297     const XMMRegister x2  = xmm2;
3298     const XMMRegister x3  = xmm3;
3299 
3300     const XMMRegister x4  = xmm4;
3301     const XMMRegister x5  = xmm5;
3302     const XMMRegister x6  = xmm6;
3303     const XMMRegister x7  = xmm7;
3304 
3305     const Register tmp   = rbx;
3306 
3307     BLOCK_COMMENT("Entry:");
3308     __ enter(); // required for proper stackwalking of RuntimeStub frame
3309     __ fast_exp(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3310     __ leave(); // required for proper stackwalking of RuntimeStub frame
3311     __ ret(0);
3312 
3313     return start;
3314 
3315   }
3316 
3317  address generate_libmLog() {
3318    StubCodeMark mark(this, "StubRoutines", "libmLog");
3319 
3320    address start = __ pc();
3321 
3322    const XMMRegister x0 = xmm0;
3323    const XMMRegister x1 = xmm1;
3324    const XMMRegister x2 = xmm2;
3325    const XMMRegister x3 = xmm3;
3326 
3327    const XMMRegister x4 = xmm4;
3328    const XMMRegister x5 = xmm5;
3329    const XMMRegister x6 = xmm6;
3330    const XMMRegister x7 = xmm7;
3331 
3332    const Register tmp = rbx;
3333 
3334    BLOCK_COMMENT("Entry:");
3335    __ enter(); // required for proper stackwalking of RuntimeStub frame
3336    __ fast_log(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3337    __ leave(); // required for proper stackwalking of RuntimeStub frame
3338    __ ret(0);
3339 
3340    return start;
3341 
3342  }
3343 
3344  address generate_libmLog10() {
3345    StubCodeMark mark(this, "StubRoutines", "libmLog10");
3346 
3347    address start = __ pc();
3348 
3349    const XMMRegister x0 = xmm0;
3350    const XMMRegister x1 = xmm1;
3351    const XMMRegister x2 = xmm2;
3352    const XMMRegister x3 = xmm3;
3353 
3354    const XMMRegister x4 = xmm4;
3355    const XMMRegister x5 = xmm5;
3356    const XMMRegister x6 = xmm6;
3357    const XMMRegister x7 = xmm7;
3358 
3359    const Register tmp = rbx;
3360 
3361    BLOCK_COMMENT("Entry:");
3362    __ enter(); // required for proper stackwalking of RuntimeStub frame
3363    __ fast_log10(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3364    __ leave(); // required for proper stackwalking of RuntimeStub frame
3365    __ ret(0);
3366 
3367    return start;
3368 
3369  }
3370 
3371  address generate_libmPow() {
3372    StubCodeMark mark(this, "StubRoutines", "libmPow");
3373 
3374    address start = __ pc();
3375 
3376    const XMMRegister x0 = xmm0;
3377    const XMMRegister x1 = xmm1;
3378    const XMMRegister x2 = xmm2;
3379    const XMMRegister x3 = xmm3;
3380 
3381    const XMMRegister x4 = xmm4;
3382    const XMMRegister x5 = xmm5;
3383    const XMMRegister x6 = xmm6;
3384    const XMMRegister x7 = xmm7;
3385 
3386    const Register tmp = rbx;
3387 
3388    BLOCK_COMMENT("Entry:");
3389    __ enter(); // required for proper stackwalking of RuntimeStub frame
3390    __ fast_pow(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3391    __ leave(); // required for proper stackwalking of RuntimeStub frame
3392    __ ret(0);
3393 
3394    return start;
3395 
3396  }
3397 
3398  address generate_libm_reduce_pi04l() {
3399    StubCodeMark mark(this, "StubRoutines", "libm_reduce_pi04l");
3400 
3401    address start = __ pc();
3402 
3403    BLOCK_COMMENT("Entry:");
3404    __ libm_reduce_pi04l(rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3405 
3406    return start;
3407 
3408  }
3409 
3410  address generate_libm_sin_cos_huge() {
3411    StubCodeMark mark(this, "StubRoutines", "libm_sin_cos_huge");
3412 
3413    address start = __ pc();
3414 
3415    const XMMRegister x0 = xmm0;
3416    const XMMRegister x1 = xmm1;
3417 
3418    BLOCK_COMMENT("Entry:");
3419    __ libm_sincos_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3420 
3421    return start;
3422 
3423  }
3424 
3425  address generate_libmSin() {
3426    StubCodeMark mark(this, "StubRoutines", "libmSin");
3427 
3428    address start = __ pc();
3429 
3430    const XMMRegister x0 = xmm0;
3431    const XMMRegister x1 = xmm1;
3432    const XMMRegister x2 = xmm2;
3433    const XMMRegister x3 = xmm3;
3434 
3435    const XMMRegister x4 = xmm4;
3436    const XMMRegister x5 = xmm5;
3437    const XMMRegister x6 = xmm6;
3438    const XMMRegister x7 = xmm7;
3439 
3440    BLOCK_COMMENT("Entry:");
3441    __ enter(); // required for proper stackwalking of RuntimeStub frame
3442    __ fast_sin(x0, x1, x2, x3, x4, x5, x6, x7, rax, rbx, rdx);
3443    __ leave(); // required for proper stackwalking of RuntimeStub frame
3444    __ ret(0);
3445 
3446    return start;
3447 
3448  }
3449 
3450  address generate_libmCos() {
3451    StubCodeMark mark(this, "StubRoutines", "libmCos");
3452 
3453    address start = __ pc();
3454 
3455    const XMMRegister x0 = xmm0;
3456    const XMMRegister x1 = xmm1;
3457    const XMMRegister x2 = xmm2;
3458    const XMMRegister x3 = xmm3;
3459 
3460    const XMMRegister x4 = xmm4;
3461    const XMMRegister x5 = xmm5;
3462    const XMMRegister x6 = xmm6;
3463    const XMMRegister x7 = xmm7;
3464 
3465    const Register tmp = rbx;
3466 
3467    BLOCK_COMMENT("Entry:");
3468    __ enter(); // required for proper stackwalking of RuntimeStub frame
3469    __ fast_cos(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3470    __ leave(); // required for proper stackwalking of RuntimeStub frame
3471    __ ret(0);
3472 
3473    return start;
3474 
3475  }
3476 
3477  address generate_libm_tan_cot_huge() {
3478    StubCodeMark mark(this, "StubRoutines", "libm_tan_cot_huge");
3479 
3480    address start = __ pc();
3481 
3482    const XMMRegister x0 = xmm0;
3483    const XMMRegister x1 = xmm1;
3484 
3485    BLOCK_COMMENT("Entry:");
3486    __ libm_tancot_huge(x0, x1, rax, rcx, rdx, rbx, rsi, rdi, rbp, rsp);
3487 
3488    return start;
3489 
3490  }
3491 
3492  address generate_libmTan() {
3493    StubCodeMark mark(this, "StubRoutines", "libmTan");
3494 
3495    address start = __ pc();
3496 
3497    const XMMRegister x0 = xmm0;
3498    const XMMRegister x1 = xmm1;
3499    const XMMRegister x2 = xmm2;
3500    const XMMRegister x3 = xmm3;
3501 
3502    const XMMRegister x4 = xmm4;
3503    const XMMRegister x5 = xmm5;
3504    const XMMRegister x6 = xmm6;
3505    const XMMRegister x7 = xmm7;
3506 
3507    const Register tmp = rbx;
3508 
3509    BLOCK_COMMENT("Entry:");
3510    __ enter(); // required for proper stackwalking of RuntimeStub frame
3511    __ fast_tan(x0, x1, x2, x3, x4, x5, x6, x7, rax, rcx, rdx, tmp);
3512    __ leave(); // required for proper stackwalking of RuntimeStub frame
3513    __ ret(0);
3514 
3515    return start;
3516 
3517  }
3518 
3519   // Safefetch stubs.
3520   void generate_safefetch(const char* name, int size, address* entry,
3521                           address* fault_pc, address* continuation_pc) {
3522     // safefetch signatures:
3523     //   int      SafeFetch32(int*      adr, int      errValue);
3524     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3525 
3526     StubCodeMark mark(this, "StubRoutines", name);
3527 
3528     // Entry point, pc or function descriptor.
3529     *entry = __ pc();
3530 
3531     __ movl(rax, Address(rsp, 0x8));
3532     __ movl(rcx, Address(rsp, 0x4));
3533     // Load *adr into eax, may fault.
3534     *fault_pc = __ pc();
3535     switch (size) {
3536       case 4:
3537         // int32_t
3538         __ movl(rax, Address(rcx, 0));
3539         break;
3540       case 8:
3541         // int64_t
3542         Unimplemented();
3543         break;
3544       default:
3545         ShouldNotReachHere();
3546     }
3547 
3548     // Return errValue or *adr.
3549     *continuation_pc = __ pc();
3550     __ ret(0);
3551   }
3552 
3553   address generate_method_entry_barrier() {
3554     __ align(CodeEntryAlignment);
3555     StubCodeMark mark(this, "StubRoutines", "nmethod_entry_barrier");
3556 
3557     Label deoptimize_label;
3558 
3559     address start = __ pc();
3560 
3561     __ push(-1); // cookie, this is used for writing the new rsp when deoptimizing
3562 
3563     BLOCK_COMMENT("Entry:");
3564     __ enter(); // save rbp
3565 
3566     // save rbx, because we want to use that value.
3567     // We could do without it but then we depend on the number of slots used by pusha
3568     __ push(rbx);
3569 
3570     __ lea(rbx, Address(rsp, wordSize * 3)); // 1 for cookie, 1 for rbp, 1 for rbx - this should be the return address
3571 
3572     __ pusha();
3573 
3574     // xmm0 and xmm1 may be used for passing float/double arguments
3575     const int xmm_size = wordSize * 2;
3576     const int xmm_spill_size = xmm_size * 2;
3577     __ subptr(rsp, xmm_spill_size);
3578     __ movdqu(Address(rsp, xmm_size * 1), xmm1);
3579     __ movdqu(Address(rsp, xmm_size * 0), xmm0);
3580 
3581     __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(address*)>(BarrierSetNMethod::nmethod_stub_entry_barrier)), rbx);
3582 
3583     __ movdqu(xmm0, Address(rsp, xmm_size * 0));
3584     __ movdqu(xmm1, Address(rsp, xmm_size * 1));
3585     __ addptr(rsp, xmm_spill_size);
3586 
3587     __ cmpl(rax, 1); // 1 means deoptimize
3588     __ jcc(Assembler::equal, deoptimize_label);
3589 
3590     __ popa();
3591     __ pop(rbx);
3592 
3593     __ leave();
3594 
3595     __ addptr(rsp, 1 * wordSize); // cookie
3596     __ ret(0);
3597 
3598     __ BIND(deoptimize_label);
3599 
3600     __ popa();
3601     __ pop(rbx);
3602 
3603     __ leave();
3604 
3605     // this can be taken out, but is good for verification purposes. getting a SIGSEGV
3606     // here while still having a correct stack is valuable
3607     __ testptr(rsp, Address(rsp, 0));
3608 
3609     __ movptr(rsp, Address(rsp, 0)); // new rsp was written in the barrier
3610     __ jmp(Address(rsp, -1 * wordSize)); // jmp target should be callers verified_entry_point
3611 
3612     return start;
3613   }
3614 
3615  public:
3616   // Information about frame layout at time of blocking runtime call.
3617   // Note that we only have to preserve callee-saved registers since
3618   // the compilers are responsible for supplying a continuation point
3619   // if they expect all registers to be preserved.
3620   enum layout {
3621     thread_off,    // last_java_sp
3622     arg1_off,
3623     arg2_off,
3624     rbp_off,       // callee saved register
3625     ret_pc,
3626     framesize
3627   };
3628 
3629  private:
3630 
3631 #undef  __
3632 #define __ masm->
3633 
3634   //------------------------------------------------------------------------------------------------------------------------
3635   // Continuation point for throwing of implicit exceptions that are not handled in
3636   // the current activation. Fabricates an exception oop and initiates normal
3637   // exception dispatching in this frame.
3638   //
3639   // Previously the compiler (c2) allowed for callee save registers on Java calls.
3640   // This is no longer true after adapter frames were removed but could possibly
3641   // be brought back in the future if the interpreter code was reworked and it
3642   // was deemed worthwhile. The comment below was left to describe what must
3643   // happen here if callee saves were resurrected. As it stands now this stub
3644   // could actually be a vanilla BufferBlob and have now oopMap at all.
3645   // Since it doesn't make much difference we've chosen to leave it the
3646   // way it was in the callee save days and keep the comment.
3647 
3648   // If we need to preserve callee-saved values we need a callee-saved oop map and
3649   // therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.
3650   // If the compiler needs all registers to be preserved between the fault
3651   // point and the exception handler then it must assume responsibility for that in
3652   // AbstractCompiler::continuation_for_implicit_null_exception or
3653   // continuation_for_implicit_division_by_zero_exception. All other implicit
3654   // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
3655   // either at call sites or otherwise assume that stack unwinding will be initiated,
3656   // so caller saved registers were assumed volatile in the compiler.
3657   address generate_throw_exception(const char* name, address runtime_entry,
3658                                    Register arg1 = noreg, Register arg2 = noreg) {
3659 
3660     int insts_size = 256;
3661     int locs_size  = 32;
3662 
3663     CodeBuffer code(name, insts_size, locs_size);
3664     OopMapSet* oop_maps  = new OopMapSet();
3665     MacroAssembler* masm = new MacroAssembler(&code);
3666 
3667     address start = __ pc();
3668 
3669     // This is an inlined and slightly modified version of call_VM
3670     // which has the ability to fetch the return PC out of
3671     // thread-local storage and also sets up last_Java_sp slightly
3672     // differently than the real call_VM
3673     Register java_thread = rbx;
3674     __ get_thread(java_thread);
3675 
3676     __ enter(); // required for proper stackwalking of RuntimeStub frame
3677 
3678     // pc and rbp, already pushed
3679     __ subptr(rsp, (framesize-2) * wordSize); // prolog
3680 
3681     // Frame is now completed as far as size and linkage.
3682 
3683     int frame_complete = __ pc() - start;
3684 
3685     // push java thread (becomes first argument of C function)
3686     __ movptr(Address(rsp, thread_off * wordSize), java_thread);
3687     if (arg1 != noreg) {
3688       __ movptr(Address(rsp, arg1_off * wordSize), arg1);
3689     }
3690     if (arg2 != noreg) {
3691       assert(arg1 != noreg, "missing reg arg");
3692       __ movptr(Address(rsp, arg2_off * wordSize), arg2);
3693     }
3694 
3695     // Set up last_Java_sp and last_Java_fp
3696     __ set_last_Java_frame(java_thread, rsp, rbp, NULL);
3697 
3698     // Call runtime
3699     BLOCK_COMMENT("call runtime_entry");
3700     __ call(RuntimeAddress(runtime_entry));
3701     // Generate oop map
3702     OopMap* map =  new OopMap(framesize, 0);
3703     oop_maps->add_gc_map(__ pc() - start, map);
3704 
3705     // restore the thread (cannot use the pushed argument since arguments
3706     // may be overwritten by C code generated by an optimizing compiler);
3707     // however can use the register value directly if it is callee saved.
3708     __ get_thread(java_thread);
3709 
3710     __ reset_last_Java_frame(java_thread, true);
3711 
3712     __ leave(); // required for proper stackwalking of RuntimeStub frame
3713 
3714     // check for pending exceptions
3715 #ifdef ASSERT
3716     Label L;
3717     __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3718     __ jcc(Assembler::notEqual, L);
3719     __ should_not_reach_here();
3720     __ bind(L);
3721 #endif /* ASSERT */
3722     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3723 
3724 
3725     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);
3726     return stub->entry_point();
3727   }
3728 
3729 
3730   void create_control_words() {
3731     // Round to nearest, 53-bit mode, exceptions masked
3732     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
3733     // Round to zero, 53-bit mode, exception mased
3734     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
3735     // Round to nearest, 24-bit mode, exceptions masked
3736     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
3737     // Round to nearest, 64-bit mode, exceptions masked
3738     StubRoutines::_mxcsr_std           = 0x1F80;
3739     // Note: the following two constants are 80-bit values
3740     //       layout is critical for correct loading by FPU.
3741     // Bias for strict fp multiply/divide
3742     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
3743     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
3744     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
3745     // Un-Bias for strict fp multiply/divide
3746     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
3747     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
3748     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
3749   }
3750 
3751   //---------------------------------------------------------------------------
3752   // Initialization
3753 
3754   void generate_initial() {
3755     // Generates all stubs and initializes the entry points
3756 
3757     //------------------------------------------------------------------------------------------------------------------------
3758     // entry points that exist in all platforms
3759     // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
3760     //       the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
3761     StubRoutines::_forward_exception_entry      = generate_forward_exception();
3762 
3763     StubRoutines::_call_stub_entry              =
3764       generate_call_stub(StubRoutines::_call_stub_return_address);
3765     // is referenced by megamorphic call
3766     StubRoutines::_catch_exception_entry        = generate_catch_exception();
3767 



3768     // platform dependent
3769     create_control_words();
3770 
3771     StubRoutines::x86::_verify_mxcsr_entry                 = generate_verify_mxcsr();
3772     StubRoutines::x86::_verify_fpu_cntrl_wrd_entry         = generate_verify_fpu_cntrl_wrd();
3773     StubRoutines::_d2i_wrapper                              = generate_d2i_wrapper(T_INT,
3774                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
3775     StubRoutines::_d2l_wrapper                              = generate_d2i_wrapper(T_LONG,
3776                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
3777 
3778     // Build this early so it's available for the interpreter
3779     StubRoutines::_throw_StackOverflowError_entry          = generate_throw_exception("StackOverflowError throw_exception",
3780                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError));
3781     StubRoutines::_throw_delayed_StackOverflowError_entry  = generate_throw_exception("delayed StackOverflowError throw_exception",
3782                                                                                       CAST_FROM_FN_PTR(address, SharedRuntime::throw_delayed_StackOverflowError));
3783 
3784     if (UseCRC32Intrinsics) {
3785       // set table address before stub generation which use it
3786       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
3787       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
3788     }
3789 
3790     if (UseCRC32CIntrinsics) {
3791       bool supports_clmul = VM_Version::supports_clmul();
3792       StubRoutines::x86::generate_CRC32C_table(supports_clmul);
3793       StubRoutines::_crc32c_table_addr = (address)StubRoutines::x86::_crc32c_table;
3794       StubRoutines::_updateBytesCRC32C = generate_updateBytesCRC32C(supports_clmul);
3795     }
3796     if (VM_Version::supports_sse2() && UseLibmIntrinsic && InlineIntrinsics) {
3797       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3798           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3799           vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3800         StubRoutines::x86::_L_2il0floatpacket_0_adr = (address)StubRoutines::x86::_L_2il0floatpacket_0;
3801         StubRoutines::x86::_Pi4Inv_adr = (address)StubRoutines::x86::_Pi4Inv;
3802         StubRoutines::x86::_Pi4x3_adr = (address)StubRoutines::x86::_Pi4x3;
3803         StubRoutines::x86::_Pi4x4_adr = (address)StubRoutines::x86::_Pi4x4;
3804         StubRoutines::x86::_ones_adr = (address)StubRoutines::x86::_ones;
3805       }
3806       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dexp)) {
3807         StubRoutines::_dexp = generate_libmExp();
3808       }
3809       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog)) {
3810         StubRoutines::_dlog = generate_libmLog();
3811       }
3812       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dlog10)) {
3813         StubRoutines::_dlog10 = generate_libmLog10();
3814       }
3815       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dpow)) {
3816         StubRoutines::_dpow = generate_libmPow();
3817       }
3818       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3819         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos) ||
3820         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3821         StubRoutines::_dlibm_reduce_pi04l = generate_libm_reduce_pi04l();
3822       }
3823       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin) ||
3824         vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3825         StubRoutines::_dlibm_sin_cos_huge = generate_libm_sin_cos_huge();
3826       }
3827       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dsin)) {
3828         StubRoutines::_dsin = generate_libmSin();
3829       }
3830       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dcos)) {
3831         StubRoutines::_dcos = generate_libmCos();
3832       }
3833       if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_dtan)) {
3834         StubRoutines::_dlibm_tan_cot_huge = generate_libm_tan_cot_huge();
3835         StubRoutines::_dtan = generate_libmTan();
3836       }
3837     }
3838 
3839     // Safefetch stubs.
3840     generate_safefetch("SafeFetch32", sizeof(int), &StubRoutines::_safefetch32_entry,
3841                                                    &StubRoutines::_safefetch32_fault_pc,
3842                                                    &StubRoutines::_safefetch32_continuation_pc);
3843     StubRoutines::_safefetchN_entry           = StubRoutines::_safefetch32_entry;
3844     StubRoutines::_safefetchN_fault_pc        = StubRoutines::_safefetch32_fault_pc;
3845     StubRoutines::_safefetchN_continuation_pc = StubRoutines::_safefetch32_continuation_pc;
3846   }
3847 
3848   void generate_all() {
3849     // Generates all stubs and initializes the entry points
3850 
3851     // These entry points require SharedInfo::stack0 to be set up in non-core builds
3852     // and need to be relocatable, so they each fabricate a RuntimeStub internally.
3853     StubRoutines::_throw_AbstractMethodError_entry         = generate_throw_exception("AbstractMethodError throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError));
3854     StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError));
3855     StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call));
3856 
3857     //------------------------------------------------------------------------------------------------------------------------
3858     // entry points that are platform specific
3859 
3860     StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF);
3861     StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x80000000);
3862     StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask_long_double("vector_double_sign_mask", 0x7FFFFFFF, 0xFFFFFFFF);
3863     StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask_long_double("vector_double_sign_flip", 0x80000000, 0x00000000);
3864     StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff);
3865     StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
3866     StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask_long_double("vector_long_sign_mask", 0x80000000, 0x00000000);
3867 
3868     // support for verify_oop (must happen after universe_init)
3869     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
3870 
3871     // arraycopy stubs used by compilers
3872     generate_arraycopy_stubs();
3873 
3874     // don't bother generating these AES intrinsic stubs unless global flag is set
3875     if (UseAESIntrinsics) {
3876       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // might be needed by the others
3877 
3878       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
3879       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
3880       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
3881       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
3882     }
3883 
3884     if (UseAESCTRIntrinsics) {
3885       StubRoutines::x86::_counter_shuffle_mask_addr = generate_counter_shuffle_mask();
3886       StubRoutines::_counterMode_AESCrypt = generate_counterMode_AESCrypt_Parallel();
3887     }
3888 
3889     if (UseSHA1Intrinsics) {
3890       StubRoutines::x86::_upper_word_mask_addr = generate_upper_word_mask();
3891       StubRoutines::x86::_shuffle_byte_flip_mask_addr = generate_shuffle_byte_flip_mask();
3892       StubRoutines::_sha1_implCompress = generate_sha1_implCompress(false, "sha1_implCompress");
3893       StubRoutines::_sha1_implCompressMB = generate_sha1_implCompress(true, "sha1_implCompressMB");
3894     }
3895     if (UseSHA256Intrinsics) {
3896       StubRoutines::x86::_k256_adr = (address)StubRoutines::x86::_k256;
3897       StubRoutines::x86::_pshuffle_byte_flip_mask_addr = generate_pshuffle_byte_flip_mask();
3898       StubRoutines::_sha256_implCompress = generate_sha256_implCompress(false, "sha256_implCompress");
3899       StubRoutines::_sha256_implCompressMB = generate_sha256_implCompress(true, "sha256_implCompressMB");
3900     }
3901 
3902     // Generate GHASH intrinsics code
3903     if (UseGHASHIntrinsics) {
3904       StubRoutines::x86::_ghash_long_swap_mask_addr = generate_ghash_long_swap_mask();
3905       StubRoutines::x86::_ghash_byte_swap_mask_addr = generate_ghash_byte_swap_mask();
3906       StubRoutines::_ghash_processBlocks = generate_ghash_processBlocks();
3907     }
3908 
3909     BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
3910     if (bs_nm != NULL) {
3911       StubRoutines::x86::_method_entry_barrier = generate_method_entry_barrier();
3912     }
3913   }
3914 
3915 
3916  public:
3917   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
3918     if (all) {
3919       generate_all();
3920     } else {
3921       generate_initial();
3922     }
3923   }
3924 }; // end class declaration
3925 
3926 #define UCM_TABLE_MAX_ENTRIES 8
3927 void StubGenerator_generate(CodeBuffer* code, bool all) {
3928   if (UnsafeCopyMemory::_table == NULL) {
3929     UnsafeCopyMemory::create_table(UCM_TABLE_MAX_ENTRIES);
3930   }
3931   StubGenerator g(code, all);
3932 }
--- EOF ---