1 /*
   2  * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "nativeInst_x86.hpp"
  30 #include "oops/instanceOop.hpp"
  31 #include "oops/method.hpp"
  32 #include "oops/objArrayKlass.hpp"
  33 #include "oops/oop.inline.hpp"
  34 #include "prims/methodHandles.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/handles.inline.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubCodeGenerator.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/thread.inline.hpp"
  41 #include "utilities/top.hpp"
  42 #ifdef COMPILER2
  43 #include "opto/runtime.hpp"
  44 #endif
  45 
  46 // Declaration and definition of StubGenerator (no .hpp file).
  47 // For a more detailed description of the stub routine structure
  48 // see the comment in stubRoutines.hpp
  49 
  50 #define __ _masm->
  51 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
  52 #define a__ ((Assembler*)_masm)->
  53 
  54 #ifdef PRODUCT
  55 #define BLOCK_COMMENT(str) /* nothing */
  56 #else
  57 #define BLOCK_COMMENT(str) __ block_comment(str)
  58 #endif
  59 
  60 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  61 const int MXCSR_MASK = 0xFFC0;  // Mask out any pending exceptions
  62 
  63 // Stub Code definitions
  64 
  65 static address handle_unsafe_access() {
  66   JavaThread* thread = JavaThread::current();
  67   address pc = thread->saved_exception_pc();
  68   // pc is the instruction which we must emulate
  69   // doing a no-op is fine:  return garbage from the load
  70   // therefore, compute npc
  71   address npc = Assembler::locate_next_instruction(pc);
  72 
  73   // request an async exception
  74   thread->set_pending_unsafe_access_error();
  75 
  76   // return address of next instruction to execute
  77   return npc;
  78 }
  79 
  80 class StubGenerator: public StubCodeGenerator {
  81  private:
  82 
  83 #ifdef PRODUCT
  84 #define inc_counter_np(counter) ((void)0)
  85 #else
  86   void inc_counter_np_(int& counter) {
  87     // This can destroy rscratch1 if counter is far from the code cache
  88     __ incrementl(ExternalAddress((address)&counter));
  89   }
  90 #define inc_counter_np(counter) \
  91   BLOCK_COMMENT("inc_counter " #counter); \
  92   inc_counter_np_(counter);
  93 #endif
  94 
  95   // Call stubs are used to call Java from C
  96   //
  97   // Linux Arguments:
  98   //    c_rarg0:   call wrapper address                   address
  99   //    c_rarg1:   result                                 address
 100   //    c_rarg2:   result type                            BasicType
 101   //    c_rarg3:   method                                 Method*
 102   //    c_rarg4:   (interpreter) entry point              address
 103   //    c_rarg5:   parameters                             intptr_t*
 104   //    16(rbp): parameter size (in words)              int
 105   //    24(rbp): thread                                 Thread*
 106   //
 107   //     [ return_from_Java     ] <--- rsp
 108   //     [ argument word n      ]
 109   //      ...
 110   // -12 [ argument word 1      ]
 111   // -11 [ saved r15            ] <--- rsp_after_call
 112   // -10 [ saved r14            ]
 113   //  -9 [ saved r13            ]
 114   //  -8 [ saved r12            ]
 115   //  -7 [ saved rbx            ]
 116   //  -6 [ call wrapper         ]
 117   //  -5 [ result               ]
 118   //  -4 [ result type          ]
 119   //  -3 [ method               ]
 120   //  -2 [ entry point          ]
 121   //  -1 [ parameters           ]
 122   //   0 [ saved rbp            ] <--- rbp
 123   //   1 [ return address       ]
 124   //   2 [ parameter size       ]
 125   //   3 [ thread               ]
 126   //
 127   // Windows Arguments:
 128   //    c_rarg0:   call wrapper address                   address
 129   //    c_rarg1:   result                                 address
 130   //    c_rarg2:   result type                            BasicType
 131   //    c_rarg3:   method                                 Method*
 132   //    48(rbp): (interpreter) entry point              address
 133   //    56(rbp): parameters                             intptr_t*
 134   //    64(rbp): parameter size (in words)              int
 135   //    72(rbp): thread                                 Thread*
 136   //
 137   //     [ return_from_Java     ] <--- rsp
 138   //     [ argument word n      ]
 139   //      ...
 140   // -28 [ argument word 1      ]
 141   // -27 [ saved xmm15          ] <--- rsp_after_call
 142   //     [ saved xmm7-xmm14     ]
 143   //  -9 [ saved xmm6           ] (each xmm register takes 2 slots)
 144   //  -7 [ saved r15            ]
 145   //  -6 [ saved r14            ]
 146   //  -5 [ saved r13            ]
 147   //  -4 [ saved r12            ]
 148   //  -3 [ saved rdi            ]
 149   //  -2 [ saved rsi            ]
 150   //  -1 [ saved rbx            ]
 151   //   0 [ saved rbp            ] <--- rbp
 152   //   1 [ return address       ]
 153   //   2 [ call wrapper         ]
 154   //   3 [ result               ]
 155   //   4 [ result type          ]
 156   //   5 [ method               ]
 157   //   6 [ entry point          ]
 158   //   7 [ parameters           ]
 159   //   8 [ parameter size       ]
 160   //   9 [ thread               ]
 161   //
 162   //    Windows reserves the callers stack space for arguments 1-4.
 163   //    We spill c_rarg0-c_rarg3 to this space.
 164 
 165   // Call stub stack layout word offsets from rbp
 166   enum call_stub_layout {
 167 #ifdef _WIN64
 168     xmm_save_first     = 6,  // save from xmm6
 169     xmm_save_last      = 15, // to xmm15
 170     xmm_save_base      = -9,
 171     rsp_after_call_off = xmm_save_base - 2 * (xmm_save_last - xmm_save_first), // -27
 172     r15_off            = -7,
 173     r14_off            = -6,
 174     r13_off            = -5,
 175     r12_off            = -4,
 176     rdi_off            = -3,
 177     rsi_off            = -2,
 178     rbx_off            = -1,
 179     rbp_off            =  0,
 180     retaddr_off        =  1,
 181     call_wrapper_off   =  2,
 182     result_off         =  3,
 183     result_type_off    =  4,
 184     method_off         =  5,
 185     entry_point_off    =  6,
 186     parameters_off     =  7,
 187     parameter_size_off =  8,
 188     thread_off         =  9
 189 #else
 190     rsp_after_call_off = -12,
 191     mxcsr_off          = rsp_after_call_off,
 192     r15_off            = -11,
 193     r14_off            = -10,
 194     r13_off            = -9,
 195     r12_off            = -8,
 196     rbx_off            = -7,
 197     call_wrapper_off   = -6,
 198     result_off         = -5,
 199     result_type_off    = -4,
 200     method_off         = -3,
 201     entry_point_off    = -2,
 202     parameters_off     = -1,
 203     rbp_off            =  0,
 204     retaddr_off        =  1,
 205     parameter_size_off =  2,
 206     thread_off         =  3
 207 #endif
 208   };
 209 
 210 #ifdef _WIN64
 211   Address xmm_save(int reg) {
 212     assert(reg >= xmm_save_first && reg <= xmm_save_last, "XMM register number out of range");
 213     return Address(rbp, (xmm_save_base - (reg - xmm_save_first) * 2) * wordSize);
 214   }
 215 #endif
 216 
 217   address generate_call_stub(address& return_address) {
 218     assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 &&
 219            (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
 220            "adjust this code");
 221     StubCodeMark mark(this, "StubRoutines", "call_stub");
 222     address start = __ pc();
 223 
 224     // same as in generate_catch_exception()!
 225     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 226 
 227     const Address call_wrapper  (rbp, call_wrapper_off   * wordSize);
 228     const Address result        (rbp, result_off         * wordSize);
 229     const Address result_type   (rbp, result_type_off    * wordSize);
 230     const Address method        (rbp, method_off         * wordSize);
 231     const Address entry_point   (rbp, entry_point_off    * wordSize);
 232     const Address parameters    (rbp, parameters_off     * wordSize);
 233     const Address parameter_size(rbp, parameter_size_off * wordSize);
 234 
 235     // same as in generate_catch_exception()!
 236     const Address thread        (rbp, thread_off         * wordSize);
 237 
 238     const Address r15_save(rbp, r15_off * wordSize);
 239     const Address r14_save(rbp, r14_off * wordSize);
 240     const Address r13_save(rbp, r13_off * wordSize);
 241     const Address r12_save(rbp, r12_off * wordSize);
 242     const Address rbx_save(rbp, rbx_off * wordSize);
 243 
 244     // stub code
 245     __ enter();
 246     __ subptr(rsp, -rsp_after_call_off * wordSize);
 247 
 248     // save register parameters
 249 #ifndef _WIN64
 250     __ movptr(parameters,   c_rarg5); // parameters
 251     __ movptr(entry_point,  c_rarg4); // entry_point
 252 #endif
 253 
 254     __ movptr(method,       c_rarg3); // method
 255     __ movl(result_type,  c_rarg2);   // result type
 256     __ movptr(result,       c_rarg1); // result
 257     __ movptr(call_wrapper, c_rarg0); // call wrapper
 258 
 259     // save regs belonging to calling function
 260     __ movptr(rbx_save, rbx);
 261     __ movptr(r12_save, r12);
 262     __ movptr(r13_save, r13);
 263     __ movptr(r14_save, r14);
 264     __ movptr(r15_save, r15);
 265 #ifdef _WIN64
 266     for (int i = 6; i <= 15; i++) {
 267       __ movdqu(xmm_save(i), as_XMMRegister(i));
 268     }
 269 
 270     const Address rdi_save(rbp, rdi_off * wordSize);
 271     const Address rsi_save(rbp, rsi_off * wordSize);
 272 
 273     __ movptr(rsi_save, rsi);
 274     __ movptr(rdi_save, rdi);
 275 #else
 276     const Address mxcsr_save(rbp, mxcsr_off * wordSize);
 277     {
 278       Label skip_ldmx;
 279       __ stmxcsr(mxcsr_save);
 280       __ movl(rax, mxcsr_save);
 281       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 282       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 283       __ cmp32(rax, mxcsr_std);
 284       __ jcc(Assembler::equal, skip_ldmx);
 285       __ ldmxcsr(mxcsr_std);
 286       __ bind(skip_ldmx);
 287     }
 288 #endif
 289 
 290     // Load up thread register
 291     __ movptr(r15_thread, thread);
 292     __ reinit_heapbase();
 293 
 294 #ifdef ASSERT
 295     // make sure we have no pending exceptions
 296     {
 297       Label L;
 298       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 299       __ jcc(Assembler::equal, L);
 300       __ stop("StubRoutines::call_stub: entered with pending exception");
 301       __ bind(L);
 302     }
 303 #endif
 304 
 305     // pass parameters if any
 306     BLOCK_COMMENT("pass parameters if any");
 307     Label parameters_done;
 308     __ movl(c_rarg3, parameter_size);
 309     __ testl(c_rarg3, c_rarg3);
 310     __ jcc(Assembler::zero, parameters_done);
 311 
 312     Label loop;
 313     __ movptr(c_rarg2, parameters);       // parameter pointer
 314     __ movl(c_rarg1, c_rarg3);            // parameter counter is in c_rarg1
 315     __ BIND(loop);
 316     __ movptr(rax, Address(c_rarg2, 0));// get parameter
 317     __ addptr(c_rarg2, wordSize);       // advance to next parameter
 318     __ decrementl(c_rarg1);             // decrement counter
 319     __ push(rax);                       // pass parameter
 320     __ jcc(Assembler::notZero, loop);
 321 
 322     // call Java function
 323     __ BIND(parameters_done);
 324     __ movptr(rbx, method);             // get Method*
 325     __ movptr(c_rarg1, entry_point);    // get entry_point
 326     __ mov(r13, rsp);                   // set sender sp
 327     BLOCK_COMMENT("call Java function");
 328     __ call(c_rarg1);
 329 
 330     BLOCK_COMMENT("call_stub_return_address:");
 331     return_address = __ pc();
 332 
 333     // store result depending on type (everything that is not
 334     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 335     __ movptr(c_rarg0, result);
 336     Label is_long, is_float, is_double, exit;
 337     __ movl(c_rarg1, result_type);
 338     __ cmpl(c_rarg1, T_OBJECT);
 339     __ jcc(Assembler::equal, is_long);
 340     __ cmpl(c_rarg1, T_LONG);
 341     __ jcc(Assembler::equal, is_long);
 342     __ cmpl(c_rarg1, T_FLOAT);
 343     __ jcc(Assembler::equal, is_float);
 344     __ cmpl(c_rarg1, T_DOUBLE);
 345     __ jcc(Assembler::equal, is_double);
 346 
 347     // handle T_INT case
 348     __ movl(Address(c_rarg0, 0), rax);
 349 
 350     __ BIND(exit);
 351 
 352     // pop parameters
 353     __ lea(rsp, rsp_after_call);
 354 
 355 #ifdef ASSERT
 356     // verify that threads correspond
 357     {
 358       Label L, S;
 359       __ cmpptr(r15_thread, thread);
 360       __ jcc(Assembler::notEqual, S);
 361       __ get_thread(rbx);
 362       __ cmpptr(r15_thread, rbx);
 363       __ jcc(Assembler::equal, L);
 364       __ bind(S);
 365       __ jcc(Assembler::equal, L);
 366       __ stop("StubRoutines::call_stub: threads must correspond");
 367       __ bind(L);
 368     }
 369 #endif
 370 
 371     // restore regs belonging to calling function
 372 #ifdef _WIN64
 373     for (int i = 15; i >= 6; i--) {
 374       __ movdqu(as_XMMRegister(i), xmm_save(i));
 375     }
 376 #endif
 377     __ movptr(r15, r15_save);
 378     __ movptr(r14, r14_save);
 379     __ movptr(r13, r13_save);
 380     __ movptr(r12, r12_save);
 381     __ movptr(rbx, rbx_save);
 382 
 383 #ifdef _WIN64
 384     __ movptr(rdi, rdi_save);
 385     __ movptr(rsi, rsi_save);
 386 #else
 387     __ ldmxcsr(mxcsr_save);
 388 #endif
 389 
 390     // restore rsp
 391     __ addptr(rsp, -rsp_after_call_off * wordSize);
 392 
 393     // return
 394     __ pop(rbp);
 395     __ ret(0);
 396 
 397     // handle return types different from T_INT
 398     __ BIND(is_long);
 399     __ movq(Address(c_rarg0, 0), rax);
 400     __ jmp(exit);
 401 
 402     __ BIND(is_float);
 403     __ movflt(Address(c_rarg0, 0), xmm0);
 404     __ jmp(exit);
 405 
 406     __ BIND(is_double);
 407     __ movdbl(Address(c_rarg0, 0), xmm0);
 408     __ jmp(exit);
 409 
 410     return start;
 411   }
 412 
 413   // Return point for a Java call if there's an exception thrown in
 414   // Java code.  The exception is caught and transformed into a
 415   // pending exception stored in JavaThread that can be tested from
 416   // within the VM.
 417   //
 418   // Note: Usually the parameters are removed by the callee. In case
 419   // of an exception crossing an activation frame boundary, that is
 420   // not the case if the callee is compiled code => need to setup the
 421   // rsp.
 422   //
 423   // rax: exception oop
 424 
 425   address generate_catch_exception() {
 426     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 427     address start = __ pc();
 428 
 429     // same as in generate_call_stub():
 430     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 431     const Address thread        (rbp, thread_off         * wordSize);
 432 
 433 #ifdef ASSERT
 434     // verify that threads correspond
 435     {
 436       Label L, S;
 437       __ cmpptr(r15_thread, thread);
 438       __ jcc(Assembler::notEqual, S);
 439       __ get_thread(rbx);
 440       __ cmpptr(r15_thread, rbx);
 441       __ jcc(Assembler::equal, L);
 442       __ bind(S);
 443       __ stop("StubRoutines::catch_exception: threads must correspond");
 444       __ bind(L);
 445     }
 446 #endif
 447 
 448     // set pending exception
 449     __ verify_oop(rax);
 450 
 451     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax);
 452     __ lea(rscratch1, ExternalAddress((address)__FILE__));
 453     __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
 454     __ movl(Address(r15_thread, Thread::exception_line_offset()), (int)  __LINE__);
 455 
 456     // complete return to VM
 457     assert(StubRoutines::_call_stub_return_address != NULL,
 458            "_call_stub_return_address must have been generated before");
 459     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 460 
 461     return start;
 462   }
 463 
 464   // Continuation point for runtime calls returning with a pending
 465   // exception.  The pending exception check happened in the runtime
 466   // or native call stub.  The pending exception in Thread is
 467   // converted into a Java-level exception.
 468   //
 469   // Contract with Java-level exception handlers:
 470   // rax: exception
 471   // rdx: throwing pc
 472   //
 473   // NOTE: At entry of this stub, exception-pc must be on stack !!
 474 
 475   address generate_forward_exception() {
 476     StubCodeMark mark(this, "StubRoutines", "forward exception");
 477     address start = __ pc();
 478 
 479     // Upon entry, the sp points to the return address returning into
 480     // Java (interpreted or compiled) code; i.e., the return address
 481     // becomes the throwing pc.
 482     //
 483     // Arguments pushed before the runtime call are still on the stack
 484     // but the exception handler will reset the stack pointer ->
 485     // ignore them.  A potential result in registers can be ignored as
 486     // well.
 487 
 488 #ifdef ASSERT
 489     // make sure this code is only executed if there is a pending exception
 490     {
 491       Label L;
 492       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL);
 493       __ jcc(Assembler::notEqual, L);
 494       __ stop("StubRoutines::forward exception: no pending exception (1)");
 495       __ bind(L);
 496     }
 497 #endif
 498 
 499     // compute exception handler into rbx
 500     __ movptr(c_rarg0, Address(rsp, 0));
 501     BLOCK_COMMENT("call exception_handler_for_return_address");
 502     __ call_VM_leaf(CAST_FROM_FN_PTR(address,
 503                          SharedRuntime::exception_handler_for_return_address),
 504                     r15_thread, c_rarg0);
 505     __ mov(rbx, rax);
 506 
 507     // setup rax & rdx, remove return address & clear pending exception
 508     __ pop(rdx);
 509     __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
 510     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 511 
 512 #ifdef ASSERT
 513     // make sure exception is set
 514     {
 515       Label L;
 516       __ testptr(rax, rax);
 517       __ jcc(Assembler::notEqual, L);
 518       __ stop("StubRoutines::forward exception: no pending exception (2)");
 519       __ bind(L);
 520     }
 521 #endif
 522 
 523     // continue at exception handler (return address removed)
 524     // rax: exception
 525     // rbx: exception handler
 526     // rdx: throwing pc
 527     __ verify_oop(rax);
 528     __ jmp(rbx);
 529 
 530     return start;
 531   }
 532 
 533   // Support for jint Atomic::xchg(jint exchange_value, volatile jint* dest)
 534   //
 535   // Arguments :
 536   //    c_rarg0: exchange_value
 537   //    c_rarg0: dest
 538   //
 539   // Result:
 540   //    *dest <- ex, return (orig *dest)
 541   address generate_atomic_xchg() {
 542     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 543     address start = __ pc();
 544 
 545     __ movl(rax, c_rarg0); // Copy to eax we need a return value anyhow
 546     __ xchgl(rax, Address(c_rarg1, 0)); // automatic LOCK
 547     __ ret(0);
 548 
 549     return start;
 550   }
 551 
 552   // Support for intptr_t Atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest)
 553   //
 554   // Arguments :
 555   //    c_rarg0: exchange_value
 556   //    c_rarg1: dest
 557   //
 558   // Result:
 559   //    *dest <- ex, return (orig *dest)
 560   address generate_atomic_xchg_ptr() {
 561     StubCodeMark mark(this, "StubRoutines", "atomic_xchg_ptr");
 562     address start = __ pc();
 563 
 564     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 565     __ xchgptr(rax, Address(c_rarg1, 0)); // automatic LOCK
 566     __ ret(0);
 567 
 568     return start;
 569   }
 570 
 571   // Support for jint Atomic::cmpxchg(jint exchange_value, volatile jint* dest,
 572   //                                  jint compare_value)
 573   //
 574   // Arguments :
 575   //    c_rarg0: exchange_value
 576   //    c_rarg1: dest
 577   //    c_rarg2: compare_value
 578   //
 579   // Result:
 580   //    if ( compare_value == *dest ) {
 581   //       *dest = exchange_value
 582   //       return compare_value;
 583   //    else
 584   //       return *dest;
 585   address generate_atomic_cmpxchg() {
 586     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg");
 587     address start = __ pc();
 588 
 589     __ movl(rax, c_rarg2);
 590    if ( os::is_MP() ) __ lock();
 591     __ cmpxchgl(c_rarg0, Address(c_rarg1, 0));
 592     __ ret(0);
 593 
 594     return start;
 595   }
 596 
 597   // Support for jbyte Atomic::cmpxchg(jbyte exchange_value, volatile jbyte* dest,
 598   //                                   jbyte compare_value)
 599   //
 600   // Arguments :
 601   //    c_rarg0: exchange_value
 602   //    c_rarg1: dest
 603   //    c_rarg2: compare_value
 604   //
 605   // Result:
 606   //    if ( compare_value == *dest ) {
 607   //       *dest = exchange_value
 608   //       return compare_value;
 609   //    else
 610   //       return *dest;
 611   address generate_atomic_cmpxchg_byte() {
 612     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_byte");
 613     address start = __ pc();
 614 
 615     __ movsbq(rax, c_rarg2);
 616    if ( os::is_MP() ) __ lock();
 617     __ cmpxchgb(c_rarg0, Address(c_rarg1, 0));
 618     __ ret(0);
 619 
 620     return start;
 621   }
 622 
 623   // Support for jlong Atomic::cmpxchg(jlong exchange_value, volatile jlong* dest,
 624   //                                   jlong compare_value)
 625   // Arguments :
 626   //    c_rarg0: exchange_value
 627   //    c_rarg1: dest
 628   //    c_rarg2: compare_value
 629   //
 630   // Result:
 631   //    if ( compare_value == *dest ) {
 632   //       *dest = exchange_value
 633   //       return compare_value;
 634   //    else
 635   //       return *dest;
 636   address generate_atomic_cmpxchg_long() {
 637     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long");
 638     address start = __ pc();
 639 
 640     __ movq(rax, c_rarg2);
 641    if ( os::is_MP() ) __ lock();
 642     __ cmpxchgq(c_rarg0, Address(c_rarg1, 0));
 643     __ ret(0);
 644 
 645     return start;
 646   }
 647 
 648   // Support for jint Atomic::add(jint add_value, volatile jint* dest)
 649   //
 650   // Arguments :
 651   //    c_rarg0: add_value
 652   //    c_rarg1: dest
 653   //
 654   // Result:
 655   //    *dest += add_value
 656   //    return *dest;
 657   address generate_atomic_add() {
 658     StubCodeMark mark(this, "StubRoutines", "atomic_add");
 659     address start = __ pc();
 660 
 661     __ movl(rax, c_rarg0);
 662    if ( os::is_MP() ) __ lock();
 663     __ xaddl(Address(c_rarg1, 0), c_rarg0);
 664     __ addl(rax, c_rarg0);
 665     __ ret(0);
 666 
 667     return start;
 668   }
 669 
 670   // Support for intptr_t Atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest)
 671   //
 672   // Arguments :
 673   //    c_rarg0: add_value
 674   //    c_rarg1: dest
 675   //
 676   // Result:
 677   //    *dest += add_value
 678   //    return *dest;
 679   address generate_atomic_add_ptr() {
 680     StubCodeMark mark(this, "StubRoutines", "atomic_add_ptr");
 681     address start = __ pc();
 682 
 683     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 684    if ( os::is_MP() ) __ lock();
 685     __ xaddptr(Address(c_rarg1, 0), c_rarg0);
 686     __ addptr(rax, c_rarg0);
 687     __ ret(0);
 688 
 689     return start;
 690   }
 691 
 692   // Support for intptr_t OrderAccess::fence()
 693   //
 694   // Arguments :
 695   //
 696   // Result:
 697   address generate_orderaccess_fence() {
 698     StubCodeMark mark(this, "StubRoutines", "orderaccess_fence");
 699     address start = __ pc();
 700     __ membar(Assembler::StoreLoad);
 701     __ ret(0);
 702 
 703     return start;
 704   }
 705 
 706   // Support for intptr_t get_previous_fp()
 707   //
 708   // This routine is used to find the previous frame pointer for the
 709   // caller (current_frame_guess). This is used as part of debugging
 710   // ps() is seemingly lost trying to find frames.
 711   // This code assumes that caller current_frame_guess) has a frame.
 712   address generate_get_previous_fp() {
 713     StubCodeMark mark(this, "StubRoutines", "get_previous_fp");
 714     const Address old_fp(rbp, 0);
 715     const Address older_fp(rax, 0);
 716     address start = __ pc();
 717 
 718     __ enter();
 719     __ movptr(rax, old_fp); // callers fp
 720     __ movptr(rax, older_fp); // the frame for ps()
 721     __ pop(rbp);
 722     __ ret(0);
 723 
 724     return start;
 725   }
 726 
 727   // Support for intptr_t get_previous_sp()
 728   //
 729   // This routine is used to find the previous stack pointer for the
 730   // caller.
 731   address generate_get_previous_sp() {
 732     StubCodeMark mark(this, "StubRoutines", "get_previous_sp");
 733     address start = __ pc();
 734 
 735     __ movptr(rax, rsp);
 736     __ addptr(rax, 8); // return address is at the top of the stack.
 737     __ ret(0);
 738 
 739     return start;
 740   }
 741 
 742   //----------------------------------------------------------------------------------------------------
 743   // Support for void verify_mxcsr()
 744   //
 745   // This routine is used with -Xcheck:jni to verify that native
 746   // JNI code does not return to Java code without restoring the
 747   // MXCSR register to our expected state.
 748 
 749   address generate_verify_mxcsr() {
 750     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 751     address start = __ pc();
 752 
 753     const Address mxcsr_save(rsp, 0);
 754 
 755     if (CheckJNICalls) {
 756       Label ok_ret;
 757       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 758       __ push(rax);
 759       __ subptr(rsp, wordSize);      // allocate a temp location
 760       __ stmxcsr(mxcsr_save);
 761       __ movl(rax, mxcsr_save);
 762       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 763       __ cmp32(rax, mxcsr_std);
 764       __ jcc(Assembler::equal, ok_ret);
 765 
 766       __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
 767 
 768       __ ldmxcsr(mxcsr_std);
 769 
 770       __ bind(ok_ret);
 771       __ addptr(rsp, wordSize);
 772       __ pop(rax);
 773     }
 774 
 775     __ ret(0);
 776 
 777     return start;
 778   }
 779 
 780   address generate_f2i_fixup() {
 781     StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
 782     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 783 
 784     address start = __ pc();
 785 
 786     Label L;
 787 
 788     __ push(rax);
 789     __ push(c_rarg3);
 790     __ push(c_rarg2);
 791     __ push(c_rarg1);
 792 
 793     __ movl(rax, 0x7f800000);
 794     __ xorl(c_rarg3, c_rarg3);
 795     __ movl(c_rarg2, inout);
 796     __ movl(c_rarg1, c_rarg2);
 797     __ andl(c_rarg1, 0x7fffffff);
 798     __ cmpl(rax, c_rarg1); // NaN? -> 0
 799     __ jcc(Assembler::negative, L);
 800     __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint
 801     __ movl(c_rarg3, 0x80000000);
 802     __ movl(rax, 0x7fffffff);
 803     __ cmovl(Assembler::positive, c_rarg3, rax);
 804 
 805     __ bind(L);
 806     __ movptr(inout, c_rarg3);
 807 
 808     __ pop(c_rarg1);
 809     __ pop(c_rarg2);
 810     __ pop(c_rarg3);
 811     __ pop(rax);
 812 
 813     __ ret(0);
 814 
 815     return start;
 816   }
 817 
 818   address generate_f2l_fixup() {
 819     StubCodeMark mark(this, "StubRoutines", "f2l_fixup");
 820     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 821     address start = __ pc();
 822 
 823     Label L;
 824 
 825     __ push(rax);
 826     __ push(c_rarg3);
 827     __ push(c_rarg2);
 828     __ push(c_rarg1);
 829 
 830     __ movl(rax, 0x7f800000);
 831     __ xorl(c_rarg3, c_rarg3);
 832     __ movl(c_rarg2, inout);
 833     __ movl(c_rarg1, c_rarg2);
 834     __ andl(c_rarg1, 0x7fffffff);
 835     __ cmpl(rax, c_rarg1); // NaN? -> 0
 836     __ jcc(Assembler::negative, L);
 837     __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong
 838     __ mov64(c_rarg3, 0x8000000000000000);
 839     __ mov64(rax, 0x7fffffffffffffff);
 840     __ cmov(Assembler::positive, c_rarg3, rax);
 841 
 842     __ bind(L);
 843     __ movptr(inout, c_rarg3);
 844 
 845     __ pop(c_rarg1);
 846     __ pop(c_rarg2);
 847     __ pop(c_rarg3);
 848     __ pop(rax);
 849 
 850     __ ret(0);
 851 
 852     return start;
 853   }
 854 
 855   address generate_d2i_fixup() {
 856     StubCodeMark mark(this, "StubRoutines", "d2i_fixup");
 857     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 858 
 859     address start = __ pc();
 860 
 861     Label L;
 862 
 863     __ push(rax);
 864     __ push(c_rarg3);
 865     __ push(c_rarg2);
 866     __ push(c_rarg1);
 867     __ push(c_rarg0);
 868 
 869     __ movl(rax, 0x7ff00000);
 870     __ movq(c_rarg2, inout);
 871     __ movl(c_rarg3, c_rarg2);
 872     __ mov(c_rarg1, c_rarg2);
 873     __ mov(c_rarg0, c_rarg2);
 874     __ negl(c_rarg3);
 875     __ shrptr(c_rarg1, 0x20);
 876     __ orl(c_rarg3, c_rarg2);
 877     __ andl(c_rarg1, 0x7fffffff);
 878     __ xorl(c_rarg2, c_rarg2);
 879     __ shrl(c_rarg3, 0x1f);
 880     __ orl(c_rarg1, c_rarg3);
 881     __ cmpl(rax, c_rarg1);
 882     __ jcc(Assembler::negative, L); // NaN -> 0
 883     __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint
 884     __ movl(c_rarg2, 0x80000000);
 885     __ movl(rax, 0x7fffffff);
 886     __ cmov(Assembler::positive, c_rarg2, rax);
 887 
 888     __ bind(L);
 889     __ movptr(inout, c_rarg2);
 890 
 891     __ pop(c_rarg0);
 892     __ pop(c_rarg1);
 893     __ pop(c_rarg2);
 894     __ pop(c_rarg3);
 895     __ pop(rax);
 896 
 897     __ ret(0);
 898 
 899     return start;
 900   }
 901 
 902   address generate_d2l_fixup() {
 903     StubCodeMark mark(this, "StubRoutines", "d2l_fixup");
 904     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 905 
 906     address start = __ pc();
 907 
 908     Label L;
 909 
 910     __ push(rax);
 911     __ push(c_rarg3);
 912     __ push(c_rarg2);
 913     __ push(c_rarg1);
 914     __ push(c_rarg0);
 915 
 916     __ movl(rax, 0x7ff00000);
 917     __ movq(c_rarg2, inout);
 918     __ movl(c_rarg3, c_rarg2);
 919     __ mov(c_rarg1, c_rarg2);
 920     __ mov(c_rarg0, c_rarg2);
 921     __ negl(c_rarg3);
 922     __ shrptr(c_rarg1, 0x20);
 923     __ orl(c_rarg3, c_rarg2);
 924     __ andl(c_rarg1, 0x7fffffff);
 925     __ xorl(c_rarg2, c_rarg2);
 926     __ shrl(c_rarg3, 0x1f);
 927     __ orl(c_rarg1, c_rarg3);
 928     __ cmpl(rax, c_rarg1);
 929     __ jcc(Assembler::negative, L); // NaN -> 0
 930     __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
 931     __ mov64(c_rarg2, 0x8000000000000000);
 932     __ mov64(rax, 0x7fffffffffffffff);
 933     __ cmovq(Assembler::positive, c_rarg2, rax);
 934 
 935     __ bind(L);
 936     __ movq(inout, c_rarg2);
 937 
 938     __ pop(c_rarg0);
 939     __ pop(c_rarg1);
 940     __ pop(c_rarg2);
 941     __ pop(c_rarg3);
 942     __ pop(rax);
 943 
 944     __ ret(0);
 945 
 946     return start;
 947   }
 948 
 949   address generate_fp_mask(const char *stub_name, int64_t mask) {
 950     __ align(CodeEntryAlignment);
 951     StubCodeMark mark(this, "StubRoutines", stub_name);
 952     address start = __ pc();
 953 
 954     __ emit_data64( mask, relocInfo::none );
 955     __ emit_data64( mask, relocInfo::none );
 956 
 957     return start;
 958   }
 959 
 960   // The following routine generates a subroutine to throw an
 961   // asynchronous UnknownError when an unsafe access gets a fault that
 962   // could not be reasonably prevented by the programmer.  (Example:
 963   // SIGBUS/OBJERR.)
 964   address generate_handler_for_unsafe_access() {
 965     StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
 966     address start = __ pc();
 967 
 968     __ push(0);                       // hole for return address-to-be
 969     __ pusha();                       // push registers
 970     Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);
 971 
 972     // FIXME: this probably needs alignment logic
 973 
 974     __ subptr(rsp, frame::arg_reg_save_area_bytes);
 975     BLOCK_COMMENT("call handle_unsafe_access");
 976     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
 977     __ addptr(rsp, frame::arg_reg_save_area_bytes);
 978 
 979     __ movptr(next_pc, rax);          // stuff next address
 980     __ popa();
 981     __ ret(0);                        // jump to next address
 982 
 983     return start;
 984   }
 985 
 986   // Non-destructive plausibility checks for oops
 987   //
 988   // Arguments:
 989   //    all args on stack!
 990   //
 991   // Stack after saving c_rarg3:
 992   //    [tos + 0]: saved c_rarg3
 993   //    [tos + 1]: saved c_rarg2
 994   //    [tos + 2]: saved r12 (several TemplateTable methods use it)
 995   //    [tos + 3]: saved flags
 996   //    [tos + 4]: return address
 997   //  * [tos + 5]: error message (char*)
 998   //  * [tos + 6]: object to verify (oop)
 999   //  * [tos + 7]: saved rax - saved by caller and bashed
1000   //  * [tos + 8]: saved r10 (rscratch1) - saved by caller
1001   //  * = popped on exit
1002   address generate_verify_oop() {
1003     StubCodeMark mark(this, "StubRoutines", "verify_oop");
1004     address start = __ pc();
1005 
1006     Label exit, error;
1007 
1008     __ pushf();
1009     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
1010 
1011     __ push(r12);
1012 
1013     // save c_rarg2 and c_rarg3
1014     __ push(c_rarg2);
1015     __ push(c_rarg3);
1016 
1017     enum {
1018            // After previous pushes.
1019            oop_to_verify = 6 * wordSize,
1020            saved_rax     = 7 * wordSize,
1021            saved_r10     = 8 * wordSize,
1022 
1023            // Before the call to MacroAssembler::debug(), see below.
1024            return_addr   = 16 * wordSize,
1025            error_msg     = 17 * wordSize
1026     };
1027 
1028     // get object
1029     __ movptr(rax, Address(rsp, oop_to_verify));
1030 
1031     // make sure object is 'reasonable'
1032     __ testptr(rax, rax);
1033     __ jcc(Assembler::zero, exit); // if obj is NULL it is OK
1034     // Check if the oop is in the right area of memory
1035     __ movptr(c_rarg2, rax);
1036     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask());
1037     __ andptr(c_rarg2, c_rarg3);
1038     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_bits());
1039     __ cmpptr(c_rarg2, c_rarg3);
1040     __ jcc(Assembler::notZero, error);
1041 
1042     // set r12 to heapbase for load_klass()
1043     __ reinit_heapbase();
1044 
1045     // make sure klass is 'reasonable', which is not zero.
1046     __ load_klass(rax, rax);  // get klass
1047     __ testptr(rax, rax);
1048     __ jcc(Assembler::zero, error); // if klass is NULL it is broken
1049 
1050     // return if everything seems ok
1051     __ bind(exit);
1052     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
1053     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1054     __ pop(c_rarg3);                             // restore c_rarg3
1055     __ pop(c_rarg2);                             // restore c_rarg2
1056     __ pop(r12);                                 // restore r12
1057     __ popf();                                   // restore flags
1058     __ ret(4 * wordSize);                        // pop caller saved stuff
1059 
1060     // handle errors
1061     __ bind(error);
1062     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
1063     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
1064     __ pop(c_rarg3);                             // get saved c_rarg3 back
1065     __ pop(c_rarg2);                             // get saved c_rarg2 back
1066     __ pop(r12);                                 // get saved r12 back
1067     __ popf();                                   // get saved flags off stack --
1068                                                  // will be ignored
1069 
1070     __ pusha();                                  // push registers
1071                                                  // (rip is already
1072                                                  // already pushed)
1073     // debug(char* msg, int64_t pc, int64_t regs[])
1074     // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and
1075     // pushed all the registers, so now the stack looks like:
1076     //     [tos +  0] 16 saved registers
1077     //     [tos + 16] return address
1078     //   * [tos + 17] error message (char*)
1079     //   * [tos + 18] object to verify (oop)
1080     //   * [tos + 19] saved rax - saved by caller and bashed
1081     //   * [tos + 20] saved r10 (rscratch1) - saved by caller
1082     //   * = popped on exit
1083 
1084     __ movptr(c_rarg0, Address(rsp, error_msg));    // pass address of error message
1085     __ movptr(c_rarg1, Address(rsp, return_addr));  // pass return address
1086     __ movq(c_rarg2, rsp);                          // pass address of regs on stack
1087     __ mov(r12, rsp);                               // remember rsp
1088     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1089     __ andptr(rsp, -16);                            // align stack as required by ABI
1090     BLOCK_COMMENT("call MacroAssembler::debug");
1091     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
1092     __ mov(rsp, r12);                               // restore rsp
1093     __ popa();                                      // pop registers (includes r12)
1094     __ ret(4 * wordSize);                           // pop caller saved stuff
1095 
1096     return start;
1097   }
1098 
1099   //
1100   // Verify that a register contains clean 32-bits positive value
1101   // (high 32-bits are 0) so it could be used in 64-bits shifts.
1102   //
1103   //  Input:
1104   //    Rint  -  32-bits value
1105   //    Rtmp  -  scratch
1106   //
1107   void assert_clean_int(Register Rint, Register Rtmp) {
1108 #ifdef ASSERT
1109     Label L;
1110     assert_different_registers(Rtmp, Rint);
1111     __ movslq(Rtmp, Rint);
1112     __ cmpq(Rtmp, Rint);
1113     __ jcc(Assembler::equal, L);
1114     __ stop("high 32-bits of int value are not 0");
1115     __ bind(L);
1116 #endif
1117   }
1118 
1119   //  Generate overlap test for array copy stubs
1120   //
1121   //  Input:
1122   //     c_rarg0 - from
1123   //     c_rarg1 - to
1124   //     c_rarg2 - element count
1125   //
1126   //  Output:
1127   //     rax   - &from[element count - 1]
1128   //
1129   void array_overlap_test(address no_overlap_target, Address::ScaleFactor sf) {
1130     assert(no_overlap_target != NULL, "must be generated");
1131     array_overlap_test(no_overlap_target, NULL, sf);
1132   }
1133   void array_overlap_test(Label& L_no_overlap, Address::ScaleFactor sf) {
1134     array_overlap_test(NULL, &L_no_overlap, sf);
1135   }
1136   void array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) {
1137     const Register from     = c_rarg0;
1138     const Register to       = c_rarg1;
1139     const Register count    = c_rarg2;
1140     const Register end_from = rax;
1141 
1142     __ cmpptr(to, from);
1143     __ lea(end_from, Address(from, count, sf, 0));
1144     if (NOLp == NULL) {
1145       ExternalAddress no_overlap(no_overlap_target);
1146       __ jump_cc(Assembler::belowEqual, no_overlap);
1147       __ cmpptr(to, end_from);
1148       __ jump_cc(Assembler::aboveEqual, no_overlap);
1149     } else {
1150       __ jcc(Assembler::belowEqual, (*NOLp));
1151       __ cmpptr(to, end_from);
1152       __ jcc(Assembler::aboveEqual, (*NOLp));
1153     }
1154   }
1155 
1156   // Shuffle first three arg regs on Windows into Linux/Solaris locations.
1157   //
1158   // Outputs:
1159   //    rdi - rcx
1160   //    rsi - rdx
1161   //    rdx - r8
1162   //    rcx - r9
1163   //
1164   // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter
1165   // are non-volatile.  r9 and r10 should not be used by the caller.
1166   //
1167   void setup_arg_regs(int nargs = 3) {
1168     const Register saved_rdi = r9;
1169     const Register saved_rsi = r10;
1170     assert(nargs == 3 || nargs == 4, "else fix");
1171 #ifdef _WIN64
1172     assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,
1173            "unexpected argument registers");
1174     if (nargs >= 4)
1175       __ mov(rax, r9);  // r9 is also saved_rdi
1176     __ movptr(saved_rdi, rdi);
1177     __ movptr(saved_rsi, rsi);
1178     __ mov(rdi, rcx); // c_rarg0
1179     __ mov(rsi, rdx); // c_rarg1
1180     __ mov(rdx, r8);  // c_rarg2
1181     if (nargs >= 4)
1182       __ mov(rcx, rax); // c_rarg3 (via rax)
1183 #else
1184     assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,
1185            "unexpected argument registers");
1186 #endif
1187   }
1188 
1189   void restore_arg_regs() {
1190     const Register saved_rdi = r9;
1191     const Register saved_rsi = r10;
1192 #ifdef _WIN64
1193     __ movptr(rdi, saved_rdi);
1194     __ movptr(rsi, saved_rsi);
1195 #endif
1196   }
1197 
1198   // Generate code for an array write pre barrier
1199   //
1200   //     addr    -  starting address
1201   //     count   -  element count
1202   //     tmp     - scratch register
1203   //
1204   //     Destroy no registers!
1205   //
1206   void  gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) {
1207     BarrierSet* bs = Universe::heap()->barrier_set();
1208     switch (bs->kind()) {
1209       case BarrierSet::G1SATBCT:
1210       case BarrierSet::G1SATBCTLogging:
1211         // With G1, don't generate the call if we statically know that the target in uninitialized
1212         if (!dest_uninitialized) {
1213            __ pusha();                      // push registers
1214            if (count == c_rarg0) {
1215              if (addr == c_rarg1) {
1216                // exactly backwards!!
1217                __ xchgptr(c_rarg1, c_rarg0);
1218              } else {
1219                __ movptr(c_rarg1, count);
1220                __ movptr(c_rarg0, addr);
1221              }
1222            } else {
1223              __ movptr(c_rarg0, addr);
1224              __ movptr(c_rarg1, count);
1225            }
1226            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
1227            __ popa();
1228         }
1229          break;
1230       case BarrierSet::CardTableModRef:
1231       case BarrierSet::CardTableExtension:
1232       case BarrierSet::ModRef:
1233         break;
1234       default:
1235         ShouldNotReachHere();
1236 
1237     }
1238   }
1239 
1240   //
1241   // Generate code for an array write post barrier
1242   //
1243   //  Input:
1244   //     start    - register containing starting address of destination array
1245   //     count    - elements count
1246   //     scratch  - scratch register
1247   //
1248   //  The input registers are overwritten.
1249   //
1250   void  gen_write_ref_array_post_barrier(Register start, Register count, Register scratch) {
1251     assert_different_registers(start, count, scratch);
1252     BarrierSet* bs = Universe::heap()->barrier_set();
1253     switch (bs->kind()) {
1254       case BarrierSet::G1SATBCT:
1255       case BarrierSet::G1SATBCTLogging:
1256         {
1257           __ pusha();             // push registers (overkill)
1258           if (c_rarg0 == count) { // On win64 c_rarg0 == rcx
1259             assert_different_registers(c_rarg1, start);
1260             __ mov(c_rarg1, count);
1261             __ mov(c_rarg0, start);
1262           } else {
1263             assert_different_registers(c_rarg0, count);
1264             __ mov(c_rarg0, start);
1265             __ mov(c_rarg1, count);
1266           }
1267           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2);
1268           __ popa();
1269         }
1270         break;
1271       case BarrierSet::CardTableModRef:
1272       case BarrierSet::CardTableExtension:
1273         {
1274           CardTableModRefBS* ct = (CardTableModRefBS*)bs;
1275           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1276 
1277           Label L_loop;
1278           const Register end = count;
1279 
1280           __ leaq(end, Address(start, count, TIMES_OOP, 0));  // end == start+count*oop_size
1281           __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
1282           __ shrptr(start, CardTableModRefBS::card_shift);
1283           __ shrptr(end,   CardTableModRefBS::card_shift);
1284           __ subptr(end, start); // end --> cards count
1285 
1286           int64_t disp = (int64_t) ct->byte_map_base;
1287           __ mov64(scratch, disp);
1288           __ addptr(start, scratch);
1289         __ BIND(L_loop);
1290           __ movb(Address(start, count, Address::times_1), 0);
1291           __ decrement(count);
1292           __ jcc(Assembler::greaterEqual, L_loop);
1293         }
1294         break;
1295       default:
1296         ShouldNotReachHere();
1297 
1298     }
1299   }
1300 
1301 
1302   // Copy big chunks forward
1303   //
1304   // Inputs:
1305   //   end_from     - source arrays end address
1306   //   end_to       - destination array end address
1307   //   qword_count  - 64-bits element count, negative
1308   //   to           - scratch
1309   //   L_copy_bytes - entry label
1310   //   L_copy_8_bytes  - exit  label
1311   //
1312   void copy_bytes_forward(Register end_from, Register end_to,
1313                              Register qword_count, Register to,
1314                              Label& L_copy_bytes, Label& L_copy_8_bytes) {
1315     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1316     Label L_loop;
1317     __ align(OptoLoopAlignment);
1318     if (UseUnalignedLoadStores) {
1319       Label L_end;
1320       // Copy 64-bytes per iteration
1321       __ BIND(L_loop);
1322       if (UseAVX >= 2) {
1323         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1324         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1325         __ vmovdqu(xmm1, Address(end_from, qword_count, Address::times_8, -24));
1326         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm1);
1327       } else {
1328         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -56));
1329         __ movdqu(Address(end_to, qword_count, Address::times_8, -56), xmm0);
1330         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, -40));
1331         __ movdqu(Address(end_to, qword_count, Address::times_8, -40), xmm1);
1332         __ movdqu(xmm2, Address(end_from, qword_count, Address::times_8, -24));
1333         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm2);
1334         __ movdqu(xmm3, Address(end_from, qword_count, Address::times_8, - 8));
1335         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm3);
1336       }
1337       __ BIND(L_copy_bytes);
1338       __ addptr(qword_count, 8);
1339       __ jcc(Assembler::lessEqual, L_loop);
1340       __ subptr(qword_count, 4);  // sub(8) and add(4)
1341       __ jccb(Assembler::greater, L_end);
1342       // Copy trailing 32 bytes
1343       if (UseAVX >= 2) {
1344         __ vmovdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1345         __ vmovdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1346       } else {
1347         __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1348         __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1349         __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, - 8));
1350         __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm1);
1351       }
1352       __ addptr(qword_count, 4);
1353       __ BIND(L_end);
1354       if (UseAVX >= 2) {
1355         // clean upper bits of YMM registers
1356         __ vzeroupper();
1357       }
1358     } else {
1359       // Copy 32-bytes per iteration
1360       __ BIND(L_loop);
1361       __ movq(to, Address(end_from, qword_count, Address::times_8, -24));
1362       __ movq(Address(end_to, qword_count, Address::times_8, -24), to);
1363       __ movq(to, Address(end_from, qword_count, Address::times_8, -16));
1364       __ movq(Address(end_to, qword_count, Address::times_8, -16), to);
1365       __ movq(to, Address(end_from, qword_count, Address::times_8, - 8));
1366       __ movq(Address(end_to, qword_count, Address::times_8, - 8), to);
1367       __ movq(to, Address(end_from, qword_count, Address::times_8, - 0));
1368       __ movq(Address(end_to, qword_count, Address::times_8, - 0), to);
1369 
1370       __ BIND(L_copy_bytes);
1371       __ addptr(qword_count, 4);
1372       __ jcc(Assembler::lessEqual, L_loop);
1373     }
1374     __ subptr(qword_count, 4);
1375     __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords
1376   }
1377 
1378   // Copy big chunks backward
1379   //
1380   // Inputs:
1381   //   from         - source arrays address
1382   //   dest         - destination array address
1383   //   qword_count  - 64-bits element count
1384   //   to           - scratch
1385   //   L_copy_bytes - entry label
1386   //   L_copy_8_bytes  - exit  label
1387   //
1388   void copy_bytes_backward(Register from, Register dest,
1389                               Register qword_count, Register to,
1390                               Label& L_copy_bytes, Label& L_copy_8_bytes) {
1391     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1392     Label L_loop;
1393     __ align(OptoLoopAlignment);
1394     if (UseUnalignedLoadStores) {
1395       Label L_end;
1396       // Copy 64-bytes per iteration
1397       __ BIND(L_loop);
1398       if (UseAVX >= 2) {
1399         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 32));
1400         __ vmovdqu(Address(dest, qword_count, Address::times_8, 32), xmm0);
1401         __ vmovdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
1402         __ vmovdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
1403       } else {
1404         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 48));
1405         __ movdqu(Address(dest, qword_count, Address::times_8, 48), xmm0);
1406         __ movdqu(xmm1, Address(from, qword_count, Address::times_8, 32));
1407         __ movdqu(Address(dest, qword_count, Address::times_8, 32), xmm1);
1408         __ movdqu(xmm2, Address(from, qword_count, Address::times_8, 16));
1409         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm2);
1410         __ movdqu(xmm3, Address(from, qword_count, Address::times_8,  0));
1411         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm3);
1412       }
1413       __ BIND(L_copy_bytes);
1414       __ subptr(qword_count, 8);
1415       __ jcc(Assembler::greaterEqual, L_loop);
1416 
1417       __ addptr(qword_count, 4);  // add(8) and sub(4)
1418       __ jccb(Assembler::less, L_end);
1419       // Copy trailing 32 bytes
1420       if (UseAVX >= 2) {
1421         __ vmovdqu(xmm0, Address(from, qword_count, Address::times_8, 0));
1422         __ vmovdqu(Address(dest, qword_count, Address::times_8, 0), xmm0);
1423       } else {
1424         __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 16));
1425         __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm0);
1426         __ movdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
1427         __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
1428       }
1429       __ subptr(qword_count, 4);
1430       __ BIND(L_end);
1431       if (UseAVX >= 2) {
1432         // clean upper bits of YMM registers
1433         __ vzeroupper();
1434       }
1435     } else {
1436       // Copy 32-bytes per iteration
1437       __ BIND(L_loop);
1438       __ movq(to, Address(from, qword_count, Address::times_8, 24));
1439       __ movq(Address(dest, qword_count, Address::times_8, 24), to);
1440       __ movq(to, Address(from, qword_count, Address::times_8, 16));
1441       __ movq(Address(dest, qword_count, Address::times_8, 16), to);
1442       __ movq(to, Address(from, qword_count, Address::times_8,  8));
1443       __ movq(Address(dest, qword_count, Address::times_8,  8), to);
1444       __ movq(to, Address(from, qword_count, Address::times_8,  0));
1445       __ movq(Address(dest, qword_count, Address::times_8,  0), to);
1446 
1447       __ BIND(L_copy_bytes);
1448       __ subptr(qword_count, 4);
1449       __ jcc(Assembler::greaterEqual, L_loop);
1450     }
1451     __ addptr(qword_count, 4);
1452     __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
1453   }
1454 
1455 
1456   // Arguments:
1457   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1458   //             ignored
1459   //   name    - stub name string
1460   //
1461   // Inputs:
1462   //   c_rarg0   - source array address
1463   //   c_rarg1   - destination array address
1464   //   c_rarg2   - element count, treated as ssize_t, can be zero
1465   //
1466   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1467   // we let the hardware handle it.  The one to eight bytes within words,
1468   // dwords or qwords that span cache line boundaries will still be loaded
1469   // and stored atomically.
1470   //
1471   // Side Effects:
1472   //   disjoint_byte_copy_entry is set to the no-overlap entry point
1473   //   used by generate_conjoint_byte_copy().
1474   //
1475   address generate_disjoint_byte_copy(bool aligned, address* entry, const char *name) {
1476     __ align(CodeEntryAlignment);
1477     StubCodeMark mark(this, "StubRoutines", name);
1478     address start = __ pc();
1479 
1480     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1481     Label L_copy_byte, L_exit;
1482     const Register from        = rdi;  // source array address
1483     const Register to          = rsi;  // destination array address
1484     const Register count       = rdx;  // elements count
1485     const Register byte_count  = rcx;
1486     const Register qword_count = count;
1487     const Register end_from    = from; // source array end address
1488     const Register end_to      = to;   // destination array end address
1489     // End pointers are inclusive, and if count is not zero they point
1490     // to the last unit copied:  end_to[0] := end_from[0]
1491 
1492     __ enter(); // required for proper stackwalking of RuntimeStub frame
1493     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1494 
1495     if (entry != NULL) {
1496       *entry = __ pc();
1497        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1498       BLOCK_COMMENT("Entry:");
1499     }
1500 
1501     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1502                       // r9 and r10 may be used to save non-volatile registers
1503 
1504     // 'from', 'to' and 'count' are now valid
1505     __ movptr(byte_count, count);
1506     __ shrptr(count, 3); // count => qword_count
1507 
1508     // Copy from low to high addresses.  Use 'to' as scratch.
1509     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1510     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1511     __ negptr(qword_count); // make the count negative
1512     __ jmp(L_copy_bytes);
1513 
1514     // Copy trailing qwords
1515   __ BIND(L_copy_8_bytes);
1516     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1517     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1518     __ increment(qword_count);
1519     __ jcc(Assembler::notZero, L_copy_8_bytes);
1520 
1521     // Check for and copy trailing dword
1522   __ BIND(L_copy_4_bytes);
1523     __ testl(byte_count, 4);
1524     __ jccb(Assembler::zero, L_copy_2_bytes);
1525     __ movl(rax, Address(end_from, 8));
1526     __ movl(Address(end_to, 8), rax);
1527 
1528     __ addptr(end_from, 4);
1529     __ addptr(end_to, 4);
1530 
1531     // Check for and copy trailing word
1532   __ BIND(L_copy_2_bytes);
1533     __ testl(byte_count, 2);
1534     __ jccb(Assembler::zero, L_copy_byte);
1535     __ movw(rax, Address(end_from, 8));
1536     __ movw(Address(end_to, 8), rax);
1537 
1538     __ addptr(end_from, 2);
1539     __ addptr(end_to, 2);
1540 
1541     // Check for and copy trailing byte
1542   __ BIND(L_copy_byte);
1543     __ testl(byte_count, 1);
1544     __ jccb(Assembler::zero, L_exit);
1545     __ movb(rax, Address(end_from, 8));
1546     __ movb(Address(end_to, 8), rax);
1547 
1548   __ BIND(L_exit);
1549     restore_arg_regs();
1550     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1551     __ xorptr(rax, rax); // return 0
1552     __ leave(); // required for proper stackwalking of RuntimeStub frame
1553     __ ret(0);
1554 
1555     // Copy in multi-bytes chunks
1556     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1557     __ jmp(L_copy_4_bytes);
1558 
1559     return start;
1560   }
1561 
1562   // Arguments:
1563   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1564   //             ignored
1565   //   name    - stub name string
1566   //
1567   // Inputs:
1568   //   c_rarg0   - source array address
1569   //   c_rarg1   - destination array address
1570   //   c_rarg2   - element count, treated as ssize_t, can be zero
1571   //
1572   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1573   // we let the hardware handle it.  The one to eight bytes within words,
1574   // dwords or qwords that span cache line boundaries will still be loaded
1575   // and stored atomically.
1576   //
1577   address generate_conjoint_byte_copy(bool aligned, address nooverlap_target,
1578                                       address* entry, const char *name) {
1579     __ align(CodeEntryAlignment);
1580     StubCodeMark mark(this, "StubRoutines", name);
1581     address start = __ pc();
1582 
1583     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1584     const Register from        = rdi;  // source array address
1585     const Register to          = rsi;  // destination array address
1586     const Register count       = rdx;  // elements count
1587     const Register byte_count  = rcx;
1588     const Register qword_count = count;
1589 
1590     __ enter(); // required for proper stackwalking of RuntimeStub frame
1591     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1592 
1593     if (entry != NULL) {
1594       *entry = __ pc();
1595       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1596       BLOCK_COMMENT("Entry:");
1597     }
1598 
1599     array_overlap_test(nooverlap_target, Address::times_1);
1600     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1601                       // r9 and r10 may be used to save non-volatile registers
1602 
1603     // 'from', 'to' and 'count' are now valid
1604     __ movptr(byte_count, count);
1605     __ shrptr(count, 3);   // count => qword_count
1606 
1607     // Copy from high to low addresses.
1608 
1609     // Check for and copy trailing byte
1610     __ testl(byte_count, 1);
1611     __ jcc(Assembler::zero, L_copy_2_bytes);
1612     __ movb(rax, Address(from, byte_count, Address::times_1, -1));
1613     __ movb(Address(to, byte_count, Address::times_1, -1), rax);
1614     __ decrement(byte_count); // Adjust for possible trailing word
1615 
1616     // Check for and copy trailing word
1617   __ BIND(L_copy_2_bytes);
1618     __ testl(byte_count, 2);
1619     __ jcc(Assembler::zero, L_copy_4_bytes);
1620     __ movw(rax, Address(from, byte_count, Address::times_1, -2));
1621     __ movw(Address(to, byte_count, Address::times_1, -2), rax);
1622 
1623     // Check for and copy trailing dword
1624   __ BIND(L_copy_4_bytes);
1625     __ testl(byte_count, 4);
1626     __ jcc(Assembler::zero, L_copy_bytes);
1627     __ movl(rax, Address(from, qword_count, Address::times_8));
1628     __ movl(Address(to, qword_count, Address::times_8), rax);
1629     __ jmp(L_copy_bytes);
1630 
1631     // Copy trailing qwords
1632   __ BIND(L_copy_8_bytes);
1633     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1634     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1635     __ decrement(qword_count);
1636     __ jcc(Assembler::notZero, L_copy_8_bytes);
1637 
1638     restore_arg_regs();
1639     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1640     __ xorptr(rax, rax); // return 0
1641     __ leave(); // required for proper stackwalking of RuntimeStub frame
1642     __ ret(0);
1643 
1644     // Copy in multi-bytes chunks
1645     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1646 
1647     restore_arg_regs();
1648     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); // Update counter after rscratch1 is free
1649     __ xorptr(rax, rax); // return 0
1650     __ leave(); // required for proper stackwalking of RuntimeStub frame
1651     __ ret(0);
1652 
1653     return start;
1654   }
1655 
1656   // Arguments:
1657   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1658   //             ignored
1659   //   name    - stub name string
1660   //
1661   // Inputs:
1662   //   c_rarg0   - source array address
1663   //   c_rarg1   - destination array address
1664   //   c_rarg2   - element count, treated as ssize_t, can be zero
1665   //
1666   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1667   // let the hardware handle it.  The two or four words within dwords
1668   // or qwords that span cache line boundaries will still be loaded
1669   // and stored atomically.
1670   //
1671   // Side Effects:
1672   //   disjoint_short_copy_entry is set to the no-overlap entry point
1673   //   used by generate_conjoint_short_copy().
1674   //
1675   address generate_disjoint_short_copy(bool aligned, address *entry, const char *name) {
1676     __ align(CodeEntryAlignment);
1677     StubCodeMark mark(this, "StubRoutines", name);
1678     address start = __ pc();
1679 
1680     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
1681     const Register from        = rdi;  // source array address
1682     const Register to          = rsi;  // destination array address
1683     const Register count       = rdx;  // elements count
1684     const Register word_count  = rcx;
1685     const Register qword_count = count;
1686     const Register end_from    = from; // source array end address
1687     const Register end_to      = to;   // destination array end address
1688     // End pointers are inclusive, and if count is not zero they point
1689     // to the last unit copied:  end_to[0] := end_from[0]
1690 
1691     __ enter(); // required for proper stackwalking of RuntimeStub frame
1692     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1693 
1694     if (entry != NULL) {
1695       *entry = __ pc();
1696       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1697       BLOCK_COMMENT("Entry:");
1698     }
1699 
1700     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1701                       // r9 and r10 may be used to save non-volatile registers
1702 
1703     // 'from', 'to' and 'count' are now valid
1704     __ movptr(word_count, count);
1705     __ shrptr(count, 2); // count => qword_count
1706 
1707     // Copy from low to high addresses.  Use 'to' as scratch.
1708     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1709     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1710     __ negptr(qword_count);
1711     __ jmp(L_copy_bytes);
1712 
1713     // Copy trailing qwords
1714   __ BIND(L_copy_8_bytes);
1715     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1716     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1717     __ increment(qword_count);
1718     __ jcc(Assembler::notZero, L_copy_8_bytes);
1719 
1720     // Original 'dest' is trashed, so we can't use it as a
1721     // base register for a possible trailing word copy
1722 
1723     // Check for and copy trailing dword
1724   __ BIND(L_copy_4_bytes);
1725     __ testl(word_count, 2);
1726     __ jccb(Assembler::zero, L_copy_2_bytes);
1727     __ movl(rax, Address(end_from, 8));
1728     __ movl(Address(end_to, 8), rax);
1729 
1730     __ addptr(end_from, 4);
1731     __ addptr(end_to, 4);
1732 
1733     // Check for and copy trailing word
1734   __ BIND(L_copy_2_bytes);
1735     __ testl(word_count, 1);
1736     __ jccb(Assembler::zero, L_exit);
1737     __ movw(rax, Address(end_from, 8));
1738     __ movw(Address(end_to, 8), rax);
1739 
1740   __ BIND(L_exit);
1741     restore_arg_regs();
1742     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1743     __ xorptr(rax, rax); // return 0
1744     __ leave(); // required for proper stackwalking of RuntimeStub frame
1745     __ ret(0);
1746 
1747     // Copy in multi-bytes chunks
1748     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1749     __ jmp(L_copy_4_bytes);
1750 
1751     return start;
1752   }
1753 
1754   address generate_fill(BasicType t, bool aligned, const char *name) {
1755     __ align(CodeEntryAlignment);
1756     StubCodeMark mark(this, "StubRoutines", name);
1757     address start = __ pc();
1758 
1759     BLOCK_COMMENT("Entry:");
1760 
1761     const Register to       = c_rarg0;  // source array address
1762     const Register value    = c_rarg1;  // value
1763     const Register count    = c_rarg2;  // elements count
1764 
1765     __ enter(); // required for proper stackwalking of RuntimeStub frame
1766 
1767     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1768 
1769     __ leave(); // required for proper stackwalking of RuntimeStub frame
1770     __ ret(0);
1771     return start;
1772   }
1773 
1774   // Arguments:
1775   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1776   //             ignored
1777   //   name    - stub name string
1778   //
1779   // Inputs:
1780   //   c_rarg0   - source array address
1781   //   c_rarg1   - destination array address
1782   //   c_rarg2   - element count, treated as ssize_t, can be zero
1783   //
1784   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1785   // let the hardware handle it.  The two or four words within dwords
1786   // or qwords that span cache line boundaries will still be loaded
1787   // and stored atomically.
1788   //
1789   address generate_conjoint_short_copy(bool aligned, address nooverlap_target,
1790                                        address *entry, const char *name) {
1791     __ align(CodeEntryAlignment);
1792     StubCodeMark mark(this, "StubRoutines", name);
1793     address start = __ pc();
1794 
1795     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes;
1796     const Register from        = rdi;  // source array address
1797     const Register to          = rsi;  // destination array address
1798     const Register count       = rdx;  // elements count
1799     const Register word_count  = rcx;
1800     const Register qword_count = count;
1801 
1802     __ enter(); // required for proper stackwalking of RuntimeStub frame
1803     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1804 
1805     if (entry != NULL) {
1806       *entry = __ pc();
1807       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1808       BLOCK_COMMENT("Entry:");
1809     }
1810 
1811     array_overlap_test(nooverlap_target, Address::times_2);
1812     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1813                       // r9 and r10 may be used to save non-volatile registers
1814 
1815     // 'from', 'to' and 'count' are now valid
1816     __ movptr(word_count, count);
1817     __ shrptr(count, 2); // count => qword_count
1818 
1819     // Copy from high to low addresses.  Use 'to' as scratch.
1820 
1821     // Check for and copy trailing word
1822     __ testl(word_count, 1);
1823     __ jccb(Assembler::zero, L_copy_4_bytes);
1824     __ movw(rax, Address(from, word_count, Address::times_2, -2));
1825     __ movw(Address(to, word_count, Address::times_2, -2), rax);
1826 
1827     // Check for and copy trailing dword
1828   __ BIND(L_copy_4_bytes);
1829     __ testl(word_count, 2);
1830     __ jcc(Assembler::zero, L_copy_bytes);
1831     __ movl(rax, Address(from, qword_count, Address::times_8));
1832     __ movl(Address(to, qword_count, Address::times_8), rax);
1833     __ jmp(L_copy_bytes);
1834 
1835     // Copy trailing qwords
1836   __ BIND(L_copy_8_bytes);
1837     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1838     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1839     __ decrement(qword_count);
1840     __ jcc(Assembler::notZero, L_copy_8_bytes);
1841 
1842     restore_arg_regs();
1843     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1844     __ xorptr(rax, rax); // return 0
1845     __ leave(); // required for proper stackwalking of RuntimeStub frame
1846     __ ret(0);
1847 
1848     // Copy in multi-bytes chunks
1849     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1850 
1851     restore_arg_regs();
1852     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); // Update counter after rscratch1 is free
1853     __ xorptr(rax, rax); // return 0
1854     __ leave(); // required for proper stackwalking of RuntimeStub frame
1855     __ ret(0);
1856 
1857     return start;
1858   }
1859 
1860   // Arguments:
1861   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1862   //             ignored
1863   //   is_oop  - true => oop array, so generate store check code
1864   //   name    - stub name string
1865   //
1866   // Inputs:
1867   //   c_rarg0   - source array address
1868   //   c_rarg1   - destination array address
1869   //   c_rarg2   - element count, treated as ssize_t, can be zero
1870   //
1871   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1872   // the hardware handle it.  The two dwords within qwords that span
1873   // cache line boundaries will still be loaded and stored atomicly.
1874   //
1875   // Side Effects:
1876   //   disjoint_int_copy_entry is set to the no-overlap entry point
1877   //   used by generate_conjoint_int_oop_copy().
1878   //
1879   address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry,
1880                                          const char *name, bool dest_uninitialized = false) {
1881     __ align(CodeEntryAlignment);
1882     StubCodeMark mark(this, "StubRoutines", name);
1883     address start = __ pc();
1884 
1885     Label L_copy_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
1886     const Register from        = rdi;  // source array address
1887     const Register to          = rsi;  // destination array address
1888     const Register count       = rdx;  // elements count
1889     const Register dword_count = rcx;
1890     const Register qword_count = count;
1891     const Register end_from    = from; // source array end address
1892     const Register end_to      = to;   // destination array end address
1893     const Register saved_to    = r11;  // saved destination array address
1894     // End pointers are inclusive, and if count is not zero they point
1895     // to the last unit copied:  end_to[0] := end_from[0]
1896 
1897     __ enter(); // required for proper stackwalking of RuntimeStub frame
1898     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1899 
1900     if (entry != NULL) {
1901       *entry = __ pc();
1902       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1903       BLOCK_COMMENT("Entry:");
1904     }
1905 
1906     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1907                       // r9 and r10 may be used to save non-volatile registers
1908     if (is_oop) {
1909       __ movq(saved_to, to);
1910       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
1911     }
1912 
1913     // 'from', 'to' and 'count' are now valid
1914     __ movptr(dword_count, count);
1915     __ shrptr(count, 1); // count => qword_count
1916 
1917     // Copy from low to high addresses.  Use 'to' as scratch.
1918     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1919     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1920     __ negptr(qword_count);
1921     __ jmp(L_copy_bytes);
1922 
1923     // Copy trailing qwords
1924   __ BIND(L_copy_8_bytes);
1925     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1926     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1927     __ increment(qword_count);
1928     __ jcc(Assembler::notZero, L_copy_8_bytes);
1929 
1930     // Check for and copy trailing dword
1931   __ BIND(L_copy_4_bytes);
1932     __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
1933     __ jccb(Assembler::zero, L_exit);
1934     __ movl(rax, Address(end_from, 8));
1935     __ movl(Address(end_to, 8), rax);
1936 
1937   __ BIND(L_exit);
1938     if (is_oop) {
1939       gen_write_ref_array_post_barrier(saved_to, dword_count, rax);
1940     }
1941     restore_arg_regs();
1942     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
1943     __ xorptr(rax, rax); // return 0
1944     __ leave(); // required for proper stackwalking of RuntimeStub frame
1945     __ ret(0);
1946 
1947     // Copy in multi-bytes chunks
1948     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
1949     __ jmp(L_copy_4_bytes);
1950 
1951     return start;
1952   }
1953 
1954   // Arguments:
1955   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1956   //             ignored
1957   //   is_oop  - true => oop array, so generate store check code
1958   //   name    - stub name string
1959   //
1960   // Inputs:
1961   //   c_rarg0   - source array address
1962   //   c_rarg1   - destination array address
1963   //   c_rarg2   - element count, treated as ssize_t, can be zero
1964   //
1965   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1966   // the hardware handle it.  The two dwords within qwords that span
1967   // cache line boundaries will still be loaded and stored atomicly.
1968   //
1969   address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, address nooverlap_target,
1970                                          address *entry, const char *name,
1971                                          bool dest_uninitialized = false) {
1972     __ align(CodeEntryAlignment);
1973     StubCodeMark mark(this, "StubRoutines", name);
1974     address start = __ pc();
1975 
1976     Label L_copy_bytes, L_copy_8_bytes, L_copy_2_bytes, L_exit;
1977     const Register from        = rdi;  // source array address
1978     const Register to          = rsi;  // destination array address
1979     const Register count       = rdx;  // elements count
1980     const Register dword_count = rcx;
1981     const Register qword_count = count;
1982 
1983     __ enter(); // required for proper stackwalking of RuntimeStub frame
1984     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1985 
1986     if (entry != NULL) {
1987       *entry = __ pc();
1988        // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1989       BLOCK_COMMENT("Entry:");
1990     }
1991 
1992     array_overlap_test(nooverlap_target, Address::times_4);
1993     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1994                       // r9 and r10 may be used to save non-volatile registers
1995 
1996     if (is_oop) {
1997       // no registers are destroyed by this call
1998       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
1999     }
2000 
2001     assert_clean_int(count, rax); // Make sure 'count' is clean int.
2002     // 'from', 'to' and 'count' are now valid
2003     __ movptr(dword_count, count);
2004     __ shrptr(count, 1); // count => qword_count
2005 
2006     // Copy from high to low addresses.  Use 'to' as scratch.
2007 
2008     // Check for and copy trailing dword
2009     __ testl(dword_count, 1);
2010     __ jcc(Assembler::zero, L_copy_bytes);
2011     __ movl(rax, Address(from, dword_count, Address::times_4, -4));
2012     __ movl(Address(to, dword_count, Address::times_4, -4), rax);
2013     __ jmp(L_copy_bytes);
2014 
2015     // Copy trailing qwords
2016   __ BIND(L_copy_8_bytes);
2017     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2018     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2019     __ decrement(qword_count);
2020     __ jcc(Assembler::notZero, L_copy_8_bytes);
2021 
2022     if (is_oop) {
2023       __ jmp(L_exit);
2024     }
2025     restore_arg_regs();
2026     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
2027     __ xorptr(rax, rax); // return 0
2028     __ leave(); // required for proper stackwalking of RuntimeStub frame
2029     __ ret(0);
2030 
2031     // Copy in multi-bytes chunks
2032     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2033 
2034   __ BIND(L_exit);
2035     if (is_oop) {
2036       gen_write_ref_array_post_barrier(to, dword_count, rax);
2037     }
2038     restore_arg_regs();
2039     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); // Update counter after rscratch1 is free
2040     __ xorptr(rax, rax); // return 0
2041     __ leave(); // required for proper stackwalking of RuntimeStub frame
2042     __ ret(0);
2043 
2044     return start;
2045   }
2046 
2047   // Arguments:
2048   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2049   //             ignored
2050   //   is_oop  - true => oop array, so generate store check code
2051   //   name    - stub name string
2052   //
2053   // Inputs:
2054   //   c_rarg0   - source array address
2055   //   c_rarg1   - destination array address
2056   //   c_rarg2   - element count, treated as ssize_t, can be zero
2057   //
2058  // Side Effects:
2059   //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
2060   //   no-overlap entry point used by generate_conjoint_long_oop_copy().
2061   //
2062   address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry,
2063                                           const char *name, bool dest_uninitialized = false) {
2064     __ align(CodeEntryAlignment);
2065     StubCodeMark mark(this, "StubRoutines", name);
2066     address start = __ pc();
2067 
2068     Label L_copy_bytes, L_copy_8_bytes, L_exit;
2069     const Register from        = rdi;  // source array address
2070     const Register to          = rsi;  // destination array address
2071     const Register qword_count = rdx;  // elements count
2072     const Register end_from    = from; // source array end address
2073     const Register end_to      = rcx;  // destination array end address
2074     const Register saved_to    = to;
2075     const Register saved_count = r11;
2076     // End pointers are inclusive, and if count is not zero they point
2077     // to the last unit copied:  end_to[0] := end_from[0]
2078 
2079     __ enter(); // required for proper stackwalking of RuntimeStub frame
2080     // Save no-overlap entry point for generate_conjoint_long_oop_copy()
2081     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2082 
2083     if (entry != NULL) {
2084       *entry = __ pc();
2085       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2086       BLOCK_COMMENT("Entry:");
2087     }
2088 
2089     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2090                       // r9 and r10 may be used to save non-volatile registers
2091     // 'from', 'to' and 'qword_count' are now valid
2092     if (is_oop) {
2093       // Save to and count for store barrier
2094       __ movptr(saved_count, qword_count);
2095       // no registers are destroyed by this call
2096       gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized);
2097     }
2098 
2099     // Copy from low to high addresses.  Use 'to' as scratch.
2100     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
2101     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
2102     __ negptr(qword_count);
2103     __ jmp(L_copy_bytes);
2104 
2105     // Copy trailing qwords
2106   __ BIND(L_copy_8_bytes);
2107     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
2108     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
2109     __ increment(qword_count);
2110     __ jcc(Assembler::notZero, L_copy_8_bytes);
2111 
2112     if (is_oop) {
2113       __ jmp(L_exit);
2114     } else {
2115       restore_arg_regs();
2116       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2117       __ xorptr(rax, rax); // return 0
2118       __ leave(); // required for proper stackwalking of RuntimeStub frame
2119       __ ret(0);
2120     }
2121 
2122     // Copy in multi-bytes chunks
2123     copy_bytes_forward(end_from, end_to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2124 
2125     if (is_oop) {
2126     __ BIND(L_exit);
2127       gen_write_ref_array_post_barrier(saved_to, saved_count, rax);
2128     }
2129     restore_arg_regs();
2130     if (is_oop) {
2131       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2132     } else {
2133       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2134     }
2135     __ xorptr(rax, rax); // return 0
2136     __ leave(); // required for proper stackwalking of RuntimeStub frame
2137     __ ret(0);
2138 
2139     return start;
2140   }
2141 
2142   // Arguments:
2143   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2144   //             ignored
2145   //   is_oop  - true => oop array, so generate store check code
2146   //   name    - stub name string
2147   //
2148   // Inputs:
2149   //   c_rarg0   - source array address
2150   //   c_rarg1   - destination array address
2151   //   c_rarg2   - element count, treated as ssize_t, can be zero
2152   //
2153   address generate_conjoint_long_oop_copy(bool aligned, bool is_oop,
2154                                           address nooverlap_target, address *entry,
2155                                           const char *name, bool dest_uninitialized = false) {
2156     __ align(CodeEntryAlignment);
2157     StubCodeMark mark(this, "StubRoutines", name);
2158     address start = __ pc();
2159 
2160     Label L_copy_bytes, L_copy_8_bytes, L_exit;
2161     const Register from        = rdi;  // source array address
2162     const Register to          = rsi;  // destination array address
2163     const Register qword_count = rdx;  // elements count
2164     const Register saved_count = rcx;
2165 
2166     __ enter(); // required for proper stackwalking of RuntimeStub frame
2167     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2168 
2169     if (entry != NULL) {
2170       *entry = __ pc();
2171       // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2172       BLOCK_COMMENT("Entry:");
2173     }
2174 
2175     array_overlap_test(nooverlap_target, Address::times_8);
2176     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2177                       // r9 and r10 may be used to save non-volatile registers
2178     // 'from', 'to' and 'qword_count' are now valid
2179     if (is_oop) {
2180       // Save to and count for store barrier
2181       __ movptr(saved_count, qword_count);
2182       // No registers are destroyed by this call
2183       gen_write_ref_array_pre_barrier(to, saved_count, dest_uninitialized);
2184     }
2185 
2186     __ jmp(L_copy_bytes);
2187 
2188     // Copy trailing qwords
2189   __ BIND(L_copy_8_bytes);
2190     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2191     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2192     __ decrement(qword_count);
2193     __ jcc(Assembler::notZero, L_copy_8_bytes);
2194 
2195     if (is_oop) {
2196       __ jmp(L_exit);
2197     } else {
2198       restore_arg_regs();
2199       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2200       __ xorptr(rax, rax); // return 0
2201       __ leave(); // required for proper stackwalking of RuntimeStub frame
2202       __ ret(0);
2203     }
2204 
2205     // Copy in multi-bytes chunks
2206     copy_bytes_backward(from, to, qword_count, rax, L_copy_bytes, L_copy_8_bytes);
2207 
2208     if (is_oop) {
2209     __ BIND(L_exit);
2210       gen_write_ref_array_post_barrier(to, saved_count, rax);
2211     }
2212     restore_arg_regs();
2213     if (is_oop) {
2214       inc_counter_np(SharedRuntime::_oop_array_copy_ctr); // Update counter after rscratch1 is free
2215     } else {
2216       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); // Update counter after rscratch1 is free
2217     }
2218     __ xorptr(rax, rax); // return 0
2219     __ leave(); // required for proper stackwalking of RuntimeStub frame
2220     __ ret(0);
2221 
2222     return start;
2223   }
2224 
2225 
2226   // Helper for generating a dynamic type check.
2227   // Smashes no registers.
2228   void generate_type_check(Register sub_klass,
2229                            Register super_check_offset,
2230                            Register super_klass,
2231                            Label& L_success) {
2232     assert_different_registers(sub_klass, super_check_offset, super_klass);
2233 
2234     BLOCK_COMMENT("type_check:");
2235 
2236     Label L_miss;
2237 
2238     __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg,        &L_success, &L_miss, NULL,
2239                                      super_check_offset);
2240     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL);
2241 
2242     // Fall through on failure!
2243     __ BIND(L_miss);
2244   }
2245 
2246   //
2247   //  Generate checkcasting array copy stub
2248   //
2249   //  Input:
2250   //    c_rarg0   - source array address
2251   //    c_rarg1   - destination array address
2252   //    c_rarg2   - element count, treated as ssize_t, can be zero
2253   //    c_rarg3   - size_t ckoff (super_check_offset)
2254   // not Win64
2255   //    c_rarg4   - oop ckval (super_klass)
2256   // Win64
2257   //    rsp+40    - oop ckval (super_klass)
2258   //
2259   //  Output:
2260   //    rax ==  0  -  success
2261   //    rax == -1^K - failure, where K is partial transfer count
2262   //
2263   address generate_checkcast_copy(const char *name, address *entry,
2264                                   bool dest_uninitialized = false) {
2265 
2266     Label L_load_element, L_store_element, L_do_card_marks, L_done;
2267 
2268     // Input registers (after setup_arg_regs)
2269     const Register from        = rdi;   // source array address
2270     const Register to          = rsi;   // destination array address
2271     const Register length      = rdx;   // elements count
2272     const Register ckoff       = rcx;   // super_check_offset
2273     const Register ckval       = r8;    // super_klass
2274 
2275     // Registers used as temps (r13, r14 are save-on-entry)
2276     const Register end_from    = from;  // source array end address
2277     const Register end_to      = r13;   // destination array end address
2278     const Register count       = rdx;   // -(count_remaining)
2279     const Register r14_length  = r14;   // saved copy of length
2280     // End pointers are inclusive, and if length is not zero they point
2281     // to the last unit copied:  end_to[0] := end_from[0]
2282 
2283     const Register rax_oop    = rax;    // actual oop copied
2284     const Register r11_klass  = r11;    // oop._klass
2285 
2286     //---------------------------------------------------------------
2287     // Assembler stub will be used for this call to arraycopy
2288     // if the two arrays are subtypes of Object[] but the
2289     // destination array type is not equal to or a supertype
2290     // of the source type.  Each element must be separately
2291     // checked.
2292 
2293     __ align(CodeEntryAlignment);
2294     StubCodeMark mark(this, "StubRoutines", name);
2295     address start = __ pc();
2296 
2297     __ enter(); // required for proper stackwalking of RuntimeStub frame
2298 
2299 #ifdef ASSERT
2300     // caller guarantees that the arrays really are different
2301     // otherwise, we would have to make conjoint checks
2302     { Label L;
2303       array_overlap_test(L, TIMES_OOP);
2304       __ stop("checkcast_copy within a single array");
2305       __ bind(L);
2306     }
2307 #endif //ASSERT
2308 
2309     setup_arg_regs(4); // from => rdi, to => rsi, length => rdx
2310                        // ckoff => rcx, ckval => r8
2311                        // r9 and r10 may be used to save non-volatile registers
2312 #ifdef _WIN64
2313     // last argument (#4) is on stack on Win64
2314     __ movptr(ckval, Address(rsp, 6 * wordSize));
2315 #endif
2316 
2317     // Caller of this entry point must set up the argument registers.
2318     if (entry != NULL) {
2319       *entry = __ pc();
2320       BLOCK_COMMENT("Entry:");
2321     }
2322 
2323     // allocate spill slots for r13, r14
2324     enum {
2325       saved_r13_offset,
2326       saved_r14_offset,
2327       saved_rbp_offset
2328     };
2329     __ subptr(rsp, saved_rbp_offset * wordSize);
2330     __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
2331     __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
2332 
2333     // check that int operands are properly extended to size_t
2334     assert_clean_int(length, rax);
2335     assert_clean_int(ckoff, rax);
2336 
2337 #ifdef ASSERT
2338     BLOCK_COMMENT("assert consistent ckoff/ckval");
2339     // The ckoff and ckval must be mutually consistent,
2340     // even though caller generates both.
2341     { Label L;
2342       int sco_offset = in_bytes(Klass::super_check_offset_offset());
2343       __ cmpl(ckoff, Address(ckval, sco_offset));
2344       __ jcc(Assembler::equal, L);
2345       __ stop("super_check_offset inconsistent");
2346       __ bind(L);
2347     }
2348 #endif //ASSERT
2349 
2350     // Loop-invariant addresses.  They are exclusive end pointers.
2351     Address end_from_addr(from, length, TIMES_OOP, 0);
2352     Address   end_to_addr(to,   length, TIMES_OOP, 0);
2353     // Loop-variant addresses.  They assume post-incremented count < 0.
2354     Address from_element_addr(end_from, count, TIMES_OOP, 0);
2355     Address   to_element_addr(end_to,   count, TIMES_OOP, 0);
2356 
2357     gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
2358 
2359     // Copy from low to high addresses, indexed from the end of each array.
2360     __ lea(end_from, end_from_addr);
2361     __ lea(end_to,   end_to_addr);
2362     __ movptr(r14_length, length);        // save a copy of the length
2363     assert(length == count, "");          // else fix next line:
2364     __ negptr(count);                     // negate and test the length
2365     __ jcc(Assembler::notZero, L_load_element);
2366 
2367     // Empty array:  Nothing to do.
2368     __ xorptr(rax, rax);                  // return 0 on (trivial) success
2369     __ jmp(L_done);
2370 
2371     // ======== begin loop ========
2372     // (Loop is rotated; its entry is L_load_element.)
2373     // Loop control:
2374     //   for (count = -count; count != 0; count++)
2375     // Base pointers src, dst are biased by 8*(count-1),to last element.
2376     __ align(OptoLoopAlignment);
2377 
2378     __ BIND(L_store_element);
2379     __ store_heap_oop(to_element_addr, rax_oop);  // store the oop
2380     __ increment(count);               // increment the count toward zero
2381     __ jcc(Assembler::zero, L_do_card_marks);
2382 
2383     // ======== loop entry is here ========
2384     __ BIND(L_load_element);
2385     __ load_heap_oop(rax_oop, from_element_addr); // load the oop
2386     __ testptr(rax_oop, rax_oop);
2387     __ jcc(Assembler::zero, L_store_element);
2388 
2389     __ load_klass(r11_klass, rax_oop);// query the object klass
2390     generate_type_check(r11_klass, ckoff, ckval, L_store_element);
2391     // ======== end loop ========
2392 
2393     // It was a real error; we must depend on the caller to finish the job.
2394     // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.
2395     // Emit GC store barriers for the oops we have copied (r14 + rdx),
2396     // and report their number to the caller.
2397     assert_different_registers(rax, r14_length, count, to, end_to, rcx, rscratch1);
2398     Label L_post_barrier;
2399     __ addptr(r14_length, count);     // K = (original - remaining) oops
2400     __ movptr(rax, r14_length);       // save the value
2401     __ notptr(rax);                   // report (-1^K) to caller (does not affect flags)
2402     __ jccb(Assembler::notZero, L_post_barrier);
2403     __ jmp(L_done); // K == 0, nothing was copied, skip post barrier
2404 
2405     // Come here on success only.
2406     __ BIND(L_do_card_marks);
2407     __ xorptr(rax, rax);              // return 0 on success
2408 
2409     __ BIND(L_post_barrier);
2410     gen_write_ref_array_post_barrier(to, r14_length, rscratch1);
2411 
2412     // Common exit point (success or failure).
2413     __ BIND(L_done);
2414     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
2415     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
2416     restore_arg_regs();
2417     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr); // Update counter after rscratch1 is free
2418     __ leave(); // required for proper stackwalking of RuntimeStub frame
2419     __ ret(0);
2420 
2421     return start;
2422   }
2423 
2424   //
2425   //  Generate 'unsafe' array copy stub
2426   //  Though just as safe as the other stubs, it takes an unscaled
2427   //  size_t argument instead of an element count.
2428   //
2429   //  Input:
2430   //    c_rarg0   - source array address
2431   //    c_rarg1   - destination array address
2432   //    c_rarg2   - byte count, treated as ssize_t, can be zero
2433   //
2434   // Examines the alignment of the operands and dispatches
2435   // to a long, int, short, or byte copy loop.
2436   //
2437   address generate_unsafe_copy(const char *name,
2438                                address byte_copy_entry, address short_copy_entry,
2439                                address int_copy_entry, address long_copy_entry) {
2440 
2441     Label L_long_aligned, L_int_aligned, L_short_aligned;
2442 
2443     // Input registers (before setup_arg_regs)
2444     const Register from        = c_rarg0;  // source array address
2445     const Register to          = c_rarg1;  // destination array address
2446     const Register size        = c_rarg2;  // byte count (size_t)
2447 
2448     // Register used as a temp
2449     const Register bits        = rax;      // test copy of low bits
2450 
2451     __ align(CodeEntryAlignment);
2452     StubCodeMark mark(this, "StubRoutines", name);
2453     address start = __ pc();
2454 
2455     __ enter(); // required for proper stackwalking of RuntimeStub frame
2456 
2457     // bump this on entry, not on exit:
2458     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2459 
2460     __ mov(bits, from);
2461     __ orptr(bits, to);
2462     __ orptr(bits, size);
2463 
2464     __ testb(bits, BytesPerLong-1);
2465     __ jccb(Assembler::zero, L_long_aligned);
2466 
2467     __ testb(bits, BytesPerInt-1);
2468     __ jccb(Assembler::zero, L_int_aligned);
2469 
2470     __ testb(bits, BytesPerShort-1);
2471     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
2472 
2473     __ BIND(L_short_aligned);
2474     __ shrptr(size, LogBytesPerShort); // size => short_count
2475     __ jump(RuntimeAddress(short_copy_entry));
2476 
2477     __ BIND(L_int_aligned);
2478     __ shrptr(size, LogBytesPerInt); // size => int_count
2479     __ jump(RuntimeAddress(int_copy_entry));
2480 
2481     __ BIND(L_long_aligned);
2482     __ shrptr(size, LogBytesPerLong); // size => qword_count
2483     __ jump(RuntimeAddress(long_copy_entry));
2484 
2485     return start;
2486   }
2487 
2488   // Perform range checks on the proposed arraycopy.
2489   // Kills temp, but nothing else.
2490   // Also, clean the sign bits of src_pos and dst_pos.
2491   void arraycopy_range_checks(Register src,     // source array oop (c_rarg0)
2492                               Register src_pos, // source position (c_rarg1)
2493                               Register dst,     // destination array oo (c_rarg2)
2494                               Register dst_pos, // destination position (c_rarg3)
2495                               Register length,
2496                               Register temp,
2497                               Label& L_failed) {
2498     BLOCK_COMMENT("arraycopy_range_checks:");
2499 
2500     //  if (src_pos + length > arrayOop(src)->length())  FAIL;
2501     __ movl(temp, length);
2502     __ addl(temp, src_pos);             // src_pos + length
2503     __ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
2504     __ jcc(Assembler::above, L_failed);
2505 
2506     //  if (dst_pos + length > arrayOop(dst)->length())  FAIL;
2507     __ movl(temp, length);
2508     __ addl(temp, dst_pos);             // dst_pos + length
2509     __ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2510     __ jcc(Assembler::above, L_failed);
2511 
2512     // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
2513     // Move with sign extension can be used since they are positive.
2514     __ movslq(src_pos, src_pos);
2515     __ movslq(dst_pos, dst_pos);
2516 
2517     BLOCK_COMMENT("arraycopy_range_checks done");
2518   }
2519 
2520   //
2521   //  Generate generic array copy stubs
2522   //
2523   //  Input:
2524   //    c_rarg0    -  src oop
2525   //    c_rarg1    -  src_pos (32-bits)
2526   //    c_rarg2    -  dst oop
2527   //    c_rarg3    -  dst_pos (32-bits)
2528   // not Win64
2529   //    c_rarg4    -  element count (32-bits)
2530   // Win64
2531   //    rsp+40     -  element count (32-bits)
2532   //
2533   //  Output:
2534   //    rax ==  0  -  success
2535   //    rax == -1^K - failure, where K is partial transfer count
2536   //
2537   address generate_generic_copy(const char *name,
2538                                 address byte_copy_entry, address short_copy_entry,
2539                                 address int_copy_entry, address oop_copy_entry,
2540                                 address long_copy_entry, address checkcast_copy_entry) {
2541 
2542     Label L_failed, L_failed_0, L_objArray;
2543     Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
2544 
2545     // Input registers
2546     const Register src        = c_rarg0;  // source array oop
2547     const Register src_pos    = c_rarg1;  // source position
2548     const Register dst        = c_rarg2;  // destination array oop
2549     const Register dst_pos    = c_rarg3;  // destination position
2550 #ifndef _WIN64
2551     const Register length     = c_rarg4;
2552 #else
2553     const Address  length(rsp, 6 * wordSize);  // elements count is on stack on Win64
2554 #endif
2555 
2556     { int modulus = CodeEntryAlignment;
2557       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
2558       int advance = target - (__ offset() % modulus);
2559       if (advance < 0)  advance += modulus;
2560       if (advance > 0)  __ nop(advance);
2561     }
2562     StubCodeMark mark(this, "StubRoutines", name);
2563 
2564     // Short-hop target to L_failed.  Makes for denser prologue code.
2565     __ BIND(L_failed_0);
2566     __ jmp(L_failed);
2567     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
2568 
2569     __ align(CodeEntryAlignment);
2570     address start = __ pc();
2571 
2572     __ enter(); // required for proper stackwalking of RuntimeStub frame
2573 
2574     // bump this on entry, not on exit:
2575     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
2576 
2577     //-----------------------------------------------------------------------
2578     // Assembler stub will be used for this call to arraycopy
2579     // if the following conditions are met:
2580     //
2581     // (1) src and dst must not be null.
2582     // (2) src_pos must not be negative.
2583     // (3) dst_pos must not be negative.
2584     // (4) length  must not be negative.
2585     // (5) src klass and dst klass should be the same and not NULL.
2586     // (6) src and dst should be arrays.
2587     // (7) src_pos + length must not exceed length of src.
2588     // (8) dst_pos + length must not exceed length of dst.
2589     //
2590 
2591     //  if (src == NULL) return -1;
2592     __ testptr(src, src);         // src oop
2593     size_t j1off = __ offset();
2594     __ jccb(Assembler::zero, L_failed_0);
2595 
2596     //  if (src_pos < 0) return -1;
2597     __ testl(src_pos, src_pos); // src_pos (32-bits)
2598     __ jccb(Assembler::negative, L_failed_0);
2599 
2600     //  if (dst == NULL) return -1;
2601     __ testptr(dst, dst);         // dst oop
2602     __ jccb(Assembler::zero, L_failed_0);
2603 
2604     //  if (dst_pos < 0) return -1;
2605     __ testl(dst_pos, dst_pos); // dst_pos (32-bits)
2606     size_t j4off = __ offset();
2607     __ jccb(Assembler::negative, L_failed_0);
2608 
2609     // The first four tests are very dense code,
2610     // but not quite dense enough to put four
2611     // jumps in a 16-byte instruction fetch buffer.
2612     // That's good, because some branch predicters
2613     // do not like jumps so close together.
2614     // Make sure of this.
2615     guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
2616 
2617     // registers used as temp
2618     const Register r11_length    = r11; // elements count to copy
2619     const Register r10_src_klass = r10; // array klass
2620 
2621     //  if (length < 0) return -1;
2622     __ movl(r11_length, length);        // length (elements count, 32-bits value)
2623     __ testl(r11_length, r11_length);
2624     __ jccb(Assembler::negative, L_failed_0);
2625 
2626     __ load_klass(r10_src_klass, src);
2627 #ifdef ASSERT
2628     //  assert(src->klass() != NULL);
2629     {
2630       BLOCK_COMMENT("assert klasses not null {");
2631       Label L1, L2;
2632       __ testptr(r10_src_klass, r10_src_klass);
2633       __ jcc(Assembler::notZero, L2);   // it is broken if klass is NULL
2634       __ bind(L1);
2635       __ stop("broken null klass");
2636       __ bind(L2);
2637       __ load_klass(rax, dst);
2638       __ cmpq(rax, 0);
2639       __ jcc(Assembler::equal, L1);     // this would be broken also
2640       BLOCK_COMMENT("} assert klasses not null done");
2641     }
2642 #endif
2643 
2644     // Load layout helper (32-bits)
2645     //
2646     //  |array_tag|     | header_size | element_type |     |log2_element_size|
2647     // 32        30    24            16              8     2                 0
2648     //
2649     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2650     //
2651 
2652     const int lh_offset = in_bytes(Klass::layout_helper_offset());
2653 
2654     // Handle objArrays completely differently...
2655     const jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2656     __ cmpl(Address(r10_src_klass, lh_offset), objArray_lh);
2657     __ jcc(Assembler::equal, L_objArray);
2658 
2659     //  if (src->klass() != dst->klass()) return -1;
2660     __ load_klass(rax, dst);
2661     __ cmpq(r10_src_klass, rax);
2662     __ jcc(Assembler::notEqual, L_failed);
2663 
2664     const Register rax_lh = rax;  // layout helper
2665     __ movl(rax_lh, Address(r10_src_klass, lh_offset));
2666 
2667     //  if (!src->is_Array()) return -1;
2668     __ cmpl(rax_lh, Klass::_lh_neutral_value);
2669     __ jcc(Assembler::greaterEqual, L_failed);
2670 
2671     // At this point, it is known to be a typeArray (array_tag 0x3).
2672 #ifdef ASSERT
2673     {
2674       BLOCK_COMMENT("assert primitive array {");
2675       Label L;
2676       __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
2677       __ jcc(Assembler::greaterEqual, L);
2678       __ stop("must be a primitive array");
2679       __ bind(L);
2680       BLOCK_COMMENT("} assert primitive array done");
2681     }
2682 #endif
2683 
2684     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2685                            r10, L_failed);
2686 
2687     // TypeArrayKlass
2688     //
2689     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2690     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2691     //
2692 
2693     const Register r10_offset = r10;    // array offset
2694     const Register rax_elsize = rax_lh; // element size
2695 
2696     __ movl(r10_offset, rax_lh);
2697     __ shrl(r10_offset, Klass::_lh_header_size_shift);
2698     __ andptr(r10_offset, Klass::_lh_header_size_mask);   // array_offset
2699     __ addptr(src, r10_offset);           // src array offset
2700     __ addptr(dst, r10_offset);           // dst array offset
2701     BLOCK_COMMENT("choose copy loop based on element size");
2702     __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
2703 
2704     // next registers should be set before the jump to corresponding stub
2705     const Register from     = c_rarg0;  // source array address
2706     const Register to       = c_rarg1;  // destination array address
2707     const Register count    = c_rarg2;  // elements count
2708 
2709     // 'from', 'to', 'count' registers should be set in such order
2710     // since they are the same as 'src', 'src_pos', 'dst'.
2711 
2712   __ BIND(L_copy_bytes);
2713     __ cmpl(rax_elsize, 0);
2714     __ jccb(Assembler::notEqual, L_copy_shorts);
2715     __ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr
2716     __ lea(to,   Address(dst, dst_pos, Address::times_1, 0));// dst_addr
2717     __ movl2ptr(count, r11_length); // length
2718     __ jump(RuntimeAddress(byte_copy_entry));
2719 
2720   __ BIND(L_copy_shorts);
2721     __ cmpl(rax_elsize, LogBytesPerShort);
2722     __ jccb(Assembler::notEqual, L_copy_ints);
2723     __ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr
2724     __ lea(to,   Address(dst, dst_pos, Address::times_2, 0));// dst_addr
2725     __ movl2ptr(count, r11_length); // length
2726     __ jump(RuntimeAddress(short_copy_entry));
2727 
2728   __ BIND(L_copy_ints);
2729     __ cmpl(rax_elsize, LogBytesPerInt);
2730     __ jccb(Assembler::notEqual, L_copy_longs);
2731     __ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr
2732     __ lea(to,   Address(dst, dst_pos, Address::times_4, 0));// dst_addr
2733     __ movl2ptr(count, r11_length); // length
2734     __ jump(RuntimeAddress(int_copy_entry));
2735 
2736   __ BIND(L_copy_longs);
2737 #ifdef ASSERT
2738     {
2739       BLOCK_COMMENT("assert long copy {");
2740       Label L;
2741       __ cmpl(rax_elsize, LogBytesPerLong);
2742       __ jcc(Assembler::equal, L);
2743       __ stop("must be long copy, but elsize is wrong");
2744       __ bind(L);
2745       BLOCK_COMMENT("} assert long copy done");
2746     }
2747 #endif
2748     __ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr
2749     __ lea(to,   Address(dst, dst_pos, Address::times_8, 0));// dst_addr
2750     __ movl2ptr(count, r11_length); // length
2751     __ jump(RuntimeAddress(long_copy_entry));
2752 
2753     // ObjArrayKlass
2754   __ BIND(L_objArray);
2755     // live at this point:  r10_src_klass, r11_length, src[_pos], dst[_pos]
2756 
2757     Label L_plain_copy, L_checkcast_copy;
2758     //  test array classes for subtyping
2759     __ load_klass(rax, dst);
2760     __ cmpq(r10_src_klass, rax); // usual case is exact equality
2761     __ jcc(Assembler::notEqual, L_checkcast_copy);
2762 
2763     // Identically typed arrays can be copied without element-wise checks.
2764     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2765                            r10, L_failed);
2766 
2767     __ lea(from, Address(src, src_pos, TIMES_OOP,
2768                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
2769     __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2770                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
2771     __ movl2ptr(count, r11_length); // length
2772   __ BIND(L_plain_copy);
2773     __ jump(RuntimeAddress(oop_copy_entry));
2774 
2775   __ BIND(L_checkcast_copy);
2776     // live at this point:  r10_src_klass, r11_length, rax (dst_klass)
2777     {
2778       // Before looking at dst.length, make sure dst is also an objArray.
2779       __ cmpl(Address(rax, lh_offset), objArray_lh);
2780       __ jcc(Assembler::notEqual, L_failed);
2781 
2782       // It is safe to examine both src.length and dst.length.
2783       arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2784                              rax, L_failed);
2785 
2786       const Register r11_dst_klass = r11;
2787       __ load_klass(r11_dst_klass, dst); // reload
2788 
2789       // Marshal the base address arguments now, freeing registers.
2790       __ lea(from, Address(src, src_pos, TIMES_OOP,
2791                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2792       __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2793                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2794       __ movl(count, length);           // length (reloaded)
2795       Register sco_temp = c_rarg3;      // this register is free now
2796       assert_different_registers(from, to, count, sco_temp,
2797                                  r11_dst_klass, r10_src_klass);
2798       assert_clean_int(count, sco_temp);
2799 
2800       // Generate the type check.
2801       const int sco_offset = in_bytes(Klass::super_check_offset_offset());
2802       __ movl(sco_temp, Address(r11_dst_klass, sco_offset));
2803       assert_clean_int(sco_temp, rax);
2804       generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy);
2805 
2806       // Fetch destination element klass from the ObjArrayKlass header.
2807       int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2808       __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
2809       __ movl(  sco_temp,      Address(r11_dst_klass, sco_offset));
2810       assert_clean_int(sco_temp, rax);
2811 
2812       // the checkcast_copy loop needs two extra arguments:
2813       assert(c_rarg3 == sco_temp, "#3 already in place");
2814       // Set up arguments for checkcast_copy_entry.
2815       setup_arg_regs(4);
2816       __ movptr(r8, r11_dst_klass);  // dst.klass.element_klass, r8 is c_rarg4 on Linux/Solaris
2817       __ jump(RuntimeAddress(checkcast_copy_entry));
2818     }
2819 
2820   __ BIND(L_failed);
2821     __ xorptr(rax, rax);
2822     __ notptr(rax); // return -1
2823     __ leave();   // required for proper stackwalking of RuntimeStub frame
2824     __ ret(0);
2825 
2826     return start;
2827   }
2828 
2829   void generate_arraycopy_stubs() {
2830     address entry;
2831     address entry_jbyte_arraycopy;
2832     address entry_jshort_arraycopy;
2833     address entry_jint_arraycopy;
2834     address entry_oop_arraycopy;
2835     address entry_jlong_arraycopy;
2836     address entry_checkcast_arraycopy;
2837 
2838     StubRoutines::_jbyte_disjoint_arraycopy  = generate_disjoint_byte_copy(false, &entry,
2839                                                                            "jbyte_disjoint_arraycopy");
2840     StubRoutines::_jbyte_arraycopy           = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy,
2841                                                                            "jbyte_arraycopy");
2842 
2843     StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry,
2844                                                                             "jshort_disjoint_arraycopy");
2845     StubRoutines::_jshort_arraycopy          = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy,
2846                                                                             "jshort_arraycopy");
2847 
2848     StubRoutines::_jint_disjoint_arraycopy   = generate_disjoint_int_oop_copy(false, false, &entry,
2849                                                                               "jint_disjoint_arraycopy");
2850     StubRoutines::_jint_arraycopy            = generate_conjoint_int_oop_copy(false, false, entry,
2851                                                                               &entry_jint_arraycopy, "jint_arraycopy");
2852 
2853     StubRoutines::_jlong_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, false, &entry,
2854                                                                                "jlong_disjoint_arraycopy");
2855     StubRoutines::_jlong_arraycopy           = generate_conjoint_long_oop_copy(false, false, entry,
2856                                                                                &entry_jlong_arraycopy, "jlong_arraycopy");
2857 
2858 
2859     if (UseCompressedOops) {
2860       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_int_oop_copy(false, true, &entry,
2861                                                                               "oop_disjoint_arraycopy");
2862       StubRoutines::_oop_arraycopy           = generate_conjoint_int_oop_copy(false, true, entry,
2863                                                                               &entry_oop_arraycopy, "oop_arraycopy");
2864       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_int_oop_copy(false, true, &entry,
2865                                                                                      "oop_disjoint_arraycopy_uninit",
2866                                                                                      /*dest_uninitialized*/true);
2867       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_int_oop_copy(false, true, entry,
2868                                                                                      NULL, "oop_arraycopy_uninit",
2869                                                                                      /*dest_uninitialized*/true);
2870     } else {
2871       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, true, &entry,
2872                                                                                "oop_disjoint_arraycopy");
2873       StubRoutines::_oop_arraycopy           = generate_conjoint_long_oop_copy(false, true, entry,
2874                                                                                &entry_oop_arraycopy, "oop_arraycopy");
2875       StubRoutines::_oop_disjoint_arraycopy_uninit  = generate_disjoint_long_oop_copy(false, true, &entry,
2876                                                                                       "oop_disjoint_arraycopy_uninit",
2877                                                                                       /*dest_uninitialized*/true);
2878       StubRoutines::_oop_arraycopy_uninit           = generate_conjoint_long_oop_copy(false, true, entry,
2879                                                                                       NULL, "oop_arraycopy_uninit",
2880                                                                                       /*dest_uninitialized*/true);
2881     }
2882 
2883     StubRoutines::_checkcast_arraycopy        = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2884     StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL,
2885                                                                         /*dest_uninitialized*/true);
2886 
2887     StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy("unsafe_arraycopy",
2888                                                               entry_jbyte_arraycopy,
2889                                                               entry_jshort_arraycopy,
2890                                                               entry_jint_arraycopy,
2891                                                               entry_jlong_arraycopy);
2892     StubRoutines::_generic_arraycopy   = generate_generic_copy("generic_arraycopy",
2893                                                                entry_jbyte_arraycopy,
2894                                                                entry_jshort_arraycopy,
2895                                                                entry_jint_arraycopy,
2896                                                                entry_oop_arraycopy,
2897                                                                entry_jlong_arraycopy,
2898                                                                entry_checkcast_arraycopy);
2899 
2900     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2901     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2902     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2903     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2904     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2905     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2906 
2907     // We don't generate specialized code for HeapWord-aligned source
2908     // arrays, so just use the code we've already generated
2909     StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = StubRoutines::_jbyte_disjoint_arraycopy;
2910     StubRoutines::_arrayof_jbyte_arraycopy           = StubRoutines::_jbyte_arraycopy;
2911 
2912     StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy;
2913     StubRoutines::_arrayof_jshort_arraycopy          = StubRoutines::_jshort_arraycopy;
2914 
2915     StubRoutines::_arrayof_jint_disjoint_arraycopy   = StubRoutines::_jint_disjoint_arraycopy;
2916     StubRoutines::_arrayof_jint_arraycopy            = StubRoutines::_jint_arraycopy;
2917 
2918     StubRoutines::_arrayof_jlong_disjoint_arraycopy  = StubRoutines::_jlong_disjoint_arraycopy;
2919     StubRoutines::_arrayof_jlong_arraycopy           = StubRoutines::_jlong_arraycopy;
2920 
2921     StubRoutines::_arrayof_oop_disjoint_arraycopy    = StubRoutines::_oop_disjoint_arraycopy;
2922     StubRoutines::_arrayof_oop_arraycopy             = StubRoutines::_oop_arraycopy;
2923 
2924     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit    = StubRoutines::_oop_disjoint_arraycopy_uninit;
2925     StubRoutines::_arrayof_oop_arraycopy_uninit             = StubRoutines::_oop_arraycopy_uninit;
2926   }
2927 
2928   void generate_math_stubs() {
2929     {
2930       StubCodeMark mark(this, "StubRoutines", "log");
2931       StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
2932 
2933       __ subq(rsp, 8);
2934       __ movdbl(Address(rsp, 0), xmm0);
2935       __ fld_d(Address(rsp, 0));
2936       __ flog();
2937       __ fstp_d(Address(rsp, 0));
2938       __ movdbl(xmm0, Address(rsp, 0));
2939       __ addq(rsp, 8);
2940       __ ret(0);
2941     }
2942     {
2943       StubCodeMark mark(this, "StubRoutines", "log10");
2944       StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
2945 
2946       __ subq(rsp, 8);
2947       __ movdbl(Address(rsp, 0), xmm0);
2948       __ fld_d(Address(rsp, 0));
2949       __ flog10();
2950       __ fstp_d(Address(rsp, 0));
2951       __ movdbl(xmm0, Address(rsp, 0));
2952       __ addq(rsp, 8);
2953       __ ret(0);
2954     }
2955     {
2956       StubCodeMark mark(this, "StubRoutines", "sin");
2957       StubRoutines::_intrinsic_sin = (double (*)(double)) __ pc();
2958 
2959       __ subq(rsp, 8);
2960       __ movdbl(Address(rsp, 0), xmm0);
2961       __ fld_d(Address(rsp, 0));
2962       __ trigfunc('s');
2963       __ fstp_d(Address(rsp, 0));
2964       __ movdbl(xmm0, Address(rsp, 0));
2965       __ addq(rsp, 8);
2966       __ ret(0);
2967     }
2968     {
2969       StubCodeMark mark(this, "StubRoutines", "cos");
2970       StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
2971 
2972       __ subq(rsp, 8);
2973       __ movdbl(Address(rsp, 0), xmm0);
2974       __ fld_d(Address(rsp, 0));
2975       __ trigfunc('c');
2976       __ fstp_d(Address(rsp, 0));
2977       __ movdbl(xmm0, Address(rsp, 0));
2978       __ addq(rsp, 8);
2979       __ ret(0);
2980     }
2981     {
2982       StubCodeMark mark(this, "StubRoutines", "tan");
2983       StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
2984 
2985       __ subq(rsp, 8);
2986       __ movdbl(Address(rsp, 0), xmm0);
2987       __ fld_d(Address(rsp, 0));
2988       __ trigfunc('t');
2989       __ fstp_d(Address(rsp, 0));
2990       __ movdbl(xmm0, Address(rsp, 0));
2991       __ addq(rsp, 8);
2992       __ ret(0);
2993     }
2994     {
2995       StubCodeMark mark(this, "StubRoutines", "exp");
2996       StubRoutines::_intrinsic_exp = (double (*)(double)) __ pc();
2997 
2998       __ subq(rsp, 8);
2999       __ movdbl(Address(rsp, 0), xmm0);
3000       __ fld_d(Address(rsp, 0));
3001       __ exp_with_fallback(0);
3002       __ fstp_d(Address(rsp, 0));
3003       __ movdbl(xmm0, Address(rsp, 0));
3004       __ addq(rsp, 8);
3005       __ ret(0);
3006     }
3007     {
3008       StubCodeMark mark(this, "StubRoutines", "pow");
3009       StubRoutines::_intrinsic_pow = (double (*)(double,double)) __ pc();
3010 
3011       __ subq(rsp, 8);
3012       __ movdbl(Address(rsp, 0), xmm1);
3013       __ fld_d(Address(rsp, 0));
3014       __ movdbl(Address(rsp, 0), xmm0);
3015       __ fld_d(Address(rsp, 0));
3016       __ pow_with_fallback(0);
3017       __ fstp_d(Address(rsp, 0));
3018       __ movdbl(xmm0, Address(rsp, 0));
3019       __ addq(rsp, 8);
3020       __ ret(0);
3021     }
3022   }
3023 
3024   // AES intrinsic stubs
3025   enum {AESBlockSize = 16};
3026 
3027   address generate_key_shuffle_mask() {
3028     __ align(16);
3029     StubCodeMark mark(this, "StubRoutines", "key_shuffle_mask");
3030     address start = __ pc();
3031     __ emit_data64( 0x0405060700010203, relocInfo::none );
3032     __ emit_data64( 0x0c0d0e0f08090a0b, relocInfo::none );
3033     return start;
3034   }
3035 
3036   // Utility routine for loading a 128-bit key word in little endian format
3037   // can optionally specify that the shuffle mask is already in an xmmregister
3038   void load_key(XMMRegister xmmdst, Register key, int offset, XMMRegister xmm_shuf_mask=NULL) {
3039     __ movdqu(xmmdst, Address(key, offset));
3040     if (xmm_shuf_mask != NULL) {
3041       __ pshufb(xmmdst, xmm_shuf_mask);
3042     } else {
3043       __ pshufb(xmmdst, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3044     }
3045   }
3046 
3047   // Arguments:
3048   //
3049   // Inputs:
3050   //   c_rarg0   - source byte array address
3051   //   c_rarg1   - destination byte array address
3052   //   c_rarg2   - K (key) in little endian int array
3053   //
3054   address generate_aescrypt_encryptBlock() {
3055     assert(UseAES, "need AES instructions and misaligned SSE support");
3056     __ align(CodeEntryAlignment);
3057     StubCodeMark mark(this, "StubRoutines", "aescrypt_encryptBlock");
3058     Label L_doLast;
3059     address start = __ pc();
3060 
3061     const Register from        = c_rarg0;  // source array address
3062     const Register to          = c_rarg1;  // destination array address
3063     const Register key         = c_rarg2;  // key array address
3064     const Register keylen      = rax;
3065 
3066     const XMMRegister xmm_result = xmm0;
3067     const XMMRegister xmm_key_shuf_mask = xmm1;
3068     // On win64 xmm6-xmm15 must be preserved so don't use them.
3069     const XMMRegister xmm_temp1  = xmm2;
3070     const XMMRegister xmm_temp2  = xmm3;
3071     const XMMRegister xmm_temp3  = xmm4;
3072     const XMMRegister xmm_temp4  = xmm5;
3073 
3074     __ enter(); // required for proper stackwalking of RuntimeStub frame
3075 
3076     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
3077     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3078 
3079     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3080     __ movdqu(xmm_result, Address(from, 0));  // get 16 bytes of input
3081 
3082     // For encryption, the java expanded key ordering is just what we need
3083     // we don't know if the key is aligned, hence not using load-execute form
3084 
3085     load_key(xmm_temp1, key, 0x00, xmm_key_shuf_mask);
3086     __ pxor(xmm_result, xmm_temp1);
3087 
3088     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
3089     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
3090     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
3091     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
3092 
3093     __ aesenc(xmm_result, xmm_temp1);
3094     __ aesenc(xmm_result, xmm_temp2);
3095     __ aesenc(xmm_result, xmm_temp3);
3096     __ aesenc(xmm_result, xmm_temp4);
3097 
3098     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
3099     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
3100     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
3101     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
3102 
3103     __ aesenc(xmm_result, xmm_temp1);
3104     __ aesenc(xmm_result, xmm_temp2);
3105     __ aesenc(xmm_result, xmm_temp3);
3106     __ aesenc(xmm_result, xmm_temp4);
3107 
3108     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
3109     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
3110 
3111     __ cmpl(keylen, 44);
3112     __ jccb(Assembler::equal, L_doLast);
3113 
3114     __ aesenc(xmm_result, xmm_temp1);
3115     __ aesenc(xmm_result, xmm_temp2);
3116 
3117     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
3118     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
3119 
3120     __ cmpl(keylen, 52);
3121     __ jccb(Assembler::equal, L_doLast);
3122 
3123     __ aesenc(xmm_result, xmm_temp1);
3124     __ aesenc(xmm_result, xmm_temp2);
3125 
3126     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3127     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3128 
3129     __ BIND(L_doLast);
3130     __ aesenc(xmm_result, xmm_temp1);
3131     __ aesenclast(xmm_result, xmm_temp2);
3132     __ movdqu(Address(to, 0), xmm_result);        // store the result
3133     __ xorptr(rax, rax); // return 0
3134     __ leave(); // required for proper stackwalking of RuntimeStub frame
3135     __ ret(0);
3136 
3137     return start;
3138   }
3139 
3140 
3141   // Arguments:
3142   //
3143   // Inputs:
3144   //   c_rarg0   - source byte array address
3145   //   c_rarg1   - destination byte array address
3146   //   c_rarg2   - K (key) in little endian int array
3147   //
3148   address generate_aescrypt_decryptBlock() {
3149     assert(UseAES, "need AES instructions and misaligned SSE support");
3150     __ align(CodeEntryAlignment);
3151     StubCodeMark mark(this, "StubRoutines", "aescrypt_decryptBlock");
3152     Label L_doLast;
3153     address start = __ pc();
3154 
3155     const Register from        = c_rarg0;  // source array address
3156     const Register to          = c_rarg1;  // destination array address
3157     const Register key         = c_rarg2;  // key array address
3158     const Register keylen      = rax;
3159 
3160     const XMMRegister xmm_result = xmm0;
3161     const XMMRegister xmm_key_shuf_mask = xmm1;
3162     // On win64 xmm6-xmm15 must be preserved so don't use them.
3163     const XMMRegister xmm_temp1  = xmm2;
3164     const XMMRegister xmm_temp2  = xmm3;
3165     const XMMRegister xmm_temp3  = xmm4;
3166     const XMMRegister xmm_temp4  = xmm5;
3167 
3168     __ enter(); // required for proper stackwalking of RuntimeStub frame
3169 
3170     // keylen could be only {11, 13, 15} * 4 = {44, 52, 60}
3171     __ movl(keylen, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3172 
3173     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3174     __ movdqu(xmm_result, Address(from, 0));
3175 
3176     // for decryption java expanded key ordering is rotated one position from what we want
3177     // so we start from 0x10 here and hit 0x00 last
3178     // we don't know if the key is aligned, hence not using load-execute form
3179     load_key(xmm_temp1, key, 0x10, xmm_key_shuf_mask);
3180     load_key(xmm_temp2, key, 0x20, xmm_key_shuf_mask);
3181     load_key(xmm_temp3, key, 0x30, xmm_key_shuf_mask);
3182     load_key(xmm_temp4, key, 0x40, xmm_key_shuf_mask);
3183 
3184     __ pxor  (xmm_result, xmm_temp1);
3185     __ aesdec(xmm_result, xmm_temp2);
3186     __ aesdec(xmm_result, xmm_temp3);
3187     __ aesdec(xmm_result, xmm_temp4);
3188 
3189     load_key(xmm_temp1, key, 0x50, xmm_key_shuf_mask);
3190     load_key(xmm_temp2, key, 0x60, xmm_key_shuf_mask);
3191     load_key(xmm_temp3, key, 0x70, xmm_key_shuf_mask);
3192     load_key(xmm_temp4, key, 0x80, xmm_key_shuf_mask);
3193 
3194     __ aesdec(xmm_result, xmm_temp1);
3195     __ aesdec(xmm_result, xmm_temp2);
3196     __ aesdec(xmm_result, xmm_temp3);
3197     __ aesdec(xmm_result, xmm_temp4);
3198 
3199     load_key(xmm_temp1, key, 0x90, xmm_key_shuf_mask);
3200     load_key(xmm_temp2, key, 0xa0, xmm_key_shuf_mask);
3201     load_key(xmm_temp3, key, 0x00, xmm_key_shuf_mask);
3202 
3203     __ cmpl(keylen, 44);
3204     __ jccb(Assembler::equal, L_doLast);
3205 
3206     __ aesdec(xmm_result, xmm_temp1);
3207     __ aesdec(xmm_result, xmm_temp2);
3208 
3209     load_key(xmm_temp1, key, 0xb0, xmm_key_shuf_mask);
3210     load_key(xmm_temp2, key, 0xc0, xmm_key_shuf_mask);
3211 
3212     __ cmpl(keylen, 52);
3213     __ jccb(Assembler::equal, L_doLast);
3214 
3215     __ aesdec(xmm_result, xmm_temp1);
3216     __ aesdec(xmm_result, xmm_temp2);
3217 
3218     load_key(xmm_temp1, key, 0xd0, xmm_key_shuf_mask);
3219     load_key(xmm_temp2, key, 0xe0, xmm_key_shuf_mask);
3220 
3221     __ BIND(L_doLast);
3222     __ aesdec(xmm_result, xmm_temp1);
3223     __ aesdec(xmm_result, xmm_temp2);
3224 
3225     // for decryption the aesdeclast operation is always on key+0x00
3226     __ aesdeclast(xmm_result, xmm_temp3);
3227     __ movdqu(Address(to, 0), xmm_result);  // store the result
3228     __ xorptr(rax, rax); // return 0
3229     __ leave(); // required for proper stackwalking of RuntimeStub frame
3230     __ ret(0);
3231 
3232     return start;
3233   }
3234 
3235 
3236   // Arguments:
3237   //
3238   // Inputs:
3239   //   c_rarg0   - source byte array address
3240   //   c_rarg1   - destination byte array address
3241   //   c_rarg2   - K (key) in little endian int array
3242   //   c_rarg3   - r vector byte array address
3243   //   c_rarg4   - input length
3244   //
3245   // Output:
3246   //   rax       - input length
3247   //
3248   address generate_cipherBlockChaining_encryptAESCrypt() {
3249     assert(UseAES, "need AES instructions and misaligned SSE support");
3250     __ align(CodeEntryAlignment);
3251     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_encryptAESCrypt");
3252     address start = __ pc();
3253 
3254     Label L_exit, L_key_192_256, L_key_256, L_loopTop_128, L_loopTop_192, L_loopTop_256;
3255     const Register from        = c_rarg0;  // source array address
3256     const Register to          = c_rarg1;  // destination array address
3257     const Register key         = c_rarg2;  // key array address
3258     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
3259                                            // and left with the results of the last encryption block
3260 #ifndef _WIN64
3261     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
3262 #else
3263     const Address  len_mem(rbp, 6 * wordSize);  // length is on stack on Win64
3264     const Register len_reg     = r10;      // pick the first volatile windows register
3265 #endif
3266     const Register pos         = rax;
3267 
3268     // xmm register assignments for the loops below
3269     const XMMRegister xmm_result = xmm0;
3270     const XMMRegister xmm_temp   = xmm1;
3271     // keys 0-10 preloaded into xmm2-xmm12
3272     const int XMM_REG_NUM_KEY_FIRST = 2;
3273     const int XMM_REG_NUM_KEY_LAST  = 15;
3274     const XMMRegister xmm_key0   = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3275     const XMMRegister xmm_key10  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+10);
3276     const XMMRegister xmm_key11  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+11);
3277     const XMMRegister xmm_key12  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+12);
3278     const XMMRegister xmm_key13  = as_XMMRegister(XMM_REG_NUM_KEY_FIRST+13);
3279 
3280     __ enter(); // required for proper stackwalking of RuntimeStub frame
3281 
3282 #ifdef _WIN64
3283     // on win64, fill len_reg from stack position
3284     __ movl(len_reg, len_mem);
3285     // save the xmm registers which must be preserved 6-15
3286     __ subptr(rsp, -rsp_after_call_off * wordSize);
3287     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
3288       __ movdqu(xmm_save(i), as_XMMRegister(i));
3289     }
3290 #else
3291     __ push(len_reg); // Save
3292 #endif
3293 
3294     const XMMRegister xmm_key_shuf_mask = xmm_temp;  // used temporarily to swap key bytes up front
3295     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3296     // load up xmm regs xmm2 thru xmm12 with key 0x00 - 0xa0
3297     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x00; rnum <= XMM_REG_NUM_KEY_FIRST+10; rnum++) {
3298       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3299       offset += 0x10;
3300     }
3301     __ movdqu(xmm_result, Address(rvec, 0x00));   // initialize xmm_result with r vec
3302 
3303     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3304     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3305     __ cmpl(rax, 44);
3306     __ jcc(Assembler::notEqual, L_key_192_256);
3307 
3308     // 128 bit code follows here
3309     __ movptr(pos, 0);
3310     __ align(OptoLoopAlignment);
3311 
3312     __ BIND(L_loopTop_128);
3313     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3314     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3315     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3316     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_FIRST + 9; rnum++) {
3317       __ aesenc(xmm_result, as_XMMRegister(rnum));
3318     }
3319     __ aesenclast(xmm_result, xmm_key10);
3320     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3321     // no need to store r to memory until we exit
3322     __ addptr(pos, AESBlockSize);
3323     __ subptr(len_reg, AESBlockSize);
3324     __ jcc(Assembler::notEqual, L_loopTop_128);
3325 
3326     __ BIND(L_exit);
3327     __ movdqu(Address(rvec, 0), xmm_result);     // final value of r stored in rvec of CipherBlockChaining object
3328 
3329 #ifdef _WIN64
3330     // restore xmm regs belonging to calling function
3331     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
3332       __ movdqu(as_XMMRegister(i), xmm_save(i));
3333     }
3334     __ movl(rax, len_mem);
3335 #else
3336     __ pop(rax); // return length
3337 #endif
3338     __ leave(); // required for proper stackwalking of RuntimeStub frame
3339     __ ret(0);
3340 
3341     __ BIND(L_key_192_256);
3342     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
3343     load_key(xmm_key11, key, 0xb0, xmm_key_shuf_mask);
3344     load_key(xmm_key12, key, 0xc0, xmm_key_shuf_mask);
3345     __ cmpl(rax, 52);
3346     __ jcc(Assembler::notEqual, L_key_256);
3347 
3348     // 192-bit code follows here (could be changed to use more xmm registers)
3349     __ movptr(pos, 0);
3350     __ align(OptoLoopAlignment);
3351 
3352     __ BIND(L_loopTop_192);
3353     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3354     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3355     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3356     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 11; rnum++) {
3357       __ aesenc(xmm_result, as_XMMRegister(rnum));
3358     }
3359     __ aesenclast(xmm_result, xmm_key12);
3360     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3361     // no need to store r to memory until we exit
3362     __ addptr(pos, AESBlockSize);
3363     __ subptr(len_reg, AESBlockSize);
3364     __ jcc(Assembler::notEqual, L_loopTop_192);
3365     __ jmp(L_exit);
3366 
3367     __ BIND(L_key_256);
3368     // 256-bit code follows here (could be changed to use more xmm registers)
3369     load_key(xmm_key13, key, 0xd0, xmm_key_shuf_mask);
3370     __ movptr(pos, 0);
3371     __ align(OptoLoopAlignment);
3372 
3373     __ BIND(L_loopTop_256);
3374     __ movdqu(xmm_temp, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of input
3375     __ pxor  (xmm_result, xmm_temp);               // xor with the current r vector
3376     __ pxor  (xmm_result, xmm_key0);               // do the aes rounds
3377     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_FIRST + 13; rnum++) {
3378       __ aesenc(xmm_result, as_XMMRegister(rnum));
3379     }
3380     load_key(xmm_temp, key, 0xe0);
3381     __ aesenclast(xmm_result, xmm_temp);
3382     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3383     // no need to store r to memory until we exit
3384     __ addptr(pos, AESBlockSize);
3385     __ subptr(len_reg, AESBlockSize);
3386     __ jcc(Assembler::notEqual, L_loopTop_256);
3387     __ jmp(L_exit);
3388 
3389     return start;
3390   }
3391 
3392   // Safefetch stubs.
3393   void generate_safefetch(const char* name, int size, address* entry,
3394                           address* fault_pc, address* continuation_pc) {
3395     // safefetch signatures:
3396     //   int      SafeFetch32(int*      adr, int      errValue);
3397     //   intptr_t SafeFetchN (intptr_t* adr, intptr_t errValue);
3398     //
3399     // arguments:
3400     //   c_rarg0 = adr
3401     //   c_rarg1 = errValue
3402     //
3403     // result:
3404     //   PPC_RET  = *adr or errValue
3405 
3406     StubCodeMark mark(this, "StubRoutines", name);
3407 
3408     // Entry point, pc or function descriptor.
3409     *entry = __ pc();
3410 
3411     // Load *adr into c_rarg1, may fault.
3412     *fault_pc = __ pc();
3413     switch (size) {
3414       case 4:
3415         // int32_t
3416         __ movl(c_rarg1, Address(c_rarg0, 0));
3417         break;
3418       case 8:
3419         // int64_t
3420         __ movq(c_rarg1, Address(c_rarg0, 0));
3421         break;
3422       default:
3423         ShouldNotReachHere();
3424     }
3425 
3426     // return errValue or *adr
3427     *continuation_pc = __ pc();
3428     __ movq(rax, c_rarg1);
3429     __ ret(0);
3430   }
3431 
3432   // This is a version of CBC/AES Decrypt which does 4 blocks in a loop at a time
3433   // to hide instruction latency
3434   //
3435   // Arguments:
3436   //
3437   // Inputs:
3438   //   c_rarg0   - source byte array address
3439   //   c_rarg1   - destination byte array address
3440   //   c_rarg2   - K (key) in little endian int array
3441   //   c_rarg3   - r vector byte array address
3442   //   c_rarg4   - input length
3443   //
3444   // Output:
3445   //   rax       - input length
3446   //
3447 
3448   address generate_cipherBlockChaining_decryptAESCrypt_Parallel() {
3449     assert(UseAES, "need AES instructions and misaligned SSE support");
3450     __ align(CodeEntryAlignment);
3451     StubCodeMark mark(this, "StubRoutines", "cipherBlockChaining_decryptAESCrypt");
3452     address start = __ pc();
3453 
3454     Label L_exit, L_key_192_256, L_key_256;
3455     Label L_singleBlock_loopTop_128, L_multiBlock_loopTop_128;
3456     Label L_singleBlock_loopTop_192, L_singleBlock_loopTop_256;
3457     const Register from        = c_rarg0;  // source array address
3458     const Register to          = c_rarg1;  // destination array address
3459     const Register key         = c_rarg2;  // key array address
3460     const Register rvec        = c_rarg3;  // r byte array initialized from initvector array address
3461                                            // and left with the results of the last encryption block
3462 #ifndef _WIN64
3463     const Register len_reg     = c_rarg4;  // src len (must be multiple of blocksize 16)
3464 #else
3465     const Address  len_mem(rbp, 6 * wordSize);  // length is on stack on Win64
3466     const Register len_reg     = r10;      // pick the first volatile windows register
3467 #endif
3468     const Register pos         = rax;
3469 
3470     // keys 0-10 preloaded into xmm2-xmm12
3471     const int XMM_REG_NUM_KEY_FIRST = 5;
3472     const int XMM_REG_NUM_KEY_LAST  = 15;
3473     const XMMRegister xmm_key_first = as_XMMRegister(XMM_REG_NUM_KEY_FIRST);
3474     const XMMRegister xmm_key_last  = as_XMMRegister(XMM_REG_NUM_KEY_LAST);
3475 
3476     __ enter(); // required for proper stackwalking of RuntimeStub frame
3477 
3478 #ifdef _WIN64
3479     // on win64, fill len_reg from stack position
3480     __ movl(len_reg, len_mem);
3481     // save the xmm registers which must be preserved 6-15
3482     __ subptr(rsp, -rsp_after_call_off * wordSize);
3483     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
3484       __ movdqu(xmm_save(i), as_XMMRegister(i));
3485     }
3486 #else
3487     __ push(len_reg); // Save
3488 #endif
3489 
3490     // the java expanded key ordering is rotated one position from what we want
3491     // so we start from 0x10 here and hit 0x00 last
3492     const XMMRegister xmm_key_shuf_mask = xmm1;  // used temporarily to swap key bytes up front
3493     __ movdqu(xmm_key_shuf_mask, ExternalAddress(StubRoutines::x86::key_shuffle_mask_addr()));
3494     // load up xmm regs 5 thru 15 with key 0x10 - 0xa0 - 0x00
3495     for (int rnum = XMM_REG_NUM_KEY_FIRST, offset = 0x10; rnum < XMM_REG_NUM_KEY_LAST; rnum++) {
3496       load_key(as_XMMRegister(rnum), key, offset, xmm_key_shuf_mask);
3497       offset += 0x10;
3498     }
3499     load_key(xmm_key_last, key, 0x00, xmm_key_shuf_mask);
3500 
3501     const XMMRegister xmm_prev_block_cipher = xmm1;  // holds cipher of previous block
3502 
3503     // registers holding the four results in the parallelized loop
3504     const XMMRegister xmm_result0 = xmm0;
3505     const XMMRegister xmm_result1 = xmm2;
3506     const XMMRegister xmm_result2 = xmm3;
3507     const XMMRegister xmm_result3 = xmm4;
3508 
3509     __ movdqu(xmm_prev_block_cipher, Address(rvec, 0x00));   // initialize with initial rvec
3510 
3511     // now split to different paths depending on the keylen (len in ints of AESCrypt.KLE array (52=192, or 60=256))
3512     __ movl(rax, Address(key, arrayOopDesc::length_offset_in_bytes() - arrayOopDesc::base_offset_in_bytes(T_INT)));
3513     __ cmpl(rax, 44);
3514     __ jcc(Assembler::notEqual, L_key_192_256);
3515 
3516 
3517     // 128-bit code follows here, parallelized
3518     __ movptr(pos, 0);
3519     __ align(OptoLoopAlignment);
3520     __ BIND(L_multiBlock_loopTop_128);
3521     __ cmpptr(len_reg, 4*AESBlockSize);           // see if at least 4 blocks left
3522     __ jcc(Assembler::less, L_singleBlock_loopTop_128);
3523 
3524     __ movdqu(xmm_result0, Address(from, pos, Address::times_1, 0*AESBlockSize));   // get next 4 blocks into xmmresult registers
3525     __ movdqu(xmm_result1, Address(from, pos, Address::times_1, 1*AESBlockSize));
3526     __ movdqu(xmm_result2, Address(from, pos, Address::times_1, 2*AESBlockSize));
3527     __ movdqu(xmm_result3, Address(from, pos, Address::times_1, 3*AESBlockSize));
3528 
3529 #define DoFour(opc, src_reg)                    \
3530     __ opc(xmm_result0, src_reg);               \
3531     __ opc(xmm_result1, src_reg);               \
3532     __ opc(xmm_result2, src_reg);               \
3533     __ opc(xmm_result3, src_reg);
3534 
3535     DoFour(pxor, xmm_key_first);
3536     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
3537       DoFour(aesdec, as_XMMRegister(rnum));
3538     }
3539     DoFour(aesdeclast, xmm_key_last);
3540     // for each result, xor with the r vector of previous cipher block
3541     __ pxor(xmm_result0, xmm_prev_block_cipher);
3542     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 0*AESBlockSize));
3543     __ pxor(xmm_result1, xmm_prev_block_cipher);
3544     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 1*AESBlockSize));
3545     __ pxor(xmm_result2, xmm_prev_block_cipher);
3546     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 2*AESBlockSize));
3547     __ pxor(xmm_result3, xmm_prev_block_cipher);
3548     __ movdqu(xmm_prev_block_cipher, Address(from, pos, Address::times_1, 3*AESBlockSize));   // this will carry over to next set of blocks
3549 
3550     __ movdqu(Address(to, pos, Address::times_1, 0*AESBlockSize), xmm_result0);     // store 4 results into the next 64 bytes of output
3551     __ movdqu(Address(to, pos, Address::times_1, 1*AESBlockSize), xmm_result1);
3552     __ movdqu(Address(to, pos, Address::times_1, 2*AESBlockSize), xmm_result2);
3553     __ movdqu(Address(to, pos, Address::times_1, 3*AESBlockSize), xmm_result3);
3554 
3555     __ addptr(pos, 4*AESBlockSize);
3556     __ subptr(len_reg, 4*AESBlockSize);
3557     __ jmp(L_multiBlock_loopTop_128);
3558 
3559     // registers used in the non-parallelized loops
3560     // xmm register assignments for the loops below
3561     const XMMRegister xmm_result = xmm0;
3562     const XMMRegister xmm_prev_block_cipher_save = xmm2;
3563     const XMMRegister xmm_key11 = xmm3;
3564     const XMMRegister xmm_key12 = xmm4;
3565     const XMMRegister xmm_temp  = xmm4;
3566 
3567     __ align(OptoLoopAlignment);
3568     __ BIND(L_singleBlock_loopTop_128);
3569     __ cmpptr(len_reg, 0);           // any blocks left??
3570     __ jcc(Assembler::equal, L_exit);
3571     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of cipher input
3572     __ movdqa(xmm_prev_block_cipher_save, xmm_result);              // save for next r vector
3573     __ pxor  (xmm_result, xmm_key_first);               // do the aes dec rounds
3574     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum  <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
3575       __ aesdec(xmm_result, as_XMMRegister(rnum));
3576     }
3577     __ aesdeclast(xmm_result, xmm_key_last);
3578     __ pxor  (xmm_result, xmm_prev_block_cipher);               // xor with the current r vector
3579     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);     // store into the next 16 bytes of output
3580     // no need to store r to memory until we exit
3581     __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save);              // set up next r vector with cipher input from this block
3582 
3583     __ addptr(pos, AESBlockSize);
3584     __ subptr(len_reg, AESBlockSize);
3585     __ jmp(L_singleBlock_loopTop_128);
3586 
3587 
3588     __ BIND(L_exit);
3589     __ movdqu(Address(rvec, 0), xmm_prev_block_cipher);     // final value of r stored in rvec of CipherBlockChaining object
3590 #ifdef _WIN64
3591     // restore regs belonging to calling function
3592     for (int i = 6; i <= XMM_REG_NUM_KEY_LAST; i++) {
3593       __ movdqu(as_XMMRegister(i), xmm_save(i));
3594     }
3595     __ movl(rax, len_mem);
3596 #else
3597     __ pop(rax); // return length
3598 #endif
3599     __ leave(); // required for proper stackwalking of RuntimeStub frame
3600     __ ret(0);
3601 
3602 
3603     __ BIND(L_key_192_256);
3604     // here rax = len in ints of AESCrypt.KLE array (52=192, or 60=256)
3605     load_key(xmm_key11, key, 0xb0);
3606     __ cmpl(rax, 52);
3607     __ jcc(Assembler::notEqual, L_key_256);
3608 
3609     // 192-bit code follows here (could be optimized to use parallelism)
3610     load_key(xmm_key12, key, 0xc0);     // 192-bit key goes up to c0
3611     __ movptr(pos, 0);
3612     __ align(OptoLoopAlignment);
3613 
3614     __ BIND(L_singleBlock_loopTop_192);
3615     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0));   // get next 16 bytes of cipher input
3616     __ movdqa(xmm_prev_block_cipher_save, xmm_result);              // save for next r vector
3617     __ pxor  (xmm_result, xmm_key_first);               // do the aes dec rounds
3618     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
3619       __ aesdec(xmm_result, as_XMMRegister(rnum));
3620     }
3621     __ aesdec(xmm_result, xmm_key11);
3622     __ aesdec(xmm_result, xmm_key12);
3623     __ aesdeclast(xmm_result, xmm_key_last);                    // xmm15 always came from key+0
3624     __ pxor  (xmm_result, xmm_prev_block_cipher);               // xor with the current r vector
3625     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);  // store into the next 16 bytes of output
3626     // no need to store r to memory until we exit
3627     __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save);  // set up next r vector with cipher input from this block
3628     __ addptr(pos, AESBlockSize);
3629     __ subptr(len_reg, AESBlockSize);
3630     __ jcc(Assembler::notEqual,L_singleBlock_loopTop_192);
3631     __ jmp(L_exit);
3632 
3633     __ BIND(L_key_256);
3634     // 256-bit code follows here (could be optimized to use parallelism)
3635     __ movptr(pos, 0);
3636     __ align(OptoLoopAlignment);
3637 
3638     __ BIND(L_singleBlock_loopTop_256);
3639     __ movdqu(xmm_result, Address(from, pos, Address::times_1, 0)); // get next 16 bytes of cipher input
3640     __ movdqa(xmm_prev_block_cipher_save, xmm_result);              // save for next r vector
3641     __ pxor  (xmm_result, xmm_key_first);               // do the aes dec rounds
3642     for (int rnum = XMM_REG_NUM_KEY_FIRST + 1; rnum <= XMM_REG_NUM_KEY_LAST - 1; rnum++) {
3643       __ aesdec(xmm_result, as_XMMRegister(rnum));
3644     }
3645     __ aesdec(xmm_result, xmm_key11);
3646     load_key(xmm_temp, key, 0xc0);
3647     __ aesdec(xmm_result, xmm_temp);
3648     load_key(xmm_temp, key, 0xd0);
3649     __ aesdec(xmm_result, xmm_temp);
3650     load_key(xmm_temp, key, 0xe0);     // 256-bit key goes up to e0
3651     __ aesdec(xmm_result, xmm_temp);
3652     __ aesdeclast(xmm_result, xmm_key_last);          // xmm15 came from key+0
3653     __ pxor  (xmm_result, xmm_prev_block_cipher);               // xor with the current r vector
3654     __ movdqu(Address(to, pos, Address::times_1, 0), xmm_result);  // store into the next 16 bytes of output
3655     // no need to store r to memory until we exit
3656     __ movdqa(xmm_prev_block_cipher, xmm_prev_block_cipher_save);  // set up next r vector with cipher input from this block
3657     __ addptr(pos, AESBlockSize);
3658     __ subptr(len_reg, AESBlockSize);
3659     __ jcc(Assembler::notEqual,L_singleBlock_loopTop_256);
3660     __ jmp(L_exit);
3661 
3662     return start;
3663   }
3664 
3665   /**
3666    *  Arguments:
3667    *
3668    * Inputs:
3669    *   c_rarg0   - int crc
3670    *   c_rarg1   - byte* buf
3671    *   c_rarg2   - int length
3672    *
3673    * Ouput:
3674    *       rax   - int crc result
3675    */
3676   address generate_updateBytesCRC32() {
3677     assert(UseCRC32Intrinsics, "need AVX and CLMUL instructions");
3678 
3679     __ align(CodeEntryAlignment);
3680     StubCodeMark mark(this, "StubRoutines", "updateBytesCRC32");
3681 
3682     address start = __ pc();
3683     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
3684     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
3685     // rscratch1: r10
3686     const Register crc   = c_rarg0;  // crc
3687     const Register buf   = c_rarg1;  // source java byte array address
3688     const Register len   = c_rarg2;  // length
3689     const Register table = c_rarg3;  // crc_table address (reuse register)
3690     const Register tmp   = r11;
3691     assert_different_registers(crc, buf, len, table, tmp, rax);
3692 
3693     BLOCK_COMMENT("Entry:");
3694     __ enter(); // required for proper stackwalking of RuntimeStub frame
3695 
3696     __ kernel_crc32(crc, buf, len, table, tmp);
3697 
3698     __ movl(rax, crc);
3699     __ leave(); // required for proper stackwalking of RuntimeStub frame
3700     __ ret(0);
3701 
3702     return start;
3703   }
3704 
3705 
3706   /**
3707    *  Arguments:
3708    *
3709    *  Input:
3710    *    c_rarg0   - x address
3711    *    c_rarg1   - x length
3712    *    c_rarg2   - y address
3713    *    c_rarg3   - y lenth
3714    * not Win64
3715    *    c_rarg4   - z address
3716    *    c_rarg5   - z length
3717    * Win64
3718    *    rsp+40    - z address
3719    *    rsp+48    - z length
3720    */
3721   address generate_multiplyToLen() {
3722     __ align(CodeEntryAlignment);
3723     StubCodeMark mark(this, "StubRoutines", "multiplyToLen");
3724 
3725     address start = __ pc();
3726     // Win64: rcx, rdx, r8, r9 (c_rarg0, c_rarg1, ...)
3727     // Unix:  rdi, rsi, rdx, rcx, r8, r9 (c_rarg0, c_rarg1, ...)
3728     const Register x     = rdi;
3729     const Register xlen  = rax;
3730     const Register y     = rsi;
3731     const Register ylen  = rcx;
3732     const Register z     = r8;
3733     const Register zlen  = r11;
3734 
3735     // Next registers will be saved on stack in multiply_to_len().
3736     const Register tmp1  = r12;
3737     const Register tmp2  = r13;
3738     const Register tmp3  = r14;
3739     const Register tmp4  = r15;
3740     const Register tmp5  = rbx;
3741 
3742     BLOCK_COMMENT("Entry:");
3743     __ enter(); // required for proper stackwalking of RuntimeStub frame
3744 
3745 #ifndef _WIN64
3746     __ movptr(zlen, r9); // Save r9 in r11 - zlen
3747 #endif
3748     setup_arg_regs(4); // x => rdi, xlen => rsi, y => rdx
3749                        // ylen => rcx, z => r8, zlen => r11
3750                        // r9 and r10 may be used to save non-volatile registers
3751 #ifdef _WIN64
3752     // last 2 arguments (#4, #5) are on stack on Win64
3753     __ movptr(z, Address(rsp, 6 * wordSize));
3754     __ movptr(zlen, Address(rsp, 7 * wordSize));
3755 #endif
3756 
3757     __ movptr(xlen, rsi);
3758     __ movptr(y,    rdx);
3759     __ multiply_to_len(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5);
3760 
3761     restore_arg_regs();
3762 
3763     __ leave(); // required for proper stackwalking of RuntimeStub frame
3764     __ ret(0);
3765 
3766     return start;
3767   }
3768 
3769 #undef __
3770 #define __ masm->
3771 
3772   // Continuation point for throwing of implicit exceptions that are
3773   // not handled in the current activation. Fabricates an exception
3774   // oop and initiates normal exception dispatching in this
3775   // frame. Since we need to preserve callee-saved values (currently
3776   // only for C2, but done for C1 as well) we need a callee-saved oop
3777   // map and therefore have to make these stubs into RuntimeStubs
3778   // rather than BufferBlobs.  If the compiler needs all registers to
3779   // be preserved between the fault point and the exception handler
3780   // then it must assume responsibility for that in
3781   // AbstractCompiler::continuation_for_implicit_null_exception or
3782   // continuation_for_implicit_division_by_zero_exception. All other
3783   // implicit exceptions (e.g., NullPointerException or
3784   // AbstractMethodError on entry) are either at call sites or
3785   // otherwise assume that stack unwinding will be initiated, so
3786   // caller saved registers were assumed volatile in the compiler.
3787   address generate_throw_exception(const char* name,
3788                                    address runtime_entry,
3789                                    Register arg1 = noreg,
3790                                    Register arg2 = noreg) {
3791     // Information about frame layout at time of blocking runtime call.
3792     // Note that we only have to preserve callee-saved registers since
3793     // the compilers are responsible for supplying a continuation point
3794     // if they expect all registers to be preserved.
3795     enum layout {
3796       rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
3797       rbp_off2,
3798       return_off,
3799       return_off2,
3800       framesize // inclusive of return address
3801     };
3802 
3803     int insts_size = 512;
3804     int locs_size  = 64;
3805 
3806     CodeBuffer code(name, insts_size, locs_size);
3807     OopMapSet* oop_maps  = new OopMapSet();
3808     MacroAssembler* masm = new MacroAssembler(&code);
3809 
3810     address start = __ pc();
3811 
3812     // This is an inlined and slightly modified version of call_VM
3813     // which has the ability to fetch the return PC out of
3814     // thread-local storage and also sets up last_Java_sp slightly
3815     // differently than the real call_VM
3816 
3817     __ enter(); // required for proper stackwalking of RuntimeStub frame
3818 
3819     assert(is_even(framesize/2), "sp not 16-byte aligned");
3820 
3821     // return address and rbp are already in place
3822     __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
3823 
3824     int frame_complete = __ pc() - start;
3825 
3826     // Set up last_Java_sp and last_Java_fp
3827     address the_pc = __ pc();
3828     __ set_last_Java_frame(rsp, rbp, the_pc);
3829     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
3830 
3831     // Call runtime
3832     if (arg1 != noreg) {
3833       assert(arg2 != c_rarg1, "clobbered");
3834       __ movptr(c_rarg1, arg1);
3835     }
3836     if (arg2 != noreg) {
3837       __ movptr(c_rarg2, arg2);
3838     }
3839     __ movptr(c_rarg0, r15_thread);
3840     BLOCK_COMMENT("call runtime_entry");
3841     __ call(RuntimeAddress(runtime_entry));
3842 
3843     // Generate oop map
3844     OopMap* map = new OopMap(framesize, 0);
3845 
3846     oop_maps->add_gc_map(the_pc - start, map);
3847 
3848     __ reset_last_Java_frame(true, true);
3849 
3850     __ leave(); // required for proper stackwalking of RuntimeStub frame
3851 
3852     // check for pending exceptions
3853 #ifdef ASSERT
3854     Label L;
3855     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()),
3856             (int32_t) NULL_WORD);
3857     __ jcc(Assembler::notEqual, L);
3858     __ should_not_reach_here();
3859     __ bind(L);
3860 #endif // ASSERT
3861     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3862 
3863 
3864     // codeBlob framesize is in words (not VMRegImpl::slot_size)
3865     RuntimeStub* stub =
3866       RuntimeStub::new_runtime_stub(name,
3867                                     &code,
3868                                     frame_complete,
3869                                     (framesize >> (LogBytesPerWord - LogBytesPerInt)),
3870                                     oop_maps, false);
3871     return stub->entry_point();
3872   }
3873 
3874   void create_control_words() {
3875     // Round to nearest, 53-bit mode, exceptions masked
3876     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
3877     // Round to zero, 53-bit mode, exception mased
3878     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
3879     // Round to nearest, 24-bit mode, exceptions masked
3880     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
3881     // Round to nearest, 64-bit mode, exceptions masked
3882     StubRoutines::_fpu_cntrl_wrd_64    = 0x037F;
3883     // Round to nearest, 64-bit mode, exceptions masked
3884     StubRoutines::_mxcsr_std           = 0x1F80;
3885     // Note: the following two constants are 80-bit values
3886     //       layout is critical for correct loading by FPU.
3887     // Bias for strict fp multiply/divide
3888     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
3889     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
3890     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
3891     // Un-Bias for strict fp multiply/divide
3892     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
3893     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
3894     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
3895   }
3896 
3897   // Initialization
3898   void generate_initial() {
3899     // Generates all stubs and initializes the entry points
3900 
3901     // This platform-specific settings are needed by generate_call_stub()
3902     create_control_words();
3903 
3904     // entry points that exist in all platforms Note: This is code
3905     // that could be shared among different platforms - however the
3906     // benefit seems to be smaller than the disadvantage of having a
3907     // much more complicated generator structure. See also comment in
3908     // stubRoutines.hpp.
3909 
3910     StubRoutines::_forward_exception_entry = generate_forward_exception();
3911 
3912     StubRoutines::_call_stub_entry =
3913       generate_call_stub(StubRoutines::_call_stub_return_address);
3914 
3915     // is referenced by megamorphic call
3916     StubRoutines::_catch_exception_entry = generate_catch_exception();
3917 
3918     // atomic calls
3919     StubRoutines::_atomic_xchg_entry         = generate_atomic_xchg();
3920     StubRoutines::_atomic_xchg_ptr_entry     = generate_atomic_xchg_ptr();
3921     StubRoutines::_atomic_cmpxchg_entry      = generate_atomic_cmpxchg();
3922     StubRoutines::_atomic_cmpxchg_byte_entry = generate_atomic_cmpxchg_byte();
3923     StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
3924     StubRoutines::_atomic_add_entry          = generate_atomic_add();
3925     StubRoutines::_atomic_add_ptr_entry      = generate_atomic_add_ptr();
3926     StubRoutines::_fence_entry               = generate_orderaccess_fence();
3927 
3928     StubRoutines::_handler_for_unsafe_access_entry =
3929       generate_handler_for_unsafe_access();
3930 
3931     // platform dependent
3932     StubRoutines::x86::_get_previous_fp_entry = generate_get_previous_fp();
3933     StubRoutines::x86::_get_previous_sp_entry = generate_get_previous_sp();
3934 
3935     StubRoutines::x86::_verify_mxcsr_entry    = generate_verify_mxcsr();
3936 
3937     // Build this early so it's available for the interpreter.
3938     StubRoutines::_throw_StackOverflowError_entry =
3939       generate_throw_exception("StackOverflowError throw_exception",
3940                                CAST_FROM_FN_PTR(address,
3941                                                 SharedRuntime::
3942                                                 throw_StackOverflowError));
3943     if (UseCRC32Intrinsics) {
3944       // set table address before stub generation which use it
3945       StubRoutines::_crc_table_adr = (address)StubRoutines::x86::_crc_table;
3946       StubRoutines::_updateBytesCRC32 = generate_updateBytesCRC32();
3947     }
3948   }
3949 
3950   void generate_all() {
3951     // Generates all stubs and initializes the entry points
3952 
3953     // These entry points require SharedInfo::stack0 to be set up in
3954     // non-core builds and need to be relocatable, so they each
3955     // fabricate a RuntimeStub internally.
3956     StubRoutines::_throw_AbstractMethodError_entry =
3957       generate_throw_exception("AbstractMethodError throw_exception",
3958                                CAST_FROM_FN_PTR(address,
3959                                                 SharedRuntime::
3960                                                 throw_AbstractMethodError));
3961 
3962     StubRoutines::_throw_IncompatibleClassChangeError_entry =
3963       generate_throw_exception("IncompatibleClassChangeError throw_exception",
3964                                CAST_FROM_FN_PTR(address,
3965                                                 SharedRuntime::
3966                                                 throw_IncompatibleClassChangeError));
3967 
3968     StubRoutines::_throw_NullPointerException_at_call_entry =
3969       generate_throw_exception("NullPointerException at call throw_exception",
3970                                CAST_FROM_FN_PTR(address,
3971                                                 SharedRuntime::
3972                                                 throw_NullPointerException_at_call));
3973 
3974     // entry points that are platform specific
3975     StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
3976     StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
3977     StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
3978     StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
3979 
3980     StubRoutines::x86::_float_sign_mask  = generate_fp_mask("float_sign_mask",  0x7FFFFFFF7FFFFFFF);
3981     StubRoutines::x86::_float_sign_flip  = generate_fp_mask("float_sign_flip",  0x8000000080000000);
3982     StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
3983     StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
3984 
3985     // support for verify_oop (must happen after universe_init)
3986     StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
3987 
3988     // arraycopy stubs used by compilers
3989     generate_arraycopy_stubs();
3990 
3991     generate_math_stubs();
3992 
3993     // don't bother generating these AES intrinsic stubs unless global flag is set
3994     if (UseAESIntrinsics) {
3995       StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask();  // needed by the others
3996 
3997       StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
3998       StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
3999       StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
4000       StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
4001     }
4002 
4003     // Safefetch stubs.
4004     generate_safefetch("SafeFetch32", sizeof(int),     &StubRoutines::_safefetch32_entry,
4005                                                        &StubRoutines::_safefetch32_fault_pc,
4006                                                        &StubRoutines::_safefetch32_continuation_pc);
4007     generate_safefetch("SafeFetchN", sizeof(intptr_t), &StubRoutines::_safefetchN_entry,
4008                                                        &StubRoutines::_safefetchN_fault_pc,
4009                                                        &StubRoutines::_safefetchN_continuation_pc);
4010 #ifdef COMPILER2
4011     if (UseMultiplyToLenIntrinsic) {
4012       StubRoutines::_multiplyToLen = generate_multiplyToLen();
4013     }
4014 #endif
4015   }
4016 
4017  public:
4018   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
4019     if (all) {
4020       generate_all();
4021     } else {
4022       generate_initial();
4023     }
4024   }
4025 }; // end class declaration
4026 
4027 void StubGenerator_generate(CodeBuffer* code, bool all) {
4028   StubGenerator g(code, all);
4029 }