1 /*
   2  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "incls/_precompiled.incl"
  26 #include "incls/_stubGenerator_x86_64.cpp.incl"
  27 
  28 // Declaration and definition of StubGenerator (no .hpp file).
  29 // For a more detailed description of the stub routine structure
  30 // see the comment in stubRoutines.hpp
  31 
  32 #define __ _masm->
  33 #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
  34 #define a__ ((Assembler*)_masm)->
  35 
  36 #ifdef PRODUCT
  37 #define BLOCK_COMMENT(str) /* nothing */
  38 #else
  39 #define BLOCK_COMMENT(str) __ block_comment(str)
  40 #endif
  41 
  42 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  43 const int MXCSR_MASK = 0xFFC0;  // Mask out any pending exceptions
  44 
  45 // Stub Code definitions
  46 
  47 static address handle_unsafe_access() {
  48   JavaThread* thread = JavaThread::current();
  49   address pc = thread->saved_exception_pc();
  50   // pc is the instruction which we must emulate
  51   // doing a no-op is fine:  return garbage from the load
  52   // therefore, compute npc
  53   address npc = Assembler::locate_next_instruction(pc);
  54 
  55   // request an async exception
  56   thread->set_pending_unsafe_access_error();
  57 
  58   // return address of next instruction to execute
  59   return npc;
  60 }
  61 
  62 class StubGenerator: public StubCodeGenerator {
  63  private:
  64 
  65 #ifdef PRODUCT
  66 #define inc_counter_np(counter) (0)
  67 #else
  68   void inc_counter_np_(int& counter) {
  69     __ incrementl(ExternalAddress((address)&counter));
  70   }
  71 #define inc_counter_np(counter) \
  72   BLOCK_COMMENT("inc_counter " #counter); \
  73   inc_counter_np_(counter);
  74 #endif
  75 
  76   // Call stubs are used to call Java from C
  77   //
  78   // Linux Arguments:
  79   //    c_rarg0:   call wrapper address                   address
  80   //    c_rarg1:   result                                 address
  81   //    c_rarg2:   result type                            BasicType
  82   //    c_rarg3:   method                                 methodOop
  83   //    c_rarg4:   (interpreter) entry point              address
  84   //    c_rarg5:   parameters                             intptr_t*
  85   //    16(rbp): parameter size (in words)              int
  86   //    24(rbp): thread                                 Thread*
  87   //
  88   //     [ return_from_Java     ] <--- rsp
  89   //     [ argument word n      ]
  90   //      ...
  91   // -12 [ argument word 1      ]
  92   // -11 [ saved r15            ] <--- rsp_after_call
  93   // -10 [ saved r14            ]
  94   //  -9 [ saved r13            ]
  95   //  -8 [ saved r12            ]
  96   //  -7 [ saved rbx            ]
  97   //  -6 [ call wrapper         ]
  98   //  -5 [ result               ]
  99   //  -4 [ result type          ]
 100   //  -3 [ method               ]
 101   //  -2 [ entry point          ]
 102   //  -1 [ parameters           ]
 103   //   0 [ saved rbp            ] <--- rbp
 104   //   1 [ return address       ]
 105   //   2 [ parameter size       ]
 106   //   3 [ thread               ]
 107   //
 108   // Windows Arguments:
 109   //    c_rarg0:   call wrapper address                   address
 110   //    c_rarg1:   result                                 address
 111   //    c_rarg2:   result type                            BasicType
 112   //    c_rarg3:   method                                 methodOop
 113   //    48(rbp): (interpreter) entry point              address
 114   //    56(rbp): parameters                             intptr_t*
 115   //    64(rbp): parameter size (in words)              int
 116   //    72(rbp): thread                                 Thread*
 117   //
 118   //     [ return_from_Java     ] <--- rsp
 119   //     [ argument word n      ]
 120   //      ...
 121   //  -8 [ argument word 1      ]
 122   //  -7 [ saved r15            ] <--- rsp_after_call
 123   //  -6 [ saved r14            ]
 124   //  -5 [ saved r13            ]
 125   //  -4 [ saved r12            ]
 126   //  -3 [ saved rdi            ]
 127   //  -2 [ saved rsi            ]
 128   //  -1 [ saved rbx            ]
 129   //   0 [ saved rbp            ] <--- rbp
 130   //   1 [ return address       ]
 131   //   2 [ call wrapper         ]
 132   //   3 [ result               ]
 133   //   4 [ result type          ]
 134   //   5 [ method               ]
 135   //   6 [ entry point          ]
 136   //   7 [ parameters           ]
 137   //   8 [ parameter size       ]
 138   //   9 [ thread               ]
 139   //
 140   //    Windows reserves the callers stack space for arguments 1-4.
 141   //    We spill c_rarg0-c_rarg3 to this space.
 142 
 143   // Call stub stack layout word offsets from rbp
 144   enum call_stub_layout {
 145 #ifdef _WIN64
 146     rsp_after_call_off = -7,
 147     r15_off            = rsp_after_call_off,
 148     r14_off            = -6,
 149     r13_off            = -5,
 150     r12_off            = -4,
 151     rdi_off            = -3,
 152     rsi_off            = -2,
 153     rbx_off            = -1,
 154     rbp_off            =  0,
 155     retaddr_off        =  1,
 156     call_wrapper_off   =  2,
 157     result_off         =  3,
 158     result_type_off    =  4,
 159     method_off         =  5,
 160     entry_point_off    =  6,
 161     parameters_off     =  7,
 162     parameter_size_off =  8,
 163     thread_off         =  9
 164 #else
 165     rsp_after_call_off = -12,
 166     mxcsr_off          = rsp_after_call_off,
 167     r15_off            = -11,
 168     r14_off            = -10,
 169     r13_off            = -9,
 170     r12_off            = -8,
 171     rbx_off            = -7,
 172     call_wrapper_off   = -6,
 173     result_off         = -5,
 174     result_type_off    = -4,
 175     method_off         = -3,
 176     entry_point_off    = -2,
 177     parameters_off     = -1,
 178     rbp_off            =  0,
 179     retaddr_off        =  1,
 180     parameter_size_off =  2,
 181     thread_off         =  3
 182 #endif
 183   };
 184 
 185   address generate_call_stub(address& return_address) {
 186     assert((int)frame::entry_frame_after_call_words == -(int)rsp_after_call_off + 1 &&
 187            (int)frame::entry_frame_call_wrapper_offset == (int)call_wrapper_off,
 188            "adjust this code");
 189     StubCodeMark mark(this, "StubRoutines", "call_stub");
 190     address start = __ pc();
 191 
 192     // same as in generate_catch_exception()!
 193     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 194 
 195     const Address call_wrapper  (rbp, call_wrapper_off   * wordSize);
 196     const Address result        (rbp, result_off         * wordSize);
 197     const Address result_type   (rbp, result_type_off    * wordSize);
 198     const Address method        (rbp, method_off         * wordSize);
 199     const Address entry_point   (rbp, entry_point_off    * wordSize);
 200     const Address parameters    (rbp, parameters_off     * wordSize);
 201     const Address parameter_size(rbp, parameter_size_off * wordSize);
 202 
 203     // same as in generate_catch_exception()!
 204     const Address thread        (rbp, thread_off         * wordSize);
 205 
 206     const Address r15_save(rbp, r15_off * wordSize);
 207     const Address r14_save(rbp, r14_off * wordSize);
 208     const Address r13_save(rbp, r13_off * wordSize);
 209     const Address r12_save(rbp, r12_off * wordSize);
 210     const Address rbx_save(rbp, rbx_off * wordSize);
 211 
 212     // stub code
 213     __ enter();
 214     __ subptr(rsp, -rsp_after_call_off * wordSize);
 215 
 216     // save register parameters
 217 #ifndef _WIN64
 218     __ movptr(parameters,   c_rarg5); // parameters
 219     __ movptr(entry_point,  c_rarg4); // entry_point
 220 #endif
 221 
 222     __ movptr(method,       c_rarg3); // method
 223     __ movl(result_type,  c_rarg2);   // result type
 224     __ movptr(result,       c_rarg1); // result
 225     __ movptr(call_wrapper, c_rarg0); // call wrapper
 226 
 227     // save regs belonging to calling function
 228     __ movptr(rbx_save, rbx);
 229     __ movptr(r12_save, r12);
 230     __ movptr(r13_save, r13);
 231     __ movptr(r14_save, r14);
 232     __ movptr(r15_save, r15);
 233 
 234 #ifdef _WIN64
 235     const Address rdi_save(rbp, rdi_off * wordSize);
 236     const Address rsi_save(rbp, rsi_off * wordSize);
 237 
 238     __ movptr(rsi_save, rsi);
 239     __ movptr(rdi_save, rdi);
 240 #else
 241     const Address mxcsr_save(rbp, mxcsr_off * wordSize);
 242     {
 243       Label skip_ldmx;
 244       __ stmxcsr(mxcsr_save);
 245       __ movl(rax, mxcsr_save);
 246       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 247       ExternalAddress mxcsr_std(StubRoutines::x86::mxcsr_std());
 248       __ cmp32(rax, mxcsr_std);
 249       __ jcc(Assembler::equal, skip_ldmx);
 250       __ ldmxcsr(mxcsr_std);
 251       __ bind(skip_ldmx);
 252     }
 253 #endif
 254 
 255     // Load up thread register
 256     __ movptr(r15_thread, thread);
 257     __ reinit_heapbase();
 258 
 259 #ifdef ASSERT
 260     // make sure we have no pending exceptions
 261     {
 262       Label L;
 263       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 264       __ jcc(Assembler::equal, L);
 265       __ stop("StubRoutines::call_stub: entered with pending exception");
 266       __ bind(L);
 267     }
 268 #endif
 269 
 270     // pass parameters if any
 271     BLOCK_COMMENT("pass parameters if any");
 272     Label parameters_done;
 273     __ movl(c_rarg3, parameter_size);
 274     __ testl(c_rarg3, c_rarg3);
 275     __ jcc(Assembler::zero, parameters_done);
 276 
 277     Label loop;
 278     __ movptr(c_rarg2, parameters);       // parameter pointer
 279     __ movl(c_rarg1, c_rarg3);            // parameter counter is in c_rarg1
 280     __ BIND(loop);
 281     __ movptr(rax, Address(c_rarg2, 0));// get parameter
 282     __ addptr(c_rarg2, wordSize);       // advance to next parameter
 283     __ decrementl(c_rarg1);             // decrement counter
 284     __ push(rax);                       // pass parameter
 285     __ jcc(Assembler::notZero, loop);
 286 
 287     // call Java function
 288     __ BIND(parameters_done);
 289     __ movptr(rbx, method);             // get methodOop
 290     __ movptr(c_rarg1, entry_point);    // get entry_point
 291     __ mov(r13, rsp);                   // set sender sp
 292     BLOCK_COMMENT("call Java function");
 293     __ call(c_rarg1);
 294 
 295     BLOCK_COMMENT("call_stub_return_address:");
 296     return_address = __ pc();
 297 
 298     // store result depending on type (everything that is not
 299     // T_OBJECT, T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 300     __ movptr(c_rarg0, result);
 301     Label is_long, is_float, is_double, exit;
 302     __ movl(c_rarg1, result_type);
 303     __ cmpl(c_rarg1, T_OBJECT);
 304     __ jcc(Assembler::equal, is_long);
 305     __ cmpl(c_rarg1, T_LONG);
 306     __ jcc(Assembler::equal, is_long);
 307     __ cmpl(c_rarg1, T_FLOAT);
 308     __ jcc(Assembler::equal, is_float);
 309     __ cmpl(c_rarg1, T_DOUBLE);
 310     __ jcc(Assembler::equal, is_double);
 311 
 312     // handle T_INT case
 313     __ movl(Address(c_rarg0, 0), rax);
 314 
 315     __ BIND(exit);
 316 
 317     // pop parameters
 318     __ lea(rsp, rsp_after_call);
 319 
 320 #ifdef ASSERT
 321     // verify that threads correspond
 322     {
 323       Label L, S;
 324       __ cmpptr(r15_thread, thread);
 325       __ jcc(Assembler::notEqual, S);
 326       __ get_thread(rbx);
 327       __ cmpptr(r15_thread, rbx);
 328       __ jcc(Assembler::equal, L);
 329       __ bind(S);
 330       __ jcc(Assembler::equal, L);
 331       __ stop("StubRoutines::call_stub: threads must correspond");
 332       __ bind(L);
 333     }
 334 #endif
 335 
 336     // restore regs belonging to calling function
 337     __ movptr(r15, r15_save);
 338     __ movptr(r14, r14_save);
 339     __ movptr(r13, r13_save);
 340     __ movptr(r12, r12_save);
 341     __ movptr(rbx, rbx_save);
 342 
 343 #ifdef _WIN64
 344     __ movptr(rdi, rdi_save);
 345     __ movptr(rsi, rsi_save);
 346 #else
 347     __ ldmxcsr(mxcsr_save);
 348 #endif
 349 
 350     // restore rsp
 351     __ addptr(rsp, -rsp_after_call_off * wordSize);
 352 
 353     // return
 354     __ pop(rbp);
 355     __ ret(0);
 356 
 357     // handle return types different from T_INT
 358     __ BIND(is_long);
 359     __ movq(Address(c_rarg0, 0), rax);
 360     __ jmp(exit);
 361 
 362     __ BIND(is_float);
 363     __ movflt(Address(c_rarg0, 0), xmm0);
 364     __ jmp(exit);
 365 
 366     __ BIND(is_double);
 367     __ movdbl(Address(c_rarg0, 0), xmm0);
 368     __ jmp(exit);
 369 
 370     return start;
 371   }
 372 
 373   // Return point for a Java call if there's an exception thrown in
 374   // Java code.  The exception is caught and transformed into a
 375   // pending exception stored in JavaThread that can be tested from
 376   // within the VM.
 377   //
 378   // Note: Usually the parameters are removed by the callee. In case
 379   // of an exception crossing an activation frame boundary, that is
 380   // not the case if the callee is compiled code => need to setup the
 381   // rsp.
 382   //
 383   // rax: exception oop
 384 
 385   address generate_catch_exception() {
 386     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 387     address start = __ pc();
 388 
 389     // same as in generate_call_stub():
 390     const Address rsp_after_call(rbp, rsp_after_call_off * wordSize);
 391     const Address thread        (rbp, thread_off         * wordSize);
 392 
 393 #ifdef ASSERT
 394     // verify that threads correspond
 395     {
 396       Label L, S;
 397       __ cmpptr(r15_thread, thread);
 398       __ jcc(Assembler::notEqual, S);
 399       __ get_thread(rbx);
 400       __ cmpptr(r15_thread, rbx);
 401       __ jcc(Assembler::equal, L);
 402       __ bind(S);
 403       __ stop("StubRoutines::catch_exception: threads must correspond");
 404       __ bind(L);
 405     }
 406 #endif
 407 
 408     // set pending exception
 409     __ verify_oop(rax);
 410 
 411     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), rax);
 412     __ lea(rscratch1, ExternalAddress((address)__FILE__));
 413     __ movptr(Address(r15_thread, Thread::exception_file_offset()), rscratch1);
 414     __ movl(Address(r15_thread, Thread::exception_line_offset()), (int)  __LINE__);
 415 
 416     // complete return to VM
 417     assert(StubRoutines::_call_stub_return_address != NULL,
 418            "_call_stub_return_address must have been generated before");
 419     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 420 
 421     return start;
 422   }
 423 
 424   // Continuation point for runtime calls returning with a pending
 425   // exception.  The pending exception check happened in the runtime
 426   // or native call stub.  The pending exception in Thread is
 427   // converted into a Java-level exception.
 428   //
 429   // Contract with Java-level exception handlers:
 430   // rax: exception
 431   // rdx: throwing pc
 432   //
 433   // NOTE: At entry of this stub, exception-pc must be on stack !!
 434 
 435   address generate_forward_exception() {
 436     StubCodeMark mark(this, "StubRoutines", "forward exception");
 437     address start = __ pc();
 438 
 439     // Upon entry, the sp points to the return address returning into
 440     // Java (interpreted or compiled) code; i.e., the return address
 441     // becomes the throwing pc.
 442     //
 443     // Arguments pushed before the runtime call are still on the stack
 444     // but the exception handler will reset the stack pointer ->
 445     // ignore them.  A potential result in registers can be ignored as
 446     // well.
 447 
 448 #ifdef ASSERT
 449     // make sure this code is only executed if there is a pending exception
 450     {
 451       Label L;
 452       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL);
 453       __ jcc(Assembler::notEqual, L);
 454       __ stop("StubRoutines::forward exception: no pending exception (1)");
 455       __ bind(L);
 456     }
 457 #endif
 458 
 459     // compute exception handler into rbx
 460     __ movptr(c_rarg0, Address(rsp, 0));
 461     BLOCK_COMMENT("call exception_handler_for_return_address");
 462     __ call_VM_leaf(CAST_FROM_FN_PTR(address,
 463                          SharedRuntime::exception_handler_for_return_address),
 464                     r15_thread, c_rarg0);
 465     __ mov(rbx, rax);
 466 
 467     // setup rax & rdx, remove return address & clear pending exception
 468     __ pop(rdx);
 469     __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
 470     __ movptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 471 
 472 #ifdef ASSERT
 473     // make sure exception is set
 474     {
 475       Label L;
 476       __ testptr(rax, rax);
 477       __ jcc(Assembler::notEqual, L);
 478       __ stop("StubRoutines::forward exception: no pending exception (2)");
 479       __ bind(L);
 480     }
 481 #endif
 482 
 483     // continue at exception handler (return address removed)
 484     // rax: exception
 485     // rbx: exception handler
 486     // rdx: throwing pc
 487     __ verify_oop(rax);
 488     __ jmp(rbx);
 489 
 490     return start;
 491   }
 492 
 493   // Support for jint atomic::xchg(jint exchange_value, volatile jint* dest)
 494   //
 495   // Arguments :
 496   //    c_rarg0: exchange_value
 497   //    c_rarg0: dest
 498   //
 499   // Result:
 500   //    *dest <- ex, return (orig *dest)
 501   address generate_atomic_xchg() {
 502     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 503     address start = __ pc();
 504 
 505     __ movl(rax, c_rarg0); // Copy to eax we need a return value anyhow
 506     __ xchgl(rax, Address(c_rarg1, 0)); // automatic LOCK
 507     __ ret(0);
 508 
 509     return start;
 510   }
 511 
 512   // Support for intptr_t atomic::xchg_ptr(intptr_t exchange_value, volatile intptr_t* dest)
 513   //
 514   // Arguments :
 515   //    c_rarg0: exchange_value
 516   //    c_rarg1: dest
 517   //
 518   // Result:
 519   //    *dest <- ex, return (orig *dest)
 520   address generate_atomic_xchg_ptr() {
 521     StubCodeMark mark(this, "StubRoutines", "atomic_xchg_ptr");
 522     address start = __ pc();
 523 
 524     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 525     __ xchgptr(rax, Address(c_rarg1, 0)); // automatic LOCK
 526     __ ret(0);
 527 
 528     return start;
 529   }
 530 
 531   // Support for jint atomic::atomic_cmpxchg(jint exchange_value, volatile jint* dest,
 532   //                                         jint compare_value)
 533   //
 534   // Arguments :
 535   //    c_rarg0: exchange_value
 536   //    c_rarg1: dest
 537   //    c_rarg2: compare_value
 538   //
 539   // Result:
 540   //    if ( compare_value == *dest ) {
 541   //       *dest = exchange_value
 542   //       return compare_value;
 543   //    else
 544   //       return *dest;
 545   address generate_atomic_cmpxchg() {
 546     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg");
 547     address start = __ pc();
 548 
 549     __ movl(rax, c_rarg2);
 550    if ( os::is_MP() ) __ lock();
 551     __ cmpxchgl(c_rarg0, Address(c_rarg1, 0));
 552     __ ret(0);
 553 
 554     return start;
 555   }
 556 
 557   // Support for jint atomic::atomic_cmpxchg_long(jlong exchange_value,
 558   //                                             volatile jlong* dest,
 559   //                                             jlong compare_value)
 560   // Arguments :
 561   //    c_rarg0: exchange_value
 562   //    c_rarg1: dest
 563   //    c_rarg2: compare_value
 564   //
 565   // Result:
 566   //    if ( compare_value == *dest ) {
 567   //       *dest = exchange_value
 568   //       return compare_value;
 569   //    else
 570   //       return *dest;
 571   address generate_atomic_cmpxchg_long() {
 572     StubCodeMark mark(this, "StubRoutines", "atomic_cmpxchg_long");
 573     address start = __ pc();
 574 
 575     __ movq(rax, c_rarg2);
 576    if ( os::is_MP() ) __ lock();
 577     __ cmpxchgq(c_rarg0, Address(c_rarg1, 0));
 578     __ ret(0);
 579 
 580     return start;
 581   }
 582 
 583   // Support for jint atomic::add(jint add_value, volatile jint* dest)
 584   //
 585   // Arguments :
 586   //    c_rarg0: add_value
 587   //    c_rarg1: dest
 588   //
 589   // Result:
 590   //    *dest += add_value
 591   //    return *dest;
 592   address generate_atomic_add() {
 593     StubCodeMark mark(this, "StubRoutines", "atomic_add");
 594     address start = __ pc();
 595 
 596     __ movl(rax, c_rarg0);
 597    if ( os::is_MP() ) __ lock();
 598     __ xaddl(Address(c_rarg1, 0), c_rarg0);
 599     __ addl(rax, c_rarg0);
 600     __ ret(0);
 601 
 602     return start;
 603   }
 604 
 605   // Support for intptr_t atomic::add_ptr(intptr_t add_value, volatile intptr_t* dest)
 606   //
 607   // Arguments :
 608   //    c_rarg0: add_value
 609   //    c_rarg1: dest
 610   //
 611   // Result:
 612   //    *dest += add_value
 613   //    return *dest;
 614   address generate_atomic_add_ptr() {
 615     StubCodeMark mark(this, "StubRoutines", "atomic_add_ptr");
 616     address start = __ pc();
 617 
 618     __ movptr(rax, c_rarg0); // Copy to eax we need a return value anyhow
 619    if ( os::is_MP() ) __ lock();
 620     __ xaddptr(Address(c_rarg1, 0), c_rarg0);
 621     __ addptr(rax, c_rarg0);
 622     __ ret(0);
 623 
 624     return start;
 625   }
 626 
 627   // Support for intptr_t OrderAccess::fence()
 628   //
 629   // Arguments :
 630   //
 631   // Result:
 632   address generate_orderaccess_fence() {
 633     StubCodeMark mark(this, "StubRoutines", "orderaccess_fence");
 634     address start = __ pc();
 635     __ membar(Assembler::StoreLoad);
 636     __ ret(0);
 637 
 638     return start;
 639   }
 640 
 641   // Support for intptr_t get_previous_fp()
 642   //
 643   // This routine is used to find the previous frame pointer for the
 644   // caller (current_frame_guess). This is used as part of debugging
 645   // ps() is seemingly lost trying to find frames.
 646   // This code assumes that caller current_frame_guess) has a frame.
 647   address generate_get_previous_fp() {
 648     StubCodeMark mark(this, "StubRoutines", "get_previous_fp");
 649     const Address old_fp(rbp, 0);
 650     const Address older_fp(rax, 0);
 651     address start = __ pc();
 652 
 653     __ enter();
 654     __ movptr(rax, old_fp); // callers fp
 655     __ movptr(rax, older_fp); // the frame for ps()
 656     __ pop(rbp);
 657     __ ret(0);
 658 
 659     return start;
 660   }
 661 
 662   //----------------------------------------------------------------------------------------------------
 663   // Support for void verify_mxcsr()
 664   //
 665   // This routine is used with -Xcheck:jni to verify that native
 666   // JNI code does not return to Java code without restoring the
 667   // MXCSR register to our expected state.
 668 
 669   address generate_verify_mxcsr() {
 670     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 671     address start = __ pc();
 672 
 673     const Address mxcsr_save(rsp, 0);
 674 
 675     if (CheckJNICalls) {
 676       Label ok_ret;
 677       __ push(rax);
 678       __ subptr(rsp, wordSize);      // allocate a temp location
 679       __ stmxcsr(mxcsr_save);
 680       __ movl(rax, mxcsr_save);
 681       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 682       __ cmpl(rax, *(int *)(StubRoutines::x86::mxcsr_std()));
 683       __ jcc(Assembler::equal, ok_ret);
 684 
 685       __ warn("MXCSR changed by native JNI code, use -XX:+RestoreMXCSROnJNICall");
 686 
 687       __ ldmxcsr(ExternalAddress(StubRoutines::x86::mxcsr_std()));
 688 
 689       __ bind(ok_ret);
 690       __ addptr(rsp, wordSize);
 691       __ pop(rax);
 692     }
 693 
 694     __ ret(0);
 695 
 696     return start;
 697   }
 698 
 699   address generate_f2i_fixup() {
 700     StubCodeMark mark(this, "StubRoutines", "f2i_fixup");
 701     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 702 
 703     address start = __ pc();
 704 
 705     Label L;
 706 
 707     __ push(rax);
 708     __ push(c_rarg3);
 709     __ push(c_rarg2);
 710     __ push(c_rarg1);
 711 
 712     __ movl(rax, 0x7f800000);
 713     __ xorl(c_rarg3, c_rarg3);
 714     __ movl(c_rarg2, inout);
 715     __ movl(c_rarg1, c_rarg2);
 716     __ andl(c_rarg1, 0x7fffffff);
 717     __ cmpl(rax, c_rarg1); // NaN? -> 0
 718     __ jcc(Assembler::negative, L);
 719     __ testl(c_rarg2, c_rarg2); // signed ? min_jint : max_jint
 720     __ movl(c_rarg3, 0x80000000);
 721     __ movl(rax, 0x7fffffff);
 722     __ cmovl(Assembler::positive, c_rarg3, rax);
 723 
 724     __ bind(L);
 725     __ movptr(inout, c_rarg3);
 726 
 727     __ pop(c_rarg1);
 728     __ pop(c_rarg2);
 729     __ pop(c_rarg3);
 730     __ pop(rax);
 731 
 732     __ ret(0);
 733 
 734     return start;
 735   }
 736 
 737   address generate_f2l_fixup() {
 738     StubCodeMark mark(this, "StubRoutines", "f2l_fixup");
 739     Address inout(rsp, 5 * wordSize); // return address + 4 saves
 740     address start = __ pc();
 741 
 742     Label L;
 743 
 744     __ push(rax);
 745     __ push(c_rarg3);
 746     __ push(c_rarg2);
 747     __ push(c_rarg1);
 748 
 749     __ movl(rax, 0x7f800000);
 750     __ xorl(c_rarg3, c_rarg3);
 751     __ movl(c_rarg2, inout);
 752     __ movl(c_rarg1, c_rarg2);
 753     __ andl(c_rarg1, 0x7fffffff);
 754     __ cmpl(rax, c_rarg1); // NaN? -> 0
 755     __ jcc(Assembler::negative, L);
 756     __ testl(c_rarg2, c_rarg2); // signed ? min_jlong : max_jlong
 757     __ mov64(c_rarg3, 0x8000000000000000);
 758     __ mov64(rax, 0x7fffffffffffffff);
 759     __ cmov(Assembler::positive, c_rarg3, rax);
 760 
 761     __ bind(L);
 762     __ movptr(inout, c_rarg3);
 763 
 764     __ pop(c_rarg1);
 765     __ pop(c_rarg2);
 766     __ pop(c_rarg3);
 767     __ pop(rax);
 768 
 769     __ ret(0);
 770 
 771     return start;
 772   }
 773 
 774   address generate_d2i_fixup() {
 775     StubCodeMark mark(this, "StubRoutines", "d2i_fixup");
 776     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 777 
 778     address start = __ pc();
 779 
 780     Label L;
 781 
 782     __ push(rax);
 783     __ push(c_rarg3);
 784     __ push(c_rarg2);
 785     __ push(c_rarg1);
 786     __ push(c_rarg0);
 787 
 788     __ movl(rax, 0x7ff00000);
 789     __ movq(c_rarg2, inout);
 790     __ movl(c_rarg3, c_rarg2);
 791     __ mov(c_rarg1, c_rarg2);
 792     __ mov(c_rarg0, c_rarg2);
 793     __ negl(c_rarg3);
 794     __ shrptr(c_rarg1, 0x20);
 795     __ orl(c_rarg3, c_rarg2);
 796     __ andl(c_rarg1, 0x7fffffff);
 797     __ xorl(c_rarg2, c_rarg2);
 798     __ shrl(c_rarg3, 0x1f);
 799     __ orl(c_rarg1, c_rarg3);
 800     __ cmpl(rax, c_rarg1);
 801     __ jcc(Assembler::negative, L); // NaN -> 0
 802     __ testptr(c_rarg0, c_rarg0); // signed ? min_jint : max_jint
 803     __ movl(c_rarg2, 0x80000000);
 804     __ movl(rax, 0x7fffffff);
 805     __ cmov(Assembler::positive, c_rarg2, rax);
 806 
 807     __ bind(L);
 808     __ movptr(inout, c_rarg2);
 809 
 810     __ pop(c_rarg0);
 811     __ pop(c_rarg1);
 812     __ pop(c_rarg2);
 813     __ pop(c_rarg3);
 814     __ pop(rax);
 815 
 816     __ ret(0);
 817 
 818     return start;
 819   }
 820 
 821   address generate_d2l_fixup() {
 822     StubCodeMark mark(this, "StubRoutines", "d2l_fixup");
 823     Address inout(rsp, 6 * wordSize); // return address + 5 saves
 824 
 825     address start = __ pc();
 826 
 827     Label L;
 828 
 829     __ push(rax);
 830     __ push(c_rarg3);
 831     __ push(c_rarg2);
 832     __ push(c_rarg1);
 833     __ push(c_rarg0);
 834 
 835     __ movl(rax, 0x7ff00000);
 836     __ movq(c_rarg2, inout);
 837     __ movl(c_rarg3, c_rarg2);
 838     __ mov(c_rarg1, c_rarg2);
 839     __ mov(c_rarg0, c_rarg2);
 840     __ negl(c_rarg3);
 841     __ shrptr(c_rarg1, 0x20);
 842     __ orl(c_rarg3, c_rarg2);
 843     __ andl(c_rarg1, 0x7fffffff);
 844     __ xorl(c_rarg2, c_rarg2);
 845     __ shrl(c_rarg3, 0x1f);
 846     __ orl(c_rarg1, c_rarg3);
 847     __ cmpl(rax, c_rarg1);
 848     __ jcc(Assembler::negative, L); // NaN -> 0
 849     __ testq(c_rarg0, c_rarg0); // signed ? min_jlong : max_jlong
 850     __ mov64(c_rarg2, 0x8000000000000000);
 851     __ mov64(rax, 0x7fffffffffffffff);
 852     __ cmovq(Assembler::positive, c_rarg2, rax);
 853 
 854     __ bind(L);
 855     __ movq(inout, c_rarg2);
 856 
 857     __ pop(c_rarg0);
 858     __ pop(c_rarg1);
 859     __ pop(c_rarg2);
 860     __ pop(c_rarg3);
 861     __ pop(rax);
 862 
 863     __ ret(0);
 864 
 865     return start;
 866   }
 867 
 868   address generate_fp_mask(const char *stub_name, int64_t mask) {
 869     __ align(CodeEntryAlignment);
 870     StubCodeMark mark(this, "StubRoutines", stub_name);
 871     address start = __ pc();
 872 
 873     __ emit_data64( mask, relocInfo::none );
 874     __ emit_data64( mask, relocInfo::none );
 875 
 876     return start;
 877   }
 878 
 879   // The following routine generates a subroutine to throw an
 880   // asynchronous UnknownError when an unsafe access gets a fault that
 881   // could not be reasonably prevented by the programmer.  (Example:
 882   // SIGBUS/OBJERR.)
 883   address generate_handler_for_unsafe_access() {
 884     StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
 885     address start = __ pc();
 886 
 887     __ push(0);                       // hole for return address-to-be
 888     __ pusha();                       // push registers
 889     Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);
 890 
 891     __ subptr(rsp, frame::arg_reg_save_area_bytes);
 892     BLOCK_COMMENT("call handle_unsafe_access");
 893     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
 894     __ addptr(rsp, frame::arg_reg_save_area_bytes);
 895 
 896     __ movptr(next_pc, rax);          // stuff next address
 897     __ popa();
 898     __ ret(0);                        // jump to next address
 899 
 900     return start;
 901   }
 902 
 903   // Non-destructive plausibility checks for oops
 904   //
 905   // Arguments:
 906   //    all args on stack!
 907   //
 908   // Stack after saving c_rarg3:
 909   //    [tos + 0]: saved c_rarg3
 910   //    [tos + 1]: saved c_rarg2
 911   //    [tos + 2]: saved r12 (several TemplateTable methods use it)
 912   //    [tos + 3]: saved flags
 913   //    [tos + 4]: return address
 914   //  * [tos + 5]: error message (char*)
 915   //  * [tos + 6]: object to verify (oop)
 916   //  * [tos + 7]: saved rax - saved by caller and bashed
 917   //  * [tos + 8]: saved r10 (rscratch1) - saved by caller
 918   //  * = popped on exit
 919   address generate_verify_oop() {
 920     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 921     address start = __ pc();
 922 
 923     Label exit, error;
 924 
 925     __ pushf();
 926     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 927 
 928     __ push(r12);
 929 
 930     // save c_rarg2 and c_rarg3
 931     __ push(c_rarg2);
 932     __ push(c_rarg3);
 933 
 934     enum {
 935            // After previous pushes.
 936            oop_to_verify = 6 * wordSize,
 937            saved_rax     = 7 * wordSize,
 938            saved_r10     = 8 * wordSize,
 939 
 940            // Before the call to MacroAssembler::debug(), see below.
 941            return_addr   = 16 * wordSize,
 942            error_msg     = 17 * wordSize
 943     };
 944 
 945     // get object
 946     __ movptr(rax, Address(rsp, oop_to_verify));
 947 
 948     // make sure object is 'reasonable'
 949     __ testptr(rax, rax);
 950     __ jcc(Assembler::zero, exit); // if obj is NULL it is OK
 951     // Check if the oop is in the right area of memory
 952     __ movptr(c_rarg2, rax);
 953     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_mask());
 954     __ andptr(c_rarg2, c_rarg3);
 955     __ movptr(c_rarg3, (intptr_t) Universe::verify_oop_bits());
 956     __ cmpptr(c_rarg2, c_rarg3);
 957     __ jcc(Assembler::notZero, error);
 958 
 959     // set r12 to heapbase for load_klass()
 960     __ reinit_heapbase();
 961 
 962     // make sure klass is 'reasonable'
 963     __ load_klass(rax, rax);  // get klass
 964     __ testptr(rax, rax);
 965     __ jcc(Assembler::zero, error); // if klass is NULL it is broken
 966     // Check if the klass is in the right area of memory
 967     __ mov(c_rarg2, rax);
 968     __ movptr(c_rarg3, (intptr_t) Universe::verify_klass_mask());
 969     __ andptr(c_rarg2, c_rarg3);
 970     __ movptr(c_rarg3, (intptr_t) Universe::verify_klass_bits());
 971     __ cmpptr(c_rarg2, c_rarg3);
 972     __ jcc(Assembler::notZero, error);
 973 
 974     // make sure klass' klass is 'reasonable'
 975     __ load_klass(rax, rax);
 976     __ testptr(rax, rax);
 977     __ jcc(Assembler::zero, error); // if klass' klass is NULL it is broken
 978     // Check if the klass' klass is in the right area of memory
 979     __ movptr(c_rarg3, (intptr_t) Universe::verify_klass_mask());
 980     __ andptr(rax, c_rarg3);
 981     __ movptr(c_rarg3, (intptr_t) Universe::verify_klass_bits());
 982     __ cmpptr(rax, c_rarg3);
 983     __ jcc(Assembler::notZero, error);
 984 
 985     // return if everything seems ok
 986     __ bind(exit);
 987     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
 988     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
 989     __ pop(c_rarg3);                             // restore c_rarg3
 990     __ pop(c_rarg2);                             // restore c_rarg2
 991     __ pop(r12);                                 // restore r12
 992     __ popf();                                   // restore flags
 993     __ ret(4 * wordSize);                        // pop caller saved stuff
 994 
 995     // handle errors
 996     __ bind(error);
 997     __ movptr(rax, Address(rsp, saved_rax));     // get saved rax back
 998     __ movptr(rscratch1, Address(rsp, saved_r10)); // get saved r10 back
 999     __ pop(c_rarg3);                             // get saved c_rarg3 back
1000     __ pop(c_rarg2);                             // get saved c_rarg2 back
1001     __ pop(r12);                                 // get saved r12 back
1002     __ popf();                                   // get saved flags off stack --
1003                                                  // will be ignored
1004 
1005     __ pusha();                                  // push registers
1006                                                  // (rip is already
1007                                                  // already pushed)
1008     // debug(char* msg, int64_t pc, int64_t regs[])
1009     // We've popped the registers we'd saved (c_rarg3, c_rarg2 and flags), and
1010     // pushed all the registers, so now the stack looks like:
1011     //     [tos +  0] 16 saved registers
1012     //     [tos + 16] return address
1013     //   * [tos + 17] error message (char*)
1014     //   * [tos + 18] object to verify (oop)
1015     //   * [tos + 19] saved rax - saved by caller and bashed
1016     //   * [tos + 20] saved r10 (rscratch1) - saved by caller
1017     //   * = popped on exit
1018 
1019     __ movptr(c_rarg0, Address(rsp, error_msg));    // pass address of error message
1020     __ movptr(c_rarg1, Address(rsp, return_addr));  // pass return address
1021     __ movq(c_rarg2, rsp);                          // pass address of regs on stack
1022     __ mov(r12, rsp);                               // remember rsp
1023     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1024     __ andptr(rsp, -16);                            // align stack as required by ABI
1025     BLOCK_COMMENT("call MacroAssembler::debug");
1026     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug64)));
1027     __ mov(rsp, r12);                               // restore rsp
1028     __ popa();                                      // pop registers (includes r12)
1029     __ ret(4 * wordSize);                           // pop caller saved stuff
1030 
1031     return start;
1032   }
1033 
1034   static address disjoint_byte_copy_entry;
1035   static address disjoint_short_copy_entry;
1036   static address disjoint_int_copy_entry;
1037   static address disjoint_long_copy_entry;
1038   static address disjoint_oop_copy_entry;
1039 
1040   static address byte_copy_entry;
1041   static address short_copy_entry;
1042   static address int_copy_entry;
1043   static address long_copy_entry;
1044   static address oop_copy_entry;
1045 
1046   static address checkcast_copy_entry;
1047 
1048   //
1049   // Verify that a register contains clean 32-bits positive value
1050   // (high 32-bits are 0) so it could be used in 64-bits shifts.
1051   //
1052   //  Input:
1053   //    Rint  -  32-bits value
1054   //    Rtmp  -  scratch
1055   //
1056   void assert_clean_int(Register Rint, Register Rtmp) {
1057 #ifdef ASSERT
1058     Label L;
1059     assert_different_registers(Rtmp, Rint);
1060     __ movslq(Rtmp, Rint);
1061     __ cmpq(Rtmp, Rint);
1062     __ jcc(Assembler::equal, L);
1063     __ stop("high 32-bits of int value are not 0");
1064     __ bind(L);
1065 #endif
1066   }
1067 
1068   //  Generate overlap test for array copy stubs
1069   //
1070   //  Input:
1071   //     c_rarg0 - from
1072   //     c_rarg1 - to
1073   //     c_rarg2 - element count
1074   //
1075   //  Output:
1076   //     rax   - &from[element count - 1]
1077   //
1078   void array_overlap_test(address no_overlap_target, Address::ScaleFactor sf) {
1079     assert(no_overlap_target != NULL, "must be generated");
1080     array_overlap_test(no_overlap_target, NULL, sf);
1081   }
1082   void array_overlap_test(Label& L_no_overlap, Address::ScaleFactor sf) {
1083     array_overlap_test(NULL, &L_no_overlap, sf);
1084   }
1085   void array_overlap_test(address no_overlap_target, Label* NOLp, Address::ScaleFactor sf) {
1086     const Register from     = c_rarg0;
1087     const Register to       = c_rarg1;
1088     const Register count    = c_rarg2;
1089     const Register end_from = rax;
1090 
1091     __ cmpptr(to, from);
1092     __ lea(end_from, Address(from, count, sf, 0));
1093     if (NOLp == NULL) {
1094       ExternalAddress no_overlap(no_overlap_target);
1095       __ jump_cc(Assembler::belowEqual, no_overlap);
1096       __ cmpptr(to, end_from);
1097       __ jump_cc(Assembler::aboveEqual, no_overlap);
1098     } else {
1099       __ jcc(Assembler::belowEqual, (*NOLp));
1100       __ cmpptr(to, end_from);
1101       __ jcc(Assembler::aboveEqual, (*NOLp));
1102     }
1103   }
1104 
1105   // Shuffle first three arg regs on Windows into Linux/Solaris locations.
1106   //
1107   // Outputs:
1108   //    rdi - rcx
1109   //    rsi - rdx
1110   //    rdx - r8
1111   //    rcx - r9
1112   //
1113   // Registers r9 and r10 are used to save rdi and rsi on Windows, which latter
1114   // are non-volatile.  r9 and r10 should not be used by the caller.
1115   //
1116   void setup_arg_regs(int nargs = 3) {
1117     const Register saved_rdi = r9;
1118     const Register saved_rsi = r10;
1119     assert(nargs == 3 || nargs == 4, "else fix");
1120 #ifdef _WIN64
1121     assert(c_rarg0 == rcx && c_rarg1 == rdx && c_rarg2 == r8 && c_rarg3 == r9,
1122            "unexpected argument registers");
1123     if (nargs >= 4)
1124       __ mov(rax, r9);  // r9 is also saved_rdi
1125     __ movptr(saved_rdi, rdi);
1126     __ movptr(saved_rsi, rsi);
1127     __ mov(rdi, rcx); // c_rarg0
1128     __ mov(rsi, rdx); // c_rarg1
1129     __ mov(rdx, r8);  // c_rarg2
1130     if (nargs >= 4)
1131       __ mov(rcx, rax); // c_rarg3 (via rax)
1132 #else
1133     assert(c_rarg0 == rdi && c_rarg1 == rsi && c_rarg2 == rdx && c_rarg3 == rcx,
1134            "unexpected argument registers");
1135 #endif
1136   }
1137 
1138   void restore_arg_regs() {
1139     const Register saved_rdi = r9;
1140     const Register saved_rsi = r10;
1141 #ifdef _WIN64
1142     __ movptr(rdi, saved_rdi);
1143     __ movptr(rsi, saved_rsi);
1144 #endif
1145   }
1146 
1147   // Generate code for an array write pre barrier
1148   //
1149   //     addr    -  starting address
1150   //     count    -  element count
1151   //
1152   //     Destroy no registers!
1153   //
1154   void  gen_write_ref_array_pre_barrier(Register addr, Register count) {
1155     BarrierSet* bs = Universe::heap()->barrier_set();
1156     switch (bs->kind()) {
1157       case BarrierSet::G1SATBCT:
1158       case BarrierSet::G1SATBCTLogging:
1159         {
1160           __ pusha();                      // push registers
1161           if (count == c_rarg0) {
1162             if (addr == c_rarg1) {
1163               // exactly backwards!!
1164               __ xchgptr(c_rarg1, c_rarg0);
1165             } else {
1166               __ movptr(c_rarg1, count);
1167               __ movptr(c_rarg0, addr);
1168             }
1169 
1170           } else {
1171             __ movptr(c_rarg0, addr);
1172             __ movptr(c_rarg1, count);
1173           }
1174           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2);
1175           __ popa();
1176         }
1177         break;
1178       case BarrierSet::CardTableModRef:
1179       case BarrierSet::CardTableExtension:
1180       case BarrierSet::ModRef:
1181         break;
1182       default:
1183         ShouldNotReachHere();
1184 
1185     }
1186   }
1187 
1188   //
1189   // Generate code for an array write post barrier
1190   //
1191   //  Input:
1192   //     start    - register containing starting address of destination array
1193   //     end      - register containing ending address of destination array
1194   //     scratch  - scratch register
1195   //
1196   //  The input registers are overwritten.
1197   //  The ending address is inclusive.
1198   void  gen_write_ref_array_post_barrier(Register start, Register end, Register scratch) {
1199     assert_different_registers(start, end, scratch);
1200     BarrierSet* bs = Universe::heap()->barrier_set();
1201     switch (bs->kind()) {
1202       case BarrierSet::G1SATBCT:
1203       case BarrierSet::G1SATBCTLogging:
1204 
1205         {
1206           __ pusha();                      // push registers (overkill)
1207           // must compute element count unless barrier set interface is changed (other platforms supply count)
1208           assert_different_registers(start, end, scratch);
1209           __ lea(scratch, Address(end, BytesPerHeapOop));
1210           __ subptr(scratch, start);               // subtract start to get #bytes
1211           __ shrptr(scratch, LogBytesPerHeapOop);  // convert to element count
1212           __ mov(c_rarg0, start);
1213           __ mov(c_rarg1, scratch);
1214           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post), 2);
1215           __ popa();
1216         }
1217         break;
1218       case BarrierSet::CardTableModRef:
1219       case BarrierSet::CardTableExtension:
1220         {
1221           CardTableModRefBS* ct = (CardTableModRefBS*)bs;
1222           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1223 
1224           Label L_loop;
1225 
1226            __ shrptr(start, CardTableModRefBS::card_shift);
1227            __ addptr(end, BytesPerHeapOop);
1228            __ shrptr(end, CardTableModRefBS::card_shift);
1229            __ subptr(end, start); // number of bytes to copy
1230 
1231           intptr_t disp = (intptr_t) ct->byte_map_base;
1232           if (__ is_simm32(disp)) {
1233             Address cardtable(noreg, noreg, Address::no_scale, disp);
1234             __ lea(scratch, cardtable);
1235           } else {
1236             ExternalAddress cardtable((address)disp);
1237             __ lea(scratch, cardtable);
1238           }
1239 
1240           const Register count = end; // 'end' register contains bytes count now
1241           __ addptr(start, scratch);
1242         __ BIND(L_loop);
1243           __ movb(Address(start, count, Address::times_1), 0);
1244           __ decrement(count);
1245           __ jcc(Assembler::greaterEqual, L_loop);
1246         }
1247         break;
1248       default:
1249         ShouldNotReachHere();
1250 
1251     }
1252   }
1253 
1254 
1255   // Copy big chunks forward
1256   //
1257   // Inputs:
1258   //   end_from     - source arrays end address
1259   //   end_to       - destination array end address
1260   //   qword_count  - 64-bits element count, negative
1261   //   to           - scratch
1262   //   L_copy_32_bytes - entry label
1263   //   L_copy_8_bytes  - exit  label
1264   //
1265   void copy_32_bytes_forward(Register end_from, Register end_to,
1266                              Register qword_count, Register to,
1267                              Label& L_copy_32_bytes, Label& L_copy_8_bytes) {
1268     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1269     Label L_loop;
1270     __ align(OptoLoopAlignment);
1271   __ BIND(L_loop);
1272     if(UseUnalignedLoadStores) {
1273       __ movdqu(xmm0, Address(end_from, qword_count, Address::times_8, -24));
1274       __ movdqu(Address(end_to, qword_count, Address::times_8, -24), xmm0);
1275       __ movdqu(xmm1, Address(end_from, qword_count, Address::times_8, - 8));
1276       __ movdqu(Address(end_to, qword_count, Address::times_8, - 8), xmm1);
1277 
1278     } else {
1279       __ movq(to, Address(end_from, qword_count, Address::times_8, -24));
1280       __ movq(Address(end_to, qword_count, Address::times_8, -24), to);
1281       __ movq(to, Address(end_from, qword_count, Address::times_8, -16));
1282       __ movq(Address(end_to, qword_count, Address::times_8, -16), to);
1283       __ movq(to, Address(end_from, qword_count, Address::times_8, - 8));
1284       __ movq(Address(end_to, qword_count, Address::times_8, - 8), to);
1285       __ movq(to, Address(end_from, qword_count, Address::times_8, - 0));
1286       __ movq(Address(end_to, qword_count, Address::times_8, - 0), to);
1287     }
1288   __ BIND(L_copy_32_bytes);
1289     __ addptr(qword_count, 4);
1290     __ jcc(Assembler::lessEqual, L_loop);
1291     __ subptr(qword_count, 4);
1292     __ jcc(Assembler::less, L_copy_8_bytes); // Copy trailing qwords
1293   }
1294 
1295 
1296   // Copy big chunks backward
1297   //
1298   // Inputs:
1299   //   from         - source arrays address
1300   //   dest         - destination array address
1301   //   qword_count  - 64-bits element count
1302   //   to           - scratch
1303   //   L_copy_32_bytes - entry label
1304   //   L_copy_8_bytes  - exit  label
1305   //
1306   void copy_32_bytes_backward(Register from, Register dest,
1307                               Register qword_count, Register to,
1308                               Label& L_copy_32_bytes, Label& L_copy_8_bytes) {
1309     DEBUG_ONLY(__ stop("enter at entry label, not here"));
1310     Label L_loop;
1311     __ align(OptoLoopAlignment);
1312   __ BIND(L_loop);
1313     if(UseUnalignedLoadStores) {
1314       __ movdqu(xmm0, Address(from, qword_count, Address::times_8, 16));
1315       __ movdqu(Address(dest, qword_count, Address::times_8, 16), xmm0);
1316       __ movdqu(xmm1, Address(from, qword_count, Address::times_8,  0));
1317       __ movdqu(Address(dest, qword_count, Address::times_8,  0), xmm1);
1318 
1319     } else {
1320       __ movq(to, Address(from, qword_count, Address::times_8, 24));
1321       __ movq(Address(dest, qword_count, Address::times_8, 24), to);
1322       __ movq(to, Address(from, qword_count, Address::times_8, 16));
1323       __ movq(Address(dest, qword_count, Address::times_8, 16), to);
1324       __ movq(to, Address(from, qword_count, Address::times_8,  8));
1325       __ movq(Address(dest, qword_count, Address::times_8,  8), to);
1326       __ movq(to, Address(from, qword_count, Address::times_8,  0));
1327       __ movq(Address(dest, qword_count, Address::times_8,  0), to);
1328     }
1329   __ BIND(L_copy_32_bytes);
1330     __ subptr(qword_count, 4);
1331     __ jcc(Assembler::greaterEqual, L_loop);
1332     __ addptr(qword_count, 4);
1333     __ jcc(Assembler::greater, L_copy_8_bytes); // Copy trailing qwords
1334   }
1335 
1336 
1337   // Arguments:
1338   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1339   //             ignored
1340   //   name    - stub name string
1341   //
1342   // Inputs:
1343   //   c_rarg0   - source array address
1344   //   c_rarg1   - destination array address
1345   //   c_rarg2   - element count, treated as ssize_t, can be zero
1346   //
1347   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1348   // we let the hardware handle it.  The one to eight bytes within words,
1349   // dwords or qwords that span cache line boundaries will still be loaded
1350   // and stored atomically.
1351   //
1352   // Side Effects:
1353   //   disjoint_byte_copy_entry is set to the no-overlap entry point
1354   //   used by generate_conjoint_byte_copy().
1355   //
1356   address generate_disjoint_byte_copy(bool aligned, const char *name) {
1357     __ align(CodeEntryAlignment);
1358     StubCodeMark mark(this, "StubRoutines", name);
1359     address start = __ pc();
1360 
1361     Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1362     Label L_copy_byte, L_exit;
1363     const Register from        = rdi;  // source array address
1364     const Register to          = rsi;  // destination array address
1365     const Register count       = rdx;  // elements count
1366     const Register byte_count  = rcx;
1367     const Register qword_count = count;
1368     const Register end_from    = from; // source array end address
1369     const Register end_to      = to;   // destination array end address
1370     // End pointers are inclusive, and if count is not zero they point
1371     // to the last unit copied:  end_to[0] := end_from[0]
1372 
1373     __ enter(); // required for proper stackwalking of RuntimeStub frame
1374     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1375 
1376     disjoint_byte_copy_entry = __ pc();
1377     BLOCK_COMMENT("Entry:");
1378     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1379 
1380     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1381                       // r9 and r10 may be used to save non-volatile registers
1382 
1383     // 'from', 'to' and 'count' are now valid
1384     __ movptr(byte_count, count);
1385     __ shrptr(count, 3); // count => qword_count
1386 
1387     // Copy from low to high addresses.  Use 'to' as scratch.
1388     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1389     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1390     __ negptr(qword_count); // make the count negative
1391     __ jmp(L_copy_32_bytes);
1392 
1393     // Copy trailing qwords
1394   __ BIND(L_copy_8_bytes);
1395     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1396     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1397     __ increment(qword_count);
1398     __ jcc(Assembler::notZero, L_copy_8_bytes);
1399 
1400     // Check for and copy trailing dword
1401   __ BIND(L_copy_4_bytes);
1402     __ testl(byte_count, 4);
1403     __ jccb(Assembler::zero, L_copy_2_bytes);
1404     __ movl(rax, Address(end_from, 8));
1405     __ movl(Address(end_to, 8), rax);
1406 
1407     __ addptr(end_from, 4);
1408     __ addptr(end_to, 4);
1409 
1410     // Check for and copy trailing word
1411   __ BIND(L_copy_2_bytes);
1412     __ testl(byte_count, 2);
1413     __ jccb(Assembler::zero, L_copy_byte);
1414     __ movw(rax, Address(end_from, 8));
1415     __ movw(Address(end_to, 8), rax);
1416 
1417     __ addptr(end_from, 2);
1418     __ addptr(end_to, 2);
1419 
1420     // Check for and copy trailing byte
1421   __ BIND(L_copy_byte);
1422     __ testl(byte_count, 1);
1423     __ jccb(Assembler::zero, L_exit);
1424     __ movb(rax, Address(end_from, 8));
1425     __ movb(Address(end_to, 8), rax);
1426 
1427   __ BIND(L_exit);
1428     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr);
1429     restore_arg_regs();
1430     __ xorptr(rax, rax); // return 0
1431     __ leave(); // required for proper stackwalking of RuntimeStub frame
1432     __ ret(0);
1433 
1434     // Copy in 32-bytes chunks
1435     copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
1436     __ jmp(L_copy_4_bytes);
1437 
1438     return start;
1439   }
1440 
1441   // Arguments:
1442   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1443   //             ignored
1444   //   name    - stub name string
1445   //
1446   // Inputs:
1447   //   c_rarg0   - source array address
1448   //   c_rarg1   - destination array address
1449   //   c_rarg2   - element count, treated as ssize_t, can be zero
1450   //
1451   // If 'from' and/or 'to' are aligned on 4-, 2-, or 1-byte boundaries,
1452   // we let the hardware handle it.  The one to eight bytes within words,
1453   // dwords or qwords that span cache line boundaries will still be loaded
1454   // and stored atomically.
1455   //
1456   address generate_conjoint_byte_copy(bool aligned, const char *name) {
1457     __ align(CodeEntryAlignment);
1458     StubCodeMark mark(this, "StubRoutines", name);
1459     address start = __ pc();
1460 
1461     Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes, L_copy_2_bytes;
1462     const Register from        = rdi;  // source array address
1463     const Register to          = rsi;  // destination array address
1464     const Register count       = rdx;  // elements count
1465     const Register byte_count  = rcx;
1466     const Register qword_count = count;
1467 
1468     __ enter(); // required for proper stackwalking of RuntimeStub frame
1469     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1470 
1471     byte_copy_entry = __ pc();
1472     BLOCK_COMMENT("Entry:");
1473     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1474 
1475     array_overlap_test(disjoint_byte_copy_entry, Address::times_1);
1476     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1477                       // r9 and r10 may be used to save non-volatile registers
1478 
1479     // 'from', 'to' and 'count' are now valid
1480     __ movptr(byte_count, count);
1481     __ shrptr(count, 3);   // count => qword_count
1482 
1483     // Copy from high to low addresses.
1484 
1485     // Check for and copy trailing byte
1486     __ testl(byte_count, 1);
1487     __ jcc(Assembler::zero, L_copy_2_bytes);
1488     __ movb(rax, Address(from, byte_count, Address::times_1, -1));
1489     __ movb(Address(to, byte_count, Address::times_1, -1), rax);
1490     __ decrement(byte_count); // Adjust for possible trailing word
1491 
1492     // Check for and copy trailing word
1493   __ BIND(L_copy_2_bytes);
1494     __ testl(byte_count, 2);
1495     __ jcc(Assembler::zero, L_copy_4_bytes);
1496     __ movw(rax, Address(from, byte_count, Address::times_1, -2));
1497     __ movw(Address(to, byte_count, Address::times_1, -2), rax);
1498 
1499     // Check for and copy trailing dword
1500   __ BIND(L_copy_4_bytes);
1501     __ testl(byte_count, 4);
1502     __ jcc(Assembler::zero, L_copy_32_bytes);
1503     __ movl(rax, Address(from, qword_count, Address::times_8));
1504     __ movl(Address(to, qword_count, Address::times_8), rax);
1505     __ jmp(L_copy_32_bytes);
1506 
1507     // Copy trailing qwords
1508   __ BIND(L_copy_8_bytes);
1509     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1510     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1511     __ decrement(qword_count);
1512     __ jcc(Assembler::notZero, L_copy_8_bytes);
1513 
1514     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr);
1515     restore_arg_regs();
1516     __ xorptr(rax, rax); // return 0
1517     __ leave(); // required for proper stackwalking of RuntimeStub frame
1518     __ ret(0);
1519 
1520     // Copy in 32-bytes chunks
1521     copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
1522 
1523     inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr);
1524     restore_arg_regs();
1525     __ xorptr(rax, rax); // return 0
1526     __ leave(); // required for proper stackwalking of RuntimeStub frame
1527     __ ret(0);
1528 
1529     return start;
1530   }
1531 
1532   // Arguments:
1533   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1534   //             ignored
1535   //   name    - stub name string
1536   //
1537   // Inputs:
1538   //   c_rarg0   - source array address
1539   //   c_rarg1   - destination array address
1540   //   c_rarg2   - element count, treated as ssize_t, can be zero
1541   //
1542   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1543   // let the hardware handle it.  The two or four words within dwords
1544   // or qwords that span cache line boundaries will still be loaded
1545   // and stored atomically.
1546   //
1547   // Side Effects:
1548   //   disjoint_short_copy_entry is set to the no-overlap entry point
1549   //   used by generate_conjoint_short_copy().
1550   //
1551   address generate_disjoint_short_copy(bool aligned, const char *name) {
1552     __ align(CodeEntryAlignment);
1553     StubCodeMark mark(this, "StubRoutines", name);
1554     address start = __ pc();
1555 
1556     Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes,L_copy_2_bytes,L_exit;
1557     const Register from        = rdi;  // source array address
1558     const Register to          = rsi;  // destination array address
1559     const Register count       = rdx;  // elements count
1560     const Register word_count  = rcx;
1561     const Register qword_count = count;
1562     const Register end_from    = from; // source array end address
1563     const Register end_to      = to;   // destination array end address
1564     // End pointers are inclusive, and if count is not zero they point
1565     // to the last unit copied:  end_to[0] := end_from[0]
1566 
1567     __ enter(); // required for proper stackwalking of RuntimeStub frame
1568     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1569 
1570     disjoint_short_copy_entry = __ pc();
1571     BLOCK_COMMENT("Entry:");
1572     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1573 
1574     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1575                       // r9 and r10 may be used to save non-volatile registers
1576 
1577     // 'from', 'to' and 'count' are now valid
1578     __ movptr(word_count, count);
1579     __ shrptr(count, 2); // count => qword_count
1580 
1581     // Copy from low to high addresses.  Use 'to' as scratch.
1582     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1583     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1584     __ negptr(qword_count);
1585     __ jmp(L_copy_32_bytes);
1586 
1587     // Copy trailing qwords
1588   __ BIND(L_copy_8_bytes);
1589     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1590     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1591     __ increment(qword_count);
1592     __ jcc(Assembler::notZero, L_copy_8_bytes);
1593 
1594     // Original 'dest' is trashed, so we can't use it as a
1595     // base register for a possible trailing word copy
1596 
1597     // Check for and copy trailing dword
1598   __ BIND(L_copy_4_bytes);
1599     __ testl(word_count, 2);
1600     __ jccb(Assembler::zero, L_copy_2_bytes);
1601     __ movl(rax, Address(end_from, 8));
1602     __ movl(Address(end_to, 8), rax);
1603 
1604     __ addptr(end_from, 4);
1605     __ addptr(end_to, 4);
1606 
1607     // Check for and copy trailing word
1608   __ BIND(L_copy_2_bytes);
1609     __ testl(word_count, 1);
1610     __ jccb(Assembler::zero, L_exit);
1611     __ movw(rax, Address(end_from, 8));
1612     __ movw(Address(end_to, 8), rax);
1613 
1614   __ BIND(L_exit);
1615     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr);
1616     restore_arg_regs();
1617     __ xorptr(rax, rax); // return 0
1618     __ leave(); // required for proper stackwalking of RuntimeStub frame
1619     __ ret(0);
1620 
1621     // Copy in 32-bytes chunks
1622     copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
1623     __ jmp(L_copy_4_bytes);
1624 
1625     return start;
1626   }
1627 
1628   address generate_fill(BasicType t, bool aligned, const char *name) {
1629     __ align(CodeEntryAlignment);
1630     StubCodeMark mark(this, "StubRoutines", name);
1631     address start = __ pc();
1632 
1633     BLOCK_COMMENT("Entry:");
1634 
1635     const Register to       = c_rarg0;  // source array address
1636     const Register value    = c_rarg1;  // value
1637     const Register count    = c_rarg2;  // elements count
1638 
1639     __ enter(); // required for proper stackwalking of RuntimeStub frame
1640 
1641     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1642 
1643     __ leave(); // required for proper stackwalking of RuntimeStub frame
1644     __ ret(0);
1645     return start;
1646   }
1647 
1648   // Arguments:
1649   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1650   //             ignored
1651   //   name    - stub name string
1652   //
1653   // Inputs:
1654   //   c_rarg0   - source array address
1655   //   c_rarg1   - destination array address
1656   //   c_rarg2   - element count, treated as ssize_t, can be zero
1657   //
1658   // If 'from' and/or 'to' are aligned on 4- or 2-byte boundaries, we
1659   // let the hardware handle it.  The two or four words within dwords
1660   // or qwords that span cache line boundaries will still be loaded
1661   // and stored atomically.
1662   //
1663   address generate_conjoint_short_copy(bool aligned, const char *name) {
1664     __ align(CodeEntryAlignment);
1665     StubCodeMark mark(this, "StubRoutines", name);
1666     address start = __ pc();
1667 
1668     Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes;
1669     const Register from        = rdi;  // source array address
1670     const Register to          = rsi;  // destination array address
1671     const Register count       = rdx;  // elements count
1672     const Register word_count  = rcx;
1673     const Register qword_count = count;
1674 
1675     __ enter(); // required for proper stackwalking of RuntimeStub frame
1676     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1677 
1678     short_copy_entry = __ pc();
1679     BLOCK_COMMENT("Entry:");
1680     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1681 
1682     array_overlap_test(disjoint_short_copy_entry, Address::times_2);
1683     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1684                       // r9 and r10 may be used to save non-volatile registers
1685 
1686     // 'from', 'to' and 'count' are now valid
1687     __ movptr(word_count, count);
1688     __ shrptr(count, 2); // count => qword_count
1689 
1690     // Copy from high to low addresses.  Use 'to' as scratch.
1691 
1692     // Check for and copy trailing word
1693     __ testl(word_count, 1);
1694     __ jccb(Assembler::zero, L_copy_4_bytes);
1695     __ movw(rax, Address(from, word_count, Address::times_2, -2));
1696     __ movw(Address(to, word_count, Address::times_2, -2), rax);
1697 
1698     // Check for and copy trailing dword
1699   __ BIND(L_copy_4_bytes);
1700     __ testl(word_count, 2);
1701     __ jcc(Assembler::zero, L_copy_32_bytes);
1702     __ movl(rax, Address(from, qword_count, Address::times_8));
1703     __ movl(Address(to, qword_count, Address::times_8), rax);
1704     __ jmp(L_copy_32_bytes);
1705 
1706     // Copy trailing qwords
1707   __ BIND(L_copy_8_bytes);
1708     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1709     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1710     __ decrement(qword_count);
1711     __ jcc(Assembler::notZero, L_copy_8_bytes);
1712 
1713     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr);
1714     restore_arg_regs();
1715     __ xorptr(rax, rax); // return 0
1716     __ leave(); // required for proper stackwalking of RuntimeStub frame
1717     __ ret(0);
1718 
1719     // Copy in 32-bytes chunks
1720     copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
1721 
1722     inc_counter_np(SharedRuntime::_jshort_array_copy_ctr);
1723     restore_arg_regs();
1724     __ xorptr(rax, rax); // return 0
1725     __ leave(); // required for proper stackwalking of RuntimeStub frame
1726     __ ret(0);
1727 
1728     return start;
1729   }
1730 
1731   // Arguments:
1732   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1733   //             ignored
1734   //   is_oop  - true => oop array, so generate store check code
1735   //   name    - stub name string
1736   //
1737   // Inputs:
1738   //   c_rarg0   - source array address
1739   //   c_rarg1   - destination array address
1740   //   c_rarg2   - element count, treated as ssize_t, can be zero
1741   //
1742   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1743   // the hardware handle it.  The two dwords within qwords that span
1744   // cache line boundaries will still be loaded and stored atomicly.
1745   //
1746   // Side Effects:
1747   //   disjoint_int_copy_entry is set to the no-overlap entry point
1748   //   used by generate_conjoint_int_oop_copy().
1749   //
1750   address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, const char *name) {
1751     __ align(CodeEntryAlignment);
1752     StubCodeMark mark(this, "StubRoutines", name);
1753     address start = __ pc();
1754 
1755     Label L_copy_32_bytes, L_copy_8_bytes, L_copy_4_bytes, L_exit;
1756     const Register from        = rdi;  // source array address
1757     const Register to          = rsi;  // destination array address
1758     const Register count       = rdx;  // elements count
1759     const Register dword_count = rcx;
1760     const Register qword_count = count;
1761     const Register end_from    = from; // source array end address
1762     const Register end_to      = to;   // destination array end address
1763     const Register saved_to    = r11;  // saved destination array address
1764     // End pointers are inclusive, and if count is not zero they point
1765     // to the last unit copied:  end_to[0] := end_from[0]
1766 
1767     __ enter(); // required for proper stackwalking of RuntimeStub frame
1768     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1769 
1770     (is_oop ? disjoint_oop_copy_entry : disjoint_int_copy_entry) = __ pc();
1771 
1772     if (is_oop) {
1773       // no registers are destroyed by this call
1774       gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2);
1775     }
1776 
1777     BLOCK_COMMENT("Entry:");
1778     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1779 
1780     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1781                       // r9 and r10 may be used to save non-volatile registers
1782 
1783     if (is_oop) {
1784       __ movq(saved_to, to);
1785     }
1786 
1787     // 'from', 'to' and 'count' are now valid
1788     __ movptr(dword_count, count);
1789     __ shrptr(count, 1); // count => qword_count
1790 
1791     // Copy from low to high addresses.  Use 'to' as scratch.
1792     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1793     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1794     __ negptr(qword_count);
1795     __ jmp(L_copy_32_bytes);
1796 
1797     // Copy trailing qwords
1798   __ BIND(L_copy_8_bytes);
1799     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1800     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1801     __ increment(qword_count);
1802     __ jcc(Assembler::notZero, L_copy_8_bytes);
1803 
1804     // Check for and copy trailing dword
1805   __ BIND(L_copy_4_bytes);
1806     __ testl(dword_count, 1); // Only byte test since the value is 0 or 1
1807     __ jccb(Assembler::zero, L_exit);
1808     __ movl(rax, Address(end_from, 8));
1809     __ movl(Address(end_to, 8), rax);
1810 
1811   __ BIND(L_exit);
1812     if (is_oop) {
1813       __ leaq(end_to, Address(saved_to, dword_count, Address::times_4, -4));
1814       gen_write_ref_array_post_barrier(saved_to, end_to, rax);
1815     }
1816     inc_counter_np(SharedRuntime::_jint_array_copy_ctr);
1817     restore_arg_regs();
1818     __ xorptr(rax, rax); // return 0
1819     __ leave(); // required for proper stackwalking of RuntimeStub frame
1820     __ ret(0);
1821 
1822     // Copy 32-bytes chunks
1823     copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
1824     __ jmp(L_copy_4_bytes);
1825 
1826     return start;
1827   }
1828 
1829   // Arguments:
1830   //   aligned - true => Input and output aligned on a HeapWord == 8-byte boundary
1831   //             ignored
1832   //   is_oop  - true => oop array, so generate store check code
1833   //   name    - stub name string
1834   //
1835   // Inputs:
1836   //   c_rarg0   - source array address
1837   //   c_rarg1   - destination array address
1838   //   c_rarg2   - element count, treated as ssize_t, can be zero
1839   //
1840   // If 'from' and/or 'to' are aligned on 4-byte boundaries, we let
1841   // the hardware handle it.  The two dwords within qwords that span
1842   // cache line boundaries will still be loaded and stored atomicly.
1843   //
1844   address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, const char *name) {
1845     __ align(CodeEntryAlignment);
1846     StubCodeMark mark(this, "StubRoutines", name);
1847     address start = __ pc();
1848 
1849     Label L_copy_32_bytes, L_copy_8_bytes, L_copy_2_bytes, L_exit;
1850     const Register from        = rdi;  // source array address
1851     const Register to          = rsi;  // destination array address
1852     const Register count       = rdx;  // elements count
1853     const Register dword_count = rcx;
1854     const Register qword_count = count;
1855 
1856     __ enter(); // required for proper stackwalking of RuntimeStub frame
1857     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1858 
1859     if (is_oop) {
1860       // no registers are destroyed by this call
1861       gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2);
1862     }
1863 
1864     (is_oop ? oop_copy_entry : int_copy_entry) = __ pc();
1865     BLOCK_COMMENT("Entry:");
1866     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1867 
1868     array_overlap_test(is_oop ? disjoint_oop_copy_entry : disjoint_int_copy_entry,
1869                        Address::times_4);
1870     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1871                       // r9 and r10 may be used to save non-volatile registers
1872 
1873     assert_clean_int(count, rax); // Make sure 'count' is clean int.
1874     // 'from', 'to' and 'count' are now valid
1875     __ movptr(dword_count, count);
1876     __ shrptr(count, 1); // count => qword_count
1877 
1878     // Copy from high to low addresses.  Use 'to' as scratch.
1879 
1880     // Check for and copy trailing dword
1881     __ testl(dword_count, 1);
1882     __ jcc(Assembler::zero, L_copy_32_bytes);
1883     __ movl(rax, Address(from, dword_count, Address::times_4, -4));
1884     __ movl(Address(to, dword_count, Address::times_4, -4), rax);
1885     __ jmp(L_copy_32_bytes);
1886 
1887     // Copy trailing qwords
1888   __ BIND(L_copy_8_bytes);
1889     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
1890     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
1891     __ decrement(qword_count);
1892     __ jcc(Assembler::notZero, L_copy_8_bytes);
1893 
1894     inc_counter_np(SharedRuntime::_jint_array_copy_ctr);
1895     if (is_oop) {
1896       __ jmp(L_exit);
1897     }
1898     restore_arg_regs();
1899     __ xorptr(rax, rax); // return 0
1900     __ leave(); // required for proper stackwalking of RuntimeStub frame
1901     __ ret(0);
1902 
1903     // Copy in 32-bytes chunks
1904     copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
1905 
1906    inc_counter_np(SharedRuntime::_jint_array_copy_ctr);
1907    __ bind(L_exit);
1908      if (is_oop) {
1909        Register end_to = rdx;
1910        __ leaq(end_to, Address(to, dword_count, Address::times_4, -4));
1911        gen_write_ref_array_post_barrier(to, end_to, rax);
1912      }
1913     restore_arg_regs();
1914     __ xorptr(rax, rax); // return 0
1915     __ leave(); // required for proper stackwalking of RuntimeStub frame
1916     __ ret(0);
1917 
1918     return start;
1919   }
1920 
1921   // Arguments:
1922   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
1923   //             ignored
1924   //   is_oop  - true => oop array, so generate store check code
1925   //   name    - stub name string
1926   //
1927   // Inputs:
1928   //   c_rarg0   - source array address
1929   //   c_rarg1   - destination array address
1930   //   c_rarg2   - element count, treated as ssize_t, can be zero
1931   //
1932  // Side Effects:
1933   //   disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the
1934   //   no-overlap entry point used by generate_conjoint_long_oop_copy().
1935   //
1936   address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, const char *name) {
1937     __ align(CodeEntryAlignment);
1938     StubCodeMark mark(this, "StubRoutines", name);
1939     address start = __ pc();
1940 
1941     Label L_copy_32_bytes, L_copy_8_bytes, L_exit;
1942     const Register from        = rdi;  // source array address
1943     const Register to          = rsi;  // destination array address
1944     const Register qword_count = rdx;  // elements count
1945     const Register end_from    = from; // source array end address
1946     const Register end_to      = rcx;  // destination array end address
1947     const Register saved_to    = to;
1948     // End pointers are inclusive, and if count is not zero they point
1949     // to the last unit copied:  end_to[0] := end_from[0]
1950 
1951     __ enter(); // required for proper stackwalking of RuntimeStub frame
1952     // Save no-overlap entry point for generate_conjoint_long_oop_copy()
1953     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
1954 
1955     if (is_oop) {
1956       disjoint_oop_copy_entry  = __ pc();
1957       // no registers are destroyed by this call
1958       gen_write_ref_array_pre_barrier(/* dest */ c_rarg1, /* count */ c_rarg2);
1959     } else {
1960       disjoint_long_copy_entry = __ pc();
1961     }
1962     BLOCK_COMMENT("Entry:");
1963     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
1964 
1965     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
1966                       // r9 and r10 may be used to save non-volatile registers
1967 
1968     // 'from', 'to' and 'qword_count' are now valid
1969 
1970     // Copy from low to high addresses.  Use 'to' as scratch.
1971     __ lea(end_from, Address(from, qword_count, Address::times_8, -8));
1972     __ lea(end_to,   Address(to,   qword_count, Address::times_8, -8));
1973     __ negptr(qword_count);
1974     __ jmp(L_copy_32_bytes);
1975 
1976     // Copy trailing qwords
1977   __ BIND(L_copy_8_bytes);
1978     __ movq(rax, Address(end_from, qword_count, Address::times_8, 8));
1979     __ movq(Address(end_to, qword_count, Address::times_8, 8), rax);
1980     __ increment(qword_count);
1981     __ jcc(Assembler::notZero, L_copy_8_bytes);
1982 
1983     if (is_oop) {
1984       __ jmp(L_exit);
1985     } else {
1986       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr);
1987       restore_arg_regs();
1988       __ xorptr(rax, rax); // return 0
1989       __ leave(); // required for proper stackwalking of RuntimeStub frame
1990       __ ret(0);
1991     }
1992 
1993     // Copy 64-byte chunks
1994     copy_32_bytes_forward(end_from, end_to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
1995 
1996     if (is_oop) {
1997     __ BIND(L_exit);
1998       gen_write_ref_array_post_barrier(saved_to, end_to, rax);
1999       inc_counter_np(SharedRuntime::_oop_array_copy_ctr);
2000     } else {
2001       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr);
2002     }
2003     restore_arg_regs();
2004     __ xorptr(rax, rax); // return 0
2005     __ leave(); // required for proper stackwalking of RuntimeStub frame
2006     __ ret(0);
2007 
2008     return start;
2009   }
2010 
2011   // Arguments:
2012   //   aligned - true => Input and output aligned on a HeapWord boundary == 8 bytes
2013   //             ignored
2014   //   is_oop  - true => oop array, so generate store check code
2015   //   name    - stub name string
2016   //
2017   // Inputs:
2018   //   c_rarg0   - source array address
2019   //   c_rarg1   - destination array address
2020   //   c_rarg2   - element count, treated as ssize_t, can be zero
2021   //
2022   address generate_conjoint_long_oop_copy(bool aligned, bool is_oop, const char *name) {
2023     __ align(CodeEntryAlignment);
2024     StubCodeMark mark(this, "StubRoutines", name);
2025     address start = __ pc();
2026 
2027     Label L_copy_32_bytes, L_copy_8_bytes, L_exit;
2028     const Register from        = rdi;  // source array address
2029     const Register to          = rsi;  // destination array address
2030     const Register qword_count = rdx;  // elements count
2031     const Register saved_count = rcx;
2032 
2033     __ enter(); // required for proper stackwalking of RuntimeStub frame
2034     assert_clean_int(c_rarg2, rax);    // Make sure 'count' is clean int.
2035 
2036     address disjoint_copy_entry = NULL;
2037     if (is_oop) {
2038       assert(!UseCompressedOops, "shouldn't be called for compressed oops");
2039       disjoint_copy_entry = disjoint_oop_copy_entry;
2040       oop_copy_entry  = __ pc();
2041       array_overlap_test(disjoint_oop_copy_entry, Address::times_8);
2042     } else {
2043       disjoint_copy_entry = disjoint_long_copy_entry;
2044       long_copy_entry = __ pc();
2045       array_overlap_test(disjoint_long_copy_entry, Address::times_8);
2046     }
2047     BLOCK_COMMENT("Entry:");
2048     // caller can pass a 64-bit byte count here (from Unsafe.copyMemory)
2049 
2050     array_overlap_test(disjoint_copy_entry, Address::times_8);
2051     setup_arg_regs(); // from => rdi, to => rsi, count => rdx
2052                       // r9 and r10 may be used to save non-volatile registers
2053 
2054     // 'from', 'to' and 'qword_count' are now valid
2055 
2056     if (is_oop) {
2057       // Save to and count for store barrier
2058       __ movptr(saved_count, qword_count);
2059       // No registers are destroyed by this call
2060       gen_write_ref_array_pre_barrier(to, saved_count);
2061     }
2062 
2063     __ jmp(L_copy_32_bytes);
2064 
2065     // Copy trailing qwords
2066   __ BIND(L_copy_8_bytes);
2067     __ movq(rax, Address(from, qword_count, Address::times_8, -8));
2068     __ movq(Address(to, qword_count, Address::times_8, -8), rax);
2069     __ decrement(qword_count);
2070     __ jcc(Assembler::notZero, L_copy_8_bytes);
2071 
2072     if (is_oop) {
2073       __ jmp(L_exit);
2074     } else {
2075       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr);
2076       restore_arg_regs();
2077       __ xorptr(rax, rax); // return 0
2078       __ leave(); // required for proper stackwalking of RuntimeStub frame
2079       __ ret(0);
2080     }
2081 
2082     // Copy in 32-bytes chunks
2083     copy_32_bytes_backward(from, to, qword_count, rax, L_copy_32_bytes, L_copy_8_bytes);
2084 
2085     if (is_oop) {
2086     __ BIND(L_exit);
2087       __ lea(rcx, Address(to, saved_count, Address::times_8, -8));
2088       gen_write_ref_array_post_barrier(to, rcx, rax);
2089       inc_counter_np(SharedRuntime::_oop_array_copy_ctr);
2090     } else {
2091       inc_counter_np(SharedRuntime::_jlong_array_copy_ctr);
2092     }
2093     restore_arg_regs();
2094     __ xorptr(rax, rax); // return 0
2095     __ leave(); // required for proper stackwalking of RuntimeStub frame
2096     __ ret(0);
2097 
2098     return start;
2099   }
2100 
2101 
2102   // Helper for generating a dynamic type check.
2103   // Smashes no registers.
2104   void generate_type_check(Register sub_klass,
2105                            Register super_check_offset,
2106                            Register super_klass,
2107                            Label& L_success) {
2108     assert_different_registers(sub_klass, super_check_offset, super_klass);
2109 
2110     BLOCK_COMMENT("type_check:");
2111 
2112     Label L_miss;
2113 
2114     __ check_klass_subtype_fast_path(sub_klass, super_klass, noreg,        &L_success, &L_miss, NULL,
2115                                      super_check_offset);
2116     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg, &L_success, NULL);
2117 
2118     // Fall through on failure!
2119     __ BIND(L_miss);
2120   }
2121 
2122   //
2123   //  Generate checkcasting array copy stub
2124   //
2125   //  Input:
2126   //    c_rarg0   - source array address
2127   //    c_rarg1   - destination array address
2128   //    c_rarg2   - element count, treated as ssize_t, can be zero
2129   //    c_rarg3   - size_t ckoff (super_check_offset)
2130   // not Win64
2131   //    c_rarg4   - oop ckval (super_klass)
2132   // Win64
2133   //    rsp+40    - oop ckval (super_klass)
2134   //
2135   //  Output:
2136   //    rax ==  0  -  success
2137   //    rax == -1^K - failure, where K is partial transfer count
2138   //
2139   address generate_checkcast_copy(const char *name) {
2140 
2141     Label L_load_element, L_store_element, L_do_card_marks, L_done;
2142 
2143     // Input registers (after setup_arg_regs)
2144     const Register from        = rdi;   // source array address
2145     const Register to          = rsi;   // destination array address
2146     const Register length      = rdx;   // elements count
2147     const Register ckoff       = rcx;   // super_check_offset
2148     const Register ckval       = r8;    // super_klass
2149 
2150     // Registers used as temps (r13, r14 are save-on-entry)
2151     const Register end_from    = from;  // source array end address
2152     const Register end_to      = r13;   // destination array end address
2153     const Register count       = rdx;   // -(count_remaining)
2154     const Register r14_length  = r14;   // saved copy of length
2155     // End pointers are inclusive, and if length is not zero they point
2156     // to the last unit copied:  end_to[0] := end_from[0]
2157 
2158     const Register rax_oop    = rax;    // actual oop copied
2159     const Register r11_klass  = r11;    // oop._klass
2160 
2161     //---------------------------------------------------------------
2162     // Assembler stub will be used for this call to arraycopy
2163     // if the two arrays are subtypes of Object[] but the
2164     // destination array type is not equal to or a supertype
2165     // of the source type.  Each element must be separately
2166     // checked.
2167 
2168     __ align(CodeEntryAlignment);
2169     StubCodeMark mark(this, "StubRoutines", name);
2170     address start = __ pc();
2171 
2172     __ enter(); // required for proper stackwalking of RuntimeStub frame
2173 
2174     checkcast_copy_entry  = __ pc();
2175     BLOCK_COMMENT("Entry:");
2176 
2177 #ifdef ASSERT
2178     // caller guarantees that the arrays really are different
2179     // otherwise, we would have to make conjoint checks
2180     { Label L;
2181       array_overlap_test(L, TIMES_OOP);
2182       __ stop("checkcast_copy within a single array");
2183       __ bind(L);
2184     }
2185 #endif //ASSERT
2186 
2187     // allocate spill slots for r13, r14
2188     enum {
2189       saved_r13_offset,
2190       saved_r14_offset,
2191       saved_rbp_offset,
2192       saved_rip_offset,
2193       saved_rarg0_offset
2194     };
2195     __ subptr(rsp, saved_rbp_offset * wordSize);
2196     __ movptr(Address(rsp, saved_r13_offset * wordSize), r13);
2197     __ movptr(Address(rsp, saved_r14_offset * wordSize), r14);
2198     setup_arg_regs(4); // from => rdi, to => rsi, length => rdx
2199                        // ckoff => rcx, ckval => r8
2200                        // r9 and r10 may be used to save non-volatile registers
2201 #ifdef _WIN64
2202     // last argument (#4) is on stack on Win64
2203     const int ckval_offset = saved_rarg0_offset + 4;
2204     __ movptr(ckval, Address(rsp, ckval_offset * wordSize));
2205 #endif
2206 
2207     // check that int operands are properly extended to size_t
2208     assert_clean_int(length, rax);
2209     assert_clean_int(ckoff, rax);
2210 
2211 #ifdef ASSERT
2212     BLOCK_COMMENT("assert consistent ckoff/ckval");
2213     // The ckoff and ckval must be mutually consistent,
2214     // even though caller generates both.
2215     { Label L;
2216       int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
2217                         Klass::super_check_offset_offset_in_bytes());
2218       __ cmpl(ckoff, Address(ckval, sco_offset));
2219       __ jcc(Assembler::equal, L);
2220       __ stop("super_check_offset inconsistent");
2221       __ bind(L);
2222     }
2223 #endif //ASSERT
2224 
2225     // Loop-invariant addresses.  They are exclusive end pointers.
2226     Address end_from_addr(from, length, TIMES_OOP, 0);
2227     Address   end_to_addr(to,   length, TIMES_OOP, 0);
2228     // Loop-variant addresses.  They assume post-incremented count < 0.
2229     Address from_element_addr(end_from, count, TIMES_OOP, 0);
2230     Address   to_element_addr(end_to,   count, TIMES_OOP, 0);
2231 
2232     gen_write_ref_array_pre_barrier(to, count);
2233 
2234     // Copy from low to high addresses, indexed from the end of each array.
2235     __ lea(end_from, end_from_addr);
2236     __ lea(end_to,   end_to_addr);
2237     __ movptr(r14_length, length);        // save a copy of the length
2238     assert(length == count, "");          // else fix next line:
2239     __ negptr(count);                     // negate and test the length
2240     __ jcc(Assembler::notZero, L_load_element);
2241 
2242     // Empty array:  Nothing to do.
2243     __ xorptr(rax, rax);                  // return 0 on (trivial) success
2244     __ jmp(L_done);
2245 
2246     // ======== begin loop ========
2247     // (Loop is rotated; its entry is L_load_element.)
2248     // Loop control:
2249     //   for (count = -count; count != 0; count++)
2250     // Base pointers src, dst are biased by 8*(count-1),to last element.
2251     __ align(OptoLoopAlignment);
2252 
2253     __ BIND(L_store_element);
2254     __ store_heap_oop(to_element_addr, rax_oop);  // store the oop
2255     __ increment(count);               // increment the count toward zero
2256     __ jcc(Assembler::zero, L_do_card_marks);
2257 
2258     // ======== loop entry is here ========
2259     __ BIND(L_load_element);
2260     __ load_heap_oop(rax_oop, from_element_addr); // load the oop
2261     __ testptr(rax_oop, rax_oop);
2262     __ jcc(Assembler::zero, L_store_element);
2263 
2264     __ load_klass(r11_klass, rax_oop);// query the object klass
2265     generate_type_check(r11_klass, ckoff, ckval, L_store_element);
2266     // ======== end loop ========
2267 
2268     // It was a real error; we must depend on the caller to finish the job.
2269     // Register rdx = -1 * number of *remaining* oops, r14 = *total* oops.
2270     // Emit GC store barriers for the oops we have copied (r14 + rdx),
2271     // and report their number to the caller.
2272     assert_different_registers(rax, r14_length, count, to, end_to, rcx);
2273     __ lea(end_to, to_element_addr);
2274     __ addptr(end_to, -heapOopSize);      // make an inclusive end pointer
2275     gen_write_ref_array_post_barrier(to, end_to, rscratch1);
2276     __ movptr(rax, r14_length);           // original oops
2277     __ addptr(rax, count);                // K = (original - remaining) oops
2278     __ notptr(rax);                       // report (-1^K) to caller
2279     __ jmp(L_done);
2280 
2281     // Come here on success only.
2282     __ BIND(L_do_card_marks);
2283     __ addptr(end_to, -heapOopSize);         // make an inclusive end pointer
2284     gen_write_ref_array_post_barrier(to, end_to, rscratch1);
2285     __ xorptr(rax, rax);                  // return 0 on success
2286 
2287     // Common exit point (success or failure).
2288     __ BIND(L_done);
2289     __ movptr(r13, Address(rsp, saved_r13_offset * wordSize));
2290     __ movptr(r14, Address(rsp, saved_r14_offset * wordSize));
2291     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
2292     restore_arg_regs();
2293     __ leave(); // required for proper stackwalking of RuntimeStub frame
2294     __ ret(0);
2295 
2296     return start;
2297   }
2298 
2299   //
2300   //  Generate 'unsafe' array copy stub
2301   //  Though just as safe as the other stubs, it takes an unscaled
2302   //  size_t argument instead of an element count.
2303   //
2304   //  Input:
2305   //    c_rarg0   - source array address
2306   //    c_rarg1   - destination array address
2307   //    c_rarg2   - byte count, treated as ssize_t, can be zero
2308   //
2309   // Examines the alignment of the operands and dispatches
2310   // to a long, int, short, or byte copy loop.
2311   //
2312   address generate_unsafe_copy(const char *name) {
2313 
2314     Label L_long_aligned, L_int_aligned, L_short_aligned;
2315 
2316     // Input registers (before setup_arg_regs)
2317     const Register from        = c_rarg0;  // source array address
2318     const Register to          = c_rarg1;  // destination array address
2319     const Register size        = c_rarg2;  // byte count (size_t)
2320 
2321     // Register used as a temp
2322     const Register bits        = rax;      // test copy of low bits
2323 
2324     __ align(CodeEntryAlignment);
2325     StubCodeMark mark(this, "StubRoutines", name);
2326     address start = __ pc();
2327 
2328     __ enter(); // required for proper stackwalking of RuntimeStub frame
2329 
2330     // bump this on entry, not on exit:
2331     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
2332 
2333     __ mov(bits, from);
2334     __ orptr(bits, to);
2335     __ orptr(bits, size);
2336 
2337     __ testb(bits, BytesPerLong-1);
2338     __ jccb(Assembler::zero, L_long_aligned);
2339 
2340     __ testb(bits, BytesPerInt-1);
2341     __ jccb(Assembler::zero, L_int_aligned);
2342 
2343     __ testb(bits, BytesPerShort-1);
2344     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
2345 
2346     __ BIND(L_short_aligned);
2347     __ shrptr(size, LogBytesPerShort); // size => short_count
2348     __ jump(RuntimeAddress(short_copy_entry));
2349 
2350     __ BIND(L_int_aligned);
2351     __ shrptr(size, LogBytesPerInt); // size => int_count
2352     __ jump(RuntimeAddress(int_copy_entry));
2353 
2354     __ BIND(L_long_aligned);
2355     __ shrptr(size, LogBytesPerLong); // size => qword_count
2356     __ jump(RuntimeAddress(long_copy_entry));
2357 
2358     return start;
2359   }
2360 
2361   // Perform range checks on the proposed arraycopy.
2362   // Kills temp, but nothing else.
2363   // Also, clean the sign bits of src_pos and dst_pos.
2364   void arraycopy_range_checks(Register src,     // source array oop (c_rarg0)
2365                               Register src_pos, // source position (c_rarg1)
2366                               Register dst,     // destination array oo (c_rarg2)
2367                               Register dst_pos, // destination position (c_rarg3)
2368                               Register length,
2369                               Register temp,
2370                               Label& L_failed) {
2371     BLOCK_COMMENT("arraycopy_range_checks:");
2372 
2373     //  if (src_pos + length > arrayOop(src)->length())  FAIL;
2374     __ movl(temp, length);
2375     __ addl(temp, src_pos);             // src_pos + length
2376     __ cmpl(temp, Address(src, arrayOopDesc::length_offset_in_bytes()));
2377     __ jcc(Assembler::above, L_failed);
2378 
2379     //  if (dst_pos + length > arrayOop(dst)->length())  FAIL;
2380     __ movl(temp, length);
2381     __ addl(temp, dst_pos);             // dst_pos + length
2382     __ cmpl(temp, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2383     __ jcc(Assembler::above, L_failed);
2384 
2385     // Have to clean up high 32-bits of 'src_pos' and 'dst_pos'.
2386     // Move with sign extension can be used since they are positive.
2387     __ movslq(src_pos, src_pos);
2388     __ movslq(dst_pos, dst_pos);
2389 
2390     BLOCK_COMMENT("arraycopy_range_checks done");
2391   }
2392 
2393   //
2394   //  Generate generic array copy stubs
2395   //
2396   //  Input:
2397   //    c_rarg0    -  src oop
2398   //    c_rarg1    -  src_pos (32-bits)
2399   //    c_rarg2    -  dst oop
2400   //    c_rarg3    -  dst_pos (32-bits)
2401   // not Win64
2402   //    c_rarg4    -  element count (32-bits)
2403   // Win64
2404   //    rsp+40     -  element count (32-bits)
2405   //
2406   //  Output:
2407   //    rax ==  0  -  success
2408   //    rax == -1^K - failure, where K is partial transfer count
2409   //
2410   address generate_generic_copy(const char *name) {
2411 
2412     Label L_failed, L_failed_0, L_objArray;
2413     Label L_copy_bytes, L_copy_shorts, L_copy_ints, L_copy_longs;
2414 
2415     // Input registers
2416     const Register src        = c_rarg0;  // source array oop
2417     const Register src_pos    = c_rarg1;  // source position
2418     const Register dst        = c_rarg2;  // destination array oop
2419     const Register dst_pos    = c_rarg3;  // destination position
2420     // elements count is on stack on Win64
2421 #ifdef _WIN64
2422 #define C_RARG4 Address(rsp, 6 * wordSize)
2423 #else
2424 #define C_RARG4 c_rarg4
2425 #endif
2426 
2427     { int modulus = CodeEntryAlignment;
2428       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
2429       int advance = target - (__ offset() % modulus);
2430       if (advance < 0)  advance += modulus;
2431       if (advance > 0)  __ nop(advance);
2432     }
2433     StubCodeMark mark(this, "StubRoutines", name);
2434 
2435     // Short-hop target to L_failed.  Makes for denser prologue code.
2436     __ BIND(L_failed_0);
2437     __ jmp(L_failed);
2438     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
2439 
2440     __ align(CodeEntryAlignment);
2441     address start = __ pc();
2442 
2443     __ enter(); // required for proper stackwalking of RuntimeStub frame
2444 
2445     // bump this on entry, not on exit:
2446     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
2447 
2448     //-----------------------------------------------------------------------
2449     // Assembler stub will be used for this call to arraycopy
2450     // if the following conditions are met:
2451     //
2452     // (1) src and dst must not be null.
2453     // (2) src_pos must not be negative.
2454     // (3) dst_pos must not be negative.
2455     // (4) length  must not be negative.
2456     // (5) src klass and dst klass should be the same and not NULL.
2457     // (6) src and dst should be arrays.
2458     // (7) src_pos + length must not exceed length of src.
2459     // (8) dst_pos + length must not exceed length of dst.
2460     //
2461 
2462     //  if (src == NULL) return -1;
2463     __ testptr(src, src);         // src oop
2464     size_t j1off = __ offset();
2465     __ jccb(Assembler::zero, L_failed_0);
2466 
2467     //  if (src_pos < 0) return -1;
2468     __ testl(src_pos, src_pos); // src_pos (32-bits)
2469     __ jccb(Assembler::negative, L_failed_0);
2470 
2471     //  if (dst == NULL) return -1;
2472     __ testptr(dst, dst);         // dst oop
2473     __ jccb(Assembler::zero, L_failed_0);
2474 
2475     //  if (dst_pos < 0) return -1;
2476     __ testl(dst_pos, dst_pos); // dst_pos (32-bits)
2477     size_t j4off = __ offset();
2478     __ jccb(Assembler::negative, L_failed_0);
2479 
2480     // The first four tests are very dense code,
2481     // but not quite dense enough to put four
2482     // jumps in a 16-byte instruction fetch buffer.
2483     // That's good, because some branch predicters
2484     // do not like jumps so close together.
2485     // Make sure of this.
2486     guarantee(((j1off ^ j4off) & ~15) != 0, "I$ line of 1st & 4th jumps");
2487 
2488     // registers used as temp
2489     const Register r11_length    = r11; // elements count to copy
2490     const Register r10_src_klass = r10; // array klass
2491     const Register r9_dst_klass  = r9;  // dest array klass
2492 
2493     //  if (length < 0) return -1;
2494     __ movl(r11_length, C_RARG4);       // length (elements count, 32-bits value)
2495     __ testl(r11_length, r11_length);
2496     __ jccb(Assembler::negative, L_failed_0);
2497 
2498     __ load_klass(r10_src_klass, src);
2499 #ifdef ASSERT
2500     //  assert(src->klass() != NULL);
2501     BLOCK_COMMENT("assert klasses not null");
2502     { Label L1, L2;
2503       __ testptr(r10_src_klass, r10_src_klass);
2504       __ jcc(Assembler::notZero, L2);   // it is broken if klass is NULL
2505       __ bind(L1);
2506       __ stop("broken null klass");
2507       __ bind(L2);
2508       __ load_klass(r9_dst_klass, dst);
2509       __ cmpq(r9_dst_klass, 0);
2510       __ jcc(Assembler::equal, L1);     // this would be broken also
2511       BLOCK_COMMENT("assert done");
2512     }
2513 #endif
2514 
2515     // Load layout helper (32-bits)
2516     //
2517     //  |array_tag|     | header_size | element_type |     |log2_element_size|
2518     // 32        30    24            16              8     2                 0
2519     //
2520     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
2521     //
2522 
2523     int lh_offset = klassOopDesc::header_size() * HeapWordSize +
2524                     Klass::layout_helper_offset_in_bytes();
2525 
2526     const Register rax_lh = rax;  // layout helper
2527 
2528     __ movl(rax_lh, Address(r10_src_klass, lh_offset));
2529 
2530     // Handle objArrays completely differently...
2531     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2532     __ cmpl(rax_lh, objArray_lh);
2533     __ jcc(Assembler::equal, L_objArray);
2534 
2535     //  if (src->klass() != dst->klass()) return -1;
2536     __ load_klass(r9_dst_klass, dst);
2537     __ cmpq(r10_src_klass, r9_dst_klass);
2538     __ jcc(Assembler::notEqual, L_failed);
2539 
2540     //  if (!src->is_Array()) return -1;
2541     __ cmpl(rax_lh, Klass::_lh_neutral_value);
2542     __ jcc(Assembler::greaterEqual, L_failed);
2543 
2544     // At this point, it is known to be a typeArray (array_tag 0x3).
2545 #ifdef ASSERT
2546     { Label L;
2547       __ cmpl(rax_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
2548       __ jcc(Assembler::greaterEqual, L);
2549       __ stop("must be a primitive array");
2550       __ bind(L);
2551     }
2552 #endif
2553 
2554     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2555                            r10, L_failed);
2556 
2557     // typeArrayKlass
2558     //
2559     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
2560     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
2561     //
2562 
2563     const Register r10_offset = r10;    // array offset
2564     const Register rax_elsize = rax_lh; // element size
2565 
2566     __ movl(r10_offset, rax_lh);
2567     __ shrl(r10_offset, Klass::_lh_header_size_shift);
2568     __ andptr(r10_offset, Klass::_lh_header_size_mask);   // array_offset
2569     __ addptr(src, r10_offset);           // src array offset
2570     __ addptr(dst, r10_offset);           // dst array offset
2571     BLOCK_COMMENT("choose copy loop based on element size");
2572     __ andl(rax_lh, Klass::_lh_log2_element_size_mask); // rax_lh -> rax_elsize
2573 
2574     // next registers should be set before the jump to corresponding stub
2575     const Register from     = c_rarg0;  // source array address
2576     const Register to       = c_rarg1;  // destination array address
2577     const Register count    = c_rarg2;  // elements count
2578 
2579     // 'from', 'to', 'count' registers should be set in such order
2580     // since they are the same as 'src', 'src_pos', 'dst'.
2581 
2582   __ BIND(L_copy_bytes);
2583     __ cmpl(rax_elsize, 0);
2584     __ jccb(Assembler::notEqual, L_copy_shorts);
2585     __ lea(from, Address(src, src_pos, Address::times_1, 0));// src_addr
2586     __ lea(to,   Address(dst, dst_pos, Address::times_1, 0));// dst_addr
2587     __ movl2ptr(count, r11_length); // length
2588     __ jump(RuntimeAddress(byte_copy_entry));
2589 
2590   __ BIND(L_copy_shorts);
2591     __ cmpl(rax_elsize, LogBytesPerShort);
2592     __ jccb(Assembler::notEqual, L_copy_ints);
2593     __ lea(from, Address(src, src_pos, Address::times_2, 0));// src_addr
2594     __ lea(to,   Address(dst, dst_pos, Address::times_2, 0));// dst_addr
2595     __ movl2ptr(count, r11_length); // length
2596     __ jump(RuntimeAddress(short_copy_entry));
2597 
2598   __ BIND(L_copy_ints);
2599     __ cmpl(rax_elsize, LogBytesPerInt);
2600     __ jccb(Assembler::notEqual, L_copy_longs);
2601     __ lea(from, Address(src, src_pos, Address::times_4, 0));// src_addr
2602     __ lea(to,   Address(dst, dst_pos, Address::times_4, 0));// dst_addr
2603     __ movl2ptr(count, r11_length); // length
2604     __ jump(RuntimeAddress(int_copy_entry));
2605 
2606   __ BIND(L_copy_longs);
2607 #ifdef ASSERT
2608     { Label L;
2609       __ cmpl(rax_elsize, LogBytesPerLong);
2610       __ jcc(Assembler::equal, L);
2611       __ stop("must be long copy, but elsize is wrong");
2612       __ bind(L);
2613     }
2614 #endif
2615     __ lea(from, Address(src, src_pos, Address::times_8, 0));// src_addr
2616     __ lea(to,   Address(dst, dst_pos, Address::times_8, 0));// dst_addr
2617     __ movl2ptr(count, r11_length); // length
2618     __ jump(RuntimeAddress(long_copy_entry));
2619 
2620     // objArrayKlass
2621   __ BIND(L_objArray);
2622     // live at this point:  r10_src_klass, src[_pos], dst[_pos]
2623 
2624     Label L_plain_copy, L_checkcast_copy;
2625     //  test array classes for subtyping
2626     __ load_klass(r9_dst_klass, dst);
2627     __ cmpq(r10_src_klass, r9_dst_klass); // usual case is exact equality
2628     __ jcc(Assembler::notEqual, L_checkcast_copy);
2629 
2630     // Identically typed arrays can be copied without element-wise checks.
2631     arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2632                            r10, L_failed);
2633 
2634     __ lea(from, Address(src, src_pos, TIMES_OOP,
2635                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
2636     __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2637                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
2638     __ movl2ptr(count, r11_length); // length
2639   __ BIND(L_plain_copy);
2640     __ jump(RuntimeAddress(oop_copy_entry));
2641 
2642   __ BIND(L_checkcast_copy);
2643     // live at this point:  r10_src_klass, !r11_length
2644     {
2645       // assert(r11_length == C_RARG4); // will reload from here
2646       Register r11_dst_klass = r11;
2647       __ load_klass(r11_dst_klass, dst);
2648 
2649       // Before looking at dst.length, make sure dst is also an objArray.
2650       __ cmpl(Address(r11_dst_klass, lh_offset), objArray_lh);
2651       __ jcc(Assembler::notEqual, L_failed);
2652 
2653       // It is safe to examine both src.length and dst.length.
2654 #ifndef _WIN64
2655       arraycopy_range_checks(src, src_pos, dst, dst_pos, C_RARG4,
2656                              rax, L_failed);
2657 #else
2658       __ movl(r11_length, C_RARG4);     // reload
2659       arraycopy_range_checks(src, src_pos, dst, dst_pos, r11_length,
2660                              rax, L_failed);
2661       __ load_klass(r11_dst_klass, dst); // reload
2662 #endif
2663 
2664       // Marshal the base address arguments now, freeing registers.
2665       __ lea(from, Address(src, src_pos, TIMES_OOP,
2666                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2667       __ lea(to,   Address(dst, dst_pos, TIMES_OOP,
2668                    arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
2669       __ movl(count, C_RARG4);          // length (reloaded)
2670       Register sco_temp = c_rarg3;      // this register is free now
2671       assert_different_registers(from, to, count, sco_temp,
2672                                  r11_dst_klass, r10_src_klass);
2673       assert_clean_int(count, sco_temp);
2674 
2675       // Generate the type check.
2676       int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
2677                         Klass::super_check_offset_offset_in_bytes());
2678       __ movl(sco_temp, Address(r11_dst_klass, sco_offset));
2679       assert_clean_int(sco_temp, rax);
2680       generate_type_check(r10_src_klass, sco_temp, r11_dst_klass, L_plain_copy);
2681 
2682       // Fetch destination element klass from the objArrayKlass header.
2683       int ek_offset = (klassOopDesc::header_size() * HeapWordSize +
2684                        objArrayKlass::element_klass_offset_in_bytes());
2685       __ movptr(r11_dst_klass, Address(r11_dst_klass, ek_offset));
2686       __ movl(sco_temp,      Address(r11_dst_klass, sco_offset));
2687       assert_clean_int(sco_temp, rax);
2688 
2689       // the checkcast_copy loop needs two extra arguments:
2690       assert(c_rarg3 == sco_temp, "#3 already in place");
2691       __ movptr(C_RARG4, r11_dst_klass);  // dst.klass.element_klass
2692       __ jump(RuntimeAddress(checkcast_copy_entry));
2693     }
2694 
2695   __ BIND(L_failed);
2696     __ xorptr(rax, rax);
2697     __ notptr(rax); // return -1
2698     __ leave();   // required for proper stackwalking of RuntimeStub frame
2699     __ ret(0);
2700 
2701     return start;
2702   }
2703 
2704 #undef length_arg
2705 
2706   void generate_arraycopy_stubs() {
2707     // Call the conjoint generation methods immediately after
2708     // the disjoint ones so that short branches from the former
2709     // to the latter can be generated.
2710     StubRoutines::_jbyte_disjoint_arraycopy  = generate_disjoint_byte_copy(false, "jbyte_disjoint_arraycopy");
2711     StubRoutines::_jbyte_arraycopy           = generate_conjoint_byte_copy(false, "jbyte_arraycopy");
2712 
2713     StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, "jshort_disjoint_arraycopy");
2714     StubRoutines::_jshort_arraycopy          = generate_conjoint_short_copy(false, "jshort_arraycopy");
2715 
2716     StubRoutines::_jint_disjoint_arraycopy   = generate_disjoint_int_oop_copy(false, false, "jint_disjoint_arraycopy");
2717     StubRoutines::_jint_arraycopy            = generate_conjoint_int_oop_copy(false, false, "jint_arraycopy");
2718 
2719     StubRoutines::_jlong_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, false, "jlong_disjoint_arraycopy");
2720     StubRoutines::_jlong_arraycopy           = generate_conjoint_long_oop_copy(false, false, "jlong_arraycopy");
2721 
2722 
2723     if (UseCompressedOops) {
2724       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_int_oop_copy(false, true, "oop_disjoint_arraycopy");
2725       StubRoutines::_oop_arraycopy           = generate_conjoint_int_oop_copy(false, true, "oop_arraycopy");
2726     } else {
2727       StubRoutines::_oop_disjoint_arraycopy  = generate_disjoint_long_oop_copy(false, true, "oop_disjoint_arraycopy");
2728       StubRoutines::_oop_arraycopy           = generate_conjoint_long_oop_copy(false, true, "oop_arraycopy");
2729     }
2730 
2731     StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy");
2732     StubRoutines::_unsafe_arraycopy    = generate_unsafe_copy("unsafe_arraycopy");
2733     StubRoutines::_generic_arraycopy   = generate_generic_copy("generic_arraycopy");
2734 
2735     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2736     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2737     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2738     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2739     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2740     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2741 
2742     // We don't generate specialized code for HeapWord-aligned source
2743     // arrays, so just use the code we've already generated
2744     StubRoutines::_arrayof_jbyte_disjoint_arraycopy  = StubRoutines::_jbyte_disjoint_arraycopy;
2745     StubRoutines::_arrayof_jbyte_arraycopy           = StubRoutines::_jbyte_arraycopy;
2746 
2747     StubRoutines::_arrayof_jshort_disjoint_arraycopy = StubRoutines::_jshort_disjoint_arraycopy;
2748     StubRoutines::_arrayof_jshort_arraycopy          = StubRoutines::_jshort_arraycopy;
2749 
2750     StubRoutines::_arrayof_jint_disjoint_arraycopy   = StubRoutines::_jint_disjoint_arraycopy;
2751     StubRoutines::_arrayof_jint_arraycopy            = StubRoutines::_jint_arraycopy;
2752 
2753     StubRoutines::_arrayof_jlong_disjoint_arraycopy  = StubRoutines::_jlong_disjoint_arraycopy;
2754     StubRoutines::_arrayof_jlong_arraycopy           = StubRoutines::_jlong_arraycopy;
2755 
2756     StubRoutines::_arrayof_oop_disjoint_arraycopy    = StubRoutines::_oop_disjoint_arraycopy;
2757     StubRoutines::_arrayof_oop_arraycopy             = StubRoutines::_oop_arraycopy;
2758   }
2759 
2760   void generate_math_stubs() {
2761     {
2762       StubCodeMark mark(this, "StubRoutines", "log");
2763       StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
2764 
2765       __ subq(rsp, 8);
2766       __ movdbl(Address(rsp, 0), xmm0);
2767       __ fld_d(Address(rsp, 0));
2768       __ flog();
2769       __ fstp_d(Address(rsp, 0));
2770       __ movdbl(xmm0, Address(rsp, 0));
2771       __ addq(rsp, 8);
2772       __ ret(0);
2773     }
2774     {
2775       StubCodeMark mark(this, "StubRoutines", "log10");
2776       StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
2777 
2778       __ subq(rsp, 8);
2779       __ movdbl(Address(rsp, 0), xmm0);
2780       __ fld_d(Address(rsp, 0));
2781       __ flog10();
2782       __ fstp_d(Address(rsp, 0));
2783       __ movdbl(xmm0, Address(rsp, 0));
2784       __ addq(rsp, 8);
2785       __ ret(0);
2786     }
2787     {
2788       StubCodeMark mark(this, "StubRoutines", "sin");
2789       StubRoutines::_intrinsic_sin = (double (*)(double)) __ pc();
2790 
2791       __ subq(rsp, 8);
2792       __ movdbl(Address(rsp, 0), xmm0);
2793       __ fld_d(Address(rsp, 0));
2794       __ trigfunc('s');
2795       __ fstp_d(Address(rsp, 0));
2796       __ movdbl(xmm0, Address(rsp, 0));
2797       __ addq(rsp, 8);
2798       __ ret(0);
2799     }
2800     {
2801       StubCodeMark mark(this, "StubRoutines", "cos");
2802       StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
2803 
2804       __ subq(rsp, 8);
2805       __ movdbl(Address(rsp, 0), xmm0);
2806       __ fld_d(Address(rsp, 0));
2807       __ trigfunc('c');
2808       __ fstp_d(Address(rsp, 0));
2809       __ movdbl(xmm0, Address(rsp, 0));
2810       __ addq(rsp, 8);
2811       __ ret(0);
2812     }
2813     {
2814       StubCodeMark mark(this, "StubRoutines", "tan");
2815       StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
2816 
2817       __ subq(rsp, 8);
2818       __ movdbl(Address(rsp, 0), xmm0);
2819       __ fld_d(Address(rsp, 0));
2820       __ trigfunc('t');
2821       __ fstp_d(Address(rsp, 0));
2822       __ movdbl(xmm0, Address(rsp, 0));
2823       __ addq(rsp, 8);
2824       __ ret(0);
2825     }
2826 
2827     // The intrinsic version of these seem to return the same value as
2828     // the strict version.
2829     StubRoutines::_intrinsic_exp = SharedRuntime::dexp;
2830     StubRoutines::_intrinsic_pow = SharedRuntime::dpow;
2831   }
2832 
2833 #undef __
2834 #define __ masm->
2835 
2836   // Continuation point for throwing of implicit exceptions that are
2837   // not handled in the current activation. Fabricates an exception
2838   // oop and initiates normal exception dispatching in this
2839   // frame. Since we need to preserve callee-saved values (currently
2840   // only for C2, but done for C1 as well) we need a callee-saved oop
2841   // map and therefore have to make these stubs into RuntimeStubs
2842   // rather than BufferBlobs.  If the compiler needs all registers to
2843   // be preserved between the fault point and the exception handler
2844   // then it must assume responsibility for that in
2845   // AbstractCompiler::continuation_for_implicit_null_exception or
2846   // continuation_for_implicit_division_by_zero_exception. All other
2847   // implicit exceptions (e.g., NullPointerException or
2848   // AbstractMethodError on entry) are either at call sites or
2849   // otherwise assume that stack unwinding will be initiated, so
2850   // caller saved registers were assumed volatile in the compiler.
2851   address generate_throw_exception(const char* name,
2852                                    address runtime_entry,
2853                                    bool restore_saved_exception_pc) {
2854     // Information about frame layout at time of blocking runtime call.
2855     // Note that we only have to preserve callee-saved registers since
2856     // the compilers are responsible for supplying a continuation point
2857     // if they expect all registers to be preserved.
2858     enum layout {
2859       rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
2860       rbp_off2,
2861       return_off,
2862       return_off2,
2863       framesize // inclusive of return address
2864     };
2865 
2866     int insts_size = 512;
2867     int locs_size  = 64;
2868 
2869     CodeBuffer code(name, insts_size, locs_size);
2870     OopMapSet* oop_maps  = new OopMapSet();
2871     MacroAssembler* masm = new MacroAssembler(&code);
2872 
2873     address start = __ pc();
2874 
2875     // This is an inlined and slightly modified version of call_VM
2876     // which has the ability to fetch the return PC out of
2877     // thread-local storage and also sets up last_Java_sp slightly
2878     // differently than the real call_VM
2879     if (restore_saved_exception_pc) {
2880       __ movptr(rax,
2881                 Address(r15_thread,
2882                         in_bytes(JavaThread::saved_exception_pc_offset())));
2883       __ push(rax);
2884     }
2885 
2886     __ enter(); // required for proper stackwalking of RuntimeStub frame
2887 
2888     assert(is_even(framesize/2), "sp not 16-byte aligned");
2889 
2890     // return address and rbp are already in place
2891     __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
2892 
2893     int frame_complete = __ pc() - start;
2894 
2895     // Set up last_Java_sp and last_Java_fp
2896     __ set_last_Java_frame(rsp, rbp, NULL);
2897 
2898     // Call runtime
2899     __ movptr(c_rarg0, r15_thread);
2900     BLOCK_COMMENT("call runtime_entry");
2901     __ call(RuntimeAddress(runtime_entry));
2902 
2903     // Generate oop map
2904     OopMap* map = new OopMap(framesize, 0);
2905 
2906     oop_maps->add_gc_map(__ pc() - start, map);
2907 
2908     __ reset_last_Java_frame(true, false);
2909 
2910     __ leave(); // required for proper stackwalking of RuntimeStub frame
2911 
2912     // check for pending exceptions
2913 #ifdef ASSERT
2914     Label L;
2915     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()),
2916             (int32_t) NULL_WORD);
2917     __ jcc(Assembler::notEqual, L);
2918     __ should_not_reach_here();
2919     __ bind(L);
2920 #endif // ASSERT
2921     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2922 
2923 
2924     // codeBlob framesize is in words (not VMRegImpl::slot_size)
2925     RuntimeStub* stub =
2926       RuntimeStub::new_runtime_stub(name,
2927                                     &code,
2928                                     frame_complete,
2929                                     (framesize >> (LogBytesPerWord - LogBytesPerInt)),
2930                                     oop_maps, false);
2931     return stub->entry_point();
2932   }
2933 
2934   // Initialization
2935   void generate_initial() {
2936     // Generates all stubs and initializes the entry points
2937 
2938     // This platform-specific stub is needed by generate_call_stub()
2939     StubRoutines::x86::_mxcsr_std        = generate_fp_mask("mxcsr_std",        0x0000000000001F80);
2940 
2941     // entry points that exist in all platforms Note: This is code
2942     // that could be shared among different platforms - however the
2943     // benefit seems to be smaller than the disadvantage of having a
2944     // much more complicated generator structure. See also comment in
2945     // stubRoutines.hpp.
2946 
2947     StubRoutines::_forward_exception_entry = generate_forward_exception();
2948 
2949     StubRoutines::_call_stub_entry =
2950       generate_call_stub(StubRoutines::_call_stub_return_address);
2951 
2952     // is referenced by megamorphic call
2953     StubRoutines::_catch_exception_entry = generate_catch_exception();
2954 
2955     // atomic calls
2956     StubRoutines::_atomic_xchg_entry         = generate_atomic_xchg();
2957     StubRoutines::_atomic_xchg_ptr_entry     = generate_atomic_xchg_ptr();
2958     StubRoutines::_atomic_cmpxchg_entry      = generate_atomic_cmpxchg();
2959     StubRoutines::_atomic_cmpxchg_long_entry = generate_atomic_cmpxchg_long();
2960     StubRoutines::_atomic_add_entry          = generate_atomic_add();
2961     StubRoutines::_atomic_add_ptr_entry      = generate_atomic_add_ptr();
2962     StubRoutines::_fence_entry               = generate_orderaccess_fence();
2963 
2964     StubRoutines::_handler_for_unsafe_access_entry =
2965       generate_handler_for_unsafe_access();
2966 
2967     // platform dependent
2968     StubRoutines::x86::_get_previous_fp_entry = generate_get_previous_fp();
2969 
2970     StubRoutines::x86::_verify_mxcsr_entry    = generate_verify_mxcsr();
2971   }
2972 
2973   void generate_all() {
2974     // Generates all stubs and initializes the entry points
2975 
2976     // These entry points require SharedInfo::stack0 to be set up in
2977     // non-core builds and need to be relocatable, so they each
2978     // fabricate a RuntimeStub internally.
2979     StubRoutines::_throw_AbstractMethodError_entry =
2980       generate_throw_exception("AbstractMethodError throw_exception",
2981                                CAST_FROM_FN_PTR(address,
2982                                                 SharedRuntime::
2983                                                 throw_AbstractMethodError),
2984                                false);
2985 
2986     StubRoutines::_throw_IncompatibleClassChangeError_entry =
2987       generate_throw_exception("IncompatibleClassChangeError throw_exception",
2988                                CAST_FROM_FN_PTR(address,
2989                                                 SharedRuntime::
2990                                                 throw_IncompatibleClassChangeError),
2991                                false);
2992 
2993     StubRoutines::_throw_ArithmeticException_entry =
2994       generate_throw_exception("ArithmeticException throw_exception",
2995                                CAST_FROM_FN_PTR(address,
2996                                                 SharedRuntime::
2997                                                 throw_ArithmeticException),
2998                                true);
2999 
3000     StubRoutines::_throw_NullPointerException_entry =
3001       generate_throw_exception("NullPointerException throw_exception",
3002                                CAST_FROM_FN_PTR(address,
3003                                                 SharedRuntime::
3004                                                 throw_NullPointerException),
3005                                true);
3006 
3007     StubRoutines::_throw_NullPointerException_at_call_entry =
3008       generate_throw_exception("NullPointerException at call throw_exception",
3009                                CAST_FROM_FN_PTR(address,
3010                                                 SharedRuntime::
3011                                                 throw_NullPointerException_at_call),
3012                                false);
3013 
3014     StubRoutines::_throw_StackOverflowError_entry =
3015       generate_throw_exception("StackOverflowError throw_exception",
3016                                CAST_FROM_FN_PTR(address,
3017                                                 SharedRuntime::
3018                                                 throw_StackOverflowError),
3019                                false);
3020 
3021     // entry points that are platform specific
3022     StubRoutines::x86::_f2i_fixup = generate_f2i_fixup();
3023     StubRoutines::x86::_f2l_fixup = generate_f2l_fixup();
3024     StubRoutines::x86::_d2i_fixup = generate_d2i_fixup();
3025     StubRoutines::x86::_d2l_fixup = generate_d2l_fixup();
3026 
3027     StubRoutines::x86::_float_sign_mask  = generate_fp_mask("float_sign_mask",  0x7FFFFFFF7FFFFFFF);
3028     StubRoutines::x86::_float_sign_flip  = generate_fp_mask("float_sign_flip",  0x8000000080000000);
3029     StubRoutines::x86::_double_sign_mask = generate_fp_mask("double_sign_mask", 0x7FFFFFFFFFFFFFFF);
3030     StubRoutines::x86::_double_sign_flip = generate_fp_mask("double_sign_flip", 0x8000000000000000);
3031 
3032     // support for verify_oop (must happen after universe_init)
3033     StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
3034 
3035     // arraycopy stubs used by compilers
3036     generate_arraycopy_stubs();
3037 
3038     generate_math_stubs();
3039   }
3040 
3041  public:
3042   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
3043     if (all) {
3044       generate_all();
3045     } else {
3046       generate_initial();
3047     }
3048   }
3049 }; // end class declaration
3050 
3051 address StubGenerator::disjoint_byte_copy_entry  = NULL;
3052 address StubGenerator::disjoint_short_copy_entry = NULL;
3053 address StubGenerator::disjoint_int_copy_entry   = NULL;
3054 address StubGenerator::disjoint_long_copy_entry  = NULL;
3055 address StubGenerator::disjoint_oop_copy_entry   = NULL;
3056 
3057 address StubGenerator::byte_copy_entry  = NULL;
3058 address StubGenerator::short_copy_entry = NULL;
3059 address StubGenerator::int_copy_entry   = NULL;
3060 address StubGenerator::long_copy_entry  = NULL;
3061 address StubGenerator::oop_copy_entry   = NULL;
3062 
3063 address StubGenerator::checkcast_copy_entry = NULL;
3064 
3065 void StubGenerator_generate(CodeBuffer* code, bool all) {
3066   StubGenerator g(code, all);
3067 }