1 /*
   2  * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "assembler_x86.inline.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "nativeInst_x86.hpp"
  30 #include "oops/instanceOop.hpp"
  31 #include "oops/methodOop.hpp"
  32 #include "oops/objArrayKlass.hpp"
  33 #include "oops/oop.inline.hpp"
  34 #include "prims/methodHandles.hpp"
  35 #include "runtime/frame.inline.hpp"
  36 #include "runtime/handles.inline.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/stubCodeGenerator.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "utilities/top.hpp"
  41 #ifdef TARGET_OS_FAMILY_linux
  42 # include "thread_linux.inline.hpp"
  43 #endif
  44 #ifdef TARGET_OS_FAMILY_solaris
  45 # include "thread_solaris.inline.hpp"
  46 #endif
  47 #ifdef TARGET_OS_FAMILY_windows
  48 # include "thread_windows.inline.hpp"
  49 #endif
  50 #ifdef COMPILER2
  51 #include "opto/runtime.hpp"
  52 #endif
  53 
  54 // Declaration and definition of StubGenerator (no .hpp file).
  55 // For a more detailed description of the stub routine structure
  56 // see the comment in stubRoutines.hpp
  57 
  58 #define __ _masm->
  59 #define a__ ((Assembler*)_masm)->
  60 
  61 #ifdef PRODUCT
  62 #define BLOCK_COMMENT(str) /* nothing */
  63 #else
  64 #define BLOCK_COMMENT(str) __ block_comment(str)
  65 #endif
  66 
  67 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  68 
  69 const int MXCSR_MASK  = 0xFFC0;  // Mask out any pending exceptions
  70 const int FPU_CNTRL_WRD_MASK = 0xFFFF;
  71 
  72 // -------------------------------------------------------------------------------------------------------------------------
  73 // Stub Code definitions
  74 
  75 static address handle_unsafe_access() {
  76   JavaThread* thread = JavaThread::current();
  77   address pc  = thread->saved_exception_pc();
  78   // pc is the instruction which we must emulate
  79   // doing a no-op is fine:  return garbage from the load
  80   // therefore, compute npc
  81   address npc = Assembler::locate_next_instruction(pc);
  82 
  83   // request an async exception
  84   thread->set_pending_unsafe_access_error();
  85 
  86   // return address of next instruction to execute
  87   return npc;
  88 }
  89 
  90 class StubGenerator: public StubCodeGenerator {
  91  private:
  92 
  93 #ifdef PRODUCT
  94 #define inc_counter_np(counter) (0)
  95 #else
  96   void inc_counter_np_(int& counter) {
  97     __ incrementl(ExternalAddress((address)&counter));
  98   }
  99 #define inc_counter_np(counter) \
 100   BLOCK_COMMENT("inc_counter " #counter); \
 101   inc_counter_np_(counter);
 102 #endif //PRODUCT
 103 
 104   void inc_copy_counter_np(BasicType t) {
 105 #ifndef PRODUCT
 106     switch (t) {
 107     case T_BYTE:    inc_counter_np(SharedRuntime::_jbyte_array_copy_ctr); return;
 108     case T_SHORT:   inc_counter_np(SharedRuntime::_jshort_array_copy_ctr); return;
 109     case T_INT:     inc_counter_np(SharedRuntime::_jint_array_copy_ctr); return;
 110     case T_LONG:    inc_counter_np(SharedRuntime::_jlong_array_copy_ctr); return;
 111     case T_OBJECT:  inc_counter_np(SharedRuntime::_oop_array_copy_ctr); return;
 112     }
 113     ShouldNotReachHere();
 114 #endif //PRODUCT
 115   }
 116 
 117   //------------------------------------------------------------------------------------------------------------------------
 118   // Call stubs are used to call Java from C
 119   //
 120   //    [ return_from_Java     ] <--- rsp
 121   //    [ argument word n      ]
 122   //      ...
 123   // -N [ argument word 1      ]
 124   // -7 [ Possible padding for stack alignment ]
 125   // -6 [ Possible padding for stack alignment ]
 126   // -5 [ Possible padding for stack alignment ]
 127   // -4 [ mxcsr save           ] <--- rsp_after_call
 128   // -3 [ saved rbx,            ]
 129   // -2 [ saved rsi            ]
 130   // -1 [ saved rdi            ]
 131   //  0 [ saved rbp,            ] <--- rbp,
 132   //  1 [ return address       ]
 133   //  2 [ ptr. to call wrapper ]
 134   //  3 [ result               ]
 135   //  4 [ result_type          ]
 136   //  5 [ method               ]
 137   //  6 [ entry_point          ]
 138   //  7 [ parameters           ]
 139   //  8 [ parameter_size       ]
 140   //  9 [ thread               ]
 141 
 142 
 143   address generate_call_stub(address& return_address) {
 144     StubCodeMark mark(this, "StubRoutines", "call_stub");
 145     address start = __ pc();
 146 
 147     // stub code parameters / addresses
 148     assert(frame::entry_frame_call_wrapper_offset == 2, "adjust this code");
 149     bool  sse_save = false;
 150     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_catch_exception()!
 151     const int     locals_count_in_bytes  (4*wordSize);
 152     const Address mxcsr_save    (rbp, -4 * wordSize);
 153     const Address saved_rbx     (rbp, -3 * wordSize);
 154     const Address saved_rsi     (rbp, -2 * wordSize);
 155     const Address saved_rdi     (rbp, -1 * wordSize);
 156     const Address result        (rbp,  3 * wordSize);
 157     const Address result_type   (rbp,  4 * wordSize);
 158     const Address method        (rbp,  5 * wordSize);
 159     const Address entry_point   (rbp,  6 * wordSize);
 160     const Address parameters    (rbp,  7 * wordSize);
 161     const Address parameter_size(rbp,  8 * wordSize);
 162     const Address thread        (rbp,  9 * wordSize); // same as in generate_catch_exception()!
 163     sse_save =  UseSSE > 0;
 164 
 165     // stub code
 166     __ enter();
 167     __ movptr(rcx, parameter_size);              // parameter counter
 168     __ shlptr(rcx, Interpreter::logStackElementSize); // convert parameter count to bytes
 169     __ addptr(rcx, locals_count_in_bytes);       // reserve space for register saves
 170     __ subptr(rsp, rcx);
 171     __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
 172 
 173     // save rdi, rsi, & rbx, according to C calling conventions
 174     __ movptr(saved_rdi, rdi);
 175     __ movptr(saved_rsi, rsi);
 176     __ movptr(saved_rbx, rbx);
 177     // save and initialize %mxcsr
 178     if (sse_save) {
 179       Label skip_ldmx;
 180       __ stmxcsr(mxcsr_save);
 181       __ movl(rax, mxcsr_save);
 182       __ andl(rax, MXCSR_MASK);    // Only check control and mask bits
 183       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 184       __ cmp32(rax, mxcsr_std);
 185       __ jcc(Assembler::equal, skip_ldmx);
 186       __ ldmxcsr(mxcsr_std);
 187       __ bind(skip_ldmx);
 188     }
 189 
 190     // make sure the control word is correct.
 191     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 192 
 193 #ifdef ASSERT
 194     // make sure we have no pending exceptions
 195     { Label L;
 196       __ movptr(rcx, thread);
 197       __ cmpptr(Address(rcx, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 198       __ jcc(Assembler::equal, L);
 199       __ stop("StubRoutines::call_stub: entered with pending exception");
 200       __ bind(L);
 201     }
 202 #endif
 203 
 204     // pass parameters if any
 205     BLOCK_COMMENT("pass parameters if any");
 206     Label parameters_done;
 207     __ movl(rcx, parameter_size);  // parameter counter
 208     __ testl(rcx, rcx);
 209     __ jcc(Assembler::zero, parameters_done);
 210 
 211     // parameter passing loop
 212 
 213     Label loop;
 214     // Copy Java parameters in reverse order (receiver last)
 215     // Note that the argument order is inverted in the process
 216     // source is rdx[rcx: N-1..0]
 217     // dest   is rsp[rbx: 0..N-1]
 218 
 219     __ movptr(rdx, parameters);          // parameter pointer
 220     __ xorptr(rbx, rbx);
 221 
 222     __ BIND(loop);
 223 
 224     // get parameter
 225     __ movptr(rax, Address(rdx, rcx, Interpreter::stackElementScale(), -wordSize));
 226     __ movptr(Address(rsp, rbx, Interpreter::stackElementScale(),
 227                     Interpreter::expr_offset_in_bytes(0)), rax);          // store parameter
 228     __ increment(rbx);
 229     __ decrement(rcx);
 230     __ jcc(Assembler::notZero, loop);
 231 
 232     // call Java function
 233     __ BIND(parameters_done);
 234     __ movptr(rbx, method);           // get methodOop
 235     __ movptr(rax, entry_point);      // get entry_point
 236     __ mov(rsi, rsp);                 // set sender sp
 237     BLOCK_COMMENT("call Java function");
 238     __ call(rax);
 239 
 240     BLOCK_COMMENT("call_stub_return_address:");
 241     return_address = __ pc();
 242 
 243 #ifdef COMPILER2
 244     {
 245       Label L_skip;
 246       if (UseSSE >= 2) {
 247         __ verify_FPU(0, "call_stub_return");
 248       } else {
 249         for (int i = 1; i < 8; i++) {
 250           __ ffree(i);
 251         }
 252 
 253         // UseSSE <= 1 so double result should be left on TOS
 254         __ movl(rsi, result_type);
 255         __ cmpl(rsi, T_DOUBLE);
 256         __ jcc(Assembler::equal, L_skip);
 257         if (UseSSE == 0) {
 258           // UseSSE == 0 so float result should be left on TOS
 259           __ cmpl(rsi, T_FLOAT);
 260           __ jcc(Assembler::equal, L_skip);
 261         }
 262         __ ffree(0);
 263       }
 264       __ BIND(L_skip);
 265     }
 266 #endif // COMPILER2
 267 
 268     // store result depending on type
 269     // (everything that is not T_LONG, T_FLOAT or T_DOUBLE is treated as T_INT)
 270     __ movptr(rdi, result);
 271     Label is_long, is_float, is_double, exit;
 272     __ movl(rsi, result_type);
 273     __ cmpl(rsi, T_LONG);
 274     __ jcc(Assembler::equal, is_long);
 275     __ cmpl(rsi, T_FLOAT);
 276     __ jcc(Assembler::equal, is_float);
 277     __ cmpl(rsi, T_DOUBLE);
 278     __ jcc(Assembler::equal, is_double);
 279 
 280     // handle T_INT case
 281     __ movl(Address(rdi, 0), rax);
 282     __ BIND(exit);
 283 
 284     // check that FPU stack is empty
 285     __ verify_FPU(0, "generate_call_stub");
 286 
 287     // pop parameters
 288     __ lea(rsp, rsp_after_call);
 289 
 290     // restore %mxcsr
 291     if (sse_save) {
 292       __ ldmxcsr(mxcsr_save);
 293     }
 294 
 295     // restore rdi, rsi and rbx,
 296     __ movptr(rbx, saved_rbx);
 297     __ movptr(rsi, saved_rsi);
 298     __ movptr(rdi, saved_rdi);
 299     __ addptr(rsp, 4*wordSize);
 300 
 301     // return
 302     __ pop(rbp);
 303     __ ret(0);
 304 
 305     // handle return types different from T_INT
 306     __ BIND(is_long);
 307     __ movl(Address(rdi, 0 * wordSize), rax);
 308     __ movl(Address(rdi, 1 * wordSize), rdx);
 309     __ jmp(exit);
 310 
 311     __ BIND(is_float);
 312     // interpreter uses xmm0 for return values
 313     if (UseSSE >= 1) {
 314       __ movflt(Address(rdi, 0), xmm0);
 315     } else {
 316       __ fstp_s(Address(rdi, 0));
 317     }
 318     __ jmp(exit);
 319 
 320     __ BIND(is_double);
 321     // interpreter uses xmm0 for return values
 322     if (UseSSE >= 2) {
 323       __ movdbl(Address(rdi, 0), xmm0);
 324     } else {
 325       __ fstp_d(Address(rdi, 0));
 326     }
 327     __ jmp(exit);
 328 
 329     return start;
 330   }
 331 
 332 
 333   //------------------------------------------------------------------------------------------------------------------------
 334   // Return point for a Java call if there's an exception thrown in Java code.
 335   // The exception is caught and transformed into a pending exception stored in
 336   // JavaThread that can be tested from within the VM.
 337   //
 338   // Note: Usually the parameters are removed by the callee. In case of an exception
 339   //       crossing an activation frame boundary, that is not the case if the callee
 340   //       is compiled code => need to setup the rsp.
 341   //
 342   // rax,: exception oop
 343 
 344   address generate_catch_exception() {
 345     StubCodeMark mark(this, "StubRoutines", "catch_exception");
 346     const Address rsp_after_call(rbp, -4 * wordSize); // same as in generate_call_stub()!
 347     const Address thread        (rbp,  9 * wordSize); // same as in generate_call_stub()!
 348     address start = __ pc();
 349 
 350     // get thread directly
 351     __ movptr(rcx, thread);
 352 #ifdef ASSERT
 353     // verify that threads correspond
 354     { Label L;
 355       __ get_thread(rbx);
 356       __ cmpptr(rbx, rcx);
 357       __ jcc(Assembler::equal, L);
 358       __ stop("StubRoutines::catch_exception: threads must correspond");
 359       __ bind(L);
 360     }
 361 #endif
 362     // set pending exception
 363     __ verify_oop(rax);
 364     __ movptr(Address(rcx, Thread::pending_exception_offset()), rax          );
 365     __ lea(Address(rcx, Thread::exception_file_offset   ()),
 366            ExternalAddress((address)__FILE__));
 367     __ movl(Address(rcx, Thread::exception_line_offset   ()), __LINE__ );
 368     // complete return to VM
 369     assert(StubRoutines::_call_stub_return_address != NULL, "_call_stub_return_address must have been generated before");
 370     __ jump(RuntimeAddress(StubRoutines::_call_stub_return_address));
 371 
 372     return start;
 373   }
 374 
 375 
 376   //------------------------------------------------------------------------------------------------------------------------
 377   // Continuation point for runtime calls returning with a pending exception.
 378   // The pending exception check happened in the runtime or native call stub.
 379   // The pending exception in Thread is converted into a Java-level exception.
 380   //
 381   // Contract with Java-level exception handlers:
 382   // rax: exception
 383   // rdx: throwing pc
 384   //
 385   // NOTE: At entry of this stub, exception-pc must be on stack !!
 386 
 387   address generate_forward_exception() {
 388     StubCodeMark mark(this, "StubRoutines", "forward exception");
 389     address start = __ pc();
 390     const Register thread = rcx;
 391 
 392     // other registers used in this stub
 393     const Register exception_oop = rax;
 394     const Register handler_addr  = rbx;
 395     const Register exception_pc  = rdx;
 396 
 397     // Upon entry, the sp points to the return address returning into Java
 398     // (interpreted or compiled) code; i.e., the return address becomes the
 399     // throwing pc.
 400     //
 401     // Arguments pushed before the runtime call are still on the stack but
 402     // the exception handler will reset the stack pointer -> ignore them.
 403     // A potential result in registers can be ignored as well.
 404 
 405 #ifdef ASSERT
 406     // make sure this code is only executed if there is a pending exception
 407     { Label L;
 408       __ get_thread(thread);
 409       __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 410       __ jcc(Assembler::notEqual, L);
 411       __ stop("StubRoutines::forward exception: no pending exception (1)");
 412       __ bind(L);
 413     }
 414 #endif
 415 
 416     // compute exception handler into rbx,
 417     __ get_thread(thread);
 418     __ movptr(exception_pc, Address(rsp, 0));
 419     BLOCK_COMMENT("call exception_handler_for_return_address");
 420     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, exception_pc);
 421     __ mov(handler_addr, rax);
 422 
 423     // setup rax & rdx, remove return address & clear pending exception
 424     __ get_thread(thread);
 425     __ pop(exception_pc);
 426     __ movptr(exception_oop, Address(thread, Thread::pending_exception_offset()));
 427     __ movptr(Address(thread, Thread::pending_exception_offset()), NULL_WORD);
 428 
 429 #ifdef ASSERT
 430     // make sure exception is set
 431     { Label L;
 432       __ testptr(exception_oop, exception_oop);
 433       __ jcc(Assembler::notEqual, L);
 434       __ stop("StubRoutines::forward exception: no pending exception (2)");
 435       __ bind(L);
 436     }
 437 #endif
 438 
 439     // Verify that there is really a valid exception in RAX.
 440     __ verify_oop(exception_oop);
 441 
 442     // continue at exception handler (return address removed)
 443     // rax: exception
 444     // rbx: exception handler
 445     // rdx: throwing pc
 446     __ jmp(handler_addr);
 447 
 448     return start;
 449   }
 450 
 451 
 452   //----------------------------------------------------------------------------------------------------
 453   // Support for jint Atomic::xchg(jint exchange_value, volatile jint* dest)
 454   //
 455   // xchg exists as far back as 8086, lock needed for MP only
 456   // Stack layout immediately after call:
 457   //
 458   // 0 [ret addr ] <--- rsp
 459   // 1 [  ex     ]
 460   // 2 [  dest   ]
 461   //
 462   // Result:   *dest <- ex, return (old *dest)
 463   //
 464   // Note: win32 does not currently use this code
 465 
 466   address generate_atomic_xchg() {
 467     StubCodeMark mark(this, "StubRoutines", "atomic_xchg");
 468     address start = __ pc();
 469 
 470     __ push(rdx);
 471     Address exchange(rsp, 2 * wordSize);
 472     Address dest_addr(rsp, 3 * wordSize);
 473     __ movl(rax, exchange);
 474     __ movptr(rdx, dest_addr);
 475     __ xchgl(rax, Address(rdx, 0));
 476     __ pop(rdx);
 477     __ ret(0);
 478 
 479     return start;
 480   }
 481 
 482   //----------------------------------------------------------------------------------------------------
 483   // Support for void verify_mxcsr()
 484   //
 485   // This routine is used with -Xcheck:jni to verify that native
 486   // JNI code does not return to Java code without restoring the
 487   // MXCSR register to our expected state.
 488 
 489 
 490   address generate_verify_mxcsr() {
 491     StubCodeMark mark(this, "StubRoutines", "verify_mxcsr");
 492     address start = __ pc();
 493 
 494     const Address mxcsr_save(rsp, 0);
 495 
 496     if (CheckJNICalls && UseSSE > 0 ) {
 497       Label ok_ret;
 498       ExternalAddress mxcsr_std(StubRoutines::addr_mxcsr_std());
 499       __ push(rax);
 500       __ subptr(rsp, wordSize);      // allocate a temp location
 501       __ stmxcsr(mxcsr_save);
 502       __ movl(rax, mxcsr_save);
 503       __ andl(rax, MXCSR_MASK);
 504       __ cmp32(rax, mxcsr_std);
 505       __ jcc(Assembler::equal, ok_ret);
 506 
 507       __ warn("MXCSR changed by native JNI code.");
 508 
 509       __ ldmxcsr(mxcsr_std);
 510 
 511       __ bind(ok_ret);
 512       __ addptr(rsp, wordSize);
 513       __ pop(rax);
 514     }
 515 
 516     __ ret(0);
 517 
 518     return start;
 519   }
 520 
 521 
 522   //---------------------------------------------------------------------------
 523   // Support for void verify_fpu_cntrl_wrd()
 524   //
 525   // This routine is used with -Xcheck:jni to verify that native
 526   // JNI code does not return to Java code without restoring the
 527   // FP control word to our expected state.
 528 
 529   address generate_verify_fpu_cntrl_wrd() {
 530     StubCodeMark mark(this, "StubRoutines", "verify_spcw");
 531     address start = __ pc();
 532 
 533     const Address fpu_cntrl_wrd_save(rsp, 0);
 534 
 535     if (CheckJNICalls) {
 536       Label ok_ret;
 537       __ push(rax);
 538       __ subptr(rsp, wordSize);      // allocate a temp location
 539       __ fnstcw(fpu_cntrl_wrd_save);
 540       __ movl(rax, fpu_cntrl_wrd_save);
 541       __ andl(rax, FPU_CNTRL_WRD_MASK);
 542       ExternalAddress fpu_std(StubRoutines::addr_fpu_cntrl_wrd_std());
 543       __ cmp32(rax, fpu_std);
 544       __ jcc(Assembler::equal, ok_ret);
 545 
 546       __ warn("Floating point control word changed by native JNI code.");
 547 
 548       __ fldcw(fpu_std);
 549 
 550       __ bind(ok_ret);
 551       __ addptr(rsp, wordSize);
 552       __ pop(rax);
 553     }
 554 
 555     __ ret(0);
 556 
 557     return start;
 558   }
 559 
 560   //---------------------------------------------------------------------------
 561   // Wrapper for slow-case handling of double-to-integer conversion
 562   // d2i or f2i fast case failed either because it is nan or because
 563   // of under/overflow.
 564   // Input:  FPU TOS: float value
 565   // Output: rax, (rdx): integer (long) result
 566 
 567   address generate_d2i_wrapper(BasicType t, address fcn) {
 568     StubCodeMark mark(this, "StubRoutines", "d2i_wrapper");
 569     address start = __ pc();
 570 
 571   // Capture info about frame layout
 572   enum layout { FPUState_off         = 0,
 573                 rbp_off              = FPUStateSizeInWords,
 574                 rdi_off,
 575                 rsi_off,
 576                 rcx_off,
 577                 rbx_off,
 578                 saved_argument_off,
 579                 saved_argument_off2, // 2nd half of double
 580                 framesize
 581   };
 582 
 583   assert(FPUStateSizeInWords == 27, "update stack layout");
 584 
 585     // Save outgoing argument to stack across push_FPU_state()
 586     __ subptr(rsp, wordSize * 2);
 587     __ fstp_d(Address(rsp, 0));
 588 
 589     // Save CPU & FPU state
 590     __ push(rbx);
 591     __ push(rcx);
 592     __ push(rsi);
 593     __ push(rdi);
 594     __ push(rbp);
 595     __ push_FPU_state();
 596 
 597     // push_FPU_state() resets the FP top of stack
 598     // Load original double into FP top of stack
 599     __ fld_d(Address(rsp, saved_argument_off * wordSize));
 600     // Store double into stack as outgoing argument
 601     __ subptr(rsp, wordSize*2);
 602     __ fst_d(Address(rsp, 0));
 603 
 604     // Prepare FPU for doing math in C-land
 605     __ empty_FPU_stack();
 606     // Call the C code to massage the double.  Result in EAX
 607     if (t == T_INT)
 608       { BLOCK_COMMENT("SharedRuntime::d2i"); }
 609     else if (t == T_LONG)
 610       { BLOCK_COMMENT("SharedRuntime::d2l"); }
 611     __ call_VM_leaf( fcn, 2 );
 612 
 613     // Restore CPU & FPU state
 614     __ pop_FPU_state();
 615     __ pop(rbp);
 616     __ pop(rdi);
 617     __ pop(rsi);
 618     __ pop(rcx);
 619     __ pop(rbx);
 620     __ addptr(rsp, wordSize * 2);
 621 
 622     __ ret(0);
 623 
 624     return start;
 625   }
 626 
 627 
 628   //---------------------------------------------------------------------------
 629   // The following routine generates a subroutine to throw an asynchronous
 630   // UnknownError when an unsafe access gets a fault that could not be
 631   // reasonably prevented by the programmer.  (Example: SIGBUS/OBJERR.)
 632   address generate_handler_for_unsafe_access() {
 633     StubCodeMark mark(this, "StubRoutines", "handler_for_unsafe_access");
 634     address start = __ pc();
 635 
 636     __ push(0);                       // hole for return address-to-be
 637     __ pusha();                       // push registers
 638     Address next_pc(rsp, RegisterImpl::number_of_registers * BytesPerWord);
 639     BLOCK_COMMENT("call handle_unsafe_access");
 640     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, handle_unsafe_access)));
 641     __ movptr(next_pc, rax);          // stuff next address
 642     __ popa();
 643     __ ret(0);                        // jump to next address
 644 
 645     return start;
 646   }
 647 
 648 
 649   //----------------------------------------------------------------------------------------------------
 650   // Non-destructive plausibility checks for oops
 651 
 652   address generate_verify_oop() {
 653     StubCodeMark mark(this, "StubRoutines", "verify_oop");
 654     address start = __ pc();
 655 
 656     // Incoming arguments on stack after saving rax,:
 657     //
 658     // [tos    ]: saved rdx
 659     // [tos + 1]: saved EFLAGS
 660     // [tos + 2]: return address
 661     // [tos + 3]: char* error message
 662     // [tos + 4]: oop   object to verify
 663     // [tos + 5]: saved rax, - saved by caller and bashed
 664 
 665     Label exit, error;
 666     __ pushf();
 667     __ incrementl(ExternalAddress((address) StubRoutines::verify_oop_count_addr()));
 668     __ push(rdx);                                // save rdx
 669     // make sure object is 'reasonable'
 670     __ movptr(rax, Address(rsp, 4 * wordSize));    // get object
 671     __ testptr(rax, rax);
 672     __ jcc(Assembler::zero, exit);               // if obj is NULL it is ok
 673 
 674     // Check if the oop is in the right area of memory
 675     const int oop_mask = Universe::verify_oop_mask();
 676     const int oop_bits = Universe::verify_oop_bits();
 677     __ mov(rdx, rax);
 678     __ andptr(rdx, oop_mask);
 679     __ cmpptr(rdx, oop_bits);
 680     __ jcc(Assembler::notZero, error);
 681 
 682     // make sure klass is 'reasonable'
 683     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass
 684     __ testptr(rax, rax);
 685     __ jcc(Assembler::zero, error);              // if klass is NULL it is broken
 686 
 687     // Check if the klass is in the right area of memory
 688     const int klass_mask = Universe::verify_klass_mask();
 689     const int klass_bits = Universe::verify_klass_bits();
 690     __ mov(rdx, rax);
 691     __ andptr(rdx, klass_mask);
 692     __ cmpptr(rdx, klass_bits);
 693     __ jcc(Assembler::notZero, error);
 694 
 695     // make sure klass' klass is 'reasonable'
 696     __ movptr(rax, Address(rax, oopDesc::klass_offset_in_bytes())); // get klass' klass
 697     __ testptr(rax, rax);
 698     __ jcc(Assembler::zero, error);              // if klass' klass is NULL it is broken
 699 
 700     __ mov(rdx, rax);
 701     __ andptr(rdx, klass_mask);
 702     __ cmpptr(rdx, klass_bits);
 703     __ jcc(Assembler::notZero, error);           // if klass not in right area
 704                                                  // of memory it is broken too.
 705 
 706     // return if everything seems ok
 707     __ bind(exit);
 708     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 709     __ pop(rdx);                                 // restore rdx
 710     __ popf();                                   // restore EFLAGS
 711     __ ret(3 * wordSize);                        // pop arguments
 712 
 713     // handle errors
 714     __ bind(error);
 715     __ movptr(rax, Address(rsp, 5 * wordSize));  // get saved rax, back
 716     __ pop(rdx);                                 // get saved rdx back
 717     __ popf();                                   // get saved EFLAGS off stack -- will be ignored
 718     __ pusha();                                  // push registers (eip = return address & msg are already pushed)
 719     BLOCK_COMMENT("call MacroAssembler::debug");
 720     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, MacroAssembler::debug32)));
 721     __ popa();
 722     __ ret(3 * wordSize);                        // pop arguments
 723     return start;
 724   }
 725 
 726   //
 727   //  Generate pre-barrier for array stores
 728   //
 729   //  Input:
 730   //     start   -  starting address
 731   //     count   -  element count
 732   void  gen_write_ref_array_pre_barrier(Register start, Register count, bool uninitialized_target) {
 733     assert_different_registers(start, count);
 734     BarrierSet* bs = Universe::heap()->barrier_set();
 735     switch (bs->kind()) {
 736       case BarrierSet::G1SATBCT:
 737       case BarrierSet::G1SATBCTLogging:
 738         // With G1, don't generate the call if we statically know that the target in uninitialized
 739         if (!uninitialized_target) {
 740            __ pusha();                      // push registers
 741            __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre),
 742                            start, count);
 743            __ popa();
 744          }
 745         break;
 746       case BarrierSet::CardTableModRef:
 747       case BarrierSet::CardTableExtension:
 748       case BarrierSet::ModRef:
 749         break;
 750       default      :
 751         ShouldNotReachHere();
 752 
 753     }
 754   }
 755 
 756 
 757   //
 758   // Generate a post-barrier for an array store
 759   //
 760   //     start    -  starting address
 761   //     count    -  element count
 762   //
 763   //  The two input registers are overwritten.
 764   //
 765   void  gen_write_ref_array_post_barrier(Register start, Register count) {
 766     BarrierSet* bs = Universe::heap()->barrier_set();
 767     assert_different_registers(start, count);
 768     switch (bs->kind()) {
 769       case BarrierSet::G1SATBCT:
 770       case BarrierSet::G1SATBCTLogging:
 771         {
 772           __ pusha();                      // push registers
 773           __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_post),
 774                           start, count);
 775           __ popa();
 776         }
 777         break;
 778 
 779       case BarrierSet::CardTableModRef:
 780       case BarrierSet::CardTableExtension:
 781         {
 782           CardTableModRefBS* ct = (CardTableModRefBS*)bs;
 783           assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
 784 
 785           Label L_loop;
 786           const Register end = count;  // elements count; end == start+count-1
 787           assert_different_registers(start, end);
 788 
 789           __ lea(end,  Address(start, count, Address::times_ptr, -wordSize));
 790           __ shrptr(start, CardTableModRefBS::card_shift);
 791           __ shrptr(end,   CardTableModRefBS::card_shift);
 792           __ subptr(end, start); // end --> count
 793         __ BIND(L_loop);
 794           intptr_t disp = (intptr_t) ct->byte_map_base;
 795           Address cardtable(start, count, Address::times_1, disp);
 796           __ movb(cardtable, 0);
 797           __ decrement(count);
 798           __ jcc(Assembler::greaterEqual, L_loop);
 799         }
 800         break;
 801       case BarrierSet::ModRef:
 802         break;
 803       default      :
 804         ShouldNotReachHere();
 805 
 806     }
 807   }
 808 
 809 
 810   // Copy 64 bytes chunks
 811   //
 812   // Inputs:
 813   //   from        - source array address
 814   //   to_from     - destination array address - from
 815   //   qword_count - 8-bytes element count, negative
 816   //
 817   void xmm_copy_forward(Register from, Register to_from, Register qword_count) {
 818     assert( UseSSE >= 2, "supported cpu only" );
 819     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 820     // Copy 64-byte chunks
 821     __ jmpb(L_copy_64_bytes);
 822     __ align(OptoLoopAlignment);
 823   __ BIND(L_copy_64_bytes_loop);
 824 
 825     if(UseUnalignedLoadStores) {
 826       __ movdqu(xmm0, Address(from, 0));
 827       __ movdqu(Address(from, to_from, Address::times_1, 0), xmm0);
 828       __ movdqu(xmm1, Address(from, 16));
 829       __ movdqu(Address(from, to_from, Address::times_1, 16), xmm1);
 830       __ movdqu(xmm2, Address(from, 32));
 831       __ movdqu(Address(from, to_from, Address::times_1, 32), xmm2);
 832       __ movdqu(xmm3, Address(from, 48));
 833       __ movdqu(Address(from, to_from, Address::times_1, 48), xmm3);
 834 
 835     } else {
 836       __ movq(xmm0, Address(from, 0));
 837       __ movq(Address(from, to_from, Address::times_1, 0), xmm0);
 838       __ movq(xmm1, Address(from, 8));
 839       __ movq(Address(from, to_from, Address::times_1, 8), xmm1);
 840       __ movq(xmm2, Address(from, 16));
 841       __ movq(Address(from, to_from, Address::times_1, 16), xmm2);
 842       __ movq(xmm3, Address(from, 24));
 843       __ movq(Address(from, to_from, Address::times_1, 24), xmm3);
 844       __ movq(xmm4, Address(from, 32));
 845       __ movq(Address(from, to_from, Address::times_1, 32), xmm4);
 846       __ movq(xmm5, Address(from, 40));
 847       __ movq(Address(from, to_from, Address::times_1, 40), xmm5);
 848       __ movq(xmm6, Address(from, 48));
 849       __ movq(Address(from, to_from, Address::times_1, 48), xmm6);
 850       __ movq(xmm7, Address(from, 56));
 851       __ movq(Address(from, to_from, Address::times_1, 56), xmm7);
 852     }
 853 
 854     __ addl(from, 64);
 855   __ BIND(L_copy_64_bytes);
 856     __ subl(qword_count, 8);
 857     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 858     __ addl(qword_count, 8);
 859     __ jccb(Assembler::zero, L_exit);
 860     //
 861     // length is too short, just copy qwords
 862     //
 863   __ BIND(L_copy_8_bytes);
 864     __ movq(xmm0, Address(from, 0));
 865     __ movq(Address(from, to_from, Address::times_1), xmm0);
 866     __ addl(from, 8);
 867     __ decrement(qword_count);
 868     __ jcc(Assembler::greater, L_copy_8_bytes);
 869   __ BIND(L_exit);
 870   }
 871 
 872   // Copy 64 bytes chunks
 873   //
 874   // Inputs:
 875   //   from        - source array address
 876   //   to_from     - destination array address - from
 877   //   qword_count - 8-bytes element count, negative
 878   //
 879   void mmx_copy_forward(Register from, Register to_from, Register qword_count) {
 880     assert( VM_Version::supports_mmx(), "supported cpu only" );
 881     Label L_copy_64_bytes_loop, L_copy_64_bytes, L_copy_8_bytes, L_exit;
 882     // Copy 64-byte chunks
 883     __ jmpb(L_copy_64_bytes);
 884     __ align(OptoLoopAlignment);
 885   __ BIND(L_copy_64_bytes_loop);
 886     __ movq(mmx0, Address(from, 0));
 887     __ movq(mmx1, Address(from, 8));
 888     __ movq(mmx2, Address(from, 16));
 889     __ movq(Address(from, to_from, Address::times_1, 0), mmx0);
 890     __ movq(mmx3, Address(from, 24));
 891     __ movq(Address(from, to_from, Address::times_1, 8), mmx1);
 892     __ movq(mmx4, Address(from, 32));
 893     __ movq(Address(from, to_from, Address::times_1, 16), mmx2);
 894     __ movq(mmx5, Address(from, 40));
 895     __ movq(Address(from, to_from, Address::times_1, 24), mmx3);
 896     __ movq(mmx6, Address(from, 48));
 897     __ movq(Address(from, to_from, Address::times_1, 32), mmx4);
 898     __ movq(mmx7, Address(from, 56));
 899     __ movq(Address(from, to_from, Address::times_1, 40), mmx5);
 900     __ movq(Address(from, to_from, Address::times_1, 48), mmx6);
 901     __ movq(Address(from, to_from, Address::times_1, 56), mmx7);
 902     __ addptr(from, 64);
 903   __ BIND(L_copy_64_bytes);
 904     __ subl(qword_count, 8);
 905     __ jcc(Assembler::greaterEqual, L_copy_64_bytes_loop);
 906     __ addl(qword_count, 8);
 907     __ jccb(Assembler::zero, L_exit);
 908     //
 909     // length is too short, just copy qwords
 910     //
 911   __ BIND(L_copy_8_bytes);
 912     __ movq(mmx0, Address(from, 0));
 913     __ movq(Address(from, to_from, Address::times_1), mmx0);
 914     __ addptr(from, 8);
 915     __ decrement(qword_count);
 916     __ jcc(Assembler::greater, L_copy_8_bytes);
 917   __ BIND(L_exit);
 918     __ emms();
 919   }
 920 
 921   address generate_disjoint_copy(BasicType t, bool aligned,
 922                                  Address::ScaleFactor sf,
 923                                  address* entry, const char *name,
 924                                  bool dest_uninitialized = false) {
 925     __ align(CodeEntryAlignment);
 926     StubCodeMark mark(this, "StubRoutines", name);
 927     address start = __ pc();
 928 
 929     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
 930     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_64_bytes;
 931 
 932     int shift = Address::times_ptr - sf;
 933 
 934     const Register from     = rsi;  // source array address
 935     const Register to       = rdi;  // destination array address
 936     const Register count    = rcx;  // elements count
 937     const Register to_from  = to;   // (to - from)
 938     const Register saved_to = rdx;  // saved destination array address
 939 
 940     __ enter(); // required for proper stackwalking of RuntimeStub frame
 941     __ push(rsi);
 942     __ push(rdi);
 943     __ movptr(from , Address(rsp, 12+ 4));
 944     __ movptr(to   , Address(rsp, 12+ 8));
 945     __ movl(count, Address(rsp, 12+ 12));
 946 
 947     if (entry != NULL) {
 948       *entry = __ pc(); // Entry point from conjoint arraycopy stub.
 949       BLOCK_COMMENT("Entry:");
 950     }
 951 
 952     if (t == T_OBJECT) {
 953       __ testl(count, count);
 954       __ jcc(Assembler::zero, L_0_count);
 955       gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
 956       __ mov(saved_to, to);          // save 'to'
 957     }
 958 
 959     __ subptr(to, from); // to --> to_from
 960     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
 961     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
 962     if (!UseUnalignedLoadStores && !aligned && (t == T_BYTE || t == T_SHORT)) {
 963       // align source address at 4 bytes address boundary
 964       if (t == T_BYTE) {
 965         // One byte misalignment happens only for byte arrays
 966         __ testl(from, 1);
 967         __ jccb(Assembler::zero, L_skip_align1);
 968         __ movb(rax, Address(from, 0));
 969         __ movb(Address(from, to_from, Address::times_1, 0), rax);
 970         __ increment(from);
 971         __ decrement(count);
 972       __ BIND(L_skip_align1);
 973       }
 974       // Two bytes misalignment happens only for byte and short (char) arrays
 975       __ testl(from, 2);
 976       __ jccb(Assembler::zero, L_skip_align2);
 977       __ movw(rax, Address(from, 0));
 978       __ movw(Address(from, to_from, Address::times_1, 0), rax);
 979       __ addptr(from, 2);
 980       __ subl(count, 1<<(shift-1));
 981     __ BIND(L_skip_align2);
 982     }
 983     if (!VM_Version::supports_mmx()) {
 984       __ mov(rax, count);      // save 'count'
 985       __ shrl(count, shift); // bytes count
 986       __ addptr(to_from, from);// restore 'to'
 987       __ rep_mov();
 988       __ subptr(to_from, from);// restore 'to_from'
 989       __ mov(count, rax);      // restore 'count'
 990       __ jmpb(L_copy_2_bytes); // all dwords were copied
 991     } else {
 992       if (!UseUnalignedLoadStores) {
 993         // align to 8 bytes, we know we are 4 byte aligned to start
 994         __ testptr(from, 4);
 995         __ jccb(Assembler::zero, L_copy_64_bytes);
 996         __ movl(rax, Address(from, 0));
 997         __ movl(Address(from, to_from, Address::times_1, 0), rax);
 998         __ addptr(from, 4);
 999         __ subl(count, 1<<shift);
1000       }
1001     __ BIND(L_copy_64_bytes);
1002       __ mov(rax, count);
1003       __ shrl(rax, shift+1);  // 8 bytes chunk count
1004       //
1005       // Copy 8-byte chunks through MMX registers, 8 per iteration of the loop
1006       //
1007       if (UseXMMForArrayCopy) {
1008         xmm_copy_forward(from, to_from, rax);
1009       } else {
1010         mmx_copy_forward(from, to_from, rax);
1011       }
1012     }
1013     // copy tailing dword
1014   __ BIND(L_copy_4_bytes);
1015     __ testl(count, 1<<shift);
1016     __ jccb(Assembler::zero, L_copy_2_bytes);
1017     __ movl(rax, Address(from, 0));
1018     __ movl(Address(from, to_from, Address::times_1, 0), rax);
1019     if (t == T_BYTE || t == T_SHORT) {
1020       __ addptr(from, 4);
1021     __ BIND(L_copy_2_bytes);
1022       // copy tailing word
1023       __ testl(count, 1<<(shift-1));
1024       __ jccb(Assembler::zero, L_copy_byte);
1025       __ movw(rax, Address(from, 0));
1026       __ movw(Address(from, to_from, Address::times_1, 0), rax);
1027       if (t == T_BYTE) {
1028         __ addptr(from, 2);
1029       __ BIND(L_copy_byte);
1030         // copy tailing byte
1031         __ testl(count, 1);
1032         __ jccb(Assembler::zero, L_exit);
1033         __ movb(rax, Address(from, 0));
1034         __ movb(Address(from, to_from, Address::times_1, 0), rax);
1035       __ BIND(L_exit);
1036       } else {
1037       __ BIND(L_copy_byte);
1038       }
1039     } else {
1040     __ BIND(L_copy_2_bytes);
1041     }
1042 
1043     if (t == T_OBJECT) {
1044       __ movl(count, Address(rsp, 12+12)); // reread 'count'
1045       __ mov(to, saved_to); // restore 'to'
1046       gen_write_ref_array_post_barrier(to, count);
1047     __ BIND(L_0_count);
1048     }
1049     inc_copy_counter_np(t);
1050     __ pop(rdi);
1051     __ pop(rsi);
1052     __ leave(); // required for proper stackwalking of RuntimeStub frame
1053     __ xorptr(rax, rax); // return 0
1054     __ ret(0);
1055     return start;
1056   }
1057 
1058 
1059   address generate_fill(BasicType t, bool aligned, const char *name) {
1060     __ align(CodeEntryAlignment);
1061     StubCodeMark mark(this, "StubRoutines", name);
1062     address start = __ pc();
1063 
1064     BLOCK_COMMENT("Entry:");
1065 
1066     const Register to       = rdi;  // source array address
1067     const Register value    = rdx;  // value
1068     const Register count    = rsi;  // elements count
1069 
1070     __ enter(); // required for proper stackwalking of RuntimeStub frame
1071     __ push(rsi);
1072     __ push(rdi);
1073     __ movptr(to   , Address(rsp, 12+ 4));
1074     __ movl(value, Address(rsp, 12+ 8));
1075     __ movl(count, Address(rsp, 12+ 12));
1076 
1077     __ generate_fill(t, aligned, to, value, count, rax, xmm0);
1078 
1079     __ pop(rdi);
1080     __ pop(rsi);
1081     __ leave(); // required for proper stackwalking of RuntimeStub frame
1082     __ ret(0);
1083     return start;
1084   }
1085 
1086   address generate_conjoint_copy(BasicType t, bool aligned,
1087                                  Address::ScaleFactor sf,
1088                                  address nooverlap_target,
1089                                  address* entry, const char *name,
1090                                  bool dest_uninitialized = false) {
1091     __ align(CodeEntryAlignment);
1092     StubCodeMark mark(this, "StubRoutines", name);
1093     address start = __ pc();
1094 
1095     Label L_0_count, L_exit, L_skip_align1, L_skip_align2, L_copy_byte;
1096     Label L_copy_2_bytes, L_copy_4_bytes, L_copy_8_bytes, L_copy_8_bytes_loop;
1097 
1098     int shift = Address::times_ptr - sf;
1099 
1100     const Register src   = rax;  // source array address
1101     const Register dst   = rdx;  // destination array address
1102     const Register from  = rsi;  // source array address
1103     const Register to    = rdi;  // destination array address
1104     const Register count = rcx;  // elements count
1105     const Register end   = rax;  // array end address
1106 
1107     __ enter(); // required for proper stackwalking of RuntimeStub frame
1108     __ push(rsi);
1109     __ push(rdi);
1110     __ movptr(src  , Address(rsp, 12+ 4));   // from
1111     __ movptr(dst  , Address(rsp, 12+ 8));   // to
1112     __ movl2ptr(count, Address(rsp, 12+12)); // count
1113 
1114     if (entry != NULL) {
1115       *entry = __ pc(); // Entry point from generic arraycopy stub.
1116       BLOCK_COMMENT("Entry:");
1117     }
1118 
1119     // nooverlap_target expects arguments in rsi and rdi.
1120     __ mov(from, src);
1121     __ mov(to  , dst);
1122 
1123     // arrays overlap test: dispatch to disjoint stub if necessary.
1124     RuntimeAddress nooverlap(nooverlap_target);
1125     __ cmpptr(dst, src);
1126     __ lea(end, Address(src, count, sf, 0)); // src + count * elem_size
1127     __ jump_cc(Assembler::belowEqual, nooverlap);
1128     __ cmpptr(dst, end);
1129     __ jump_cc(Assembler::aboveEqual, nooverlap);
1130 
1131     if (t == T_OBJECT) {
1132       __ testl(count, count);
1133       __ jcc(Assembler::zero, L_0_count);
1134       gen_write_ref_array_pre_barrier(dst, count, dest_uninitialized);
1135     }
1136 
1137     // copy from high to low
1138     __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1139     __ jcc(Assembler::below, L_copy_4_bytes); // use unsigned cmp
1140     if (t == T_BYTE || t == T_SHORT) {
1141       // Align the end of destination array at 4 bytes address boundary
1142       __ lea(end, Address(dst, count, sf, 0));
1143       if (t == T_BYTE) {
1144         // One byte misalignment happens only for byte arrays
1145         __ testl(end, 1);
1146         __ jccb(Assembler::zero, L_skip_align1);
1147         __ decrement(count);
1148         __ movb(rdx, Address(from, count, sf, 0));
1149         __ movb(Address(to, count, sf, 0), rdx);
1150       __ BIND(L_skip_align1);
1151       }
1152       // Two bytes misalignment happens only for byte and short (char) arrays
1153       __ testl(end, 2);
1154       __ jccb(Assembler::zero, L_skip_align2);
1155       __ subptr(count, 1<<(shift-1));
1156       __ movw(rdx, Address(from, count, sf, 0));
1157       __ movw(Address(to, count, sf, 0), rdx);
1158     __ BIND(L_skip_align2);
1159       __ cmpl(count, 2<<shift); // Short arrays (< 8 bytes) copy by element
1160       __ jcc(Assembler::below, L_copy_4_bytes);
1161     }
1162 
1163     if (!VM_Version::supports_mmx()) {
1164       __ std();
1165       __ mov(rax, count); // Save 'count'
1166       __ mov(rdx, to);    // Save 'to'
1167       __ lea(rsi, Address(from, count, sf, -4));
1168       __ lea(rdi, Address(to  , count, sf, -4));
1169       __ shrptr(count, shift); // bytes count
1170       __ rep_mov();
1171       __ cld();
1172       __ mov(count, rax); // restore 'count'
1173       __ andl(count, (1<<shift)-1);      // mask the number of rest elements
1174       __ movptr(from, Address(rsp, 12+4)); // reread 'from'
1175       __ mov(to, rdx);   // restore 'to'
1176       __ jmpb(L_copy_2_bytes); // all dword were copied
1177    } else {
1178       // Align to 8 bytes the end of array. It is aligned to 4 bytes already.
1179       __ testptr(end, 4);
1180       __ jccb(Assembler::zero, L_copy_8_bytes);
1181       __ subl(count, 1<<shift);
1182       __ movl(rdx, Address(from, count, sf, 0));
1183       __ movl(Address(to, count, sf, 0), rdx);
1184       __ jmpb(L_copy_8_bytes);
1185 
1186       __ align(OptoLoopAlignment);
1187       // Move 8 bytes
1188     __ BIND(L_copy_8_bytes_loop);
1189       if (UseXMMForArrayCopy) {
1190         __ movq(xmm0, Address(from, count, sf, 0));
1191         __ movq(Address(to, count, sf, 0), xmm0);
1192       } else {
1193         __ movq(mmx0, Address(from, count, sf, 0));
1194         __ movq(Address(to, count, sf, 0), mmx0);
1195       }
1196     __ BIND(L_copy_8_bytes);
1197       __ subl(count, 2<<shift);
1198       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1199       __ addl(count, 2<<shift);
1200       if (!UseXMMForArrayCopy) {
1201         __ emms();
1202       }
1203     }
1204   __ BIND(L_copy_4_bytes);
1205     // copy prefix qword
1206     __ testl(count, 1<<shift);
1207     __ jccb(Assembler::zero, L_copy_2_bytes);
1208     __ movl(rdx, Address(from, count, sf, -4));
1209     __ movl(Address(to, count, sf, -4), rdx);
1210 
1211     if (t == T_BYTE || t == T_SHORT) {
1212         __ subl(count, (1<<shift));
1213       __ BIND(L_copy_2_bytes);
1214         // copy prefix dword
1215         __ testl(count, 1<<(shift-1));
1216         __ jccb(Assembler::zero, L_copy_byte);
1217         __ movw(rdx, Address(from, count, sf, -2));
1218         __ movw(Address(to, count, sf, -2), rdx);
1219         if (t == T_BYTE) {
1220           __ subl(count, 1<<(shift-1));
1221         __ BIND(L_copy_byte);
1222           // copy prefix byte
1223           __ testl(count, 1);
1224           __ jccb(Assembler::zero, L_exit);
1225           __ movb(rdx, Address(from, 0));
1226           __ movb(Address(to, 0), rdx);
1227         __ BIND(L_exit);
1228         } else {
1229         __ BIND(L_copy_byte);
1230         }
1231     } else {
1232     __ BIND(L_copy_2_bytes);
1233     }
1234     if (t == T_OBJECT) {
1235       __ movl2ptr(count, Address(rsp, 12+12)); // reread count
1236       gen_write_ref_array_post_barrier(to, count);
1237     __ BIND(L_0_count);
1238     }
1239     inc_copy_counter_np(t);
1240     __ pop(rdi);
1241     __ pop(rsi);
1242     __ leave(); // required for proper stackwalking of RuntimeStub frame
1243     __ xorptr(rax, rax); // return 0
1244     __ ret(0);
1245     return start;
1246   }
1247 
1248 
1249   address generate_disjoint_long_copy(address* entry, const char *name) {
1250     __ align(CodeEntryAlignment);
1251     StubCodeMark mark(this, "StubRoutines", name);
1252     address start = __ pc();
1253 
1254     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1255     const Register from       = rax;  // source array address
1256     const Register to         = rdx;  // destination array address
1257     const Register count      = rcx;  // elements count
1258     const Register to_from    = rdx;  // (to - from)
1259 
1260     __ enter(); // required for proper stackwalking of RuntimeStub frame
1261     __ movptr(from , Address(rsp, 8+0));       // from
1262     __ movptr(to   , Address(rsp, 8+4));       // to
1263     __ movl2ptr(count, Address(rsp, 8+8));     // count
1264 
1265     *entry = __ pc(); // Entry point from conjoint arraycopy stub.
1266     BLOCK_COMMENT("Entry:");
1267 
1268     __ subptr(to, from); // to --> to_from
1269     if (VM_Version::supports_mmx()) {
1270       if (UseXMMForArrayCopy) {
1271         xmm_copy_forward(from, to_from, count);
1272       } else {
1273         mmx_copy_forward(from, to_from, count);
1274       }
1275     } else {
1276       __ jmpb(L_copy_8_bytes);
1277       __ align(OptoLoopAlignment);
1278     __ BIND(L_copy_8_bytes_loop);
1279       __ fild_d(Address(from, 0));
1280       __ fistp_d(Address(from, to_from, Address::times_1));
1281       __ addptr(from, 8);
1282     __ BIND(L_copy_8_bytes);
1283       __ decrement(count);
1284       __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1285     }
1286     inc_copy_counter_np(T_LONG);
1287     __ leave(); // required for proper stackwalking of RuntimeStub frame
1288     __ xorptr(rax, rax); // return 0
1289     __ ret(0);
1290     return start;
1291   }
1292 
1293   address generate_conjoint_long_copy(address nooverlap_target,
1294                                       address* entry, const char *name) {
1295     __ align(CodeEntryAlignment);
1296     StubCodeMark mark(this, "StubRoutines", name);
1297     address start = __ pc();
1298 
1299     Label L_copy_8_bytes, L_copy_8_bytes_loop;
1300     const Register from       = rax;  // source array address
1301     const Register to         = rdx;  // destination array address
1302     const Register count      = rcx;  // elements count
1303     const Register end_from   = rax;  // source array end address
1304 
1305     __ enter(); // required for proper stackwalking of RuntimeStub frame
1306     __ movptr(from , Address(rsp, 8+0));       // from
1307     __ movptr(to   , Address(rsp, 8+4));       // to
1308     __ movl2ptr(count, Address(rsp, 8+8));     // count
1309 
1310     *entry = __ pc(); // Entry point from generic arraycopy stub.
1311     BLOCK_COMMENT("Entry:");
1312 
1313     // arrays overlap test
1314     __ cmpptr(to, from);
1315     RuntimeAddress nooverlap(nooverlap_target);
1316     __ jump_cc(Assembler::belowEqual, nooverlap);
1317     __ lea(end_from, Address(from, count, Address::times_8, 0));
1318     __ cmpptr(to, end_from);
1319     __ movptr(from, Address(rsp, 8));  // from
1320     __ jump_cc(Assembler::aboveEqual, nooverlap);
1321 
1322     __ jmpb(L_copy_8_bytes);
1323 
1324     __ align(OptoLoopAlignment);
1325   __ BIND(L_copy_8_bytes_loop);
1326     if (VM_Version::supports_mmx()) {
1327       if (UseXMMForArrayCopy) {
1328         __ movq(xmm0, Address(from, count, Address::times_8));
1329         __ movq(Address(to, count, Address::times_8), xmm0);
1330       } else {
1331         __ movq(mmx0, Address(from, count, Address::times_8));
1332         __ movq(Address(to, count, Address::times_8), mmx0);
1333       }
1334     } else {
1335       __ fild_d(Address(from, count, Address::times_8));
1336       __ fistp_d(Address(to, count, Address::times_8));
1337     }
1338   __ BIND(L_copy_8_bytes);
1339     __ decrement(count);
1340     __ jcc(Assembler::greaterEqual, L_copy_8_bytes_loop);
1341 
1342     if (VM_Version::supports_mmx() && !UseXMMForArrayCopy) {
1343       __ emms();
1344     }
1345     inc_copy_counter_np(T_LONG);
1346     __ leave(); // required for proper stackwalking of RuntimeStub frame
1347     __ xorptr(rax, rax); // return 0
1348     __ ret(0);
1349     return start;
1350   }
1351 
1352 
1353   // Helper for generating a dynamic type check.
1354   // The sub_klass must be one of {rbx, rdx, rsi}.
1355   // The temp is killed.
1356   void generate_type_check(Register sub_klass,
1357                            Address& super_check_offset_addr,
1358                            Address& super_klass_addr,
1359                            Register temp,
1360                            Label* L_success, Label* L_failure) {
1361     BLOCK_COMMENT("type_check:");
1362 
1363     Label L_fallthrough;
1364 #define LOCAL_JCC(assembler_con, label_ptr)                             \
1365     if (label_ptr != NULL)  __ jcc(assembler_con, *(label_ptr));        \
1366     else                    __ jcc(assembler_con, L_fallthrough) /*omit semi*/
1367 
1368     // The following is a strange variation of the fast path which requires
1369     // one less register, because needed values are on the argument stack.
1370     // __ check_klass_subtype_fast_path(sub_klass, *super_klass*, temp,
1371     //                                  L_success, L_failure, NULL);
1372     assert_different_registers(sub_klass, temp);
1373 
1374     int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
1375                      Klass::secondary_super_cache_offset_in_bytes());
1376 
1377     // if the pointers are equal, we are done (e.g., String[] elements)
1378     __ cmpptr(sub_klass, super_klass_addr);
1379     LOCAL_JCC(Assembler::equal, L_success);
1380 
1381     // check the supertype display:
1382     __ movl2ptr(temp, super_check_offset_addr);
1383     Address super_check_addr(sub_klass, temp, Address::times_1, 0);
1384     __ movptr(temp, super_check_addr); // load displayed supertype
1385     __ cmpptr(temp, super_klass_addr); // test the super type
1386     LOCAL_JCC(Assembler::equal, L_success);
1387 
1388     // if it was a primary super, we can just fail immediately
1389     __ cmpl(super_check_offset_addr, sc_offset);
1390     LOCAL_JCC(Assembler::notEqual, L_failure);
1391 
1392     // The repne_scan instruction uses fixed registers, which will get spilled.
1393     // We happen to know this works best when super_klass is in rax.
1394     Register super_klass = temp;
1395     __ movptr(super_klass, super_klass_addr);
1396     __ check_klass_subtype_slow_path(sub_klass, super_klass, noreg, noreg,
1397                                      L_success, L_failure);
1398 
1399     __ bind(L_fallthrough);
1400 
1401     if (L_success == NULL) { BLOCK_COMMENT("L_success:"); }
1402     if (L_failure == NULL) { BLOCK_COMMENT("L_failure:"); }
1403 
1404 #undef LOCAL_JCC
1405   }
1406 
1407   //
1408   //  Generate checkcasting array copy stub
1409   //
1410   //  Input:
1411   //    4(rsp)   - source array address
1412   //    8(rsp)   - destination array address
1413   //   12(rsp)   - element count, can be zero
1414   //   16(rsp)   - size_t ckoff (super_check_offset)
1415   //   20(rsp)   - oop ckval (super_klass)
1416   //
1417   //  Output:
1418   //    rax, ==  0  -  success
1419   //    rax, == -1^K - failure, where K is partial transfer count
1420   //
1421   address generate_checkcast_copy(const char *name, address* entry, bool dest_uninitialized = false) {
1422     __ align(CodeEntryAlignment);
1423     StubCodeMark mark(this, "StubRoutines", name);
1424     address start = __ pc();
1425 
1426     Label L_load_element, L_store_element, L_do_card_marks, L_done;
1427 
1428     // register use:
1429     //  rax, rdx, rcx -- loop control (end_from, end_to, count)
1430     //  rdi, rsi      -- element access (oop, klass)
1431     //  rbx,           -- temp
1432     const Register from       = rax;    // source array address
1433     const Register to         = rdx;    // destination array address
1434     const Register length     = rcx;    // elements count
1435     const Register elem       = rdi;    // each oop copied
1436     const Register elem_klass = rsi;    // each elem._klass (sub_klass)
1437     const Register temp       = rbx;    // lone remaining temp
1438 
1439     __ enter(); // required for proper stackwalking of RuntimeStub frame
1440 
1441     __ push(rsi);
1442     __ push(rdi);
1443     __ push(rbx);
1444 
1445     Address   from_arg(rsp, 16+ 4);     // from
1446     Address     to_arg(rsp, 16+ 8);     // to
1447     Address length_arg(rsp, 16+12);     // elements count
1448     Address  ckoff_arg(rsp, 16+16);     // super_check_offset
1449     Address  ckval_arg(rsp, 16+20);     // super_klass
1450 
1451     // Load up:
1452     __ movptr(from,     from_arg);
1453     __ movptr(to,         to_arg);
1454     __ movl2ptr(length, length_arg);
1455 
1456     if (entry != NULL) {
1457       *entry = __ pc(); // Entry point from generic arraycopy stub.
1458       BLOCK_COMMENT("Entry:");
1459     }
1460 
1461     //---------------------------------------------------------------
1462     // Assembler stub will be used for this call to arraycopy
1463     // if the two arrays are subtypes of Object[] but the
1464     // destination array type is not equal to or a supertype
1465     // of the source type.  Each element must be separately
1466     // checked.
1467 
1468     // Loop-invariant addresses.  They are exclusive end pointers.
1469     Address end_from_addr(from, length, Address::times_ptr, 0);
1470     Address   end_to_addr(to,   length, Address::times_ptr, 0);
1471 
1472     Register end_from = from;           // re-use
1473     Register end_to   = to;             // re-use
1474     Register count    = length;         // re-use
1475 
1476     // Loop-variant addresses.  They assume post-incremented count < 0.
1477     Address from_element_addr(end_from, count, Address::times_ptr, 0);
1478     Address   to_element_addr(end_to,   count, Address::times_ptr, 0);
1479     Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes());
1480 
1481     // Copy from low to high addresses, indexed from the end of each array.
1482     gen_write_ref_array_pre_barrier(to, count, dest_uninitialized);
1483     __ lea(end_from, end_from_addr);
1484     __ lea(end_to,   end_to_addr);
1485     assert(length == count, "");        // else fix next line:
1486     __ negptr(count);                   // negate and test the length
1487     __ jccb(Assembler::notZero, L_load_element);
1488 
1489     // Empty array:  Nothing to do.
1490     __ xorptr(rax, rax);                  // return 0 on (trivial) success
1491     __ jmp(L_done);
1492 
1493     // ======== begin loop ========
1494     // (Loop is rotated; its entry is L_load_element.)
1495     // Loop control:
1496     //   for (count = -count; count != 0; count++)
1497     // Base pointers src, dst are biased by 8*count,to last element.
1498     __ align(OptoLoopAlignment);
1499 
1500     __ BIND(L_store_element);
1501     __ movptr(to_element_addr, elem);     // store the oop
1502     __ increment(count);                // increment the count toward zero
1503     __ jccb(Assembler::zero, L_do_card_marks);
1504 
1505     // ======== loop entry is here ========
1506     __ BIND(L_load_element);
1507     __ movptr(elem, from_element_addr);   // load the oop
1508     __ testptr(elem, elem);
1509     __ jccb(Assembler::zero, L_store_element);
1510 
1511     // (Could do a trick here:  Remember last successful non-null
1512     // element stored and make a quick oop equality check on it.)
1513 
1514     __ movptr(elem_klass, elem_klass_addr); // query the object klass
1515     generate_type_check(elem_klass, ckoff_arg, ckval_arg, temp,
1516                         &L_store_element, NULL);
1517       // (On fall-through, we have failed the element type check.)
1518     // ======== end loop ========
1519 
1520     // It was a real error; we must depend on the caller to finish the job.
1521     // Register "count" = -1 * number of *remaining* oops, length_arg = *total* oops.
1522     // Emit GC store barriers for the oops we have copied (length_arg + count),
1523     // and report their number to the caller.
1524     __ addl(count, length_arg);         // transfers = (length - remaining)
1525     __ movl2ptr(rax, count);            // save the value
1526     __ notptr(rax);                     // report (-1^K) to caller
1527     __ movptr(to, to_arg);              // reload
1528     assert_different_registers(to, count, rax);
1529     gen_write_ref_array_post_barrier(to, count);
1530     __ jmpb(L_done);
1531 
1532     // Come here on success only.
1533     __ BIND(L_do_card_marks);
1534     __ movl2ptr(count, length_arg);
1535     __ movptr(to, to_arg);                // reload
1536     gen_write_ref_array_post_barrier(to, count);
1537     __ xorptr(rax, rax);                  // return 0 on success
1538 
1539     // Common exit point (success or failure).
1540     __ BIND(L_done);
1541     __ pop(rbx);
1542     __ pop(rdi);
1543     __ pop(rsi);
1544     inc_counter_np(SharedRuntime::_checkcast_array_copy_ctr);
1545     __ leave(); // required for proper stackwalking of RuntimeStub frame
1546     __ ret(0);
1547 
1548     return start;
1549   }
1550 
1551   //
1552   //  Generate 'unsafe' array copy stub
1553   //  Though just as safe as the other stubs, it takes an unscaled
1554   //  size_t argument instead of an element count.
1555   //
1556   //  Input:
1557   //    4(rsp)   - source array address
1558   //    8(rsp)   - destination array address
1559   //   12(rsp)   - byte count, can be zero
1560   //
1561   //  Output:
1562   //    rax, ==  0  -  success
1563   //    rax, == -1  -  need to call System.arraycopy
1564   //
1565   // Examines the alignment of the operands and dispatches
1566   // to a long, int, short, or byte copy loop.
1567   //
1568   address generate_unsafe_copy(const char *name,
1569                                address byte_copy_entry,
1570                                address short_copy_entry,
1571                                address int_copy_entry,
1572                                address long_copy_entry) {
1573 
1574     Label L_long_aligned, L_int_aligned, L_short_aligned;
1575 
1576     __ align(CodeEntryAlignment);
1577     StubCodeMark mark(this, "StubRoutines", name);
1578     address start = __ pc();
1579 
1580     const Register from       = rax;  // source array address
1581     const Register to         = rdx;  // destination array address
1582     const Register count      = rcx;  // elements count
1583 
1584     __ enter(); // required for proper stackwalking of RuntimeStub frame
1585     __ push(rsi);
1586     __ push(rdi);
1587     Address  from_arg(rsp, 12+ 4);      // from
1588     Address    to_arg(rsp, 12+ 8);      // to
1589     Address count_arg(rsp, 12+12);      // byte count
1590 
1591     // Load up:
1592     __ movptr(from ,  from_arg);
1593     __ movptr(to   ,    to_arg);
1594     __ movl2ptr(count, count_arg);
1595 
1596     // bump this on entry, not on exit:
1597     inc_counter_np(SharedRuntime::_unsafe_array_copy_ctr);
1598 
1599     const Register bits = rsi;
1600     __ mov(bits, from);
1601     __ orptr(bits, to);
1602     __ orptr(bits, count);
1603 
1604     __ testl(bits, BytesPerLong-1);
1605     __ jccb(Assembler::zero, L_long_aligned);
1606 
1607     __ testl(bits, BytesPerInt-1);
1608     __ jccb(Assembler::zero, L_int_aligned);
1609 
1610     __ testl(bits, BytesPerShort-1);
1611     __ jump_cc(Assembler::notZero, RuntimeAddress(byte_copy_entry));
1612 
1613     __ BIND(L_short_aligned);
1614     __ shrptr(count, LogBytesPerShort); // size => short_count
1615     __ movl(count_arg, count);          // update 'count'
1616     __ jump(RuntimeAddress(short_copy_entry));
1617 
1618     __ BIND(L_int_aligned);
1619     __ shrptr(count, LogBytesPerInt); // size => int_count
1620     __ movl(count_arg, count);          // update 'count'
1621     __ jump(RuntimeAddress(int_copy_entry));
1622 
1623     __ BIND(L_long_aligned);
1624     __ shrptr(count, LogBytesPerLong); // size => qword_count
1625     __ movl(count_arg, count);          // update 'count'
1626     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1627     __ pop(rsi);
1628     __ jump(RuntimeAddress(long_copy_entry));
1629 
1630     return start;
1631   }
1632 
1633 
1634   // Perform range checks on the proposed arraycopy.
1635   // Smashes src_pos and dst_pos.  (Uses them up for temps.)
1636   void arraycopy_range_checks(Register src,
1637                               Register src_pos,
1638                               Register dst,
1639                               Register dst_pos,
1640                               Address& length,
1641                               Label& L_failed) {
1642     BLOCK_COMMENT("arraycopy_range_checks:");
1643     const Register src_end = src_pos;   // source array end position
1644     const Register dst_end = dst_pos;   // destination array end position
1645     __ addl(src_end, length); // src_pos + length
1646     __ addl(dst_end, length); // dst_pos + length
1647 
1648     //  if (src_pos + length > arrayOop(src)->length() ) FAIL;
1649     __ cmpl(src_end, Address(src, arrayOopDesc::length_offset_in_bytes()));
1650     __ jcc(Assembler::above, L_failed);
1651 
1652     //  if (dst_pos + length > arrayOop(dst)->length() ) FAIL;
1653     __ cmpl(dst_end, Address(dst, arrayOopDesc::length_offset_in_bytes()));
1654     __ jcc(Assembler::above, L_failed);
1655 
1656     BLOCK_COMMENT("arraycopy_range_checks done");
1657   }
1658 
1659 
1660   //
1661   //  Generate generic array copy stubs
1662   //
1663   //  Input:
1664   //     4(rsp)    -  src oop
1665   //     8(rsp)    -  src_pos
1666   //    12(rsp)    -  dst oop
1667   //    16(rsp)    -  dst_pos
1668   //    20(rsp)    -  element count
1669   //
1670   //  Output:
1671   //    rax, ==  0  -  success
1672   //    rax, == -1^K - failure, where K is partial transfer count
1673   //
1674   address generate_generic_copy(const char *name,
1675                                 address entry_jbyte_arraycopy,
1676                                 address entry_jshort_arraycopy,
1677                                 address entry_jint_arraycopy,
1678                                 address entry_oop_arraycopy,
1679                                 address entry_jlong_arraycopy,
1680                                 address entry_checkcast_arraycopy) {
1681     Label L_failed, L_failed_0, L_objArray;
1682 
1683     { int modulus = CodeEntryAlignment;
1684       int target  = modulus - 5; // 5 = sizeof jmp(L_failed)
1685       int advance = target - (__ offset() % modulus);
1686       if (advance < 0)  advance += modulus;
1687       if (advance > 0)  __ nop(advance);
1688     }
1689     StubCodeMark mark(this, "StubRoutines", name);
1690 
1691     // Short-hop target to L_failed.  Makes for denser prologue code.
1692     __ BIND(L_failed_0);
1693     __ jmp(L_failed);
1694     assert(__ offset() % CodeEntryAlignment == 0, "no further alignment needed");
1695 
1696     __ align(CodeEntryAlignment);
1697     address start = __ pc();
1698 
1699     __ enter(); // required for proper stackwalking of RuntimeStub frame
1700     __ push(rsi);
1701     __ push(rdi);
1702 
1703     // bump this on entry, not on exit:
1704     inc_counter_np(SharedRuntime::_generic_array_copy_ctr);
1705 
1706     // Input values
1707     Address SRC     (rsp, 12+ 4);
1708     Address SRC_POS (rsp, 12+ 8);
1709     Address DST     (rsp, 12+12);
1710     Address DST_POS (rsp, 12+16);
1711     Address LENGTH  (rsp, 12+20);
1712 
1713     //-----------------------------------------------------------------------
1714     // Assembler stub will be used for this call to arraycopy
1715     // if the following conditions are met:
1716     //
1717     // (1) src and dst must not be null.
1718     // (2) src_pos must not be negative.
1719     // (3) dst_pos must not be negative.
1720     // (4) length  must not be negative.
1721     // (5) src klass and dst klass should be the same and not NULL.
1722     // (6) src and dst should be arrays.
1723     // (7) src_pos + length must not exceed length of src.
1724     // (8) dst_pos + length must not exceed length of dst.
1725     //
1726 
1727     const Register src     = rax;       // source array oop
1728     const Register src_pos = rsi;
1729     const Register dst     = rdx;       // destination array oop
1730     const Register dst_pos = rdi;
1731     const Register length  = rcx;       // transfer count
1732 
1733     //  if (src == NULL) return -1;
1734     __ movptr(src, SRC);      // src oop
1735     __ testptr(src, src);
1736     __ jccb(Assembler::zero, L_failed_0);
1737 
1738     //  if (src_pos < 0) return -1;
1739     __ movl2ptr(src_pos, SRC_POS);  // src_pos
1740     __ testl(src_pos, src_pos);
1741     __ jccb(Assembler::negative, L_failed_0);
1742 
1743     //  if (dst == NULL) return -1;
1744     __ movptr(dst, DST);      // dst oop
1745     __ testptr(dst, dst);
1746     __ jccb(Assembler::zero, L_failed_0);
1747 
1748     //  if (dst_pos < 0) return -1;
1749     __ movl2ptr(dst_pos, DST_POS);  // dst_pos
1750     __ testl(dst_pos, dst_pos);
1751     __ jccb(Assembler::negative, L_failed_0);
1752 
1753     //  if (length < 0) return -1;
1754     __ movl2ptr(length, LENGTH);   // length
1755     __ testl(length, length);
1756     __ jccb(Assembler::negative, L_failed_0);
1757 
1758     //  if (src->klass() == NULL) return -1;
1759     Address src_klass_addr(src, oopDesc::klass_offset_in_bytes());
1760     Address dst_klass_addr(dst, oopDesc::klass_offset_in_bytes());
1761     const Register rcx_src_klass = rcx;    // array klass
1762     __ movptr(rcx_src_klass, Address(src, oopDesc::klass_offset_in_bytes()));
1763 
1764 #ifdef ASSERT
1765     //  assert(src->klass() != NULL);
1766     BLOCK_COMMENT("assert klasses not null");
1767     { Label L1, L2;
1768       __ testptr(rcx_src_klass, rcx_src_klass);
1769       __ jccb(Assembler::notZero, L2);   // it is broken if klass is NULL
1770       __ bind(L1);
1771       __ stop("broken null klass");
1772       __ bind(L2);
1773       __ cmpptr(dst_klass_addr, (int32_t)NULL_WORD);
1774       __ jccb(Assembler::equal, L1);      // this would be broken also
1775       BLOCK_COMMENT("assert done");
1776     }
1777 #endif //ASSERT
1778 
1779     // Load layout helper (32-bits)
1780     //
1781     //  |array_tag|     | header_size | element_type |     |log2_element_size|
1782     // 32        30    24            16              8     2                 0
1783     //
1784     //   array_tag: typeArray = 0x3, objArray = 0x2, non-array = 0x0
1785     //
1786 
1787     int lh_offset = klassOopDesc::header_size() * HeapWordSize +
1788                     Klass::layout_helper_offset_in_bytes();
1789     Address src_klass_lh_addr(rcx_src_klass, lh_offset);
1790 
1791     // Handle objArrays completely differently...
1792     jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
1793     __ cmpl(src_klass_lh_addr, objArray_lh);
1794     __ jcc(Assembler::equal, L_objArray);
1795 
1796     //  if (src->klass() != dst->klass()) return -1;
1797     __ cmpptr(rcx_src_klass, dst_klass_addr);
1798     __ jccb(Assembler::notEqual, L_failed_0);
1799 
1800     const Register rcx_lh = rcx;  // layout helper
1801     assert(rcx_lh == rcx_src_klass, "known alias");
1802     __ movl(rcx_lh, src_klass_lh_addr);
1803 
1804     //  if (!src->is_Array()) return -1;
1805     __ cmpl(rcx_lh, Klass::_lh_neutral_value);
1806     __ jcc(Assembler::greaterEqual, L_failed_0); // signed cmp
1807 
1808     // At this point, it is known to be a typeArray (array_tag 0x3).
1809 #ifdef ASSERT
1810     { Label L;
1811       __ cmpl(rcx_lh, (Klass::_lh_array_tag_type_value << Klass::_lh_array_tag_shift));
1812       __ jcc(Assembler::greaterEqual, L); // signed cmp
1813       __ stop("must be a primitive array");
1814       __ bind(L);
1815     }
1816 #endif
1817 
1818     assert_different_registers(src, src_pos, dst, dst_pos, rcx_lh);
1819     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1820 
1821     // typeArrayKlass
1822     //
1823     // src_addr = (src + array_header_in_bytes()) + (src_pos << log2elemsize);
1824     // dst_addr = (dst + array_header_in_bytes()) + (dst_pos << log2elemsize);
1825     //
1826     const Register rsi_offset = rsi; // array offset
1827     const Register src_array  = src; // src array offset
1828     const Register dst_array  = dst; // dst array offset
1829     const Register rdi_elsize = rdi; // log2 element size
1830 
1831     __ mov(rsi_offset, rcx_lh);
1832     __ shrptr(rsi_offset, Klass::_lh_header_size_shift);
1833     __ andptr(rsi_offset, Klass::_lh_header_size_mask);   // array_offset
1834     __ addptr(src_array, rsi_offset);  // src array offset
1835     __ addptr(dst_array, rsi_offset);  // dst array offset
1836     __ andptr(rcx_lh, Klass::_lh_log2_element_size_mask); // log2 elsize
1837 
1838     // next registers should be set before the jump to corresponding stub
1839     const Register from       = src; // source array address
1840     const Register to         = dst; // destination array address
1841     const Register count      = rcx; // elements count
1842     // some of them should be duplicated on stack
1843 #define FROM   Address(rsp, 12+ 4)
1844 #define TO     Address(rsp, 12+ 8)   // Not used now
1845 #define COUNT  Address(rsp, 12+12)   // Only for oop arraycopy
1846 
1847     BLOCK_COMMENT("scale indexes to element size");
1848     __ movl2ptr(rsi, SRC_POS);  // src_pos
1849     __ shlptr(rsi);             // src_pos << rcx (log2 elsize)
1850     assert(src_array == from, "");
1851     __ addptr(from, rsi);       // from = src_array + SRC_POS << log2 elsize
1852     __ movl2ptr(rdi, DST_POS);  // dst_pos
1853     __ shlptr(rdi);             // dst_pos << rcx (log2 elsize)
1854     assert(dst_array == to, "");
1855     __ addptr(to,  rdi);        // to   = dst_array + DST_POS << log2 elsize
1856     __ movptr(FROM, from);      // src_addr
1857     __ mov(rdi_elsize, rcx_lh); // log2 elsize
1858     __ movl2ptr(count, LENGTH); // elements count
1859 
1860     BLOCK_COMMENT("choose copy loop based on element size");
1861     __ cmpl(rdi_elsize, 0);
1862 
1863     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jbyte_arraycopy));
1864     __ cmpl(rdi_elsize, LogBytesPerShort);
1865     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jshort_arraycopy));
1866     __ cmpl(rdi_elsize, LogBytesPerInt);
1867     __ jump_cc(Assembler::equal, RuntimeAddress(entry_jint_arraycopy));
1868 #ifdef ASSERT
1869     __ cmpl(rdi_elsize, LogBytesPerLong);
1870     __ jccb(Assembler::notEqual, L_failed);
1871 #endif
1872     __ pop(rdi); // Do pops here since jlong_arraycopy stub does not do it.
1873     __ pop(rsi);
1874     __ jump(RuntimeAddress(entry_jlong_arraycopy));
1875 
1876   __ BIND(L_failed);
1877     __ xorptr(rax, rax);
1878     __ notptr(rax); // return -1
1879     __ pop(rdi);
1880     __ pop(rsi);
1881     __ leave(); // required for proper stackwalking of RuntimeStub frame
1882     __ ret(0);
1883 
1884     // objArrayKlass
1885   __ BIND(L_objArray);
1886     // live at this point:  rcx_src_klass, src[_pos], dst[_pos]
1887 
1888     Label L_plain_copy, L_checkcast_copy;
1889     //  test array classes for subtyping
1890     __ cmpptr(rcx_src_klass, dst_klass_addr); // usual case is exact equality
1891     __ jccb(Assembler::notEqual, L_checkcast_copy);
1892 
1893     // Identically typed arrays can be copied without element-wise checks.
1894     assert_different_registers(src, src_pos, dst, dst_pos, rcx_src_klass);
1895     arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1896 
1897   __ BIND(L_plain_copy);
1898     __ movl2ptr(count, LENGTH); // elements count
1899     __ movl2ptr(src_pos, SRC_POS);  // reload src_pos
1900     __ lea(from, Address(src, src_pos, Address::times_ptr,
1901                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // src_addr
1902     __ movl2ptr(dst_pos, DST_POS);  // reload dst_pos
1903     __ lea(to,   Address(dst, dst_pos, Address::times_ptr,
1904                  arrayOopDesc::base_offset_in_bytes(T_OBJECT))); // dst_addr
1905     __ movptr(FROM,  from);   // src_addr
1906     __ movptr(TO,    to);     // dst_addr
1907     __ movl(COUNT, count);  // count
1908     __ jump(RuntimeAddress(entry_oop_arraycopy));
1909 
1910   __ BIND(L_checkcast_copy);
1911     // live at this point:  rcx_src_klass, dst[_pos], src[_pos]
1912     {
1913       // Handy offsets:
1914       int  ek_offset = (klassOopDesc::header_size() * HeapWordSize +
1915                         objArrayKlass::element_klass_offset_in_bytes());
1916       int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
1917                         Klass::super_check_offset_offset_in_bytes());
1918 
1919       Register rsi_dst_klass = rsi;
1920       Register rdi_temp      = rdi;
1921       assert(rsi_dst_klass == src_pos, "expected alias w/ src_pos");
1922       assert(rdi_temp      == dst_pos, "expected alias w/ dst_pos");
1923       Address dst_klass_lh_addr(rsi_dst_klass, lh_offset);
1924 
1925       // Before looking at dst.length, make sure dst is also an objArray.
1926       __ movptr(rsi_dst_klass, dst_klass_addr);
1927       __ cmpl(dst_klass_lh_addr, objArray_lh);
1928       __ jccb(Assembler::notEqual, L_failed);
1929 
1930       // It is safe to examine both src.length and dst.length.
1931       __ movl2ptr(src_pos, SRC_POS);        // reload rsi
1932       arraycopy_range_checks(src, src_pos, dst, dst_pos, LENGTH, L_failed);
1933       // (Now src_pos and dst_pos are killed, but not src and dst.)
1934 
1935       // We'll need this temp (don't forget to pop it after the type check).
1936       __ push(rbx);
1937       Register rbx_src_klass = rbx;
1938 
1939       __ mov(rbx_src_klass, rcx_src_klass); // spill away from rcx
1940       __ movptr(rsi_dst_klass, dst_klass_addr);
1941       Address super_check_offset_addr(rsi_dst_klass, sco_offset);
1942       Label L_fail_array_check;
1943       generate_type_check(rbx_src_klass,
1944                           super_check_offset_addr, dst_klass_addr,
1945                           rdi_temp, NULL, &L_fail_array_check);
1946       // (On fall-through, we have passed the array type check.)
1947       __ pop(rbx);
1948       __ jmp(L_plain_copy);
1949 
1950       __ BIND(L_fail_array_check);
1951       // Reshuffle arguments so we can call checkcast_arraycopy:
1952 
1953       // match initial saves for checkcast_arraycopy
1954       // push(rsi);    // already done; see above
1955       // push(rdi);    // already done; see above
1956       // push(rbx);    // already done; see above
1957 
1958       // Marshal outgoing arguments now, freeing registers.
1959       Address   from_arg(rsp, 16+ 4);   // from
1960       Address     to_arg(rsp, 16+ 8);   // to
1961       Address length_arg(rsp, 16+12);   // elements count
1962       Address  ckoff_arg(rsp, 16+16);   // super_check_offset
1963       Address  ckval_arg(rsp, 16+20);   // super_klass
1964 
1965       Address SRC_POS_arg(rsp, 16+ 8);
1966       Address DST_POS_arg(rsp, 16+16);
1967       Address  LENGTH_arg(rsp, 16+20);
1968       // push rbx, changed the incoming offsets (why not just use rbp,??)
1969       // assert(SRC_POS_arg.disp() == SRC_POS.disp() + 4, "");
1970 
1971       __ movptr(rbx, Address(rsi_dst_klass, ek_offset));
1972       __ movl2ptr(length, LENGTH_arg);    // reload elements count
1973       __ movl2ptr(src_pos, SRC_POS_arg);  // reload src_pos
1974       __ movl2ptr(dst_pos, DST_POS_arg);  // reload dst_pos
1975 
1976       __ movptr(ckval_arg, rbx);          // destination element type
1977       __ movl(rbx, Address(rbx, sco_offset));
1978       __ movl(ckoff_arg, rbx);          // corresponding class check offset
1979 
1980       __ movl(length_arg, length);      // outgoing length argument
1981 
1982       __ lea(from, Address(src, src_pos, Address::times_ptr,
1983                             arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1984       __ movptr(from_arg, from);
1985 
1986       __ lea(to, Address(dst, dst_pos, Address::times_ptr,
1987                           arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
1988       __ movptr(to_arg, to);
1989       __ jump(RuntimeAddress(entry_checkcast_arraycopy));
1990     }
1991 
1992     return start;
1993   }
1994 
1995   void generate_arraycopy_stubs() {
1996     address entry;
1997     address entry_jbyte_arraycopy;
1998     address entry_jshort_arraycopy;
1999     address entry_jint_arraycopy;
2000     address entry_oop_arraycopy;
2001     address entry_jlong_arraycopy;
2002     address entry_checkcast_arraycopy;
2003 
2004     StubRoutines::_arrayof_jbyte_disjoint_arraycopy =
2005         generate_disjoint_copy(T_BYTE,  true, Address::times_1, &entry,
2006                                "arrayof_jbyte_disjoint_arraycopy");
2007     StubRoutines::_arrayof_jbyte_arraycopy =
2008         generate_conjoint_copy(T_BYTE,  true, Address::times_1,  entry,
2009                                NULL, "arrayof_jbyte_arraycopy");
2010     StubRoutines::_jbyte_disjoint_arraycopy =
2011         generate_disjoint_copy(T_BYTE, false, Address::times_1, &entry,
2012                                "jbyte_disjoint_arraycopy");
2013     StubRoutines::_jbyte_arraycopy =
2014         generate_conjoint_copy(T_BYTE, false, Address::times_1,  entry,
2015                                &entry_jbyte_arraycopy, "jbyte_arraycopy");
2016 
2017     StubRoutines::_arrayof_jshort_disjoint_arraycopy =
2018         generate_disjoint_copy(T_SHORT,  true, Address::times_2, &entry,
2019                                "arrayof_jshort_disjoint_arraycopy");
2020     StubRoutines::_arrayof_jshort_arraycopy =
2021         generate_conjoint_copy(T_SHORT,  true, Address::times_2,  entry,
2022                                NULL, "arrayof_jshort_arraycopy");
2023     StubRoutines::_jshort_disjoint_arraycopy =
2024         generate_disjoint_copy(T_SHORT, false, Address::times_2, &entry,
2025                                "jshort_disjoint_arraycopy");
2026     StubRoutines::_jshort_arraycopy =
2027         generate_conjoint_copy(T_SHORT, false, Address::times_2,  entry,
2028                                &entry_jshort_arraycopy, "jshort_arraycopy");
2029 
2030     // Next arrays are always aligned on 4 bytes at least.
2031     StubRoutines::_jint_disjoint_arraycopy =
2032         generate_disjoint_copy(T_INT, true, Address::times_4, &entry,
2033                                "jint_disjoint_arraycopy");
2034     StubRoutines::_jint_arraycopy =
2035         generate_conjoint_copy(T_INT, true, Address::times_4,  entry,
2036                                &entry_jint_arraycopy, "jint_arraycopy");
2037 
2038     StubRoutines::_oop_disjoint_arraycopy =
2039         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
2040                                "oop_disjoint_arraycopy");
2041     StubRoutines::_oop_arraycopy =
2042         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
2043                                &entry_oop_arraycopy, "oop_arraycopy");
2044 
2045     StubRoutines::_oop_disjoint_arraycopy_uninit =
2046         generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry,
2047                                "oop_disjoint_arraycopy_uninit",
2048                                /*dest_uninitialized*/true);
2049     StubRoutines::_oop_arraycopy_uninit =
2050         generate_conjoint_copy(T_OBJECT, true, Address::times_ptr,  entry,
2051                                NULL, "oop_arraycopy_uninit",
2052                                /*dest_uninitialized*/true);
2053 
2054     StubRoutines::_jlong_disjoint_arraycopy =
2055         generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy");
2056     StubRoutines::_jlong_arraycopy =
2057         generate_conjoint_long_copy(entry, &entry_jlong_arraycopy,
2058                                     "jlong_arraycopy");
2059 
2060     StubRoutines::_jbyte_fill = generate_fill(T_BYTE, false, "jbyte_fill");
2061     StubRoutines::_jshort_fill = generate_fill(T_SHORT, false, "jshort_fill");
2062     StubRoutines::_jint_fill = generate_fill(T_INT, false, "jint_fill");
2063     StubRoutines::_arrayof_jbyte_fill = generate_fill(T_BYTE, true, "arrayof_jbyte_fill");
2064     StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill");
2065     StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill");
2066 
2067     StubRoutines::_arrayof_jint_disjoint_arraycopy       = StubRoutines::_jint_disjoint_arraycopy;
2068     StubRoutines::_arrayof_oop_disjoint_arraycopy        = StubRoutines::_oop_disjoint_arraycopy;
2069     StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit;
2070     StubRoutines::_arrayof_jlong_disjoint_arraycopy      = StubRoutines::_jlong_disjoint_arraycopy;
2071 
2072     StubRoutines::_arrayof_jint_arraycopy       = StubRoutines::_jint_arraycopy;
2073     StubRoutines::_arrayof_oop_arraycopy        = StubRoutines::_oop_arraycopy;
2074     StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit;
2075     StubRoutines::_arrayof_jlong_arraycopy      = StubRoutines::_jlong_arraycopy;
2076 
2077     StubRoutines::_checkcast_arraycopy =
2078         generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy);
2079     StubRoutines::_checkcast_arraycopy_uninit =
2080         generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, /*dest_uninitialized*/true);
2081 
2082     StubRoutines::_unsafe_arraycopy =
2083         generate_unsafe_copy("unsafe_arraycopy",
2084                                entry_jbyte_arraycopy,
2085                                entry_jshort_arraycopy,
2086                                entry_jint_arraycopy,
2087                                entry_jlong_arraycopy);
2088 
2089     StubRoutines::_generic_arraycopy =
2090         generate_generic_copy("generic_arraycopy",
2091                                entry_jbyte_arraycopy,
2092                                entry_jshort_arraycopy,
2093                                entry_jint_arraycopy,
2094                                entry_oop_arraycopy,
2095                                entry_jlong_arraycopy,
2096                                entry_checkcast_arraycopy);
2097   }
2098 
2099   void generate_math_stubs() {
2100     {
2101       StubCodeMark mark(this, "StubRoutines", "log");
2102       StubRoutines::_intrinsic_log = (double (*)(double)) __ pc();
2103 
2104       __ fld_d(Address(rsp, 4));
2105       __ flog();
2106       __ ret(0);
2107     }
2108     {
2109       StubCodeMark mark(this, "StubRoutines", "log10");
2110       StubRoutines::_intrinsic_log10 = (double (*)(double)) __ pc();
2111 
2112       __ fld_d(Address(rsp, 4));
2113       __ flog10();
2114       __ ret(0);
2115     }
2116     {
2117       StubCodeMark mark(this, "StubRoutines", "sin");
2118       StubRoutines::_intrinsic_sin = (double (*)(double))  __ pc();
2119 
2120       __ fld_d(Address(rsp, 4));
2121       __ trigfunc('s');
2122       __ ret(0);
2123     }
2124     {
2125       StubCodeMark mark(this, "StubRoutines", "cos");
2126       StubRoutines::_intrinsic_cos = (double (*)(double)) __ pc();
2127 
2128       __ fld_d(Address(rsp, 4));
2129       __ trigfunc('c');
2130       __ ret(0);
2131     }
2132     {
2133       StubCodeMark mark(this, "StubRoutines", "tan");
2134       StubRoutines::_intrinsic_tan = (double (*)(double)) __ pc();
2135 
2136       __ fld_d(Address(rsp, 4));
2137       __ trigfunc('t');
2138       __ ret(0);
2139     }
2140 
2141     // The intrinsic version of these seem to return the same value as
2142     // the strict version.
2143     StubRoutines::_intrinsic_exp = SharedRuntime::dexp;
2144     StubRoutines::_intrinsic_pow = SharedRuntime::dpow;
2145   }
2146 
2147  public:
2148   // Information about frame layout at time of blocking runtime call.
2149   // Note that we only have to preserve callee-saved registers since
2150   // the compilers are responsible for supplying a continuation point
2151   // if they expect all registers to be preserved.
2152   enum layout {
2153     thread_off,    // last_java_sp
2154     rbp_off,       // callee saved register
2155     ret_pc,
2156     framesize
2157   };
2158 
2159  private:
2160 
2161 #undef  __
2162 #define __ masm->
2163 
2164   //------------------------------------------------------------------------------------------------------------------------
2165   // Continuation point for throwing of implicit exceptions that are not handled in
2166   // the current activation. Fabricates an exception oop and initiates normal
2167   // exception dispatching in this frame.
2168   //
2169   // Previously the compiler (c2) allowed for callee save registers on Java calls.
2170   // This is no longer true after adapter frames were removed but could possibly
2171   // be brought back in the future if the interpreter code was reworked and it
2172   // was deemed worthwhile. The comment below was left to describe what must
2173   // happen here if callee saves were resurrected. As it stands now this stub
2174   // could actually be a vanilla BufferBlob and have now oopMap at all.
2175   // Since it doesn't make much difference we've chosen to leave it the
2176   // way it was in the callee save days and keep the comment.
2177 
2178   // If we need to preserve callee-saved values we need a callee-saved oop map and
2179   // therefore have to make these stubs into RuntimeStubs rather than BufferBlobs.
2180   // If the compiler needs all registers to be preserved between the fault
2181   // point and the exception handler then it must assume responsibility for that in
2182   // AbstractCompiler::continuation_for_implicit_null_exception or
2183   // continuation_for_implicit_division_by_zero_exception. All other implicit
2184   // exceptions (e.g., NullPointerException or AbstractMethodError on entry) are
2185   // either at call sites or otherwise assume that stack unwinding will be initiated,
2186   // so caller saved registers were assumed volatile in the compiler.
2187   address generate_throw_exception(const char* name, address runtime_entry,
2188                                    bool restore_saved_exception_pc) {
2189 
2190     int insts_size = 256;
2191     int locs_size  = 32;
2192 
2193     CodeBuffer code(name, insts_size, locs_size);
2194     OopMapSet* oop_maps  = new OopMapSet();
2195     MacroAssembler* masm = new MacroAssembler(&code);
2196 
2197     address start = __ pc();
2198 
2199     // This is an inlined and slightly modified version of call_VM
2200     // which has the ability to fetch the return PC out of
2201     // thread-local storage and also sets up last_Java_sp slightly
2202     // differently than the real call_VM
2203     Register java_thread = rbx;
2204     __ get_thread(java_thread);
2205     if (restore_saved_exception_pc) {
2206       __ movptr(rax, Address(java_thread, in_bytes(JavaThread::saved_exception_pc_offset())));
2207       __ push(rax);
2208     }
2209 
2210     __ enter(); // required for proper stackwalking of RuntimeStub frame
2211 
2212     // pc and rbp, already pushed
2213     __ subptr(rsp, (framesize-2) * wordSize); // prolog
2214 
2215     // Frame is now completed as far as size and linkage.
2216 
2217     int frame_complete = __ pc() - start;
2218 
2219     // push java thread (becomes first argument of C function)
2220     __ movptr(Address(rsp, thread_off * wordSize), java_thread);
2221 
2222     // Set up last_Java_sp and last_Java_fp
2223     __ set_last_Java_frame(java_thread, rsp, rbp, NULL);
2224 
2225     // Call runtime
2226     BLOCK_COMMENT("call runtime_entry");
2227     __ call(RuntimeAddress(runtime_entry));
2228     // Generate oop map
2229     OopMap* map =  new OopMap(framesize, 0);
2230     oop_maps->add_gc_map(__ pc() - start, map);
2231 
2232     // restore the thread (cannot use the pushed argument since arguments
2233     // may be overwritten by C code generated by an optimizing compiler);
2234     // however can use the register value directly if it is callee saved.
2235     __ get_thread(java_thread);
2236 
2237     __ reset_last_Java_frame(java_thread, true, false);
2238 
2239     __ leave(); // required for proper stackwalking of RuntimeStub frame
2240 
2241     // check for pending exceptions
2242 #ifdef ASSERT
2243     Label L;
2244     __ cmpptr(Address(java_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
2245     __ jcc(Assembler::notEqual, L);
2246     __ should_not_reach_here();
2247     __ bind(L);
2248 #endif /* ASSERT */
2249     __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2250 
2251 
2252     RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete, framesize, oop_maps, false);
2253     return stub->entry_point();
2254   }
2255 
2256 
2257   void create_control_words() {
2258     // Round to nearest, 53-bit mode, exceptions masked
2259     StubRoutines::_fpu_cntrl_wrd_std   = 0x027F;
2260     // Round to zero, 53-bit mode, exception mased
2261     StubRoutines::_fpu_cntrl_wrd_trunc = 0x0D7F;
2262     // Round to nearest, 24-bit mode, exceptions masked
2263     StubRoutines::_fpu_cntrl_wrd_24    = 0x007F;
2264     // Round to nearest, 64-bit mode, exceptions masked
2265     StubRoutines::_fpu_cntrl_wrd_64    = 0x037F;
2266     // Round to nearest, 64-bit mode, exceptions masked
2267     StubRoutines::_mxcsr_std           = 0x1F80;
2268     // Note: the following two constants are 80-bit values
2269     //       layout is critical for correct loading by FPU.
2270     // Bias for strict fp multiply/divide
2271     StubRoutines::_fpu_subnormal_bias1[0]= 0x00000000; // 2^(-15360) == 0x03ff 8000 0000 0000 0000
2272     StubRoutines::_fpu_subnormal_bias1[1]= 0x80000000;
2273     StubRoutines::_fpu_subnormal_bias1[2]= 0x03ff;
2274     // Un-Bias for strict fp multiply/divide
2275     StubRoutines::_fpu_subnormal_bias2[0]= 0x00000000; // 2^(+15360) == 0x7bff 8000 0000 0000 0000
2276     StubRoutines::_fpu_subnormal_bias2[1]= 0x80000000;
2277     StubRoutines::_fpu_subnormal_bias2[2]= 0x7bff;
2278   }
2279 
2280   //---------------------------------------------------------------------------
2281   // Initialization
2282 
2283   void generate_initial() {
2284     // Generates all stubs and initializes the entry points
2285 
2286     //------------------------------------------------------------------------------------------------------------------------
2287     // entry points that exist in all platforms
2288     // Note: This is code that could be shared among different platforms - however the benefit seems to be smaller than
2289     //       the disadvantage of having a much more complicated generator structure. See also comment in stubRoutines.hpp.
2290     StubRoutines::_forward_exception_entry      = generate_forward_exception();
2291 
2292     StubRoutines::_call_stub_entry              =
2293       generate_call_stub(StubRoutines::_call_stub_return_address);
2294     // is referenced by megamorphic call
2295     StubRoutines::_catch_exception_entry        = generate_catch_exception();
2296 
2297     // These are currently used by Solaris/Intel
2298     StubRoutines::_atomic_xchg_entry            = generate_atomic_xchg();
2299 
2300     StubRoutines::_handler_for_unsafe_access_entry =
2301       generate_handler_for_unsafe_access();
2302 
2303     // platform dependent
2304     create_control_words();
2305 
2306     StubRoutines::x86::_verify_mxcsr_entry                 = generate_verify_mxcsr();
2307     StubRoutines::x86::_verify_fpu_cntrl_wrd_entry         = generate_verify_fpu_cntrl_wrd();
2308     StubRoutines::_d2i_wrapper                              = generate_d2i_wrapper(T_INT,
2309                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2i));
2310     StubRoutines::_d2l_wrapper                              = generate_d2i_wrapper(T_LONG,
2311                                                                                    CAST_FROM_FN_PTR(address, SharedRuntime::d2l));
2312   }
2313 
2314 
2315   void generate_all() {
2316     // Generates all stubs and initializes the entry points
2317 
2318     // These entry points require SharedInfo::stack0 to be set up in non-core builds
2319     // and need to be relocatable, so they each fabricate a RuntimeStub internally.
2320     StubRoutines::_throw_AbstractMethodError_entry         = generate_throw_exception("AbstractMethodError throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_AbstractMethodError),  false);
2321     StubRoutines::_throw_IncompatibleClassChangeError_entry= generate_throw_exception("IncompatibleClassChangeError throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_IncompatibleClassChangeError),  false);
2322     StubRoutines::_throw_ArithmeticException_entry         = generate_throw_exception("ArithmeticException throw_exception",          CAST_FROM_FN_PTR(address, SharedRuntime::throw_ArithmeticException),  true);
2323     StubRoutines::_throw_NullPointerException_entry        = generate_throw_exception("NullPointerException throw_exception",         CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException), true);
2324     StubRoutines::_throw_NullPointerException_at_call_entry= generate_throw_exception("NullPointerException at call throw_exception", CAST_FROM_FN_PTR(address, SharedRuntime::throw_NullPointerException_at_call), false);
2325     StubRoutines::_throw_StackOverflowError_entry          = generate_throw_exception("StackOverflowError throw_exception",           CAST_FROM_FN_PTR(address, SharedRuntime::throw_StackOverflowError),   false);
2326 
2327     //------------------------------------------------------------------------------------------------------------------------
2328     // entry points that are platform specific
2329 
2330     // support for verify_oop (must happen after universe_init)
2331     StubRoutines::_verify_oop_subroutine_entry     = generate_verify_oop();
2332 
2333     // arraycopy stubs used by compilers
2334     generate_arraycopy_stubs();
2335 
2336     generate_math_stubs();
2337   }
2338 
2339 
2340  public:
2341   StubGenerator(CodeBuffer* code, bool all) : StubCodeGenerator(code) {
2342     if (all) {
2343       generate_all();
2344     } else {
2345       generate_initial();
2346     }
2347   }
2348 }; // end class declaration
2349 
2350 
2351 void StubGenerator_generate(CodeBuffer* code, bool all) {
2352   StubGenerator g(code, all);
2353 }