1 /*
   2  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "incls/_precompiled.incl"
  26 #include "incls/_interpreter_x86_64.cpp.incl"
  27 
  28 #define __ _masm->
  29 
  30 #ifndef CC_INTERP
  31 
  32 const int method_offset = frame::interpreter_frame_method_offset * wordSize;
  33 const int bci_offset    = frame::interpreter_frame_bcx_offset    * wordSize;
  34 const int locals_offset = frame::interpreter_frame_locals_offset * wordSize;
  35 
  36 //-----------------------------------------------------------------------------
  37 
  38 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
  39   address entry = __ pc();
  40 
  41 #ifdef ASSERT
  42   {
  43     Label L;
  44     __ lea(rax, Address(rbp,
  45                         frame::interpreter_frame_monitor_block_top_offset *
  46                         wordSize));
  47     __ cmpptr(rax, rsp); // rax = maximal rsp for current rbp (stack
  48                          // grows negative)
  49     __ jcc(Assembler::aboveEqual, L); // check if frame is complete
  50     __ stop ("interpreter frame not set up");
  51     __ bind(L);
  52   }
  53 #endif // ASSERT
  54   // Restore bcp under the assumption that the current frame is still
  55   // interpreted
  56   __ restore_bcp();
  57 
  58   // expression stack must be empty before entering the VM if an
  59   // exception happened
  60   __ empty_expression_stack();
  61   // throw exception
  62   __ call_VM(noreg,
  63              CAST_FROM_FN_PTR(address,
  64                               InterpreterRuntime::throw_StackOverflowError));
  65   return entry;
  66 }
  67 
  68 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler(
  69         const char* name) {
  70   address entry = __ pc();
  71   // expression stack must be empty before entering the VM if an
  72   // exception happened
  73   __ empty_expression_stack();
  74   // setup parameters
  75   // ??? convention: expect aberrant index in register ebx
  76   __ lea(c_rarg1, ExternalAddress((address)name));
  77   __ call_VM(noreg,
  78              CAST_FROM_FN_PTR(address,
  79                               InterpreterRuntime::
  80                               throw_ArrayIndexOutOfBoundsException),
  81              c_rarg1, rbx);
  82   return entry;
  83 }
  84 
  85 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
  86   address entry = __ pc();
  87 
  88   // object is at TOS
  89   __ pop(c_rarg1);
  90 
  91   // expression stack must be empty before entering the VM if an
  92   // exception happened
  93   __ empty_expression_stack();
  94 
  95   __ call_VM(noreg,
  96              CAST_FROM_FN_PTR(address,
  97                               InterpreterRuntime::
  98                               throw_ClassCastException),
  99              c_rarg1);
 100   return entry;
 101 }
 102 
 103 // Arguments are: required type at TOS+8, failing object (or NULL) at TOS+4.
 104 address TemplateInterpreterGenerator::generate_WrongMethodType_handler() {
 105   address entry = __ pc();
 106 
 107   __ pop(c_rarg2);              // failing object is at TOS
 108   __ pop(c_rarg1);              // required type is at TOS+8
 109 
 110   __ verify_oop(c_rarg1);
 111   __ verify_oop(c_rarg2);
 112 
 113   // Various method handle types use interpreter registers as temps.
 114   __ restore_bcp();
 115   __ restore_locals();
 116 
 117   // Expression stack must be empty before entering the VM for an exception.
 118   __ empty_expression_stack();
 119 
 120   __ call_VM(noreg,
 121              CAST_FROM_FN_PTR(address,
 122                               InterpreterRuntime::throw_WrongMethodTypeException),
 123              // pass required type, failing object (or NULL)
 124              c_rarg1, c_rarg2);
 125   return entry;
 126 }
 127 
 128 address TemplateInterpreterGenerator::generate_exception_handler_common(
 129         const char* name, const char* message, bool pass_oop) {
 130   assert(!pass_oop || message == NULL, "either oop or message but not both");
 131   address entry = __ pc();
 132   if (pass_oop) {
 133     // object is at TOS
 134     __ pop(c_rarg2);
 135   }
 136   // expression stack must be empty before entering the VM if an
 137   // exception happened
 138   __ empty_expression_stack();
 139   // setup parameters
 140   __ lea(c_rarg1, ExternalAddress((address)name));
 141   if (pass_oop) {
 142     __ call_VM(rax, CAST_FROM_FN_PTR(address,
 143                                      InterpreterRuntime::
 144                                      create_klass_exception),
 145                c_rarg1, c_rarg2);
 146   } else {
 147     // kind of lame ExternalAddress can't take NULL because
 148     // external_word_Relocation will assert.
 149     if (message != NULL) {
 150       __ lea(c_rarg2, ExternalAddress((address)message));
 151     } else {
 152       __ movptr(c_rarg2, NULL_WORD);
 153     }
 154     __ call_VM(rax,
 155                CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception),
 156                c_rarg1, c_rarg2);
 157   }
 158   // throw exception
 159   __ jump(ExternalAddress(Interpreter::throw_exception_entry()));
 160   return entry;
 161 }
 162 
 163 
 164 address TemplateInterpreterGenerator::generate_continuation_for(TosState state) {
 165   address entry = __ pc();
 166   // NULL last_sp until next java call
 167   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
 168   __ dispatch_next(state);
 169   return entry;
 170 }
 171 
 172 
 173 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state,
 174                                                                 int step) {
 175 
 176   // amd64 doesn't need to do anything special about compiled returns
 177   // to the interpreter so the code that exists on x86 to place a sentinel
 178   // here and the specialized cleanup code is not needed here.
 179 
 180   address entry = __ pc();
 181 
 182   // Restore stack bottom in case i2c adjusted stack
 183   __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
 184   // and NULL it as marker that esp is now tos until next java call
 185   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
 186 
 187   __ restore_bcp();
 188   __ restore_locals();
 189 
 190   Label L_got_cache, L_giant_index;
 191   if (EnableInvokeDynamic) {
 192     __ cmpb(Address(r13, 0), Bytecodes::_invokedynamic);
 193     __ jcc(Assembler::equal, L_giant_index);
 194   }
 195   __ get_cache_and_index_at_bcp(rbx, rcx, 1, sizeof(u2));
 196   __ bind(L_got_cache);
 197   __ movl(rbx, Address(rbx, rcx,
 198                        Address::times_ptr,
 199                        in_bytes(constantPoolCacheOopDesc::base_offset()) +
 200                        3 * wordSize));
 201   __ andl(rbx, 0xFF);
 202   __ lea(rsp, Address(rsp, rbx, Address::times_8));
 203   __ dispatch_next(state, step);
 204 
 205   // out of the main line of code...
 206   if (EnableInvokeDynamic) {
 207     __ bind(L_giant_index);
 208     __ get_cache_and_index_at_bcp(rbx, rcx, 1, sizeof(u4));
 209     __ jmp(L_got_cache);
 210   }
 211 
 212   return entry;
 213 }
 214 
 215 
 216 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state,
 217                                                                int step) {
 218   address entry = __ pc();
 219   // NULL last_sp until next java call
 220   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
 221   __ restore_bcp();
 222   __ restore_locals();
 223   // handle exceptions
 224   {
 225     Label L;
 226     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
 227     __ jcc(Assembler::zero, L);
 228     __ call_VM(noreg,
 229                CAST_FROM_FN_PTR(address,
 230                                 InterpreterRuntime::throw_pending_exception));
 231     __ should_not_reach_here();
 232     __ bind(L);
 233   }
 234   __ dispatch_next(state, step);
 235   return entry;
 236 }
 237 
 238 int AbstractInterpreter::BasicType_as_index(BasicType type) {
 239   int i = 0;
 240   switch (type) {
 241     case T_BOOLEAN: i = 0; break;
 242     case T_CHAR   : i = 1; break;
 243     case T_BYTE   : i = 2; break;
 244     case T_SHORT  : i = 3; break;
 245     case T_INT    : i = 4; break;
 246     case T_LONG   : i = 5; break;
 247     case T_VOID   : i = 6; break;
 248     case T_FLOAT  : i = 7; break;
 249     case T_DOUBLE : i = 8; break;
 250     case T_OBJECT : i = 9; break;
 251     case T_ARRAY  : i = 9; break;
 252     default       : ShouldNotReachHere();
 253   }
 254   assert(0 <= i && i < AbstractInterpreter::number_of_result_handlers,
 255          "index out of bounds");
 256   return i;
 257 }
 258 
 259 
 260 address TemplateInterpreterGenerator::generate_result_handler_for(
 261         BasicType type) {
 262   address entry = __ pc();
 263   switch (type) {
 264   case T_BOOLEAN: __ c2bool(rax);            break;
 265   case T_CHAR   : __ movzwl(rax, rax);       break;
 266   case T_BYTE   : __ sign_extend_byte(rax);  break;
 267   case T_SHORT  : __ sign_extend_short(rax); break;
 268   case T_INT    : /* nothing to do */        break;
 269   case T_LONG   : /* nothing to do */        break;
 270   case T_VOID   : /* nothing to do */        break;
 271   case T_FLOAT  : /* nothing to do */        break;
 272   case T_DOUBLE : /* nothing to do */        break;
 273   case T_OBJECT :
 274     // retrieve result from frame
 275     __ movptr(rax, Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize));
 276     // and verify it
 277     __ verify_oop(rax);
 278     break;
 279   default       : ShouldNotReachHere();
 280   }
 281   __ ret(0);                                   // return from result handler
 282   return entry;
 283 }
 284 
 285 address TemplateInterpreterGenerator::generate_safept_entry_for(
 286         TosState state,
 287         address runtime_entry) {
 288   address entry = __ pc();
 289   __ push(state);
 290   __ call_VM(noreg, runtime_entry);
 291   __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));
 292   return entry;
 293 }
 294 
 295 
 296 
 297 // Helpers for commoning out cases in the various type of method entries.
 298 //
 299 
 300 
 301 // increment invocation count & check for overflow
 302 //
 303 // Note: checking for negative value instead of overflow
 304 //       so we have a 'sticky' overflow test
 305 //
 306 // rbx: method
 307 // ecx: invocation counter
 308 //
 309 void InterpreterGenerator::generate_counter_incr(
 310         Label* overflow,
 311         Label* profile_method,
 312         Label* profile_method_continue) {
 313   const Address invocation_counter(rbx, in_bytes(methodOopDesc::invocation_counter_offset()) +
 314                                         in_bytes(InvocationCounter::counter_offset()));
 315   // Note: In tiered we increment either counters in methodOop or in MDO depending if we're profiling or not.
 316   if (TieredCompilation) {
 317     int increment = InvocationCounter::count_increment;
 318     int mask = ((1 << Tier0InvokeNotifyFreqLog)  - 1) << InvocationCounter::count_shift;
 319     Label no_mdo, done;
 320     if (ProfileInterpreter) {
 321       // Are we profiling?
 322       __ movptr(rax, Address(rbx, methodOopDesc::method_data_offset()));
 323       __ testptr(rax, rax);
 324       __ jccb(Assembler::zero, no_mdo);
 325       // Increment counter in the MDO
 326       const Address mdo_invocation_counter(rax, in_bytes(methodDataOopDesc::invocation_counter_offset()) +
 327                                                 in_bytes(InvocationCounter::counter_offset()));
 328       __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, rcx, false, Assembler::zero, overflow);
 329       __ jmpb(done);
 330     }
 331     __ bind(no_mdo);
 332     // Increment counter in methodOop (we don't need to load it, it's in ecx).
 333     __ increment_mask_and_jump(invocation_counter, increment, mask, rcx, true, Assembler::zero, overflow);
 334     __ bind(done);
 335   } else {
 336     const Address backedge_counter(rbx,
 337                                    methodOopDesc::backedge_counter_offset() +
 338                                    InvocationCounter::counter_offset());
 339 
 340     if (ProfileInterpreter) { // %%% Merge this into methodDataOop
 341       __ incrementl(Address(rbx,
 342                             methodOopDesc::interpreter_invocation_counter_offset()));
 343     }
 344     // Update standard invocation counters
 345     __ movl(rax, backedge_counter);   // load backedge counter
 346 
 347     __ incrementl(rcx, InvocationCounter::count_increment);
 348     __ andl(rax, InvocationCounter::count_mask_value); // mask out the status bits
 349 
 350     __ movl(invocation_counter, rcx); // save invocation count
 351     __ addl(rcx, rax);                // add both counters
 352 
 353     // profile_method is non-null only for interpreted method so
 354     // profile_method != NULL == !native_call
 355 
 356     if (ProfileInterpreter && profile_method != NULL) {
 357       // Test to see if we should create a method data oop
 358       __ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterProfileLimit));
 359       __ jcc(Assembler::less, *profile_method_continue);
 360 
 361       // if no method data exists, go to profile_method
 362       __ test_method_data_pointer(rax, *profile_method);
 363     }
 364 
 365     __ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterInvocationLimit));
 366     __ jcc(Assembler::aboveEqual, *overflow);
 367   }
 368 }
 369 
 370 void InterpreterGenerator::generate_counter_overflow(Label* do_continue) {
 371 
 372   // Asm interpreter on entry
 373   // r14 - locals
 374   // r13 - bcp
 375   // rbx - method
 376   // edx - cpool --- DOES NOT APPEAR TO BE TRUE
 377   // rbp - interpreter frame
 378 
 379   // On return (i.e. jump to entry_point) [ back to invocation of interpreter ]
 380   // Everything as it was on entry
 381   // rdx is not restored. Doesn't appear to really be set.
 382 
 383   const Address size_of_parameters(rbx,
 384                                    methodOopDesc::size_of_parameters_offset());
 385 
 386   // InterpreterRuntime::frequency_counter_overflow takes two
 387   // arguments, the first (thread) is passed by call_VM, the second
 388   // indicates if the counter overflow occurs at a backwards branch
 389   // (NULL bcp).  We pass zero for it.  The call returns the address
 390   // of the verified entry point for the method or NULL if the
 391   // compilation did not complete (either went background or bailed
 392   // out).
 393   __ movl(c_rarg1, 0);
 394   __ call_VM(noreg,
 395              CAST_FROM_FN_PTR(address,
 396                               InterpreterRuntime::frequency_counter_overflow),
 397              c_rarg1);
 398 
 399   __ movptr(rbx, Address(rbp, method_offset));   // restore methodOop
 400   // Preserve invariant that r13/r14 contain bcp/locals of sender frame
 401   // and jump to the interpreted entry.
 402   __ jmp(*do_continue, relocInfo::none);
 403 }
 404 
 405 // See if we've got enough room on the stack for locals plus overhead.
 406 // The expression stack grows down incrementally, so the normal guard
 407 // page mechanism will work for that.
 408 //
 409 // NOTE: Since the additional locals are also always pushed (wasn't
 410 // obvious in generate_method_entry) so the guard should work for them
 411 // too.
 412 //
 413 // Args:
 414 //      rdx: number of additional locals this frame needs (what we must check)
 415 //      rbx: methodOop
 416 //
 417 // Kills:
 418 //      rax
 419 void InterpreterGenerator::generate_stack_overflow_check(void) {
 420 
 421   // monitor entry size: see picture of stack set
 422   // (generate_method_entry) and frame_amd64.hpp
 423   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 424 
 425   // total overhead size: entry_size + (saved rbp through expr stack
 426   // bottom).  be sure to change this if you add/subtract anything
 427   // to/from the overhead area
 428   const int overhead_size =
 429     -(frame::interpreter_frame_initial_sp_offset * wordSize) + entry_size;
 430 
 431   const int page_size = os::vm_page_size();
 432 
 433   Label after_frame_check;
 434 
 435   // see if the frame is greater than one page in size. If so,
 436   // then we need to verify there is enough stack space remaining
 437   // for the additional locals.
 438   __ cmpl(rdx, (page_size - overhead_size) / Interpreter::stackElementSize);
 439   __ jcc(Assembler::belowEqual, after_frame_check);
 440 
 441   // compute rsp as if this were going to be the last frame on
 442   // the stack before the red zone
 443 
 444   const Address stack_base(r15_thread, Thread::stack_base_offset());
 445   const Address stack_size(r15_thread, Thread::stack_size_offset());
 446 
 447   // locals + overhead, in bytes
 448   __ mov(rax, rdx);
 449   __ shlptr(rax, Interpreter::logStackElementSize);  // 2 slots per parameter.
 450   __ addptr(rax, overhead_size);
 451 
 452 #ifdef ASSERT
 453   Label stack_base_okay, stack_size_okay;
 454   // verify that thread stack base is non-zero
 455   __ cmpptr(stack_base, (int32_t)NULL_WORD);
 456   __ jcc(Assembler::notEqual, stack_base_okay);
 457   __ stop("stack base is zero");
 458   __ bind(stack_base_okay);
 459   // verify that thread stack size is non-zero
 460   __ cmpptr(stack_size, 0);
 461   __ jcc(Assembler::notEqual, stack_size_okay);
 462   __ stop("stack size is zero");
 463   __ bind(stack_size_okay);
 464 #endif
 465 
 466   // Add stack base to locals and subtract stack size
 467   __ addptr(rax, stack_base);
 468   __ subptr(rax, stack_size);
 469 
 470   // Use the maximum number of pages we might bang.
 471   const int max_pages = StackShadowPages > (StackRedPages+StackYellowPages) ? StackShadowPages :
 472                                                                               (StackRedPages+StackYellowPages);
 473 
 474   // add in the red and yellow zone sizes
 475   __ addptr(rax, max_pages * page_size);
 476 
 477   // check against the current stack bottom
 478   __ cmpptr(rsp, rax);
 479   __ jcc(Assembler::above, after_frame_check);
 480 
 481   __ pop(rax); // get return address
 482   __ jump(ExternalAddress(Interpreter::throw_StackOverflowError_entry()));
 483 
 484   // all done with frame size check
 485   __ bind(after_frame_check);
 486 }
 487 
 488 // Allocate monitor and lock method (asm interpreter)
 489 //
 490 // Args:
 491 //      rbx: methodOop
 492 //      r14: locals
 493 //
 494 // Kills:
 495 //      rax
 496 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, ...(param regs)
 497 //      rscratch1, rscratch2 (scratch regs)
 498 void InterpreterGenerator::lock_method(void) {
 499   // synchronize method
 500   const Address access_flags(rbx, methodOopDesc::access_flags_offset());
 501   const Address monitor_block_top(
 502         rbp,
 503         frame::interpreter_frame_monitor_block_top_offset * wordSize);
 504   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 505 
 506 #ifdef ASSERT
 507   {
 508     Label L;
 509     __ movl(rax, access_flags);
 510     __ testl(rax, JVM_ACC_SYNCHRONIZED);
 511     __ jcc(Assembler::notZero, L);
 512     __ stop("method doesn't need synchronization");
 513     __ bind(L);
 514   }
 515 #endif // ASSERT
 516 
 517   // get synchronization object
 518   {
 519     const int mirror_offset = klassOopDesc::klass_part_offset_in_bytes() +
 520                               Klass::java_mirror_offset_in_bytes();
 521     Label done;
 522     __ movl(rax, access_flags);
 523     __ testl(rax, JVM_ACC_STATIC);
 524     // get receiver (assume this is frequent case)
 525     __ movptr(rax, Address(r14, Interpreter::local_offset_in_bytes(0)));
 526     __ jcc(Assembler::zero, done);
 527     __ movptr(rax, Address(rbx, methodOopDesc::constants_offset()));
 528     __ movptr(rax, Address(rax,
 529                            constantPoolOopDesc::pool_holder_offset_in_bytes()));
 530     __ movptr(rax, Address(rax, mirror_offset));
 531 
 532 #ifdef ASSERT
 533     {
 534       Label L;
 535       __ testptr(rax, rax);
 536       __ jcc(Assembler::notZero, L);
 537       __ stop("synchronization object is NULL");
 538       __ bind(L);
 539     }
 540 #endif // ASSERT
 541 
 542     __ bind(done);
 543   }
 544 
 545   // add space for monitor & lock
 546   __ subptr(rsp, entry_size); // add space for a monitor entry
 547   __ movptr(monitor_block_top, rsp);  // set new monitor block top
 548   // store object
 549   __ movptr(Address(rsp, BasicObjectLock::obj_offset_in_bytes()), rax);
 550   __ movptr(c_rarg1, rsp); // object address
 551   __ lock_object(c_rarg1);
 552 }
 553 
 554 // Generate a fixed interpreter frame. This is identical setup for
 555 // interpreted methods and for native methods hence the shared code.
 556 //
 557 // Args:
 558 //      rax: return address
 559 //      rbx: methodOop
 560 //      r14: pointer to locals
 561 //      r13: sender sp
 562 //      rdx: cp cache
 563 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
 564   // initialize fixed part of activation frame
 565   __ push(rax);        // save return address
 566   __ enter();          // save old & set new rbp
 567   __ push(r13);        // set sender sp
 568   __ push((int)NULL_WORD); // leave last_sp as null
 569   __ movptr(r13, Address(rbx, methodOopDesc::const_offset()));      // get constMethodOop
 570   __ lea(r13, Address(r13, constMethodOopDesc::codes_offset())); // get codebase
 571   __ push(rbx);        // save methodOop
 572   if (ProfileInterpreter) {
 573     Label method_data_continue;
 574     __ movptr(rdx, Address(rbx, in_bytes(methodOopDesc::method_data_offset())));
 575     __ testptr(rdx, rdx);
 576     __ jcc(Assembler::zero, method_data_continue);
 577     __ addptr(rdx, in_bytes(methodDataOopDesc::data_offset()));
 578     __ bind(method_data_continue);
 579     __ push(rdx);      // set the mdp (method data pointer)
 580   } else {
 581     __ push(0);
 582   }
 583 
 584   __ movptr(rdx, Address(rbx, methodOopDesc::constants_offset()));
 585   __ movptr(rdx, Address(rdx, constantPoolOopDesc::cache_offset_in_bytes()));
 586   __ push(rdx); // set constant pool cache
 587   __ push(r14); // set locals pointer
 588   if (native_call) {
 589     __ push(0); // no bcp
 590   } else {
 591     __ push(r13); // set bcp
 592   }
 593   __ push(0); // reserve word for pointer to expression stack bottom
 594   __ movptr(Address(rsp, 0), rsp); // set expression stack bottom
 595 }
 596 
 597 // End of helpers
 598 
 599 // Various method entries
 600 //------------------------------------------------------------------------------------------------------------------------
 601 //
 602 //
 603 
 604 // Call an accessor method (assuming it is resolved, otherwise drop
 605 // into vanilla (slow path) entry
 606 address InterpreterGenerator::generate_accessor_entry(void) {
 607   // rbx: methodOop
 608 
 609   // r13: senderSP must preserver for slow path, set SP to it on fast path
 610 
 611   address entry_point = __ pc();
 612   Label xreturn_path;
 613 
 614   // do fastpath for resolved accessor methods
 615   if (UseFastAccessorMethods) {
 616     // Code: _aload_0, _(i|a)getfield, _(i|a)return or any rewrites
 617     //       thereof; parameter size = 1
 618     // Note: We can only use this code if the getfield has been resolved
 619     //       and if we don't have a null-pointer exception => check for
 620     //       these conditions first and use slow path if necessary.
 621     Label slow_path;
 622     // If we need a safepoint check, generate full interpreter entry.
 623     __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
 624              SafepointSynchronize::_not_synchronized);
 625 
 626     __ jcc(Assembler::notEqual, slow_path);
 627     // rbx: method
 628     __ movptr(rax, Address(rsp, wordSize));
 629 
 630     // check if local 0 != NULL and read field
 631     __ testptr(rax, rax);
 632     __ jcc(Assembler::zero, slow_path);
 633 
 634     __ movptr(rdi, Address(rbx, methodOopDesc::constants_offset()));
 635     // read first instruction word and extract bytecode @ 1 and index @ 2
 636     __ movptr(rdx, Address(rbx, methodOopDesc::const_offset()));
 637     __ movl(rdx, Address(rdx, constMethodOopDesc::codes_offset()));
 638     // Shift codes right to get the index on the right.
 639     // The bytecode fetched looks like <index><0xb4><0x2a>
 640     __ shrl(rdx, 2 * BitsPerByte);
 641     __ shll(rdx, exact_log2(in_words(ConstantPoolCacheEntry::size())));
 642     __ movptr(rdi, Address(rdi, constantPoolOopDesc::cache_offset_in_bytes()));
 643 
 644     // rax: local 0
 645     // rbx: method
 646     // rdx: constant pool cache index
 647     // rdi: constant pool cache
 648 
 649     // check if getfield has been resolved and read constant pool cache entry
 650     // check the validity of the cache entry by testing whether _indices field
 651     // contains Bytecode::_getfield in b1 byte.
 652     assert(in_words(ConstantPoolCacheEntry::size()) == 4,
 653            "adjust shift below");
 654     __ movl(rcx,
 655             Address(rdi,
 656                     rdx,
 657                     Address::times_8,
 658                     constantPoolCacheOopDesc::base_offset() +
 659                     ConstantPoolCacheEntry::indices_offset()));
 660     __ shrl(rcx, 2 * BitsPerByte);
 661     __ andl(rcx, 0xFF);
 662     __ cmpl(rcx, Bytecodes::_getfield);
 663     __ jcc(Assembler::notEqual, slow_path);
 664 
 665     // Note: constant pool entry is not valid before bytecode is resolved
 666     __ movptr(rcx,
 667               Address(rdi,
 668                       rdx,
 669                       Address::times_8,
 670                       constantPoolCacheOopDesc::base_offset() +
 671                       ConstantPoolCacheEntry::f2_offset()));
 672     // edx: flags
 673     __ movl(rdx,
 674             Address(rdi,
 675                     rdx,
 676                     Address::times_8,
 677                     constantPoolCacheOopDesc::base_offset() +
 678                     ConstantPoolCacheEntry::flags_offset()));
 679 
 680     Label notObj, notInt, notByte, notShort;
 681     const Address field_address(rax, rcx, Address::times_1);
 682 
 683     // Need to differentiate between igetfield, agetfield, bgetfield etc.
 684     // because they are different sizes.
 685     // Use the type from the constant pool cache
 686     __ shrl(rdx, ConstantPoolCacheEntry::tosBits);
 687     // Make sure we don't need to mask edx for tosBits after the above shift
 688     ConstantPoolCacheEntry::verify_tosBits();
 689 
 690     __ cmpl(rdx, atos);
 691     __ jcc(Assembler::notEqual, notObj);
 692     // atos
 693     __ load_heap_oop(rax, field_address);
 694     __ jmp(xreturn_path);
 695 
 696     __ bind(notObj);
 697     __ cmpl(rdx, itos);
 698     __ jcc(Assembler::notEqual, notInt);
 699     // itos
 700     __ movl(rax, field_address);
 701     __ jmp(xreturn_path);
 702 
 703     __ bind(notInt);
 704     __ cmpl(rdx, btos);
 705     __ jcc(Assembler::notEqual, notByte);
 706     // btos
 707     __ load_signed_byte(rax, field_address);
 708     __ jmp(xreturn_path);
 709 
 710     __ bind(notByte);
 711     __ cmpl(rdx, stos);
 712     __ jcc(Assembler::notEqual, notShort);
 713     // stos
 714     __ load_signed_short(rax, field_address);
 715     __ jmp(xreturn_path);
 716 
 717     __ bind(notShort);
 718 #ifdef ASSERT
 719     Label okay;
 720     __ cmpl(rdx, ctos);
 721     __ jcc(Assembler::equal, okay);
 722     __ stop("what type is this?");
 723     __ bind(okay);
 724 #endif
 725     // ctos
 726     __ load_unsigned_short(rax, field_address);
 727 
 728     __ bind(xreturn_path);
 729 
 730     // _ireturn/_areturn
 731     __ pop(rdi);
 732     __ mov(rsp, r13);
 733     __ jmp(rdi);
 734     __ ret(0);
 735 
 736     // generate a vanilla interpreter entry as the slow path
 737     __ bind(slow_path);
 738     (void) generate_normal_entry(false);
 739   } else {
 740     (void) generate_normal_entry(false);
 741   }
 742 
 743   return entry_point;
 744 }
 745 
 746 // Interpreter stub for calling a native method. (asm interpreter)
 747 // This sets up a somewhat different looking stack for calling the
 748 // native method than the typical interpreter frame setup.
 749 address InterpreterGenerator::generate_native_entry(bool synchronized) {
 750   // determine code generation flags
 751   bool inc_counter  = UseCompiler || CountCompiledCalls;
 752 
 753   // rbx: methodOop
 754   // r13: sender sp
 755 
 756   address entry_point = __ pc();
 757 
 758   const Address size_of_parameters(rbx, methodOopDesc::
 759                                         size_of_parameters_offset());
 760   const Address invocation_counter(rbx, methodOopDesc::
 761                                         invocation_counter_offset() +
 762                                         InvocationCounter::counter_offset());
 763   const Address access_flags      (rbx, methodOopDesc::access_flags_offset());
 764 
 765   // get parameter size (always needed)
 766   __ load_unsigned_short(rcx, size_of_parameters);
 767 
 768   // native calls don't need the stack size check since they have no
 769   // expression stack and the arguments are already on the stack and
 770   // we only add a handful of words to the stack
 771 
 772   // rbx: methodOop
 773   // rcx: size of parameters
 774   // r13: sender sp
 775   __ pop(rax);                                       // get return address
 776 
 777   // for natives the size of locals is zero
 778 
 779   // compute beginning of parameters (r14)
 780   __ lea(r14, Address(rsp, rcx, Address::times_8, -wordSize));
 781 
 782   // add 2 zero-initialized slots for native calls
 783   // initialize result_handler slot
 784   __ push((int) NULL_WORD);
 785   // slot for oop temp
 786   // (static native method holder mirror/jni oop result)
 787   __ push((int) NULL_WORD);
 788 
 789   if (inc_counter) {
 790     __ movl(rcx, invocation_counter);  // (pre-)fetch invocation count
 791   }
 792 
 793   // initialize fixed part of activation frame
 794   generate_fixed_frame(true);
 795 
 796   // make sure method is native & not abstract
 797 #ifdef ASSERT
 798   __ movl(rax, access_flags);
 799   {
 800     Label L;
 801     __ testl(rax, JVM_ACC_NATIVE);
 802     __ jcc(Assembler::notZero, L);
 803     __ stop("tried to execute non-native method as native");
 804     __ bind(L);
 805   }
 806   {
 807     Label L;
 808     __ testl(rax, JVM_ACC_ABSTRACT);
 809     __ jcc(Assembler::zero, L);
 810     __ stop("tried to execute abstract method in interpreter");
 811     __ bind(L);
 812   }
 813 #endif
 814 
 815   // Since at this point in the method invocation the exception handler
 816   // would try to exit the monitor of synchronized methods which hasn't
 817   // been entered yet, we set the thread local variable
 818   // _do_not_unlock_if_synchronized to true. The remove_activation will
 819   // check this flag.
 820 
 821   const Address do_not_unlock_if_synchronized(r15_thread,
 822         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
 823   __ movbool(do_not_unlock_if_synchronized, true);
 824 
 825   // increment invocation count & check for overflow
 826   Label invocation_counter_overflow;
 827   if (inc_counter) {
 828     generate_counter_incr(&invocation_counter_overflow, NULL, NULL);
 829   }
 830 
 831   Label continue_after_compile;
 832   __ bind(continue_after_compile);
 833 
 834   bang_stack_shadow_pages(true);
 835 
 836   // reset the _do_not_unlock_if_synchronized flag
 837   __ movbool(do_not_unlock_if_synchronized, false);
 838 
 839   // check for synchronized methods
 840   // Must happen AFTER invocation_counter check and stack overflow check,
 841   // so method is not locked if overflows.
 842   if (synchronized) {
 843     lock_method();
 844   } else {
 845     // no synchronization necessary
 846 #ifdef ASSERT
 847     {
 848       Label L;
 849       __ movl(rax, access_flags);
 850       __ testl(rax, JVM_ACC_SYNCHRONIZED);
 851       __ jcc(Assembler::zero, L);
 852       __ stop("method needs synchronization");
 853       __ bind(L);
 854     }
 855 #endif
 856   }
 857 
 858   // start execution
 859 #ifdef ASSERT
 860   {
 861     Label L;
 862     const Address monitor_block_top(rbp,
 863                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
 864     __ movptr(rax, monitor_block_top);
 865     __ cmpptr(rax, rsp);
 866     __ jcc(Assembler::equal, L);
 867     __ stop("broken stack frame setup in interpreter");
 868     __ bind(L);
 869   }
 870 #endif
 871 
 872   // jvmti support
 873   __ notify_method_entry();
 874 
 875   // work registers
 876   const Register method = rbx;
 877   const Register t      = r11;
 878 
 879   // allocate space for parameters
 880   __ get_method(method);
 881   __ verify_oop(method);
 882   __ load_unsigned_short(t,
 883                          Address(method,
 884                                  methodOopDesc::size_of_parameters_offset()));
 885   __ shll(t, Interpreter::logStackElementSize);
 886 
 887   __ subptr(rsp, t);
 888   __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
 889   __ andptr(rsp, -16); // must be 16 byte boundary (see amd64 ABI)
 890 
 891   // get signature handler
 892   {
 893     Label L;
 894     __ movptr(t, Address(method, methodOopDesc::signature_handler_offset()));
 895     __ testptr(t, t);
 896     __ jcc(Assembler::notZero, L);
 897     __ call_VM(noreg,
 898                CAST_FROM_FN_PTR(address,
 899                                 InterpreterRuntime::prepare_native_call),
 900                method);
 901     __ get_method(method);
 902     __ movptr(t, Address(method, methodOopDesc::signature_handler_offset()));
 903     __ bind(L);
 904   }
 905 
 906   // call signature handler
 907   assert(InterpreterRuntime::SignatureHandlerGenerator::from() == r14,
 908          "adjust this code");
 909   assert(InterpreterRuntime::SignatureHandlerGenerator::to() == rsp,
 910          "adjust this code");
 911   assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == rscratch1,
 912           "adjust this code");
 913 
 914   // The generated handlers do not touch RBX (the method oop).
 915   // However, large signatures cannot be cached and are generated
 916   // each time here.  The slow-path generator can do a GC on return,
 917   // so we must reload it after the call.
 918   __ call(t);
 919   __ get_method(method);        // slow path can do a GC, reload RBX
 920 
 921 
 922   // result handler is in rax
 923   // set result handler
 924   __ movptr(Address(rbp,
 925                     (frame::interpreter_frame_result_handler_offset) * wordSize),
 926             rax);
 927 
 928   // pass mirror handle if static call
 929   {
 930     Label L;
 931     const int mirror_offset = klassOopDesc::klass_part_offset_in_bytes() +
 932                               Klass::java_mirror_offset_in_bytes();
 933     __ movl(t, Address(method, methodOopDesc::access_flags_offset()));
 934     __ testl(t, JVM_ACC_STATIC);
 935     __ jcc(Assembler::zero, L);
 936     // get mirror
 937     __ movptr(t, Address(method, methodOopDesc::constants_offset()));
 938     __ movptr(t, Address(t, constantPoolOopDesc::pool_holder_offset_in_bytes()));
 939     __ movptr(t, Address(t, mirror_offset));
 940     // copy mirror into activation frame
 941     __ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize),
 942             t);
 943     // pass handle to mirror
 944     __ lea(c_rarg1,
 945            Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize));
 946     __ bind(L);
 947   }
 948 
 949   // get native function entry point
 950   {
 951     Label L;
 952     __ movptr(rax, Address(method, methodOopDesc::native_function_offset()));
 953     ExternalAddress unsatisfied(SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
 954     __ movptr(rscratch2, unsatisfied.addr());
 955     __ cmpptr(rax, rscratch2);
 956     __ jcc(Assembler::notEqual, L);
 957     __ call_VM(noreg,
 958                CAST_FROM_FN_PTR(address,
 959                                 InterpreterRuntime::prepare_native_call),
 960                method);
 961     __ get_method(method);
 962     __ verify_oop(method);
 963     __ movptr(rax, Address(method, methodOopDesc::native_function_offset()));
 964     __ bind(L);
 965   }
 966 
 967   // pass JNIEnv
 968   __ lea(c_rarg0, Address(r15_thread, JavaThread::jni_environment_offset()));
 969 
 970   // It is enough that the pc() points into the right code
 971   // segment. It does not have to be the correct return pc.
 972   __ set_last_Java_frame(rsp, rbp, (address) __ pc());
 973 
 974   // change thread state
 975 #ifdef ASSERT
 976   {
 977     Label L;
 978     __ movl(t, Address(r15_thread, JavaThread::thread_state_offset()));
 979     __ cmpl(t, _thread_in_Java);
 980     __ jcc(Assembler::equal, L);
 981     __ stop("Wrong thread state in native stub");
 982     __ bind(L);
 983   }
 984 #endif
 985 
 986   // Change state to native
 987 
 988   __ movl(Address(r15_thread, JavaThread::thread_state_offset()),
 989           _thread_in_native);
 990 
 991   // Call the native method.
 992   __ call(rax);
 993   // result potentially in rax or xmm0
 994 
 995   // Depending on runtime options, either restore the MXCSR
 996   // register after returning from the JNI Call or verify that
 997   // it wasn't changed during -Xcheck:jni.
 998   if (RestoreMXCSROnJNICalls) {
 999     __ ldmxcsr(ExternalAddress(StubRoutines::x86::mxcsr_std()));
1000   }
1001   else if (CheckJNICalls) {
1002     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::verify_mxcsr_entry())));
1003   }
1004 
1005   // NOTE: The order of these pushes is known to frame::interpreter_frame_result
1006   // in order to extract the result of a method call. If the order of these
1007   // pushes change or anything else is added to the stack then the code in
1008   // interpreter_frame_result must also change.
1009 
1010   __ push(dtos);
1011   __ push(ltos);
1012 
1013   // change thread state
1014   __ movl(Address(r15_thread, JavaThread::thread_state_offset()),
1015           _thread_in_native_trans);
1016 
1017   if (os::is_MP()) {
1018     if (UseMembar) {
1019       // Force this write out before the read below
1020       __ membar(Assembler::Membar_mask_bits(
1021            Assembler::LoadLoad | Assembler::LoadStore |
1022            Assembler::StoreLoad | Assembler::StoreStore));
1023     } else {
1024       // Write serialization page so VM thread can do a pseudo remote membar.
1025       // We use the current thread pointer to calculate a thread specific
1026       // offset to write to within the page. This minimizes bus traffic
1027       // due to cache line collision.
1028       __ serialize_memory(r15_thread, rscratch2);
1029     }
1030   }
1031 
1032   // check for safepoint operation in progress and/or pending suspend requests
1033   {
1034     Label Continue;
1035     __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
1036              SafepointSynchronize::_not_synchronized);
1037 
1038     Label L;
1039     __ jcc(Assembler::notEqual, L);
1040     __ cmpl(Address(r15_thread, JavaThread::suspend_flags_offset()), 0);
1041     __ jcc(Assembler::equal, Continue);
1042     __ bind(L);
1043 
1044     // Don't use call_VM as it will see a possible pending exception
1045     // and forward it and never return here preventing us from
1046     // clearing _last_native_pc down below.  Also can't use
1047     // call_VM_leaf either as it will check to see if r13 & r14 are
1048     // preserved and correspond to the bcp/locals pointers. So we do a
1049     // runtime call by hand.
1050     //
1051     __ mov(c_rarg0, r15_thread);
1052     __ mov(r12, rsp); // remember sp
1053     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1054     __ andptr(rsp, -16); // align stack as required by ABI
1055     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
1056     __ mov(rsp, r12); // restore sp
1057     __ reinit_heapbase();
1058     __ bind(Continue);
1059   }
1060 
1061   // change thread state
1062   __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
1063 
1064   // reset_last_Java_frame
1065   __ reset_last_Java_frame(true, true);
1066 
1067   // reset handle block
1068   __ movptr(t, Address(r15_thread, JavaThread::active_handles_offset()));
1069   __ movptr(Address(t, JNIHandleBlock::top_offset_in_bytes()), (int32_t)NULL_WORD);
1070 
1071   // If result is an oop unbox and store it in frame where gc will see it
1072   // and result handler will pick it up
1073 
1074   {
1075     Label no_oop, store_result;
1076     __ lea(t, ExternalAddress(AbstractInterpreter::result_handler(T_OBJECT)));
1077     __ cmpptr(t, Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize));
1078     __ jcc(Assembler::notEqual, no_oop);
1079     // retrieve result
1080     __ pop(ltos);
1081     __ testptr(rax, rax);
1082     __ jcc(Assembler::zero, store_result);
1083     __ movptr(rax, Address(rax, 0));
1084     __ bind(store_result);
1085     __ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize), rax);
1086     // keep stack depth as expected by pushing oop which will eventually be discarde
1087     __ push(ltos);
1088     __ bind(no_oop);
1089   }
1090 
1091 
1092   {
1093     Label no_reguard;
1094     __ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()),
1095             JavaThread::stack_guard_yellow_disabled);
1096     __ jcc(Assembler::notEqual, no_reguard);
1097 
1098     __ pusha(); // XXX only save smashed registers
1099     __ mov(r12, rsp); // remember sp
1100     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1101     __ andptr(rsp, -16); // align stack as required by ABI
1102     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
1103     __ mov(rsp, r12); // restore sp
1104     __ popa(); // XXX only restore smashed registers
1105     __ reinit_heapbase();
1106 
1107     __ bind(no_reguard);
1108   }
1109 
1110 
1111   // The method register is junk from after the thread_in_native transition
1112   // until here.  Also can't call_VM until the bcp has been
1113   // restored.  Need bcp for throwing exception below so get it now.
1114   __ get_method(method);
1115   __ verify_oop(method);
1116 
1117   // restore r13 to have legal interpreter frame, i.e., bci == 0 <=>
1118   // r13 == code_base()
1119   __ movptr(r13, Address(method, methodOopDesc::const_offset()));   // get constMethodOop
1120   __ lea(r13, Address(r13, constMethodOopDesc::codes_offset()));    // get codebase
1121   // handle exceptions (exception handling will handle unlocking!)
1122   {
1123     Label L;
1124     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
1125     __ jcc(Assembler::zero, L);
1126     // Note: At some point we may want to unify this with the code
1127     // used in call_VM_base(); i.e., we should use the
1128     // StubRoutines::forward_exception code. For now this doesn't work
1129     // here because the rsp is not correctly set at this point.
1130     __ MacroAssembler::call_VM(noreg,
1131                                CAST_FROM_FN_PTR(address,
1132                                InterpreterRuntime::throw_pending_exception));
1133     __ should_not_reach_here();
1134     __ bind(L);
1135   }
1136 
1137   // do unlocking if necessary
1138   {
1139     Label L;
1140     __ movl(t, Address(method, methodOopDesc::access_flags_offset()));
1141     __ testl(t, JVM_ACC_SYNCHRONIZED);
1142     __ jcc(Assembler::zero, L);
1143     // the code below should be shared with interpreter macro
1144     // assembler implementation
1145     {
1146       Label unlock;
1147       // BasicObjectLock will be first in list, since this is a
1148       // synchronized method. However, need to check that the object
1149       // has not been unlocked by an explicit monitorexit bytecode.
1150       const Address monitor(rbp,
1151                             (intptr_t)(frame::interpreter_frame_initial_sp_offset *
1152                                        wordSize - sizeof(BasicObjectLock)));
1153 
1154       // monitor expect in c_rarg1 for slow unlock path
1155       __ lea(c_rarg1, monitor); // address of first monitor
1156 
1157       __ movptr(t, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
1158       __ testptr(t, t);
1159       __ jcc(Assembler::notZero, unlock);
1160 
1161       // Entry already unlocked, need to throw exception
1162       __ MacroAssembler::call_VM(noreg,
1163                                  CAST_FROM_FN_PTR(address,
1164                    InterpreterRuntime::throw_illegal_monitor_state_exception));
1165       __ should_not_reach_here();
1166 
1167       __ bind(unlock);
1168       __ unlock_object(c_rarg1);
1169     }
1170     __ bind(L);
1171   }
1172 
1173   // jvmti support
1174   // Note: This must happen _after_ handling/throwing any exceptions since
1175   //       the exception handler code notifies the runtime of method exits
1176   //       too. If this happens before, method entry/exit notifications are
1177   //       not properly paired (was bug - gri 11/22/99).
1178   __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI);
1179 
1180   // restore potential result in edx:eax, call result handler to
1181   // restore potential result in ST0 & handle result
1182 
1183   __ pop(ltos);
1184   __ pop(dtos);
1185 
1186   __ movptr(t, Address(rbp,
1187                        (frame::interpreter_frame_result_handler_offset) * wordSize));
1188   __ call(t);
1189 
1190   // remove activation
1191   __ movptr(t, Address(rbp,
1192                        frame::interpreter_frame_sender_sp_offset *
1193                        wordSize)); // get sender sp
1194   __ leave();                                // remove frame anchor
1195   __ pop(rdi);                               // get return address
1196   __ mov(rsp, t);                            // set sp to sender sp
1197   __ jmp(rdi);
1198 
1199   if (inc_counter) {
1200     // Handle overflow of counter and compile method
1201     __ bind(invocation_counter_overflow);
1202     generate_counter_overflow(&continue_after_compile);
1203   }
1204 
1205   return entry_point;
1206 }
1207 
1208 //
1209 // Generic interpreted method entry to (asm) interpreter
1210 //
1211 address InterpreterGenerator::generate_normal_entry(bool synchronized) {
1212   // determine code generation flags
1213   bool inc_counter  = UseCompiler || CountCompiledCalls;
1214 
1215   // ebx: methodOop
1216   // r13: sender sp
1217   address entry_point = __ pc();
1218 
1219   const Address size_of_parameters(rbx,
1220                                    methodOopDesc::size_of_parameters_offset());
1221   const Address size_of_locals(rbx, methodOopDesc::size_of_locals_offset());
1222   const Address invocation_counter(rbx,
1223                                    methodOopDesc::invocation_counter_offset() +
1224                                    InvocationCounter::counter_offset());
1225   const Address access_flags(rbx, methodOopDesc::access_flags_offset());
1226 
1227   // get parameter size (always needed)
1228   __ load_unsigned_short(rcx, size_of_parameters);
1229 
1230   // rbx: methodOop
1231   // rcx: size of parameters
1232   // r13: sender_sp (could differ from sp+wordSize if we were called via c2i )
1233 
1234   __ load_unsigned_short(rdx, size_of_locals); // get size of locals in words
1235   __ subl(rdx, rcx); // rdx = no. of additional locals
1236 
1237   // YYY
1238 //   __ incrementl(rdx);
1239 //   __ andl(rdx, -2);
1240 
1241   // see if we've got enough room on the stack for locals plus overhead.
1242   generate_stack_overflow_check();
1243 
1244   // get return address
1245   __ pop(rax);
1246 
1247   // compute beginning of parameters (r14)
1248   __ lea(r14, Address(rsp, rcx, Address::times_8, -wordSize));
1249 
1250   // rdx - # of additional locals
1251   // allocate space for locals
1252   // explicitly initialize locals
1253   {
1254     Label exit, loop;
1255     __ testl(rdx, rdx);
1256     __ jcc(Assembler::lessEqual, exit); // do nothing if rdx <= 0
1257     __ bind(loop);
1258     __ push((int) NULL_WORD); // initialize local variables
1259     __ decrementl(rdx); // until everything initialized
1260     __ jcc(Assembler::greater, loop);
1261     __ bind(exit);
1262   }
1263 
1264   // (pre-)fetch invocation count
1265   if (inc_counter) {
1266     __ movl(rcx, invocation_counter);
1267   }
1268   // initialize fixed part of activation frame
1269   generate_fixed_frame(false);
1270 
1271   // make sure method is not native & not abstract
1272 #ifdef ASSERT
1273   __ movl(rax, access_flags);
1274   {
1275     Label L;
1276     __ testl(rax, JVM_ACC_NATIVE);
1277     __ jcc(Assembler::zero, L);
1278     __ stop("tried to execute native method as non-native");
1279     __ bind(L);
1280   }
1281   {
1282     Label L;
1283     __ testl(rax, JVM_ACC_ABSTRACT);
1284     __ jcc(Assembler::zero, L);
1285     __ stop("tried to execute abstract method in interpreter");
1286     __ bind(L);
1287   }
1288 #endif
1289 
1290   // Since at this point in the method invocation the exception
1291   // handler would try to exit the monitor of synchronized methods
1292   // which hasn't been entered yet, we set the thread local variable
1293   // _do_not_unlock_if_synchronized to true. The remove_activation
1294   // will check this flag.
1295 
1296   const Address do_not_unlock_if_synchronized(r15_thread,
1297         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1298   __ movbool(do_not_unlock_if_synchronized, true);
1299 
1300   // increment invocation count & check for overflow
1301   Label invocation_counter_overflow;
1302   Label profile_method;
1303   Label profile_method_continue;
1304   if (inc_counter) {
1305     generate_counter_incr(&invocation_counter_overflow,
1306                           &profile_method,
1307                           &profile_method_continue);
1308     if (ProfileInterpreter) {
1309       __ bind(profile_method_continue);
1310     }
1311   }
1312 
1313   Label continue_after_compile;
1314   __ bind(continue_after_compile);
1315 
1316   // check for synchronized interpreted methods
1317   bang_stack_shadow_pages(false);
1318 
1319   // reset the _do_not_unlock_if_synchronized flag
1320   __ movbool(do_not_unlock_if_synchronized, false);
1321 
1322   // check for synchronized methods
1323   // Must happen AFTER invocation_counter check and stack overflow check,
1324   // so method is not locked if overflows.
1325   if (synchronized) {
1326     // Allocate monitor and lock method
1327     lock_method();
1328   } else {
1329     // no synchronization necessary
1330 #ifdef ASSERT
1331     {
1332       Label L;
1333       __ movl(rax, access_flags);
1334       __ testl(rax, JVM_ACC_SYNCHRONIZED);
1335       __ jcc(Assembler::zero, L);
1336       __ stop("method needs synchronization");
1337       __ bind(L);
1338     }
1339 #endif
1340   }
1341 
1342   // start execution
1343 #ifdef ASSERT
1344   {
1345     Label L;
1346      const Address monitor_block_top (rbp,
1347                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
1348     __ movptr(rax, monitor_block_top);
1349     __ cmpptr(rax, rsp);
1350     __ jcc(Assembler::equal, L);
1351     __ stop("broken stack frame setup in interpreter");
1352     __ bind(L);
1353   }
1354 #endif
1355 
1356   // jvmti support
1357   __ notify_method_entry();
1358 
1359   __ dispatch_next(vtos);
1360 
1361   // invocation counter overflow
1362   if (inc_counter) {
1363     if (ProfileInterpreter) {
1364       // We have decided to profile this method in the interpreter
1365       __ bind(profile_method);
1366 
1367       __ call_VM(noreg,
1368                  CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method),
1369                  r13, true);
1370 
1371       __ movptr(rbx, Address(rbp, method_offset)); // restore methodOop
1372       __ movptr(rax, Address(rbx,
1373                              in_bytes(methodOopDesc::method_data_offset())));
1374       __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize),
1375                 rax);
1376       __ test_method_data_pointer(rax, profile_method_continue);
1377       __ addptr(rax, in_bytes(methodDataOopDesc::data_offset()));
1378       __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize),
1379               rax);
1380       __ jmp(profile_method_continue);
1381     }
1382     // Handle overflow of counter and compile method
1383     __ bind(invocation_counter_overflow);
1384     generate_counter_overflow(&continue_after_compile);
1385   }
1386 
1387   return entry_point;
1388 }
1389 
1390 // Entry points
1391 //
1392 // Here we generate the various kind of entries into the interpreter.
1393 // The two main entry type are generic bytecode methods and native
1394 // call method.  These both come in synchronized and non-synchronized
1395 // versions but the frame layout they create is very similar. The
1396 // other method entry types are really just special purpose entries
1397 // that are really entry and interpretation all in one. These are for
1398 // trivial methods like accessor, empty, or special math methods.
1399 //
1400 // When control flow reaches any of the entry types for the interpreter
1401 // the following holds ->
1402 //
1403 // Arguments:
1404 //
1405 // rbx: methodOop
1406 //
1407 // Stack layout immediately at entry
1408 //
1409 // [ return address     ] <--- rsp
1410 // [ parameter n        ]
1411 //   ...
1412 // [ parameter 1        ]
1413 // [ expression stack   ] (caller's java expression stack)
1414 
1415 // Assuming that we don't go to one of the trivial specialized entries
1416 // the stack will look like below when we are ready to execute the
1417 // first bytecode (or call the native routine). The register usage
1418 // will be as the template based interpreter expects (see
1419 // interpreter_amd64.hpp).
1420 //
1421 // local variables follow incoming parameters immediately; i.e.
1422 // the return address is moved to the end of the locals).
1423 //
1424 // [ monitor entry      ] <--- rsp
1425 //   ...
1426 // [ monitor entry      ]
1427 // [ expr. stack bottom ]
1428 // [ saved r13          ]
1429 // [ current r14        ]
1430 // [ methodOop          ]
1431 // [ saved ebp          ] <--- rbp
1432 // [ return address     ]
1433 // [ local variable m   ]
1434 //   ...
1435 // [ local variable 1   ]
1436 // [ parameter n        ]
1437 //   ...
1438 // [ parameter 1        ] <--- r14
1439 
1440 address AbstractInterpreterGenerator::generate_method_entry(
1441                                         AbstractInterpreter::MethodKind kind) {
1442   // determine code generation flags
1443   bool synchronized = false;
1444   address entry_point = NULL;
1445 
1446   switch (kind) {
1447   case Interpreter::zerolocals             :                                                                             break;
1448   case Interpreter::zerolocals_synchronized: synchronized = true;                                                        break;
1449   case Interpreter::native                 : entry_point = ((InterpreterGenerator*) this)->generate_native_entry(false); break;
1450   case Interpreter::native_synchronized    : entry_point = ((InterpreterGenerator*) this)->generate_native_entry(true);  break;
1451   case Interpreter::empty                  : entry_point = ((InterpreterGenerator*) this)->generate_empty_entry();       break;
1452   case Interpreter::accessor               : entry_point = ((InterpreterGenerator*) this)->generate_accessor_entry();    break;
1453   case Interpreter::abstract               : entry_point = ((InterpreterGenerator*) this)->generate_abstract_entry();    break;
1454   case Interpreter::method_handle          : entry_point = ((InterpreterGenerator*) this)->generate_method_handle_entry();break;
1455 
1456   case Interpreter::java_lang_math_sin     : // fall thru
1457   case Interpreter::java_lang_math_cos     : // fall thru
1458   case Interpreter::java_lang_math_tan     : // fall thru
1459   case Interpreter::java_lang_math_abs     : // fall thru
1460   case Interpreter::java_lang_math_log     : // fall thru
1461   case Interpreter::java_lang_math_log10   : // fall thru
1462   case Interpreter::java_lang_math_sqrt    : entry_point = ((InterpreterGenerator*) this)->generate_math_entry(kind);    break;
1463   default                                  : ShouldNotReachHere();                                                       break;
1464   }
1465 
1466   if (entry_point) {
1467     return entry_point;
1468   }
1469 
1470   return ((InterpreterGenerator*) this)->
1471                                 generate_normal_entry(synchronized);
1472 }
1473 
1474 // These should never be compiled since the interpreter will prefer
1475 // the compiled version to the intrinsic version.
1476 bool AbstractInterpreter::can_be_compiled(methodHandle m) {
1477   switch (method_kind(m)) {
1478     case Interpreter::java_lang_math_sin     : // fall thru
1479     case Interpreter::java_lang_math_cos     : // fall thru
1480     case Interpreter::java_lang_math_tan     : // fall thru
1481     case Interpreter::java_lang_math_abs     : // fall thru
1482     case Interpreter::java_lang_math_log     : // fall thru
1483     case Interpreter::java_lang_math_log10   : // fall thru
1484     case Interpreter::java_lang_math_sqrt    :
1485       return false;
1486     default:
1487       return true;
1488   }
1489 }
1490 
1491 // How much stack a method activation needs in words.
1492 int AbstractInterpreter::size_top_interpreter_activation(methodOop method) {
1493   const int entry_size = frame::interpreter_frame_monitor_size();
1494 
1495   // total overhead size: entry_size + (saved rbp thru expr stack
1496   // bottom).  be sure to change this if you add/subtract anything
1497   // to/from the overhead area
1498   const int overhead_size =
1499     -(frame::interpreter_frame_initial_sp_offset) + entry_size;
1500 
1501   const int stub_code = frame::entry_frame_after_call_words;
1502   const int extra_stack = methodOopDesc::extra_stack_entries();
1503   const int method_stack = (method->max_locals() + method->max_stack() + extra_stack) *
1504                            Interpreter::stackElementWords;
1505   return (overhead_size + method_stack + stub_code);
1506 }
1507 
1508 int AbstractInterpreter::layout_activation(methodOop method,
1509                                            int tempcount,
1510                                            int popframe_extra_args,
1511                                            int moncount,
1512                                            int callee_param_count,
1513                                            int callee_locals,
1514                                            frame* caller,
1515                                            frame* interpreter_frame,
1516                                            bool is_top_frame) {
1517   // Note: This calculation must exactly parallel the frame setup
1518   // in AbstractInterpreterGenerator::generate_method_entry.
1519   // If interpreter_frame!=NULL, set up the method, locals, and monitors.
1520   // The frame interpreter_frame, if not NULL, is guaranteed to be the
1521   // right size, as determined by a previous call to this method.
1522   // It is also guaranteed to be walkable even though it is in a skeletal state
1523 
1524   // fixed size of an interpreter frame:
1525   int max_locals = method->max_locals() * Interpreter::stackElementWords;
1526   int extra_locals = (method->max_locals() - method->size_of_parameters()) *
1527                      Interpreter::stackElementWords;
1528 
1529   int overhead = frame::sender_sp_offset -
1530                  frame::interpreter_frame_initial_sp_offset;
1531   // Our locals were accounted for by the caller (or last_frame_adjust
1532   // on the transistion) Since the callee parameters already account
1533   // for the callee's params we only need to account for the extra
1534   // locals.
1535   int size = overhead +
1536          (callee_locals - callee_param_count)*Interpreter::stackElementWords +
1537          moncount * frame::interpreter_frame_monitor_size() +
1538          tempcount* Interpreter::stackElementWords + popframe_extra_args;
1539   if (interpreter_frame != NULL) {
1540 #ifdef ASSERT
1541     if (!EnableMethodHandles)
1542       // @@@ FIXME: Should we correct interpreter_frame_sender_sp in the calling sequences?
1543       // Probably, since deoptimization doesn't work yet.
1544       assert(caller->unextended_sp() == interpreter_frame->interpreter_frame_sender_sp(), "Frame not properly walkable");
1545     assert(caller->sp() == interpreter_frame->sender_sp(), "Frame not properly walkable(2)");
1546 #endif
1547 
1548     interpreter_frame->interpreter_frame_set_method(method);
1549     // NOTE the difference in using sender_sp and
1550     // interpreter_frame_sender_sp interpreter_frame_sender_sp is
1551     // the original sp of the caller (the unextended_sp) and
1552     // sender_sp is fp+16 XXX
1553     intptr_t* locals = interpreter_frame->sender_sp() + max_locals - 1;
1554 
1555     interpreter_frame->interpreter_frame_set_locals(locals);
1556     BasicObjectLock* montop = interpreter_frame->interpreter_frame_monitor_begin();
1557     BasicObjectLock* monbot = montop - moncount;
1558     interpreter_frame->interpreter_frame_set_monitor_end(monbot);
1559 
1560     // Set last_sp
1561     intptr_t*  esp = (intptr_t*) monbot -
1562                      tempcount*Interpreter::stackElementWords -
1563                      popframe_extra_args;
1564     interpreter_frame->interpreter_frame_set_last_sp(esp);
1565 
1566     // All frames but the initial (oldest) interpreter frame we fill in have
1567     // a value for sender_sp that allows walking the stack but isn't
1568     // truly correct. Correct the value here.
1569     if (extra_locals != 0 &&
1570         interpreter_frame->sender_sp() ==
1571         interpreter_frame->interpreter_frame_sender_sp()) {
1572       interpreter_frame->set_interpreter_frame_sender_sp(caller->sp() +
1573                                                          extra_locals);
1574     }
1575     *interpreter_frame->interpreter_frame_cache_addr() =
1576       method->constants()->cache();
1577   }
1578   return size;
1579 }
1580 
1581 //-----------------------------------------------------------------------------
1582 // Exceptions
1583 
1584 void TemplateInterpreterGenerator::generate_throw_exception() {
1585   // Entry point in previous activation (i.e., if the caller was
1586   // interpreted)
1587   Interpreter::_rethrow_exception_entry = __ pc();
1588   // Restore sp to interpreter_frame_last_sp even though we are going
1589   // to empty the expression stack for the exception processing.
1590   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
1591   // rax: exception
1592   // rdx: return address/pc that threw exception
1593   __ restore_bcp();    // r13 points to call/send
1594   __ restore_locals();
1595   __ reinit_heapbase();  // restore r12 as heapbase.
1596   // Entry point for exceptions thrown within interpreter code
1597   Interpreter::_throw_exception_entry = __ pc();
1598   // expression stack is undefined here
1599   // rax: exception
1600   // r13: exception bcp
1601   __ verify_oop(rax);
1602   __ mov(c_rarg1, rax);
1603 
1604   // expression stack must be empty before entering the VM in case of
1605   // an exception
1606   __ empty_expression_stack();
1607   // find exception handler address and preserve exception oop
1608   __ call_VM(rdx,
1609              CAST_FROM_FN_PTR(address,
1610                           InterpreterRuntime::exception_handler_for_exception),
1611              c_rarg1);
1612   // rax: exception handler entry point
1613   // rdx: preserved exception oop
1614   // r13: bcp for exception handler
1615   __ push_ptr(rdx); // push exception which is now the only value on the stack
1616   __ jmp(rax); // jump to exception handler (may be _remove_activation_entry!)
1617 
1618   // If the exception is not handled in the current frame the frame is
1619   // removed and the exception is rethrown (i.e. exception
1620   // continuation is _rethrow_exception).
1621   //
1622   // Note: At this point the bci is still the bxi for the instruction
1623   // which caused the exception and the expression stack is
1624   // empty. Thus, for any VM calls at this point, GC will find a legal
1625   // oop map (with empty expression stack).
1626 
1627   // In current activation
1628   // tos: exception
1629   // esi: exception bcp
1630 
1631   //
1632   // JVMTI PopFrame support
1633   //
1634 
1635   Interpreter::_remove_activation_preserving_args_entry = __ pc();
1636   __ empty_expression_stack();
1637   // Set the popframe_processing bit in pending_popframe_condition
1638   // indicating that we are currently handling popframe, so that
1639   // call_VMs that may happen later do not trigger new popframe
1640   // handling cycles.
1641   __ movl(rdx, Address(r15_thread, JavaThread::popframe_condition_offset()));
1642   __ orl(rdx, JavaThread::popframe_processing_bit);
1643   __ movl(Address(r15_thread, JavaThread::popframe_condition_offset()), rdx);
1644 
1645   {
1646     // Check to see whether we are returning to a deoptimized frame.
1647     // (The PopFrame call ensures that the caller of the popped frame is
1648     // either interpreted or compiled and deoptimizes it if compiled.)
1649     // In this case, we can't call dispatch_next() after the frame is
1650     // popped, but instead must save the incoming arguments and restore
1651     // them after deoptimization has occurred.
1652     //
1653     // Note that we don't compare the return PC against the
1654     // deoptimization blob's unpack entry because of the presence of
1655     // adapter frames in C2.
1656     Label caller_not_deoptimized;
1657     __ movptr(c_rarg1, Address(rbp, frame::return_addr_offset * wordSize));
1658     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1659                                InterpreterRuntime::interpreter_contains), c_rarg1);
1660     __ testl(rax, rax);
1661     __ jcc(Assembler::notZero, caller_not_deoptimized);
1662 
1663     // Compute size of arguments for saving when returning to
1664     // deoptimized caller
1665     __ get_method(rax);
1666     __ load_unsigned_short(rax, Address(rax, in_bytes(methodOopDesc::
1667                                                 size_of_parameters_offset())));
1668     __ shll(rax, Interpreter::logStackElementSize);
1669     __ restore_locals(); // XXX do we need this?
1670     __ subptr(r14, rax);
1671     __ addptr(r14, wordSize);
1672     // Save these arguments
1673     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1674                                            Deoptimization::
1675                                            popframe_preserve_args),
1676                           r15_thread, rax, r14);
1677 
1678     __ remove_activation(vtos, rdx,
1679                          /* throw_monitor_exception */ false,
1680                          /* install_monitor_exception */ false,
1681                          /* notify_jvmdi */ false);
1682 
1683     // Inform deoptimization that it is responsible for restoring
1684     // these arguments
1685     __ movl(Address(r15_thread, JavaThread::popframe_condition_offset()),
1686             JavaThread::popframe_force_deopt_reexecution_bit);
1687 
1688     // Continue in deoptimization handler
1689     __ jmp(rdx);
1690 
1691     __ bind(caller_not_deoptimized);
1692   }
1693 
1694   __ remove_activation(vtos, rdx, /* rdx result (retaddr) is not used */
1695                        /* throw_monitor_exception */ false,
1696                        /* install_monitor_exception */ false,
1697                        /* notify_jvmdi */ false);
1698 
1699   // Finish with popframe handling
1700   // A previous I2C followed by a deoptimization might have moved the
1701   // outgoing arguments further up the stack. PopFrame expects the
1702   // mutations to those outgoing arguments to be preserved and other
1703   // constraints basically require this frame to look exactly as
1704   // though it had previously invoked an interpreted activation with
1705   // no space between the top of the expression stack (current
1706   // last_sp) and the top of stack. Rather than force deopt to
1707   // maintain this kind of invariant all the time we call a small
1708   // fixup routine to move the mutated arguments onto the top of our
1709   // expression stack if necessary.
1710   __ mov(c_rarg1, rsp);
1711   __ movptr(c_rarg2, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1712   // PC must point into interpreter here
1713   __ set_last_Java_frame(noreg, rbp, __ pc());
1714   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), r15_thread, c_rarg1, c_rarg2);
1715   __ reset_last_Java_frame(true, true);
1716   // Restore the last_sp and null it out
1717   __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1718   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
1719 
1720   __ restore_bcp();  // XXX do we need this?
1721   __ restore_locals(); // XXX do we need this?
1722   // The method data pointer was incremented already during
1723   // call profiling. We have to restore the mdp for the current bcp.
1724   if (ProfileInterpreter) {
1725     __ set_method_data_pointer_for_bcp();
1726   }
1727 
1728   // Clear the popframe condition flag
1729   __ movl(Address(r15_thread, JavaThread::popframe_condition_offset()),
1730           JavaThread::popframe_inactive);
1731 
1732   __ dispatch_next(vtos);
1733   // end of PopFrame support
1734 
1735   Interpreter::_remove_activation_entry = __ pc();
1736 
1737   // preserve exception over this code sequence
1738   __ pop_ptr(rax);
1739   __ movptr(Address(r15_thread, JavaThread::vm_result_offset()), rax);
1740   // remove the activation (without doing throws on illegalMonitorExceptions)
1741   __ remove_activation(vtos, rdx, false, true, false);
1742   // restore exception
1743   __ movptr(rax, Address(r15_thread, JavaThread::vm_result_offset()));
1744   __ movptr(Address(r15_thread, JavaThread::vm_result_offset()), (int32_t)NULL_WORD);
1745   __ verify_oop(rax);
1746 
1747   // In between activations - previous activation type unknown yet
1748   // compute continuation point - the continuation point expects the
1749   // following registers set up:
1750   //
1751   // rax: exception
1752   // rdx: return address/pc that threw exception
1753   // rsp: expression stack of caller
1754   // rbp: ebp of caller
1755   __ push(rax);                                  // save exception
1756   __ push(rdx);                                  // save return address
1757   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1758                           SharedRuntime::exception_handler_for_return_address),
1759                         r15_thread, rdx);
1760   __ mov(rbx, rax);                              // save exception handler
1761   __ pop(rdx);                                   // restore return address
1762   __ pop(rax);                                   // restore exception
1763   // Note that an "issuing PC" is actually the next PC after the call
1764   __ jmp(rbx);                                   // jump to exception
1765                                                  // handler of caller
1766 }
1767 
1768 
1769 //
1770 // JVMTI ForceEarlyReturn support
1771 //
1772 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1773   address entry = __ pc();
1774 
1775   __ restore_bcp();
1776   __ restore_locals();
1777   __ empty_expression_stack();
1778   __ load_earlyret_value(state);
1779 
1780   __ movptr(rdx, Address(r15_thread, JavaThread::jvmti_thread_state_offset()));
1781   Address cond_addr(rdx, JvmtiThreadState::earlyret_state_offset());
1782 
1783   // Clear the earlyret state
1784   __ movl(cond_addr, JvmtiThreadState::earlyret_inactive);
1785 
1786   __ remove_activation(state, rsi,
1787                        false, /* throw_monitor_exception */
1788                        false, /* install_monitor_exception */
1789                        true); /* notify_jvmdi */
1790   __ jmp(rsi);
1791 
1792   return entry;
1793 } // end of ForceEarlyReturn support
1794 
1795 
1796 //-----------------------------------------------------------------------------
1797 // Helper for vtos entry point generation
1798 
1799 void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t,
1800                                                          address& bep,
1801                                                          address& cep,
1802                                                          address& sep,
1803                                                          address& aep,
1804                                                          address& iep,
1805                                                          address& lep,
1806                                                          address& fep,
1807                                                          address& dep,
1808                                                          address& vep) {
1809   assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1810   Label L;
1811   aep = __ pc();  __ push_ptr();  __ jmp(L);
1812   fep = __ pc();  __ push_f();    __ jmp(L);
1813   dep = __ pc();  __ push_d();    __ jmp(L);
1814   lep = __ pc();  __ push_l();    __ jmp(L);
1815   bep = cep = sep =
1816   iep = __ pc();  __ push_i();
1817   vep = __ pc();
1818   __ bind(L);
1819   generate_and_dispatch(t);
1820 }
1821 
1822 
1823 //-----------------------------------------------------------------------------
1824 // Generation of individual instructions
1825 
1826 // helpers for generate_and_dispatch
1827 
1828 
1829 InterpreterGenerator::InterpreterGenerator(StubQueue* code)
1830   : TemplateInterpreterGenerator(code) {
1831    generate_all(); // down here so it can be "virtual"
1832 }
1833 
1834 //-----------------------------------------------------------------------------
1835 
1836 // Non-product code
1837 #ifndef PRODUCT
1838 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1839   address entry = __ pc();
1840 
1841   __ push(state);
1842   __ push(c_rarg0);
1843   __ push(c_rarg1);
1844   __ push(c_rarg2);
1845   __ push(c_rarg3);
1846   __ mov(c_rarg2, rax);  // Pass itos
1847 #ifdef _WIN64
1848   __ movflt(xmm3, xmm0); // Pass ftos
1849 #endif
1850   __ call_VM(noreg,
1851              CAST_FROM_FN_PTR(address, SharedRuntime::trace_bytecode),
1852              c_rarg1, c_rarg2, c_rarg3);
1853   __ pop(c_rarg3);
1854   __ pop(c_rarg2);
1855   __ pop(c_rarg1);
1856   __ pop(c_rarg0);
1857   __ pop(state);
1858   __ ret(0);                                   // return from result handler
1859 
1860   return entry;
1861 }
1862 
1863 void TemplateInterpreterGenerator::count_bytecode() {
1864   __ incrementl(ExternalAddress((address) &BytecodeCounter::_counter_value));
1865 }
1866 
1867 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
1868   __ incrementl(ExternalAddress((address) &BytecodeHistogram::_counters[t->bytecode()]));
1869 }
1870 
1871 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
1872   __ mov32(rbx, ExternalAddress((address) &BytecodePairHistogram::_index));
1873   __ shrl(rbx, BytecodePairHistogram::log2_number_of_codes);
1874   __ orl(rbx,
1875          ((int) t->bytecode()) <<
1876          BytecodePairHistogram::log2_number_of_codes);
1877   __ mov32(ExternalAddress((address) &BytecodePairHistogram::_index), rbx);
1878   __ lea(rscratch1, ExternalAddress((address) BytecodePairHistogram::_counters));
1879   __ incrementl(Address(rscratch1, rbx, Address::times_4));
1880 }
1881 
1882 
1883 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
1884   // Call a little run-time stub to avoid blow-up for each bytecode.
1885   // The run-time runtime saves the right registers, depending on
1886   // the tosca in-state for the given template.
1887 
1888   assert(Interpreter::trace_code(t->tos_in()) != NULL,
1889          "entry must have been generated");
1890   __ mov(r12, rsp); // remember sp
1891   __ andptr(rsp, -16); // align stack as required by ABI
1892   __ call(RuntimeAddress(Interpreter::trace_code(t->tos_in())));
1893   __ mov(rsp, r12); // restore sp
1894   __ reinit_heapbase();
1895 }
1896 
1897 
1898 void TemplateInterpreterGenerator::stop_interpreter_at() {
1899   Label L;
1900   __ cmp32(ExternalAddress((address) &BytecodeCounter::_counter_value),
1901            StopInterpreterAt);
1902   __ jcc(Assembler::notEqual, L);
1903   __ int3();
1904   __ bind(L);
1905 }
1906 #endif // !PRODUCT
1907 #endif // ! CC_INTERP