1 /*
   2  * Copyright (c) 1997, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "interpreter/bytecodeHistogram.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "interpreter/interpreterGenerator.hpp"
  30 #include "interpreter/interpreterRuntime.hpp"
  31 #include "interpreter/interp_masm.hpp"
  32 #include "interpreter/templateTable.hpp"
  33 #include "oops/arrayOop.hpp"
  34 #include "oops/methodData.hpp"
  35 #include "oops/method.hpp"
  36 #include "oops/oop.inline.hpp"
  37 #include "prims/jvmtiExport.hpp"
  38 #include "prims/jvmtiThreadState.hpp"
  39 #include "runtime/arguments.hpp"
  40 #include "runtime/deoptimization.hpp"
  41 #include "runtime/frame.inline.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "runtime/stubRoutines.hpp"
  44 #include "runtime/synchronizer.hpp"
  45 #include "runtime/timer.hpp"
  46 #include "runtime/vframeArray.hpp"
  47 #include "utilities/debug.hpp"
  48 #include "utilities/macros.hpp"
  49 
  50 #define __ _masm->
  51 
  52 
  53 #ifndef CC_INTERP
  54 const int method_offset = frame::interpreter_frame_method_offset * wordSize;
  55 const int bcp_offset    = frame::interpreter_frame_bcp_offset    * wordSize;
  56 const int locals_offset = frame::interpreter_frame_locals_offset * wordSize;
  57 
  58 //------------------------------------------------------------------------------------------------------------------------
  59 
  60 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
  61   address entry = __ pc();
  62 
  63   // Note: There should be a minimal interpreter frame set up when stack
  64   // overflow occurs since we check explicitly for it now.
  65   //
  66 #ifdef ASSERT
  67   { Label L;
  68     __ lea(rax, Address(rbp,
  69                 frame::interpreter_frame_monitor_block_top_offset * wordSize));
  70     __ cmpptr(rax, rsp);  // rax, = maximal rsp for current rbp,
  71                         //  (stack grows negative)
  72     __ jcc(Assembler::aboveEqual, L); // check if frame is complete
  73     __ stop ("interpreter frame not set up");
  74     __ bind(L);
  75   }
  76 #endif // ASSERT
  77   // Restore bcp under the assumption that the current frame is still
  78   // interpreted
  79   __ restore_bcp();
  80 
  81   // expression stack must be empty before entering the VM if an exception
  82   // happened
  83   __ empty_expression_stack();
  84   __ empty_FPU_stack();
  85   // throw exception
  86   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_StackOverflowError));
  87   return entry;
  88 }
  89 
  90 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler(const char* name) {
  91   address entry = __ pc();
  92   // expression stack must be empty before entering the VM if an exception happened
  93   __ empty_expression_stack();
  94   __ empty_FPU_stack();
  95   // setup parameters
  96   // ??? convention: expect aberrant index in register rbx,
  97   __ lea(rax, ExternalAddress((address)name));
  98   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_ArrayIndexOutOfBoundsException), rax, rbx);
  99   return entry;
 100 }
 101 
 102 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
 103   address entry = __ pc();
 104   // object is at TOS
 105   __ pop(rax);
 106   // expression stack must be empty before entering the VM if an exception
 107   // happened
 108   __ empty_expression_stack();
 109   __ empty_FPU_stack();
 110   __ call_VM(noreg,
 111              CAST_FROM_FN_PTR(address,
 112                               InterpreterRuntime::throw_ClassCastException),
 113              rax);
 114   return entry;
 115 }
 116 
 117 address TemplateInterpreterGenerator::generate_exception_handler_common(const char* name, const char* message, bool pass_oop) {
 118   assert(!pass_oop || message == NULL, "either oop or message but not both");
 119   address entry = __ pc();
 120   if (pass_oop) {
 121     // object is at TOS
 122     __ pop(rbx);
 123   }
 124   // expression stack must be empty before entering the VM if an exception happened
 125   __ empty_expression_stack();
 126   __ empty_FPU_stack();
 127   // setup parameters
 128   __ lea(rax, ExternalAddress((address)name));
 129   if (pass_oop) {
 130     __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_klass_exception), rax, rbx);
 131   } else {
 132     if (message != NULL) {
 133       __ lea(rbx, ExternalAddress((address)message));
 134     } else {
 135       __ movptr(rbx, NULL_WORD);
 136     }
 137     __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception), rax, rbx);
 138   }
 139   // throw exception
 140   __ jump(ExternalAddress(Interpreter::throw_exception_entry()));
 141   return entry;
 142 }
 143 
 144 
 145 address TemplateInterpreterGenerator::generate_continuation_for(TosState state) {
 146   address entry = __ pc();
 147   // NULL last_sp until next java call
 148   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
 149   __ dispatch_next(state);
 150   return entry;
 151 }
 152 
 153 
 154 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {
 155   address entry = __ pc();
 156 
 157 #ifdef COMPILER2
 158   // The FPU stack is clean if UseSSE >= 2 but must be cleaned in other cases
 159   if ((state == ftos && UseSSE < 1) || (state == dtos && UseSSE < 2)) {
 160     for (int i = 1; i < 8; i++) {
 161         __ ffree(i);
 162     }
 163   } else if (UseSSE < 2) {
 164     __ empty_FPU_stack();
 165   }
 166 #endif
 167   if ((state == ftos && UseSSE < 1) || (state == dtos && UseSSE < 2)) {
 168     __ MacroAssembler::verify_FPU(1, "generate_return_entry_for compiled");
 169   } else {
 170     __ MacroAssembler::verify_FPU(0, "generate_return_entry_for compiled");
 171   }
 172 
 173   if (state == ftos) {
 174     __ MacroAssembler::verify_FPU(UseSSE >= 1 ? 0 : 1, "generate_return_entry_for in interpreter");
 175   } else if (state == dtos) {
 176     __ MacroAssembler::verify_FPU(UseSSE >= 2 ? 0 : 1, "generate_return_entry_for in interpreter");
 177   }
 178 
 179   // Restore stack bottom in case i2c adjusted stack
 180   __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
 181   // and NULL it as marker that rsp is now tos until next java call
 182   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
 183 
 184   __ restore_bcp();
 185   __ restore_locals();
 186 
 187   if (state == atos) {
 188     Register mdp = rbx;
 189     Register tmp = rcx;
 190     __ profile_return_type(mdp, rax, tmp);
 191   }
 192 
 193   const Register cache = rbx;
 194   const Register index = rcx;
 195   __ get_cache_and_index_at_bcp(cache, index, 1, index_size);
 196 
 197   const Register flags = cache;
 198   __ movl(flags, Address(cache, index, Address::times_ptr, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));
 199   __ andl(flags, ConstantPoolCacheEntry::parameter_size_mask);
 200   __ lea(rsp, Address(rsp, flags, Interpreter::stackElementScale()));
 201   __ dispatch_next(state, step);
 202 
 203   return entry;
 204 }
 205 
 206 
 207 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, int step) {
 208   address entry = __ pc();
 209 
 210   if (state == ftos) {
 211     __ MacroAssembler::verify_FPU(UseSSE >= 1 ? 0 : 1, "generate_deopt_entry_for in interpreter");
 212   } else if (state == dtos) {
 213     __ MacroAssembler::verify_FPU(UseSSE >= 2 ? 0 : 1, "generate_deopt_entry_for in interpreter");
 214   }
 215 
 216   // The stack is not extended by deopt but we must NULL last_sp as this
 217   // entry is like a "return".
 218   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
 219   __ restore_bcp();
 220   __ restore_locals();
 221   // handle exceptions
 222   { Label L;
 223     const Register thread = rcx;
 224     __ get_thread(thread);
 225     __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 226     __ jcc(Assembler::zero, L);
 227     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));
 228     __ should_not_reach_here();
 229     __ bind(L);
 230   }
 231   __ dispatch_next(state, step);
 232   return entry;
 233 }
 234 
 235 
 236 int AbstractInterpreter::BasicType_as_index(BasicType type) {
 237   int i = 0;
 238   switch (type) {
 239     case T_BOOLEAN: i = 0; break;
 240     case T_CHAR   : i = 1; break;
 241     case T_BYTE   : i = 2; break;
 242     case T_SHORT  : i = 3; break;
 243     case T_INT    : // fall through
 244     case T_LONG   : // fall through
 245     case T_VOID   : i = 4; break;
 246     case T_FLOAT  : i = 5; break;  // have to treat float and double separately for SSE
 247     case T_DOUBLE : i = 6; break;
 248     case T_OBJECT : // fall through
 249     case T_ARRAY  : i = 7; break;
 250     default       : ShouldNotReachHere();
 251   }
 252   assert(0 <= i && i < AbstractInterpreter::number_of_result_handlers, "index out of bounds");
 253   return i;
 254 }
 255 
 256 
 257 address TemplateInterpreterGenerator::generate_result_handler_for(BasicType type) {
 258   address entry = __ pc();
 259   switch (type) {
 260     case T_BOOLEAN: __ c2bool(rax);            break;
 261     case T_CHAR   : __ andptr(rax, 0xFFFF);    break;
 262     case T_BYTE   : __ sign_extend_byte (rax); break;
 263     case T_SHORT  : __ sign_extend_short(rax); break;
 264     case T_INT    : /* nothing to do */        break;
 265     case T_LONG   : /* nothing to do */        break;
 266     case T_VOID   : /* nothing to do */        break;
 267     case T_DOUBLE :
 268     case T_FLOAT  :
 269       { const Register t = InterpreterRuntime::SignatureHandlerGenerator::temp();
 270         __ pop(t);                            // remove return address first
 271         // Must return a result for interpreter or compiler. In SSE
 272         // mode, results are returned in xmm0 and the FPU stack must
 273         // be empty.
 274         if (type == T_FLOAT && UseSSE >= 1) {
 275           // Load ST0
 276           __ fld_d(Address(rsp, 0));
 277           // Store as float and empty fpu stack
 278           __ fstp_s(Address(rsp, 0));
 279           // and reload
 280           __ movflt(xmm0, Address(rsp, 0));
 281         } else if (type == T_DOUBLE && UseSSE >= 2 ) {
 282           __ movdbl(xmm0, Address(rsp, 0));
 283         } else {
 284           // restore ST0
 285           __ fld_d(Address(rsp, 0));
 286         }
 287         // and pop the temp
 288         __ addptr(rsp, 2 * wordSize);
 289         __ push(t);                           // restore return address
 290       }
 291       break;
 292     case T_OBJECT :
 293       // retrieve result from frame
 294       __ movptr(rax, Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize));
 295       // and verify it
 296       __ verify_oop(rax);
 297       break;
 298     default       : ShouldNotReachHere();
 299   }
 300   __ ret(0);                                   // return from result handler
 301   return entry;
 302 }
 303 
 304 address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, address runtime_entry) {
 305   address entry = __ pc();
 306   __ push(state);
 307   __ call_VM(noreg, runtime_entry);
 308   __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));
 309   return entry;
 310 }
 311 
 312 
 313 // Helpers for commoning out cases in the various type of method entries.
 314 //
 315 
 316 // increment invocation count & check for overflow
 317 //
 318 // Note: checking for negative value instead of overflow
 319 //       so we have a 'sticky' overflow test
 320 //
 321 // rbx,: method
 322 // rcx: invocation counter
 323 //
 324 void InterpreterGenerator::generate_counter_incr(Label* overflow, Label* profile_method, Label* profile_method_continue) {
 325   Label done;
 326   // Note: In tiered we increment either counters in MethodCounters* or in MDO
 327   // depending if we're profiling or not.
 328   if (TieredCompilation) {
 329     int increment = InvocationCounter::count_increment;
 330     Label no_mdo;
 331     if (ProfileInterpreter) {
 332       // Are we profiling?
 333       __ movptr(rax, Address(rbx, Method::method_data_offset()));
 334       __ testptr(rax, rax);
 335       __ jccb(Assembler::zero, no_mdo);
 336       // Increment counter in the MDO
 337       const Address mdo_invocation_counter(rax, in_bytes(MethodData::invocation_counter_offset()) +
 338                                                 in_bytes(InvocationCounter::counter_offset()));
 339       const Address mask(rax, in_bytes(MethodData::invoke_mask_offset()));
 340       __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, rcx, false, Assembler::zero, overflow);
 341       __ jmp(done);
 342     }
 343     __ bind(no_mdo);
 344     // Increment counter in MethodCounters
 345     const Address invocation_counter(rax,
 346                   MethodCounters::invocation_counter_offset() +
 347                   InvocationCounter::counter_offset());
 348 
 349     __ get_method_counters(rbx, rax, done);
 350     const Address mask(rax, in_bytes(MethodCounters::invoke_mask_offset()));
 351     __ increment_mask_and_jump(invocation_counter, increment, mask,
 352                                rcx, false, Assembler::zero, overflow);
 353     __ bind(done);
 354   } else { // not TieredCompilation
 355     const Address backedge_counter(rax,
 356                   MethodCounters::backedge_counter_offset() +
 357                   InvocationCounter::counter_offset());
 358     const Address invocation_counter(rax,
 359                   MethodCounters::invocation_counter_offset() +
 360                   InvocationCounter::counter_offset());
 361 
 362     __ get_method_counters(rbx, rax, done);
 363 
 364     if (ProfileInterpreter) {
 365       __ incrementl(Address(rax,
 366               MethodCounters::interpreter_invocation_counter_offset()));
 367     }
 368 
 369     // Update standard invocation counters
 370     __ movl(rcx, invocation_counter);
 371     __ incrementl(rcx, InvocationCounter::count_increment);
 372     __ movl(invocation_counter, rcx);             // save invocation count
 373 
 374     __ movl(rax, backedge_counter);               // load backedge counter
 375     __ andl(rax, InvocationCounter::count_mask_value);  // mask out the status bits
 376 
 377     __ addl(rcx, rax);                            // add both counters
 378 
 379     // profile_method is non-null only for interpreted method so
 380     // profile_method != NULL == !native_call
 381     // BytecodeInterpreter only calls for native so code is elided.
 382 
 383     if (ProfileInterpreter && profile_method != NULL) {
 384       // Test to see if we should create a method data oop
 385       __ movptr(rax, Address(rbx, Method::method_counters_offset()));
 386       __ cmp32(rcx, Address(rax, in_bytes(MethodCounters::interpreter_profile_limit_offset())));
 387       __ jcc(Assembler::less, *profile_method_continue);
 388 
 389       // if no method data exists, go to profile_method
 390       __ test_method_data_pointer(rax, *profile_method);
 391     }
 392 
 393     __ movptr(rax, Address(rbx, Method::method_counters_offset()));
 394     __ cmp32(rcx, Address(rax, in_bytes(MethodCounters::interpreter_invocation_limit_offset())));
 395     __ jcc(Assembler::aboveEqual, *overflow);
 396     __ bind(done);
 397   }
 398 }
 399 
 400 void InterpreterGenerator::generate_counter_overflow(Label* do_continue) {
 401 
 402   // Asm interpreter on entry
 403   // rdi - locals
 404   // rsi - bcp
 405   // rbx, - method
 406   // rdx - cpool
 407   // rbp, - interpreter frame
 408 
 409   // C++ interpreter on entry
 410   // rsi - new interpreter state pointer
 411   // rbp - interpreter frame pointer
 412   // rbx - method
 413 
 414   // On return (i.e. jump to entry_point) [ back to invocation of interpreter ]
 415   // rbx, - method
 416   // rcx - rcvr (assuming there is one)
 417   // top of stack return address of interpreter caller
 418   // rsp - sender_sp
 419 
 420   // C++ interpreter only
 421   // rsi - previous interpreter state pointer
 422 
 423   // InterpreterRuntime::frequency_counter_overflow takes one argument
 424   // indicating if the counter overflow occurs at a backwards branch (non-NULL bcp).
 425   // The call returns the address of the verified entry point for the method or NULL
 426   // if the compilation did not complete (either went background or bailed out).
 427   __ movptr(rax, (intptr_t)false);
 428   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), rax);
 429 
 430   __ movptr(rbx, Address(rbp, method_offset));   // restore Method*
 431 
 432   // Preserve invariant that rsi/rdi contain bcp/locals of sender frame
 433   // and jump to the interpreted entry.
 434   __ jmp(*do_continue, relocInfo::none);
 435 
 436 }
 437 
 438 void InterpreterGenerator::generate_stack_overflow_check(void) {
 439   // see if we've got enough room on the stack for locals plus overhead.
 440   // the expression stack grows down incrementally, so the normal guard
 441   // page mechanism will work for that.
 442   //
 443   // Registers live on entry:
 444   //
 445   // Asm interpreter
 446   // rdx: number of additional locals this frame needs (what we must check)
 447   // rbx,: Method*
 448 
 449   // destroyed on exit
 450   // rax,
 451 
 452   // NOTE:  since the additional locals are also always pushed (wasn't obvious in
 453   // generate_fixed_frame) so the guard should work for them too.
 454   //
 455 
 456   // monitor entry size: see picture of stack in frame_x86.hpp
 457   const int entry_size    = frame::interpreter_frame_monitor_size() * wordSize;
 458 
 459   // total overhead size: entry_size + (saved rbp, thru expr stack bottom).
 460   // be sure to change this if you add/subtract anything to/from the overhead area
 461   const int overhead_size = -(frame::interpreter_frame_initial_sp_offset*wordSize) + entry_size;
 462 
 463   const int page_size = os::vm_page_size();
 464 
 465   Label after_frame_check;
 466 
 467   // see if the frame is greater than one page in size. If so,
 468   // then we need to verify there is enough stack space remaining
 469   // for the additional locals.
 470   __ cmpl(rdx, (page_size - overhead_size)/Interpreter::stackElementSize);
 471   __ jcc(Assembler::belowEqual, after_frame_check);
 472 
 473   // compute rsp as if this were going to be the last frame on
 474   // the stack before the red zone
 475 
 476   Label after_frame_check_pop;
 477 
 478   __ push(rsi);
 479 
 480   const Register thread = rsi;
 481 
 482   __ get_thread(thread);
 483 
 484   const Address stack_base(thread, Thread::stack_base_offset());
 485   const Address stack_size(thread, Thread::stack_size_offset());
 486 
 487   // locals + overhead, in bytes
 488   __ lea(rax, Address(noreg, rdx, Interpreter::stackElementScale(), overhead_size));
 489 
 490 #ifdef ASSERT
 491   Label stack_base_okay, stack_size_okay;
 492   // verify that thread stack base is non-zero
 493   __ cmpptr(stack_base, (int32_t)NULL_WORD);
 494   __ jcc(Assembler::notEqual, stack_base_okay);
 495   __ stop("stack base is zero");
 496   __ bind(stack_base_okay);
 497   // verify that thread stack size is non-zero
 498   __ cmpptr(stack_size, 0);
 499   __ jcc(Assembler::notEqual, stack_size_okay);
 500   __ stop("stack size is zero");
 501   __ bind(stack_size_okay);
 502 #endif
 503 
 504   // Add stack base to locals and subtract stack size
 505   __ addptr(rax, stack_base);
 506   __ subptr(rax, stack_size);
 507 
 508   // Use the maximum number of pages we might bang.
 509   const int max_pages = StackShadowPages > (StackRedPages+StackYellowPages) ? StackShadowPages :
 510                                                                               (StackRedPages+StackYellowPages);
 511   __ addptr(rax, max_pages * page_size);
 512 
 513   // check against the current stack bottom
 514   __ cmpptr(rsp, rax);
 515   __ jcc(Assembler::above, after_frame_check_pop);
 516 
 517   __ pop(rsi);  // get saved bcp / (c++ prev state ).
 518 
 519   // Restore sender's sp as SP. This is necessary if the sender's
 520   // frame is an extended compiled frame (see gen_c2i_adapter())
 521   // and safer anyway in case of JSR292 adaptations.
 522 
 523   __ pop(rax); // return address must be moved if SP is changed
 524   __ mov(rsp, rsi);
 525   __ push(rax);
 526 
 527   // Note: the restored frame is not necessarily interpreted.
 528   // Use the shared runtime version of the StackOverflowError.
 529   assert(StubRoutines::throw_StackOverflowError_entry() != NULL, "stub not yet generated");
 530   __ jump(ExternalAddress(StubRoutines::throw_StackOverflowError_entry()));
 531   // all done with frame size check
 532   __ bind(after_frame_check_pop);
 533   __ pop(rsi);
 534 
 535   __ bind(after_frame_check);
 536 }
 537 
 538 // Allocate monitor and lock method (asm interpreter)
 539 // rbx, - Method*
 540 //
 541 void InterpreterGenerator::lock_method(void) {
 542   // synchronize method
 543   const Address access_flags      (rbx, Method::access_flags_offset());
 544   const Address monitor_block_top (rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
 545   const int entry_size            = frame::interpreter_frame_monitor_size() * wordSize;
 546 
 547   #ifdef ASSERT
 548     { Label L;
 549       __ movl(rax, access_flags);
 550       __ testl(rax, JVM_ACC_SYNCHRONIZED);
 551       __ jcc(Assembler::notZero, L);
 552       __ stop("method doesn't need synchronization");
 553       __ bind(L);
 554     }
 555   #endif // ASSERT
 556   // get synchronization object
 557   { Label done;
 558     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 559     __ movl(rax, access_flags);
 560     __ testl(rax, JVM_ACC_STATIC);
 561     __ movptr(rax, Address(rdi, Interpreter::local_offset_in_bytes(0)));  // get receiver (assume this is frequent case)
 562     __ jcc(Assembler::zero, done);
 563     __ movptr(rax, Address(rbx, Method::const_offset()));
 564     __ movptr(rax, Address(rax, ConstMethod::constants_offset()));
 565     __ movptr(rax, Address(rax, ConstantPool::pool_holder_offset_in_bytes()));
 566     __ movptr(rax, Address(rax, mirror_offset));
 567     __ bind(done);
 568   }
 569   // add space for monitor & lock
 570   __ subptr(rsp, entry_size);                                           // add space for a monitor entry
 571   __ movptr(monitor_block_top, rsp);                                    // set new monitor block top
 572   __ movptr(Address(rsp, BasicObjectLock::obj_offset_in_bytes()), rax); // store object
 573   __ mov(rdx, rsp);                                                    // object address
 574   __ lock_object(rdx);
 575 }
 576 
 577 //
 578 // Generate a fixed interpreter frame. This is identical setup for interpreted methods
 579 // and for native methods hence the shared code.
 580 
 581 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
 582   // initialize fixed part of activation frame
 583   __ push(rax);                                       // save return address
 584   __ enter();                                         // save old & set new rbp,
 585 
 586 
 587   __ push(rsi);                                       // set sender sp
 588   __ push((int32_t)NULL_WORD);                        // leave last_sp as null
 589   __ movptr(rsi, Address(rbx,Method::const_offset())); // get ConstMethod*
 590   __ lea(rsi, Address(rsi,ConstMethod::codes_offset())); // get codebase
 591   __ push(rbx);                                      // save Method*
 592   if (ProfileInterpreter) {
 593     Label method_data_continue;
 594     __ movptr(rdx, Address(rbx, in_bytes(Method::method_data_offset())));
 595     __ testptr(rdx, rdx);
 596     __ jcc(Assembler::zero, method_data_continue);
 597     __ addptr(rdx, in_bytes(MethodData::data_offset()));
 598     __ bind(method_data_continue);
 599     __ push(rdx);                                       // set the mdp (method data pointer)
 600   } else {
 601     __ push(0);
 602   }
 603 
 604   __ movptr(rdx, Address(rbx, Method::const_offset()));
 605   __ movptr(rdx, Address(rdx, ConstMethod::constants_offset()));
 606   __ movptr(rdx, Address(rdx, ConstantPool::cache_offset_in_bytes()));
 607   __ push(rdx);                                       // set constant pool cache
 608   __ push(rdi);                                       // set locals pointer
 609   if (native_call) {
 610     __ push(0);                                       // no bcp
 611   } else {
 612     __ push(rsi);                                     // set bcp
 613     }
 614   __ push(0);                                         // reserve word for pointer to expression stack bottom
 615   __ movptr(Address(rsp, 0), rsp);                    // set expression stack bottom
 616 }
 617 
 618 
 619 // Method entry for java.lang.ref.Reference.get.
 620 address InterpreterGenerator::generate_Reference_get_entry(void) {
 621 #if INCLUDE_ALL_GCS
 622   // Code: _aload_0, _getfield, _areturn
 623   // parameter size = 1
 624   //
 625   // The code that gets generated by this routine is split into 2 parts:
 626   //    1. The "intrinsified" code for G1 (or any SATB based GC),
 627   //    2. The slow path - which is an expansion of the regular method entry.
 628   //
 629   // Notes:-
 630   // * In the G1 code we do not check whether we need to block for
 631   //   a safepoint. If G1 is enabled then we must execute the specialized
 632   //   code for Reference.get (except when the Reference object is null)
 633   //   so that we can log the value in the referent field with an SATB
 634   //   update buffer.
 635   //   If the code for the getfield template is modified so that the
 636   //   G1 pre-barrier code is executed when the current method is
 637   //   Reference.get() then going through the normal method entry
 638   //   will be fine.
 639   // * The G1 code below can, however, check the receiver object (the instance
 640   //   of java.lang.Reference) and jump to the slow path if null. If the
 641   //   Reference object is null then we obviously cannot fetch the referent
 642   //   and so we don't need to call the G1 pre-barrier. Thus we can use the
 643   //   regular method entry code to generate the NPE.
 644   //
 645   // This code is based on generate_accessor_enty.
 646 
 647   // rbx,: Method*
 648   // rcx: receiver (preserve for slow entry into asm interpreter)
 649 
 650   // rsi: senderSP must preserved for slow path, set SP to it on fast path
 651 
 652   address entry = __ pc();
 653 
 654   const int referent_offset = java_lang_ref_Reference::referent_offset;
 655   guarantee(referent_offset > 0, "referent offset not initialized");
 656 
 657   if (UseG1GC) {
 658     Label slow_path;
 659 
 660     // Check if local 0 != NULL
 661     // If the receiver is null then it is OK to jump to the slow path.
 662     __ movptr(rax, Address(rsp, wordSize));
 663     __ testptr(rax, rax);
 664     __ jcc(Assembler::zero, slow_path);
 665 
 666     // rax: local 0 (must be preserved across the G1 barrier call)
 667     //
 668     // rbx: method (at this point it's scratch)
 669     // rcx: receiver (at this point it's scratch)
 670     // rdx: scratch
 671     // rdi: scratch
 672     //
 673     // rsi: sender sp
 674 
 675     // Preserve the sender sp in case the pre-barrier
 676     // calls the runtime
 677     __ push(rsi);
 678 
 679     // Load the value of the referent field.
 680     const Address field_address(rax, referent_offset);
 681     __ movptr(rax, field_address);
 682 
 683     // Generate the G1 pre-barrier code to log the value of
 684     // the referent field in an SATB buffer.
 685     __ get_thread(rcx);
 686     __ g1_write_barrier_pre(noreg /* obj */,
 687                             rax /* pre_val */,
 688                             rcx /* thread */,
 689                             rbx /* tmp */,
 690                             true /* tosca_save */,
 691                             true /* expand_call */);
 692 
 693     // _areturn
 694     __ pop(rsi);                // get sender sp
 695     __ pop(rdi);                // get return address
 696     __ mov(rsp, rsi);           // set sp to sender sp
 697     __ jmp(rdi);
 698 
 699     __ bind(slow_path);
 700     __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals));
 701     return entry;
 702   }
 703 #endif // INCLUDE_ALL_GCS
 704 
 705   // If G1 is not enabled then attempt to go through the accessor entry point
 706   // Reference.get is an accessor
 707   return NULL;
 708 }
 709 
 710 /**
 711  * Method entry for static native methods:
 712  *   int java.util.zip.CRC32.update(int crc, int b)
 713  */
 714 address InterpreterGenerator::generate_CRC32_update_entry() {
 715   if (UseCRC32Intrinsics) {
 716     address entry = __ pc();
 717 
 718     // rbx: Method*
 719     // rsi: senderSP must preserved for slow path, set SP to it on fast path
 720     // rdx: scratch
 721     // rdi: scratch
 722 
 723     Label slow_path;
 724     // If we need a safepoint check, generate full interpreter entry.
 725     ExternalAddress state(SafepointSynchronize::address_of_state());
 726     __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
 727              SafepointSynchronize::_not_synchronized);
 728     __ jcc(Assembler::notEqual, slow_path);
 729 
 730     // We don't generate local frame and don't align stack because
 731     // we call stub code and there is no safepoint on this path.
 732 
 733     // Load parameters
 734     const Register crc = rax;  // crc
 735     const Register val = rdx;  // source java byte value
 736     const Register tbl = rdi;  // scratch
 737 
 738     // Arguments are reversed on java expression stack
 739     __ movl(val, Address(rsp,   wordSize)); // byte value
 740     __ movl(crc, Address(rsp, 2*wordSize)); // Initial CRC
 741 
 742     __ lea(tbl, ExternalAddress(StubRoutines::crc_table_addr()));
 743     __ notl(crc); // ~crc
 744     __ update_byte_crc32(crc, val, tbl);
 745     __ notl(crc); // ~crc
 746     // result in rax
 747 
 748     // _areturn
 749     __ pop(rdi);                // get return address
 750     __ mov(rsp, rsi);           // set sp to sender sp
 751     __ jmp(rdi);
 752 
 753     // generate a vanilla native entry as the slow path
 754     __ bind(slow_path);
 755     __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::native));
 756     return entry;
 757   }
 758   return NULL;
 759 }
 760 
 761 /**
 762  * Method entry for static native methods:
 763  *   int java.util.zip.CRC32.updateBytes(int crc, byte[] b, int off, int len)
 764  *   int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
 765  */
 766 address InterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) {
 767   if (UseCRC32Intrinsics) {
 768     address entry = __ pc();
 769 
 770     // rbx,: Method*
 771     // rsi: senderSP must preserved for slow path, set SP to it on fast path
 772     // rdx: scratch
 773     // rdi: scratch
 774 
 775     Label slow_path;
 776     // If we need a safepoint check, generate full interpreter entry.
 777     ExternalAddress state(SafepointSynchronize::address_of_state());
 778     __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
 779              SafepointSynchronize::_not_synchronized);
 780     __ jcc(Assembler::notEqual, slow_path);
 781 
 782     // We don't generate local frame and don't align stack because
 783     // we call stub code and there is no safepoint on this path.
 784 
 785     // Load parameters
 786     const Register crc = rax;  // crc
 787     const Register buf = rdx;  // source java byte array address
 788     const Register len = rdi;  // length
 789 
 790     // value              x86_32
 791     // interp. arg ptr    ESP + 4
 792     // int java.util.zip.CRC32.updateBytes(int crc, byte[] b, int off, int len)
 793     //                                         3           2      1        0
 794     // int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
 795     //                                              4         2,3      1        0
 796 
 797     // Arguments are reversed on java expression stack
 798     __ movl(len,   Address(rsp,   4 + 0)); // Length
 799     // Calculate address of start element
 800     if (kind == Interpreter::java_util_zip_CRC32_updateByteBuffer) {
 801       __ movptr(buf, Address(rsp, 4 + 2 * wordSize)); // long buf
 802       __ addptr(buf, Address(rsp, 4 + 1 * wordSize)); // + offset
 803       __ movl(crc,   Address(rsp, 4 + 4 * wordSize)); // Initial CRC
 804     } else {
 805       __ movptr(buf, Address(rsp, 4 + 2 * wordSize)); // byte[] array
 806       __ addptr(buf, arrayOopDesc::base_offset_in_bytes(T_BYTE)); // + header size
 807       __ addptr(buf, Address(rsp, 4 + 1 * wordSize)); // + offset
 808       __ movl(crc,   Address(rsp, 4 + 3 * wordSize)); // Initial CRC
 809     }
 810 
 811     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, StubRoutines::updateBytesCRC32()), crc, buf, len);
 812     // result in rax
 813 
 814     // _areturn
 815     __ pop(rdi);                // get return address
 816     __ mov(rsp, rsi);           // set sp to sender sp
 817     __ jmp(rdi);
 818 
 819     // generate a vanilla native entry as the slow path
 820     __ bind(slow_path);
 821     __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::native));
 822     return entry;
 823   }
 824   return NULL;
 825 }
 826 
 827 /**
 828 * Method entry for static native methods:
 829 *   int java.util.zip.CRC32C.updateBytes(int crc, byte[] b, int off, int end)
 830 *   int java.util.zip.CRC32C.updateByteBuffer(int crc, long address, int off, int end)
 831 */
 832 address InterpreterGenerator::generate_CRC32C_updateBytes_entry(AbstractInterpreter::MethodKind kind) {
 833   if (UseCRC32CIntrinsics) {
 834     address entry = __ pc();
 835     // Load parameters
 836     const Register crc = rax;  // crc
 837     const Register buf = rcx;  // source java byte array address
 838     const Register len = rdx;  // length
 839     const Register end = len;
 840 
 841     // value              x86_32
 842     // interp. arg ptr    ESP + 4
 843     // int java.util.zip.CRC32.updateBytes(int crc, byte[] b, int off, int end)
 844     //                                         3           2      1        0
 845     // int java.util.zip.CRC32.updateByteBuffer(int crc, long address, int off, int end)
 846     //                                              4         2,3          1        0
 847 
 848     // Arguments are reversed on java expression stack
 849     __ movl(end, Address(rsp, 4 + 0)); // end
 850     __ subl(len, Address(rsp, 4 + 1 * wordSize));  // end - offset == length
 851     // Calculate address of start element
 852     if (kind == Interpreter::java_util_zip_CRC32_updateByteBuffer) {
 853       __ movptr(buf, Address(rsp, 4 + 2 * wordSize)); // long address
 854       __ addptr(buf, Address(rsp, 4 + 1 * wordSize)); // + offset
 855       __ movl(crc, Address(rsp, 4 + 4 * wordSize)); // Initial CRC
 856     } else {
 857       __ movptr(buf, Address(rsp, 4 + 2 * wordSize)); // byte[] array
 858       __ addptr(buf, arrayOopDesc::base_offset_in_bytes(T_BYTE)); // + header size
 859       __ addptr(buf, Address(rsp, 4 + 1 * wordSize)); // + offset
 860       __ movl(crc, Address(rsp, 4 + 3 * wordSize)); // Initial CRC
 861     }
 862     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, StubRoutines::updateBytesCRC32C()), crc, buf, len);
 863     // result in rax
 864     // _areturn
 865     __ pop(rdi);                // get return address
 866     __ mov(rsp, rsi);           // set sp to sender sp
 867     __ jmp(rdi);
 868 
 869     return entry;
 870   }
 871   return NULL;
 872 }
 873 
 874 /**
 875  * Method entry for static native method:
 876  *    java.lang.Float.intBitsToFloat(int bits)
 877  */
 878 address InterpreterGenerator::generate_Float_intBitsToFloat_entry() {
 879   if (UseSSE >= 1) {
 880     address entry = __ pc();
 881 
 882     // rsi: the sender's SP
 883 
 884     // Skip safepoint check (compiler intrinsic versions of this method
 885     // do not perform safepoint checks either).
 886 
 887     // Load 'bits' into xmm0 (interpreter returns results in xmm0)
 888     __ movflt(xmm0, Address(rsp, wordSize));
 889 
 890     // Return
 891     __ pop(rdi); // get return address
 892     __ mov(rsp, rsi); // set rsp to the sender's SP
 893     __ jmp(rdi);
 894     return entry;
 895   }
 896 
 897   return NULL;
 898 }
 899 
 900 /**
 901  * Method entry for static native method:
 902  *    java.lang.Float.floatToRawIntBits(float value)
 903  */
 904 address InterpreterGenerator::generate_Float_floatToRawIntBits_entry() {
 905   if (UseSSE >= 1) {
 906     address entry = __ pc();
 907 
 908     // rsi: the sender's SP
 909 
 910     // Skip safepoint check (compiler intrinsic versions of this method
 911     // do not perform safepoint checks either).
 912 
 913     // Load the parameter (a floating-point value) into rax.
 914     __ movl(rax, Address(rsp, wordSize));
 915 
 916     // Return
 917     __ pop(rdi); // get return address
 918     __ mov(rsp, rsi); // set rsp to the sender's SP
 919     __ jmp(rdi);
 920     return entry;
 921   }
 922 
 923   return NULL;
 924 }
 925 
 926 
 927 /**
 928  * Method entry for static native method:
 929  *    java.lang.Double.longBitsToDouble(long bits)
 930  */
 931 address InterpreterGenerator::generate_Double_longBitsToDouble_entry() {
 932    if (UseSSE >= 2) {
 933      address entry = __ pc();
 934 
 935      // rsi: the sender's SP
 936 
 937      // Skip safepoint check (compiler intrinsic versions of this method
 938      // do not perform safepoint checks either).
 939 
 940      // Load 'bits' into xmm0 (interpreter returns results in xmm0)
 941      __ movdbl(xmm0, Address(rsp, wordSize));
 942 
 943      // Return
 944      __ pop(rdi); // get return address
 945      __ mov(rsp, rsi); // set rsp to the sender's SP
 946      __ jmp(rdi);
 947      return entry;
 948    }
 949 
 950    return NULL;
 951 }
 952 
 953 /**
 954  * Method entry for static native method:
 955  *    java.lang.Double.doubleToRawLongBits(double value)
 956  */
 957 address InterpreterGenerator::generate_Double_doubleToRawLongBits_entry() {
 958   if (UseSSE >= 2) {
 959     address entry = __ pc();
 960 
 961     // rsi: the sender's SP
 962 
 963     // Skip safepoint check (compiler intrinsic versions of this method
 964     // do not perform safepoint checks either).
 965 
 966     // Load the parameter (a floating-point value) into rax.
 967     __ movl(rdx, Address(rsp, 2*wordSize));
 968     __ movl(rax, Address(rsp, wordSize));
 969 
 970     // Return
 971     __ pop(rdi); // get return address
 972     __ mov(rsp, rsi); // set rsp to the sender's SP
 973     __ jmp(rdi);
 974     return entry;
 975   }
 976 
 977   return NULL;
 978 }
 979 
 980 //
 981 // Interpreter stub for calling a native method. (asm interpreter)
 982 // This sets up a somewhat different looking stack for calling the native method
 983 // than the typical interpreter frame setup.
 984 //
 985 
 986 address InterpreterGenerator::generate_native_entry(bool synchronized) {
 987   // determine code generation flags
 988   bool inc_counter  = UseCompiler || CountCompiledCalls || LogTouchedMethods;
 989 
 990   // rbx,: Method*
 991   // rsi: sender sp
 992   // rsi: previous interpreter state (C++ interpreter) must preserve
 993   address entry_point = __ pc();
 994 
 995   const Address constMethod       (rbx, Method::const_offset());
 996   const Address access_flags      (rbx, Method::access_flags_offset());
 997   const Address size_of_parameters(rcx, ConstMethod::size_of_parameters_offset());
 998 
 999   // get parameter size (always needed)
1000   __ movptr(rcx, constMethod);
1001   __ load_unsigned_short(rcx, size_of_parameters);
1002 
1003   // native calls don't need the stack size check since they have no expression stack
1004   // and the arguments are already on the stack and we only add a handful of words
1005   // to the stack
1006 
1007   // rbx,: Method*
1008   // rcx: size of parameters
1009   // rsi: sender sp
1010 
1011   __ pop(rax);                                       // get return address
1012   // for natives the size of locals is zero
1013 
1014   // compute beginning of parameters (rdi)
1015   __ lea(rdi, Address(rsp, rcx, Interpreter::stackElementScale(), -wordSize));
1016 
1017 
1018   // add 2 zero-initialized slots for native calls
1019   // NULL result handler
1020   __ push((int32_t)NULL_WORD);
1021   // NULL oop temp (mirror or jni oop result)
1022   __ push((int32_t)NULL_WORD);
1023 
1024   // initialize fixed part of activation frame
1025   generate_fixed_frame(true);
1026 
1027   // make sure method is native & not abstract
1028 #ifdef ASSERT
1029   __ movl(rax, access_flags);
1030   {
1031     Label L;
1032     __ testl(rax, JVM_ACC_NATIVE);
1033     __ jcc(Assembler::notZero, L);
1034     __ stop("tried to execute non-native method as native");
1035     __ bind(L);
1036   }
1037   { Label L;
1038     __ testl(rax, JVM_ACC_ABSTRACT);
1039     __ jcc(Assembler::zero, L);
1040     __ stop("tried to execute abstract method in interpreter");
1041     __ bind(L);
1042   }
1043 #endif
1044 
1045   // Since at this point in the method invocation the exception handler
1046   // would try to exit the monitor of synchronized methods which hasn't
1047   // been entered yet, we set the thread local variable
1048   // _do_not_unlock_if_synchronized to true. The remove_activation will
1049   // check this flag.
1050 
1051   __ get_thread(rax);
1052   const Address do_not_unlock_if_synchronized(rax,
1053         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1054   __ movbool(do_not_unlock_if_synchronized, true);
1055 
1056   // increment invocation count & check for overflow
1057   Label invocation_counter_overflow;
1058   if (inc_counter) {
1059     generate_counter_incr(&invocation_counter_overflow, NULL, NULL);
1060   }
1061 
1062   Label continue_after_compile;
1063   __ bind(continue_after_compile);
1064 
1065   bang_stack_shadow_pages(true);
1066 
1067   // reset the _do_not_unlock_if_synchronized flag
1068   __ get_thread(rax);
1069   __ movbool(do_not_unlock_if_synchronized, false);
1070 
1071   // check for synchronized methods
1072   // Must happen AFTER invocation_counter check and stack overflow check,
1073   // so method is not locked if overflows.
1074   //
1075   if (synchronized) {
1076     lock_method();
1077   } else {
1078     // no synchronization necessary
1079 #ifdef ASSERT
1080       { Label L;
1081         __ movl(rax, access_flags);
1082         __ testl(rax, JVM_ACC_SYNCHRONIZED);
1083         __ jcc(Assembler::zero, L);
1084         __ stop("method needs synchronization");
1085         __ bind(L);
1086       }
1087 #endif
1088   }
1089 
1090   // start execution
1091 #ifdef ASSERT
1092   { Label L;
1093     const Address monitor_block_top (rbp,
1094                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
1095     __ movptr(rax, monitor_block_top);
1096     __ cmpptr(rax, rsp);
1097     __ jcc(Assembler::equal, L);
1098     __ stop("broken stack frame setup in interpreter");
1099     __ bind(L);
1100   }
1101 #endif
1102 
1103   // jvmti/dtrace support
1104   __ notify_method_entry();
1105 
1106   // work registers
1107   const Register method = rbx;
1108   const Register thread = rdi;
1109   const Register t      = rcx;
1110 
1111   // allocate space for parameters
1112   __ get_method(method);
1113   __ movptr(t, Address(method, Method::const_offset()));
1114   __ load_unsigned_short(t, Address(t, ConstMethod::size_of_parameters_offset()));
1115 
1116   __ shlptr(t, Interpreter::logStackElementSize);
1117   __ addptr(t, 2*wordSize);     // allocate two more slots for JNIEnv and possible mirror
1118   __ subptr(rsp, t);
1119   __ andptr(rsp, -(StackAlignmentInBytes)); // gcc needs 16 byte aligned stacks to do XMM intrinsics
1120 
1121   // get signature handler
1122   { Label L;
1123     __ movptr(t, Address(method, Method::signature_handler_offset()));
1124     __ testptr(t, t);
1125     __ jcc(Assembler::notZero, L);
1126     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), method);
1127     __ get_method(method);
1128     __ movptr(t, Address(method, Method::signature_handler_offset()));
1129     __ bind(L);
1130   }
1131 
1132   // call signature handler
1133   assert(InterpreterRuntime::SignatureHandlerGenerator::from() == rdi, "adjust this code");
1134   assert(InterpreterRuntime::SignatureHandlerGenerator::to  () == rsp, "adjust this code");
1135   assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == t  , "adjust this code");
1136   // The generated handlers do not touch RBX (the method oop).
1137   // However, large signatures cannot be cached and are generated
1138   // each time here.  The slow-path generator will blow RBX
1139   // sometime, so we must reload it after the call.
1140   __ call(t);
1141   __ get_method(method);        // slow path call blows RBX on DevStudio 5.0
1142 
1143   // result handler is in rax,
1144   // set result handler
1145   __ movptr(Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize), rax);
1146 
1147   // pass mirror handle if static call
1148   { Label L;
1149     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
1150     __ movl(t, Address(method, Method::access_flags_offset()));
1151     __ testl(t, JVM_ACC_STATIC);
1152     __ jcc(Assembler::zero, L);
1153     // get mirror
1154     __ movptr(t, Address(method, Method:: const_offset()));
1155     __ movptr(t, Address(t, ConstMethod::constants_offset()));
1156     __ movptr(t, Address(t, ConstantPool::pool_holder_offset_in_bytes()));
1157     __ movptr(t, Address(t, mirror_offset));
1158     // copy mirror into activation frame
1159     __ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize), t);
1160     // pass handle to mirror
1161     __ lea(t, Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize));
1162     __ movptr(Address(rsp, wordSize), t);
1163     __ bind(L);
1164   }
1165 
1166   // get native function entry point
1167   { Label L;
1168     __ movptr(rax, Address(method, Method::native_function_offset()));
1169     ExternalAddress unsatisfied(SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
1170     __ cmpptr(rax, unsatisfied.addr());
1171     __ jcc(Assembler::notEqual, L);
1172     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), method);
1173     __ get_method(method);
1174     __ movptr(rax, Address(method, Method::native_function_offset()));
1175     __ bind(L);
1176   }
1177 
1178   // pass JNIEnv
1179   __ get_thread(thread);
1180   __ lea(t, Address(thread, JavaThread::jni_environment_offset()));
1181   __ movptr(Address(rsp, 0), t);
1182 
1183   // set_last_Java_frame_before_call
1184   // It is enough that the pc()
1185   // points into the right code segment. It does not have to be the correct return pc.
1186   __ set_last_Java_frame(thread, noreg, rbp, __ pc());
1187 
1188   // change thread state
1189 #ifdef ASSERT
1190   { Label L;
1191     __ movl(t, Address(thread, JavaThread::thread_state_offset()));
1192     __ cmpl(t, _thread_in_Java);
1193     __ jcc(Assembler::equal, L);
1194     __ stop("Wrong thread state in native stub");
1195     __ bind(L);
1196   }
1197 #endif
1198 
1199   // Change state to native
1200   __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_native);
1201   __ call(rax);
1202 
1203   // result potentially in rdx:rax or ST0
1204 
1205   // Verify or restore cpu control state after JNI call
1206   __ restore_cpu_control_state_after_jni();
1207 
1208   // save potential result in ST(0) & rdx:rax
1209   // (if result handler is the T_FLOAT or T_DOUBLE handler, result must be in ST0 -
1210   // the check is necessary to avoid potential Intel FPU overflow problems by saving/restoring 'empty' FPU registers)
1211   // It is safe to do this push because state is _thread_in_native and return address will be found
1212   // via _last_native_pc and not via _last_jave_sp
1213 
1214   // NOTE: the order of theses push(es) is known to frame::interpreter_frame_result.
1215   // If the order changes or anything else is added to the stack the code in
1216   // interpreter_frame_result will have to be changed.
1217 
1218   { Label L;
1219     Label push_double;
1220     ExternalAddress float_handler(AbstractInterpreter::result_handler(T_FLOAT));
1221     ExternalAddress double_handler(AbstractInterpreter::result_handler(T_DOUBLE));
1222     __ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize),
1223               float_handler.addr());
1224     __ jcc(Assembler::equal, push_double);
1225     __ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize),
1226               double_handler.addr());
1227     __ jcc(Assembler::notEqual, L);
1228     __ bind(push_double);
1229     __ push_d(); // FP values are returned using the FPU, so push FPU contents (even if UseSSE > 0).
1230     __ bind(L);
1231   }
1232   __ push(ltos);
1233 
1234   // change thread state
1235   __ get_thread(thread);
1236   __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
1237   if(os::is_MP()) {
1238     if (UseMembar) {
1239       // Force this write out before the read below
1240       __ membar(Assembler::Membar_mask_bits(
1241            Assembler::LoadLoad | Assembler::LoadStore |
1242            Assembler::StoreLoad | Assembler::StoreStore));
1243     } else {
1244       // Write serialization page so VM thread can do a pseudo remote membar.
1245       // We use the current thread pointer to calculate a thread specific
1246       // offset to write to within the page. This minimizes bus traffic
1247       // due to cache line collision.
1248       __ serialize_memory(thread, rcx);
1249     }
1250   }
1251 
1252   if (AlwaysRestoreFPU) {
1253     //  Make sure the control word is correct.
1254     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1255   }
1256 
1257   // check for safepoint operation in progress and/or pending suspend requests
1258   { Label Continue;
1259 
1260     __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
1261              SafepointSynchronize::_not_synchronized);
1262 
1263     Label L;
1264     __ jcc(Assembler::notEqual, L);
1265     __ cmpl(Address(thread, JavaThread::suspend_flags_offset()), 0);
1266     __ jcc(Assembler::equal, Continue);
1267     __ bind(L);
1268 
1269     // Don't use call_VM as it will see a possible pending exception and forward it
1270     // and never return here preventing us from clearing _last_native_pc down below.
1271     // Also can't use call_VM_leaf either as it will check to see if rsi & rdi are
1272     // preserved and correspond to the bcp/locals pointers. So we do a runtime call
1273     // by hand.
1274     //
1275     __ push(thread);
1276     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address,
1277                                             JavaThread::check_special_condition_for_native_trans)));
1278     __ increment(rsp, wordSize);
1279     __ get_thread(thread);
1280 
1281     __ bind(Continue);
1282   }
1283 
1284   // change thread state
1285   __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_Java);
1286 
1287   __ reset_last_Java_frame(thread, true, true);
1288 
1289   // reset handle block
1290   __ movptr(t, Address(thread, JavaThread::active_handles_offset()));
1291   __ movl(Address(t, JNIHandleBlock::top_offset_in_bytes()), NULL_WORD);
1292 
1293   // If result was an oop then unbox and save it in the frame
1294   { Label L;
1295     Label no_oop, store_result;
1296     ExternalAddress handler(AbstractInterpreter::result_handler(T_OBJECT));
1297     __ cmpptr(Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize),
1298               handler.addr());
1299     __ jcc(Assembler::notEqual, no_oop);
1300     __ cmpptr(Address(rsp, 0), (int32_t)NULL_WORD);
1301     __ pop(ltos);
1302     __ testptr(rax, rax);
1303     __ jcc(Assembler::zero, store_result);
1304     // unbox
1305     __ movptr(rax, Address(rax, 0));
1306     __ bind(store_result);
1307     __ movptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset)*wordSize), rax);
1308     // keep stack depth as expected by pushing oop which will eventually be discarded
1309     __ push(ltos);
1310     __ bind(no_oop);
1311   }
1312 
1313   {
1314      Label no_reguard;
1315      __ cmpl(Address(thread, JavaThread::stack_guard_state_offset()), JavaThread::stack_guard_yellow_disabled);
1316      __ jcc(Assembler::notEqual, no_reguard);
1317 
1318      __ pusha();
1319      __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
1320      __ popa();
1321 
1322      __ bind(no_reguard);
1323    }
1324 
1325   // restore rsi to have legal interpreter frame,
1326   // i.e., bci == 0 <=> rsi == code_base()
1327   // Can't call_VM until bcp is within reasonable.
1328   __ get_method(method);      // method is junk from thread_in_native to now.
1329   __ movptr(rsi, Address(method,Method::const_offset()));   // get ConstMethod*
1330   __ lea(rsi, Address(rsi,ConstMethod::codes_offset()));    // get codebase
1331 
1332   // handle exceptions (exception handling will handle unlocking!)
1333   { Label L;
1334     __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
1335     __ jcc(Assembler::zero, L);
1336     // Note: At some point we may want to unify this with the code used in call_VM_base();
1337     //       i.e., we should use the StubRoutines::forward_exception code. For now this
1338     //       doesn't work here because the rsp is not correctly set at this point.
1339     __ MacroAssembler::call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));
1340     __ should_not_reach_here();
1341     __ bind(L);
1342   }
1343 
1344   // do unlocking if necessary
1345   { Label L;
1346     __ movl(t, Address(method, Method::access_flags_offset()));
1347     __ testl(t, JVM_ACC_SYNCHRONIZED);
1348     __ jcc(Assembler::zero, L);
1349     // the code below should be shared with interpreter macro assembler implementation
1350     { Label unlock;
1351       // BasicObjectLock will be first in list, since this is a synchronized method. However, need
1352       // to check that the object has not been unlocked by an explicit monitorexit bytecode.
1353       const Address monitor(rbp, frame::interpreter_frame_initial_sp_offset * wordSize - (int)sizeof(BasicObjectLock));
1354 
1355       __ lea(rdx, monitor);                   // address of first monitor
1356 
1357       __ movptr(t, Address(rdx, BasicObjectLock::obj_offset_in_bytes()));
1358       __ testptr(t, t);
1359       __ jcc(Assembler::notZero, unlock);
1360 
1361       // Entry already unlocked, need to throw exception
1362       __ MacroAssembler::call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
1363       __ should_not_reach_here();
1364 
1365       __ bind(unlock);
1366       __ unlock_object(rdx);
1367     }
1368     __ bind(L);
1369   }
1370 
1371   // jvmti/dtrace support
1372   // Note: This must happen _after_ handling/throwing any exceptions since
1373   //       the exception handler code notifies the runtime of method exits
1374   //       too. If this happens before, method entry/exit notifications are
1375   //       not properly paired (was bug - gri 11/22/99).
1376   __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI);
1377 
1378   // restore potential result in rdx:rax, call result handler to restore potential result in ST0 & handle result
1379   __ pop(ltos);
1380   __ movptr(t, Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize));
1381   __ call(t);
1382 
1383   // remove activation
1384   __ movptr(t, Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize)); // get sender sp
1385   __ leave();                                // remove frame anchor
1386   __ pop(rdi);                               // get return address
1387   __ mov(rsp, t);                            // set sp to sender sp
1388   __ jmp(rdi);
1389 
1390   if (inc_counter) {
1391     // Handle overflow of counter and compile method
1392     __ bind(invocation_counter_overflow);
1393     generate_counter_overflow(&continue_after_compile);
1394   }
1395 
1396   return entry_point;
1397 }
1398 
1399 //
1400 // Generic interpreted method entry to (asm) interpreter
1401 //
1402 address InterpreterGenerator::generate_normal_entry(bool synchronized) {
1403   // determine code generation flags
1404   bool inc_counter  = UseCompiler || CountCompiledCalls || LogTouchedMethods;
1405 
1406   // rbx,: Method*
1407   // rsi: sender sp
1408   address entry_point = __ pc();
1409 
1410   const Address constMethod       (rbx, Method::const_offset());
1411   const Address access_flags      (rbx, Method::access_flags_offset());
1412   const Address size_of_parameters(rdx, ConstMethod::size_of_parameters_offset());
1413   const Address size_of_locals    (rdx, ConstMethod::size_of_locals_offset());
1414 
1415   // get parameter size (always needed)
1416   __ movptr(rdx, constMethod);
1417   __ load_unsigned_short(rcx, size_of_parameters);
1418 
1419   // rbx,: Method*
1420   // rcx: size of parameters
1421 
1422   // rsi: sender_sp (could differ from sp+wordSize if we were called via c2i )
1423 
1424   __ load_unsigned_short(rdx, size_of_locals);       // get size of locals in words
1425   __ subl(rdx, rcx);                                // rdx = no. of additional locals
1426 
1427   // see if we've got enough room on the stack for locals plus overhead.
1428   generate_stack_overflow_check();
1429 
1430   // get return address
1431   __ pop(rax);
1432 
1433   // compute beginning of parameters (rdi)
1434   __ lea(rdi, Address(rsp, rcx, Interpreter::stackElementScale(), -wordSize));
1435 
1436   // rdx - # of additional locals
1437   // allocate space for locals
1438   // explicitly initialize locals
1439   {
1440     Label exit, loop;
1441     __ testl(rdx, rdx);
1442     __ jcc(Assembler::lessEqual, exit);               // do nothing if rdx <= 0
1443     __ bind(loop);
1444     __ push((int32_t)NULL_WORD);                      // initialize local variables
1445     __ decrement(rdx);                                // until everything initialized
1446     __ jcc(Assembler::greater, loop);
1447     __ bind(exit);
1448   }
1449 
1450   // initialize fixed part of activation frame
1451   generate_fixed_frame(false);
1452 
1453   // make sure method is not native & not abstract
1454 #ifdef ASSERT
1455   __ movl(rax, access_flags);
1456   {
1457     Label L;
1458     __ testl(rax, JVM_ACC_NATIVE);
1459     __ jcc(Assembler::zero, L);
1460     __ stop("tried to execute native method as non-native");
1461     __ bind(L);
1462   }
1463   { Label L;
1464     __ testl(rax, JVM_ACC_ABSTRACT);
1465     __ jcc(Assembler::zero, L);
1466     __ stop("tried to execute abstract method in interpreter");
1467     __ bind(L);
1468   }
1469 #endif
1470 
1471   // Since at this point in the method invocation the exception handler
1472   // would try to exit the monitor of synchronized methods which hasn't
1473   // been entered yet, we set the thread local variable
1474   // _do_not_unlock_if_synchronized to true. The remove_activation will
1475   // check this flag.
1476 
1477   __ get_thread(rax);
1478   const Address do_not_unlock_if_synchronized(rax,
1479         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1480   __ movbool(do_not_unlock_if_synchronized, true);
1481 
1482   __ profile_parameters_type(rax, rcx, rdx);
1483   // increment invocation count & check for overflow
1484   Label invocation_counter_overflow;
1485   Label profile_method;
1486   Label profile_method_continue;
1487   if (inc_counter) {
1488     generate_counter_incr(&invocation_counter_overflow, &profile_method, &profile_method_continue);
1489     if (ProfileInterpreter) {
1490       __ bind(profile_method_continue);
1491     }
1492   }
1493   Label continue_after_compile;
1494   __ bind(continue_after_compile);
1495 
1496   bang_stack_shadow_pages(false);
1497 
1498   // reset the _do_not_unlock_if_synchronized flag
1499   __ get_thread(rax);
1500   __ movbool(do_not_unlock_if_synchronized, false);
1501 
1502   // check for synchronized methods
1503   // Must happen AFTER invocation_counter check and stack overflow check,
1504   // so method is not locked if overflows.
1505   //
1506   if (synchronized) {
1507     // Allocate monitor and lock method
1508     lock_method();
1509   } else {
1510     // no synchronization necessary
1511 #ifdef ASSERT
1512       { Label L;
1513         __ movl(rax, access_flags);
1514         __ testl(rax, JVM_ACC_SYNCHRONIZED);
1515         __ jcc(Assembler::zero, L);
1516         __ stop("method needs synchronization");
1517         __ bind(L);
1518       }
1519 #endif
1520   }
1521 
1522   // start execution
1523 #ifdef ASSERT
1524   { Label L;
1525      const Address monitor_block_top (rbp,
1526                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
1527     __ movptr(rax, monitor_block_top);
1528     __ cmpptr(rax, rsp);
1529     __ jcc(Assembler::equal, L);
1530     __ stop("broken stack frame setup in interpreter");
1531     __ bind(L);
1532   }
1533 #endif
1534 
1535   // jvmti support
1536   __ notify_method_entry();
1537 
1538   __ dispatch_next(vtos);
1539 
1540   // invocation counter overflow
1541   if (inc_counter) {
1542     if (ProfileInterpreter) {
1543       // We have decided to profile this method in the interpreter
1544       __ bind(profile_method);
1545       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1546       __ set_method_data_pointer_for_bcp();
1547       __ get_method(rbx);
1548       __ jmp(profile_method_continue);
1549     }
1550     // Handle overflow of counter and compile method
1551     __ bind(invocation_counter_overflow);
1552     generate_counter_overflow(&continue_after_compile);
1553   }
1554 
1555   return entry_point;
1556 }
1557 
1558 
1559 // These should never be compiled since the interpreter will prefer
1560 // the compiled version to the intrinsic version.
1561 bool AbstractInterpreter::can_be_compiled(methodHandle m) {
1562   switch (method_kind(m)) {
1563     case Interpreter::java_lang_math_sin     : // fall thru
1564     case Interpreter::java_lang_math_cos     : // fall thru
1565     case Interpreter::java_lang_math_tan     : // fall thru
1566     case Interpreter::java_lang_math_abs     : // fall thru
1567     case Interpreter::java_lang_math_log     : // fall thru
1568     case Interpreter::java_lang_math_log10   : // fall thru
1569     case Interpreter::java_lang_math_sqrt    : // fall thru
1570     case Interpreter::java_lang_math_pow     : // fall thru
1571     case Interpreter::java_lang_math_exp     :
1572       return false;
1573     default:
1574       return true;
1575   }
1576 }
1577 
1578 // How much stack a method activation needs in words.
1579 int AbstractInterpreter::size_top_interpreter_activation(Method* method) {
1580 
1581   const int stub_code = 4;  // see generate_call_stub
1582   // Save space for one monitor to get into the interpreted method in case
1583   // the method is synchronized
1584   int monitor_size    = method->is_synchronized() ?
1585                                 1*frame::interpreter_frame_monitor_size() : 0;
1586 
1587   // total overhead size: entry_size + (saved rbp, thru expr stack bottom).
1588   // be sure to change this if you add/subtract anything to/from the overhead area
1589   const int overhead_size = -frame::interpreter_frame_initial_sp_offset;
1590 
1591   const int method_stack = (method->max_locals() + method->max_stack()) *
1592                            Interpreter::stackElementWords;
1593   return overhead_size + method_stack + stub_code;
1594 }
1595 
1596 //------------------------------------------------------------------------------------------------------------------------
1597 // Exceptions
1598 
1599 void TemplateInterpreterGenerator::generate_throw_exception() {
1600   // Entry point in previous activation (i.e., if the caller was interpreted)
1601   Interpreter::_rethrow_exception_entry = __ pc();
1602   const Register thread = rcx;
1603 
1604   // Restore sp to interpreter_frame_last_sp even though we are going
1605   // to empty the expression stack for the exception processing.
1606   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
1607   // rax,: exception
1608   // rdx: return address/pc that threw exception
1609   __ restore_bcp();                              // rsi points to call/send
1610   __ restore_locals();
1611 
1612   // Entry point for exceptions thrown within interpreter code
1613   Interpreter::_throw_exception_entry = __ pc();
1614   // expression stack is undefined here
1615   // rax,: exception
1616   // rsi: exception bcp
1617   __ verify_oop(rax);
1618 
1619   // expression stack must be empty before entering the VM in case of an exception
1620   __ empty_expression_stack();
1621   __ empty_FPU_stack();
1622   // find exception handler address and preserve exception oop
1623   __ call_VM(rdx, CAST_FROM_FN_PTR(address, InterpreterRuntime::exception_handler_for_exception), rax);
1624   // rax,: exception handler entry point
1625   // rdx: preserved exception oop
1626   // rsi: bcp for exception handler
1627   __ push_ptr(rdx);                              // push exception which is now the only value on the stack
1628   __ jmp(rax);                                   // jump to exception handler (may be _remove_activation_entry!)
1629 
1630   // If the exception is not handled in the current frame the frame is removed and
1631   // the exception is rethrown (i.e. exception continuation is _rethrow_exception).
1632   //
1633   // Note: At this point the bci is still the bxi for the instruction which caused
1634   //       the exception and the expression stack is empty. Thus, for any VM calls
1635   //       at this point, GC will find a legal oop map (with empty expression stack).
1636 
1637   // In current activation
1638   // tos: exception
1639   // rsi: exception bcp
1640 
1641   //
1642   // JVMTI PopFrame support
1643   //
1644 
1645    Interpreter::_remove_activation_preserving_args_entry = __ pc();
1646   __ empty_expression_stack();
1647   __ empty_FPU_stack();
1648   // Set the popframe_processing bit in pending_popframe_condition indicating that we are
1649   // currently handling popframe, so that call_VMs that may happen later do not trigger new
1650   // popframe handling cycles.
1651   __ get_thread(thread);
1652   __ movl(rdx, Address(thread, JavaThread::popframe_condition_offset()));
1653   __ orl(rdx, JavaThread::popframe_processing_bit);
1654   __ movl(Address(thread, JavaThread::popframe_condition_offset()), rdx);
1655 
1656   {
1657     // Check to see whether we are returning to a deoptimized frame.
1658     // (The PopFrame call ensures that the caller of the popped frame is
1659     // either interpreted or compiled and deoptimizes it if compiled.)
1660     // In this case, we can't call dispatch_next() after the frame is
1661     // popped, but instead must save the incoming arguments and restore
1662     // them after deoptimization has occurred.
1663     //
1664     // Note that we don't compare the return PC against the
1665     // deoptimization blob's unpack entry because of the presence of
1666     // adapter frames in C2.
1667     Label caller_not_deoptimized;
1668     __ movptr(rdx, Address(rbp, frame::return_addr_offset * wordSize));
1669     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::interpreter_contains), rdx);
1670     __ testl(rax, rax);
1671     __ jcc(Assembler::notZero, caller_not_deoptimized);
1672 
1673     // Compute size of arguments for saving when returning to deoptimized caller
1674     __ get_method(rax);
1675     __ movptr(rax, Address(rax, Method::const_offset()));
1676     __ load_unsigned_short(rax, Address(rax, ConstMethod::size_of_parameters_offset()));
1677     __ shlptr(rax, Interpreter::logStackElementSize);
1678     __ restore_locals();
1679     __ subptr(rdi, rax);
1680     __ addptr(rdi, wordSize);
1681     // Save these arguments
1682     __ get_thread(thread);
1683     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::popframe_preserve_args), thread, rax, rdi);
1684 
1685     __ remove_activation(vtos, rdx,
1686                          /* throw_monitor_exception */ false,
1687                          /* install_monitor_exception */ false,
1688                          /* notify_jvmdi */ false);
1689 
1690     // Inform deoptimization that it is responsible for restoring these arguments
1691     __ get_thread(thread);
1692     __ movl(Address(thread, JavaThread::popframe_condition_offset()), JavaThread::popframe_force_deopt_reexecution_bit);
1693 
1694     // Continue in deoptimization handler
1695     __ jmp(rdx);
1696 
1697     __ bind(caller_not_deoptimized);
1698   }
1699 
1700   __ remove_activation(vtos, rdx,
1701                        /* throw_monitor_exception */ false,
1702                        /* install_monitor_exception */ false,
1703                        /* notify_jvmdi */ false);
1704 
1705   // Finish with popframe handling
1706   // A previous I2C followed by a deoptimization might have moved the
1707   // outgoing arguments further up the stack. PopFrame expects the
1708   // mutations to those outgoing arguments to be preserved and other
1709   // constraints basically require this frame to look exactly as
1710   // though it had previously invoked an interpreted activation with
1711   // no space between the top of the expression stack (current
1712   // last_sp) and the top of stack. Rather than force deopt to
1713   // maintain this kind of invariant all the time we call a small
1714   // fixup routine to move the mutated arguments onto the top of our
1715   // expression stack if necessary.
1716   __ mov(rax, rsp);
1717   __ movptr(rbx, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1718   __ get_thread(thread);
1719   // PC must point into interpreter here
1720   __ set_last_Java_frame(thread, noreg, rbp, __ pc());
1721   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), thread, rax, rbx);
1722   __ get_thread(thread);
1723   __ reset_last_Java_frame(thread, true, true);
1724   // Restore the last_sp and null it out
1725   __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1726   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
1727 
1728   __ restore_bcp();
1729   __ restore_locals();
1730   // The method data pointer was incremented already during
1731   // call profiling. We have to restore the mdp for the current bcp.
1732   if (ProfileInterpreter) {
1733     __ set_method_data_pointer_for_bcp();
1734   }
1735 
1736   // Clear the popframe condition flag
1737   __ get_thread(thread);
1738   __ movl(Address(thread, JavaThread::popframe_condition_offset()), JavaThread::popframe_inactive);
1739 
1740 #if INCLUDE_JVMTI
1741   {
1742     Label L_done;
1743     const Register local0 = rdi;
1744 
1745     __ cmpb(Address(rsi, 0), Bytecodes::_invokestatic);
1746     __ jcc(Assembler::notEqual, L_done);
1747 
1748     // The member name argument must be restored if _invokestatic is re-executed after a PopFrame call.
1749     // Detect such a case in the InterpreterRuntime function and return the member name argument, or NULL.
1750 
1751     __ get_method(rdx);
1752     __ movptr(rax, Address(local0, 0));
1753     __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null), rax, rdx, rsi);
1754 
1755     __ testptr(rax, rax);
1756     __ jcc(Assembler::zero, L_done);
1757 
1758     __ movptr(Address(rbx, 0), rax);
1759     __ bind(L_done);
1760   }
1761 #endif // INCLUDE_JVMTI
1762 
1763   __ dispatch_next(vtos);
1764   // end of PopFrame support
1765 
1766   Interpreter::_remove_activation_entry = __ pc();
1767 
1768   // preserve exception over this code sequence
1769   __ pop_ptr(rax);
1770   __ get_thread(thread);
1771   __ movptr(Address(thread, JavaThread::vm_result_offset()), rax);
1772   // remove the activation (without doing throws on illegalMonitorExceptions)
1773   __ remove_activation(vtos, rdx, false, true, false);
1774   // restore exception
1775   __ get_thread(thread);
1776   __ get_vm_result(rax, thread);
1777 
1778   // Inbetween activations - previous activation type unknown yet
1779   // compute continuation point - the continuation point expects
1780   // the following registers set up:
1781   //
1782   // rax: exception
1783   // rdx: return address/pc that threw exception
1784   // rsp: expression stack of caller
1785   // rbp: rbp, of caller
1786   __ push(rax);                                  // save exception
1787   __ push(rdx);                                  // save return address
1788   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, rdx);
1789   __ mov(rbx, rax);                              // save exception handler
1790   __ pop(rdx);                                   // restore return address
1791   __ pop(rax);                                   // restore exception
1792   // Note that an "issuing PC" is actually the next PC after the call
1793   __ jmp(rbx);                                   // jump to exception handler of caller
1794 }
1795 
1796 
1797 //
1798 // JVMTI ForceEarlyReturn support
1799 //
1800 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1801   address entry = __ pc();
1802   const Register thread = rcx;
1803 
1804   __ restore_bcp();
1805   __ restore_locals();
1806   __ empty_expression_stack();
1807   __ empty_FPU_stack();
1808   __ load_earlyret_value(state);
1809 
1810   __ get_thread(thread);
1811   __ movptr(rcx, Address(thread, JavaThread::jvmti_thread_state_offset()));
1812   const Address cond_addr(rcx, JvmtiThreadState::earlyret_state_offset());
1813 
1814   // Clear the earlyret state
1815   __ movl(cond_addr, JvmtiThreadState::earlyret_inactive);
1816 
1817   __ remove_activation(state, rsi,
1818                        false, /* throw_monitor_exception */
1819                        false, /* install_monitor_exception */
1820                        true); /* notify_jvmdi */
1821   __ jmp(rsi);
1822   return entry;
1823 } // end of ForceEarlyReturn support
1824 
1825 
1826 //------------------------------------------------------------------------------------------------------------------------
1827 // Helper for vtos entry point generation
1828 
1829 void TemplateInterpreterGenerator::set_vtos_entry_points (Template* t, address& bep, address& cep, address& sep, address& aep, address& iep, address& lep, address& fep, address& dep, address& vep) {
1830   assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1831   Label L;
1832   fep = __ pc(); __ push(ftos); __ jmp(L);
1833   dep = __ pc(); __ push(dtos); __ jmp(L);
1834   lep = __ pc(); __ push(ltos); __ jmp(L);
1835   aep = __ pc(); __ push(atos); __ jmp(L);
1836   bep = cep = sep =             // fall through
1837   iep = __ pc(); __ push(itos); // fall through
1838   vep = __ pc(); __ bind(L);    // fall through
1839   generate_and_dispatch(t);
1840 }
1841 
1842 //------------------------------------------------------------------------------------------------------------------------
1843 // Generation of individual instructions
1844 
1845 // helpers for generate_and_dispatch
1846 
1847 
1848 
1849 InterpreterGenerator::InterpreterGenerator(StubQueue* code)
1850  : TemplateInterpreterGenerator(code) {
1851    generate_all(); // down here so it can be "virtual"
1852 }
1853 
1854 //------------------------------------------------------------------------------------------------------------------------
1855 
1856 // Non-product code
1857 #ifndef PRODUCT
1858 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1859   address entry = __ pc();
1860 
1861   // prepare expression stack
1862   __ pop(rcx);          // pop return address so expression stack is 'pure'
1863   __ push(state);       // save tosca
1864 
1865   // pass tosca registers as arguments & call tracer
1866   __ call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::trace_bytecode), rcx, rax, rdx);
1867   __ mov(rcx, rax);     // make sure return address is not destroyed by pop(state)
1868   __ pop(state);        // restore tosca
1869 
1870   // return
1871   __ jmp(rcx);
1872 
1873   return entry;
1874 }
1875 
1876 
1877 void TemplateInterpreterGenerator::count_bytecode() {
1878   __ incrementl(ExternalAddress((address) &BytecodeCounter::_counter_value));
1879 }
1880 
1881 
1882 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
1883   __ incrementl(ExternalAddress((address) &BytecodeHistogram::_counters[t->bytecode()]));
1884 }
1885 
1886 
1887 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
1888   __ mov32(ExternalAddress((address) &BytecodePairHistogram::_index), rbx);
1889   __ shrl(rbx, BytecodePairHistogram::log2_number_of_codes);
1890   __ orl(rbx, ((int)t->bytecode()) << BytecodePairHistogram::log2_number_of_codes);
1891   ExternalAddress table((address) BytecodePairHistogram::_counters);
1892   Address index(noreg, rbx, Address::times_4);
1893   __ incrementl(ArrayAddress(table, index));
1894 }
1895 
1896 
1897 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
1898   // Call a little run-time stub to avoid blow-up for each bytecode.
1899   // The run-time runtime saves the right registers, depending on
1900   // the tosca in-state for the given template.
1901   assert(Interpreter::trace_code(t->tos_in()) != NULL,
1902          "entry must have been generated");
1903   __ call(RuntimeAddress(Interpreter::trace_code(t->tos_in())));
1904 }
1905 
1906 
1907 void TemplateInterpreterGenerator::stop_interpreter_at() {
1908   Label L;
1909   __ cmp32(ExternalAddress((address) &BytecodeCounter::_counter_value),
1910            StopInterpreterAt);
1911   __ jcc(Assembler::notEqual, L);
1912   __ int3();
1913   __ bind(L);
1914 }
1915 #endif // !PRODUCT
1916 #endif // CC_INTERP