1 /*
   2  * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.hpp"
  27 #include "interpreter/bytecodeHistogram.hpp"
  28 #include "interpreter/interpreter.hpp"
  29 #include "interpreter/interpreterGenerator.hpp"
  30 #include "interpreter/interpreterRuntime.hpp"
  31 #include "interpreter/templateTable.hpp"
  32 #include "oops/arrayOop.hpp"
  33 #include "oops/methodDataOop.hpp"
  34 #include "oops/methodOop.hpp"
  35 #include "oops/oop.inline.hpp"
  36 #include "prims/jvmtiExport.hpp"
  37 #include "prims/jvmtiThreadState.hpp"
  38 #include "runtime/arguments.hpp"
  39 #include "runtime/deoptimization.hpp"
  40 #include "runtime/frame.inline.hpp"
  41 #include "runtime/sharedRuntime.hpp"
  42 #include "runtime/stubRoutines.hpp"
  43 #include "runtime/synchronizer.hpp"
  44 #include "runtime/timer.hpp"
  45 #include "runtime/vframeArray.hpp"
  46 #include "utilities/debug.hpp"
  47 
  48 #define __ _masm->
  49 
  50 
  51 #ifndef CC_INTERP
  52 const int method_offset = frame::interpreter_frame_method_offset * wordSize;
  53 const int bci_offset    = frame::interpreter_frame_bcx_offset    * wordSize;
  54 const int locals_offset = frame::interpreter_frame_locals_offset * wordSize;
  55 
  56 //------------------------------------------------------------------------------------------------------------------------
  57 
  58 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
  59   address entry = __ pc();
  60 
  61   // Note: There should be a minimal interpreter frame set up when stack
  62   // overflow occurs since we check explicitly for it now.
  63   //
  64 #ifdef ASSERT
  65   { Label L;
  66     __ lea(rax, Address(rbp,
  67                 frame::interpreter_frame_monitor_block_top_offset * wordSize));
  68     __ cmpptr(rax, rsp);  // rax, = maximal rsp for current rbp,
  69                         //  (stack grows negative)
  70     __ jcc(Assembler::aboveEqual, L); // check if frame is complete
  71     __ stop ("interpreter frame not set up");
  72     __ bind(L);
  73   }
  74 #endif // ASSERT
  75   // Restore bcp under the assumption that the current frame is still
  76   // interpreted
  77   __ restore_bcp();
  78 
  79   // expression stack must be empty before entering the VM if an exception
  80   // happened
  81   __ empty_expression_stack();
  82   __ empty_FPU_stack();
  83   // throw exception
  84   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_StackOverflowError));
  85   return entry;
  86 }
  87 
  88 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler(const char* name) {
  89   address entry = __ pc();
  90   // expression stack must be empty before entering the VM if an exception happened
  91   __ empty_expression_stack();
  92   __ empty_FPU_stack();
  93   // setup parameters
  94   // ??? convention: expect aberrant index in register rbx,
  95   __ lea(rax, ExternalAddress((address)name));
  96   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_ArrayIndexOutOfBoundsException), rax, rbx);
  97   return entry;
  98 }
  99 
 100 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
 101   address entry = __ pc();
 102   // object is at TOS
 103   __ pop(rax);
 104   // expression stack must be empty before entering the VM if an exception
 105   // happened
 106   __ empty_expression_stack();
 107   __ empty_FPU_stack();
 108   __ call_VM(noreg,
 109              CAST_FROM_FN_PTR(address,
 110                               InterpreterRuntime::throw_ClassCastException),
 111              rax);
 112   return entry;
 113 }
 114 
 115 // Arguments are: required type at TOS+4, failing object (or NULL) at TOS.
 116 address TemplateInterpreterGenerator::generate_WrongMethodType_handler() {
 117   address entry = __ pc();
 118 
 119   __ pop(rbx);                  // actual failing object is at TOS
 120   __ pop(rax);                  // required type is at TOS+4
 121 
 122   __ verify_oop(rbx);
 123   __ verify_oop(rax);
 124 
 125   // Various method handle types use interpreter registers as temps.
 126   __ restore_bcp();
 127   __ restore_locals();
 128 
 129   // Expression stack must be empty before entering the VM for an exception.
 130   __ empty_expression_stack();
 131   __ empty_FPU_stack();
 132   __ call_VM(noreg,
 133              CAST_FROM_FN_PTR(address,
 134                               InterpreterRuntime::throw_WrongMethodTypeException),
 135              // pass required type, failing object (or NULL)
 136              rax, rbx);
 137   return entry;
 138 }
 139 
 140 
 141 address TemplateInterpreterGenerator::generate_exception_handler_common(const char* name, const char* message, bool pass_oop) {
 142   assert(!pass_oop || message == NULL, "either oop or message but not both");
 143   address entry = __ pc();
 144   if (pass_oop) {
 145     // object is at TOS
 146     __ pop(rbx);
 147   }
 148   // expression stack must be empty before entering the VM if an exception happened
 149   __ empty_expression_stack();
 150   __ empty_FPU_stack();
 151   // setup parameters
 152   __ lea(rax, ExternalAddress((address)name));
 153   if (pass_oop) {
 154     __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_klass_exception), rax, rbx);
 155   } else {
 156     if (message != NULL) {
 157       __ lea(rbx, ExternalAddress((address)message));
 158     } else {
 159       __ movptr(rbx, NULL_WORD);
 160     }
 161     __ call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception), rax, rbx);
 162   }
 163   // throw exception
 164   __ jump(ExternalAddress(Interpreter::throw_exception_entry()));
 165   return entry;
 166 }
 167 
 168 
 169 address TemplateInterpreterGenerator::generate_continuation_for(TosState state) {
 170   address entry = __ pc();
 171   // NULL last_sp until next java call
 172   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
 173   __ dispatch_next(state);
 174   return entry;
 175 }
 176 
 177 
 178 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step) {
 179   TosState incoming_state = state;
 180   address entry = __ pc();
 181 
 182 #ifdef COMPILER2
 183   // The FPU stack is clean if UseSSE >= 2 but must be cleaned in other cases
 184   if ((incoming_state == ftos && UseSSE < 1) || (incoming_state == dtos && UseSSE < 2)) {
 185     for (int i = 1; i < 8; i++) {
 186         __ ffree(i);
 187     }
 188   } else if (UseSSE < 2) {
 189     __ empty_FPU_stack();
 190   }
 191 #endif
 192   if ((incoming_state == ftos && UseSSE < 1) || (incoming_state == dtos && UseSSE < 2)) {
 193     __ MacroAssembler::verify_FPU(1, "generate_return_entry_for compiled");
 194   } else {
 195     __ MacroAssembler::verify_FPU(0, "generate_return_entry_for compiled");
 196   }
 197 
 198   // In SSE mode, interpreter returns FP results in xmm0 but they need
 199   // to end up back on the FPU so it can operate on them.
 200   if (incoming_state == ftos && UseSSE >= 1) {
 201     __ subptr(rsp, wordSize);
 202     __ movflt(Address(rsp, 0), xmm0);
 203     __ fld_s(Address(rsp, 0));
 204     __ addptr(rsp, wordSize);
 205   } else if (incoming_state == dtos && UseSSE >= 2) {
 206     __ subptr(rsp, 2*wordSize);
 207     __ movdbl(Address(rsp, 0), xmm0);
 208     __ fld_d(Address(rsp, 0));
 209     __ addptr(rsp, 2*wordSize);
 210   }
 211 
 212   __ MacroAssembler::verify_FPU(state == ftos || state == dtos ? 1 : 0, "generate_return_entry_for in interpreter");
 213 
 214   // Restore stack bottom in case i2c adjusted stack
 215   __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
 216   // and NULL it as marker that rsp is now tos until next java call
 217   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
 218 
 219   __ restore_bcp();
 220   __ restore_locals();
 221 
 222   Label L_got_cache, L_giant_index;
 223   if (EnableInvokeDynamic) {
 224     __ cmpb(Address(rsi, 0), Bytecodes::_invokedynamic);
 225     __ jcc(Assembler::equal, L_giant_index);
 226   }
 227   __ get_cache_and_index_at_bcp(rbx, rcx, 1, sizeof(u2));
 228   __ bind(L_got_cache);
 229   __ movl(rbx, Address(rbx, rcx,
 230                     Address::times_ptr, constantPoolCacheOopDesc::base_offset() +
 231                     ConstantPoolCacheEntry::flags_offset()));
 232   __ andptr(rbx, 0xFF);
 233   __ lea(rsp, Address(rsp, rbx, Interpreter::stackElementScale()));
 234   __ dispatch_next(state, step);
 235 
 236   // out of the main line of code...
 237   if (EnableInvokeDynamic) {
 238     __ bind(L_giant_index);
 239     __ get_cache_and_index_at_bcp(rbx, rcx, 1, sizeof(u4));
 240     __ jmp(L_got_cache);
 241   }
 242 
 243   return entry;
 244 }
 245 
 246 
 247 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, int step) {
 248   address entry = __ pc();
 249 
 250   // In SSE mode, FP results are in xmm0
 251   if (state == ftos && UseSSE > 0) {
 252     __ subptr(rsp, wordSize);
 253     __ movflt(Address(rsp, 0), xmm0);
 254     __ fld_s(Address(rsp, 0));
 255     __ addptr(rsp, wordSize);
 256   } else if (state == dtos && UseSSE >= 2) {
 257     __ subptr(rsp, 2*wordSize);
 258     __ movdbl(Address(rsp, 0), xmm0);
 259     __ fld_d(Address(rsp, 0));
 260     __ addptr(rsp, 2*wordSize);
 261   }
 262 
 263   __ MacroAssembler::verify_FPU(state == ftos || state == dtos ? 1 : 0, "generate_deopt_entry_for in interpreter");
 264 
 265   // The stack is not extended by deopt but we must NULL last_sp as this
 266   // entry is like a "return".
 267   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
 268   __ restore_bcp();
 269   __ restore_locals();
 270   // handle exceptions
 271   { Label L;
 272     const Register thread = rcx;
 273     __ get_thread(thread);
 274     __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
 275     __ jcc(Assembler::zero, L);
 276     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));
 277     __ should_not_reach_here();
 278     __ bind(L);
 279   }
 280   __ dispatch_next(state, step);
 281   return entry;
 282 }
 283 
 284 
 285 int AbstractInterpreter::BasicType_as_index(BasicType type) {
 286   int i = 0;
 287   switch (type) {
 288     case T_BOOLEAN: i = 0; break;
 289     case T_CHAR   : i = 1; break;
 290     case T_BYTE   : i = 2; break;
 291     case T_SHORT  : i = 3; break;
 292     case T_INT    : // fall through
 293     case T_LONG   : // fall through
 294     case T_VOID   : i = 4; break;
 295     case T_FLOAT  : i = 5; break;  // have to treat float and double separately for SSE
 296     case T_DOUBLE : i = 6; break;
 297     case T_OBJECT : // fall through
 298     case T_ARRAY  : i = 7; break;
 299     default       : ShouldNotReachHere();
 300   }
 301   assert(0 <= i && i < AbstractInterpreter::number_of_result_handlers, "index out of bounds");
 302   return i;
 303 }
 304 
 305 
 306 address TemplateInterpreterGenerator::generate_result_handler_for(BasicType type) {
 307   address entry = __ pc();
 308   switch (type) {
 309     case T_BOOLEAN: __ c2bool(rax);            break;
 310     case T_CHAR   : __ andptr(rax, 0xFFFF);    break;
 311     case T_BYTE   : __ sign_extend_byte (rax); break;
 312     case T_SHORT  : __ sign_extend_short(rax); break;
 313     case T_INT    : /* nothing to do */        break;
 314     case T_DOUBLE :
 315     case T_FLOAT  :
 316       { const Register t = InterpreterRuntime::SignatureHandlerGenerator::temp();
 317         __ pop(t);                            // remove return address first
 318         // Must return a result for interpreter or compiler. In SSE
 319         // mode, results are returned in xmm0 and the FPU stack must
 320         // be empty.
 321         if (type == T_FLOAT && UseSSE >= 1) {
 322           // Load ST0
 323           __ fld_d(Address(rsp, 0));
 324           // Store as float and empty fpu stack
 325           __ fstp_s(Address(rsp, 0));
 326           // and reload
 327           __ movflt(xmm0, Address(rsp, 0));
 328         } else if (type == T_DOUBLE && UseSSE >= 2 ) {
 329           __ movdbl(xmm0, Address(rsp, 0));
 330         } else {
 331           // restore ST0
 332           __ fld_d(Address(rsp, 0));
 333         }
 334         // and pop the temp
 335         __ addptr(rsp, 2 * wordSize);
 336         __ push(t);                           // restore return address
 337       }
 338       break;
 339     case T_OBJECT :
 340       // retrieve result from frame
 341       __ movptr(rax, Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize));
 342       // and verify it
 343       __ verify_oop(rax);
 344       break;
 345     default       : ShouldNotReachHere();
 346   }
 347   __ ret(0);                                   // return from result handler
 348   return entry;
 349 }
 350 
 351 address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, address runtime_entry) {
 352   address entry = __ pc();
 353   __ push(state);
 354   __ call_VM(noreg, runtime_entry);
 355   __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));
 356   return entry;
 357 }
 358 
 359 
 360 // Helpers for commoning out cases in the various type of method entries.
 361 //
 362 
 363 // increment invocation count & check for overflow
 364 //
 365 // Note: checking for negative value instead of overflow
 366 //       so we have a 'sticky' overflow test
 367 //
 368 // rbx,: method
 369 // rcx: invocation counter
 370 //
 371 void InterpreterGenerator::generate_counter_incr(Label* overflow, Label* profile_method, Label* profile_method_continue) {
 372   const Address invocation_counter(rbx, in_bytes(methodOopDesc::invocation_counter_offset()) +
 373                                         in_bytes(InvocationCounter::counter_offset()));
 374   // Note: In tiered we increment either counters in methodOop or in MDO depending if we're profiling or not.
 375   if (TieredCompilation) {
 376     int increment = InvocationCounter::count_increment;
 377     int mask = ((1 << Tier0InvokeNotifyFreqLog)  - 1) << InvocationCounter::count_shift;
 378     Label no_mdo, done;
 379     if (ProfileInterpreter) {
 380       // Are we profiling?
 381       __ movptr(rax, Address(rbx, methodOopDesc::method_data_offset()));
 382       __ testptr(rax, rax);
 383       __ jccb(Assembler::zero, no_mdo);
 384       // Increment counter in the MDO
 385       const Address mdo_invocation_counter(rax, in_bytes(methodDataOopDesc::invocation_counter_offset()) +
 386                                                 in_bytes(InvocationCounter::counter_offset()));
 387       __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, rcx, false, Assembler::zero, overflow);
 388       __ jmpb(done);
 389     }
 390     __ bind(no_mdo);
 391     // Increment counter in methodOop (we don't need to load it, it's in rcx).
 392     __ increment_mask_and_jump(invocation_counter, increment, mask, rcx, true, Assembler::zero, overflow);
 393     __ bind(done);
 394   } else {
 395     const Address backedge_counter  (rbx, methodOopDesc::backedge_counter_offset() +
 396                                           InvocationCounter::counter_offset());
 397 
 398     if (ProfileInterpreter) { // %%% Merge this into methodDataOop
 399       __ incrementl(Address(rbx,methodOopDesc::interpreter_invocation_counter_offset()));
 400     }
 401     // Update standard invocation counters
 402     __ movl(rax, backedge_counter);               // load backedge counter
 403 
 404     __ incrementl(rcx, InvocationCounter::count_increment);
 405     __ andl(rax, InvocationCounter::count_mask_value);  // mask out the status bits
 406 
 407     __ movl(invocation_counter, rcx);             // save invocation count
 408     __ addl(rcx, rax);                            // add both counters
 409 
 410     // profile_method is non-null only for interpreted method so
 411     // profile_method != NULL == !native_call
 412     // BytecodeInterpreter only calls for native so code is elided.
 413 
 414     if (ProfileInterpreter && profile_method != NULL) {
 415       // Test to see if we should create a method data oop
 416       __ cmp32(rcx,
 417                ExternalAddress((address)&InvocationCounter::InterpreterProfileLimit));
 418       __ jcc(Assembler::less, *profile_method_continue);
 419 
 420       // if no method data exists, go to profile_method
 421       __ test_method_data_pointer(rax, *profile_method);
 422     }
 423 
 424     __ cmp32(rcx,
 425              ExternalAddress((address)&InvocationCounter::InterpreterInvocationLimit));
 426     __ jcc(Assembler::aboveEqual, *overflow);
 427   }
 428 }
 429 
 430 void InterpreterGenerator::generate_counter_overflow(Label* do_continue) {
 431 
 432   // Asm interpreter on entry
 433   // rdi - locals
 434   // rsi - bcp
 435   // rbx, - method
 436   // rdx - cpool
 437   // rbp, - interpreter frame
 438 
 439   // C++ interpreter on entry
 440   // rsi - new interpreter state pointer
 441   // rbp - interpreter frame pointer
 442   // rbx - method
 443 
 444   // On return (i.e. jump to entry_point) [ back to invocation of interpreter ]
 445   // rbx, - method
 446   // rcx - rcvr (assuming there is one)
 447   // top of stack return address of interpreter caller
 448   // rsp - sender_sp
 449 
 450   // C++ interpreter only
 451   // rsi - previous interpreter state pointer
 452 
 453   const Address size_of_parameters(rbx, methodOopDesc::size_of_parameters_offset());
 454 
 455   // InterpreterRuntime::frequency_counter_overflow takes one argument
 456   // indicating if the counter overflow occurs at a backwards branch (non-NULL bcp).
 457   // The call returns the address of the verified entry point for the method or NULL
 458   // if the compilation did not complete (either went background or bailed out).
 459   __ movptr(rax, (intptr_t)false);
 460   __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), rax);
 461 
 462   __ movptr(rbx, Address(rbp, method_offset));   // restore methodOop
 463 
 464   // Preserve invariant that rsi/rdi contain bcp/locals of sender frame
 465   // and jump to the interpreted entry.
 466   __ jmp(*do_continue, relocInfo::none);
 467 
 468 }
 469 
 470 void InterpreterGenerator::generate_stack_overflow_check(void) {
 471   // see if we've got enough room on the stack for locals plus overhead.
 472   // the expression stack grows down incrementally, so the normal guard
 473   // page mechanism will work for that.
 474   //
 475   // Registers live on entry:
 476   //
 477   // Asm interpreter
 478   // rdx: number of additional locals this frame needs (what we must check)
 479   // rbx,: methodOop
 480 
 481   // destroyed on exit
 482   // rax,
 483 
 484   // NOTE:  since the additional locals are also always pushed (wasn't obvious in
 485   // generate_method_entry) so the guard should work for them too.
 486   //
 487 
 488   // monitor entry size: see picture of stack set (generate_method_entry) and frame_x86.hpp
 489   const int entry_size    = frame::interpreter_frame_monitor_size() * wordSize;
 490 
 491   // total overhead size: entry_size + (saved rbp, thru expr stack bottom).
 492   // be sure to change this if you add/subtract anything to/from the overhead area
 493   const int overhead_size = -(frame::interpreter_frame_initial_sp_offset*wordSize) + entry_size;
 494 
 495   const int page_size = os::vm_page_size();
 496 
 497   Label after_frame_check;
 498 
 499   // see if the frame is greater than one page in size. If so,
 500   // then we need to verify there is enough stack space remaining
 501   // for the additional locals.
 502   __ cmpl(rdx, (page_size - overhead_size)/Interpreter::stackElementSize);
 503   __ jcc(Assembler::belowEqual, after_frame_check);
 504 
 505   // compute rsp as if this were going to be the last frame on
 506   // the stack before the red zone
 507 
 508   Label after_frame_check_pop;
 509 
 510   __ push(rsi);
 511 
 512   const Register thread = rsi;
 513 
 514   __ get_thread(thread);
 515 
 516   const Address stack_base(thread, Thread::stack_base_offset());
 517   const Address stack_size(thread, Thread::stack_size_offset());
 518 
 519   // locals + overhead, in bytes
 520   __ lea(rax, Address(noreg, rdx, Interpreter::stackElementScale(), overhead_size));
 521 
 522 #ifdef ASSERT
 523   Label stack_base_okay, stack_size_okay;
 524   // verify that thread stack base is non-zero
 525   __ cmpptr(stack_base, (int32_t)NULL_WORD);
 526   __ jcc(Assembler::notEqual, stack_base_okay);
 527   __ stop("stack base is zero");
 528   __ bind(stack_base_okay);
 529   // verify that thread stack size is non-zero
 530   __ cmpptr(stack_size, 0);
 531   __ jcc(Assembler::notEqual, stack_size_okay);
 532   __ stop("stack size is zero");
 533   __ bind(stack_size_okay);
 534 #endif
 535 
 536   // Add stack base to locals and subtract stack size
 537   __ addptr(rax, stack_base);
 538   __ subptr(rax, stack_size);
 539 
 540   // Use the maximum number of pages we might bang.
 541   const int max_pages = StackShadowPages > (StackRedPages+StackYellowPages) ? StackShadowPages :
 542                                                                               (StackRedPages+StackYellowPages);
 543   __ addptr(rax, max_pages * page_size);
 544 
 545   // check against the current stack bottom
 546   __ cmpptr(rsp, rax);
 547   __ jcc(Assembler::above, after_frame_check_pop);
 548 
 549   __ pop(rsi);  // get saved bcp / (c++ prev state ).
 550 
 551   __ pop(rax);  // get return address
 552   __ jump(ExternalAddress(Interpreter::throw_StackOverflowError_entry()));
 553 
 554   // all done with frame size check
 555   __ bind(after_frame_check_pop);
 556   __ pop(rsi);
 557 
 558   __ bind(after_frame_check);
 559 }
 560 
 561 // Allocate monitor and lock method (asm interpreter)
 562 // rbx, - methodOop
 563 //
 564 void InterpreterGenerator::lock_method(void) {
 565   // synchronize method
 566   const Address access_flags      (rbx, methodOopDesc::access_flags_offset());
 567   const Address monitor_block_top (rbp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
 568   const int entry_size            = frame::interpreter_frame_monitor_size() * wordSize;
 569 
 570   #ifdef ASSERT
 571     { Label L;
 572       __ movl(rax, access_flags);
 573       __ testl(rax, JVM_ACC_SYNCHRONIZED);
 574       __ jcc(Assembler::notZero, L);
 575       __ stop("method doesn't need synchronization");
 576       __ bind(L);
 577     }
 578   #endif // ASSERT
 579   // get synchronization object
 580   { Label done;
 581     const int mirror_offset = klassOopDesc::klass_part_offset_in_bytes() + Klass::java_mirror_offset_in_bytes();
 582     __ movl(rax, access_flags);
 583     __ testl(rax, JVM_ACC_STATIC);
 584     __ movptr(rax, Address(rdi, Interpreter::local_offset_in_bytes(0)));  // get receiver (assume this is frequent case)
 585     __ jcc(Assembler::zero, done);
 586     __ movptr(rax, Address(rbx, methodOopDesc::constants_offset()));
 587     __ movptr(rax, Address(rax, constantPoolOopDesc::pool_holder_offset_in_bytes()));
 588     __ movptr(rax, Address(rax, mirror_offset));
 589     __ bind(done);
 590   }
 591   // add space for monitor & lock
 592   __ subptr(rsp, entry_size);                                           // add space for a monitor entry
 593   __ movptr(monitor_block_top, rsp);                                    // set new monitor block top
 594   __ movptr(Address(rsp, BasicObjectLock::obj_offset_in_bytes()), rax); // store object
 595   __ mov(rdx, rsp);                                                    // object address
 596   __ lock_object(rdx);
 597 }
 598 
 599 //
 600 // Generate a fixed interpreter frame. This is identical setup for interpreted methods
 601 // and for native methods hence the shared code.
 602 
 603 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
 604   // initialize fixed part of activation frame
 605   __ push(rax);                                       // save return address
 606   __ enter();                                         // save old & set new rbp,
 607 
 608 
 609   __ push(rsi);                                       // set sender sp
 610   __ push((int32_t)NULL_WORD);                        // leave last_sp as null
 611   __ movptr(rsi, Address(rbx,methodOopDesc::const_offset())); // get constMethodOop
 612   __ lea(rsi, Address(rsi,constMethodOopDesc::codes_offset())); // get codebase
 613   __ push(rbx);                                      // save methodOop
 614   if (ProfileInterpreter) {
 615     Label method_data_continue;
 616     __ movptr(rdx, Address(rbx, in_bytes(methodOopDesc::method_data_offset())));
 617     __ testptr(rdx, rdx);
 618     __ jcc(Assembler::zero, method_data_continue);
 619     __ addptr(rdx, in_bytes(methodDataOopDesc::data_offset()));
 620     __ bind(method_data_continue);
 621     __ push(rdx);                                       // set the mdp (method data pointer)
 622   } else {
 623     __ push(0);
 624   }
 625 
 626   __ movptr(rdx, Address(rbx, methodOopDesc::constants_offset()));
 627   __ movptr(rdx, Address(rdx, constantPoolOopDesc::cache_offset_in_bytes()));
 628   __ push(rdx);                                       // set constant pool cache
 629   __ push(rdi);                                       // set locals pointer
 630   if (native_call) {
 631     __ push(0);                                       // no bcp
 632   } else {
 633     __ push(rsi);                                     // set bcp
 634     }
 635   __ push(0);                                         // reserve word for pointer to expression stack bottom
 636   __ movptr(Address(rsp, 0), rsp);                    // set expression stack bottom
 637 }
 638 
 639 // End of helpers
 640 
 641 //
 642 // Various method entries
 643 //------------------------------------------------------------------------------------------------------------------------
 644 //
 645 //
 646 
 647 // Call an accessor method (assuming it is resolved, otherwise drop into vanilla (slow path) entry
 648 
 649 address InterpreterGenerator::generate_accessor_entry(void) {
 650 
 651   // rbx,: methodOop
 652   // rcx: receiver (preserve for slow entry into asm interpreter)
 653 
 654   // rsi: senderSP must preserved for slow path, set SP to it on fast path
 655 
 656   address entry_point = __ pc();
 657   Label xreturn_path;
 658 
 659   // do fastpath for resolved accessor methods
 660   if (UseFastAccessorMethods) {
 661     Label slow_path;
 662     // If we need a safepoint check, generate full interpreter entry.
 663     ExternalAddress state(SafepointSynchronize::address_of_state());
 664     __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
 665              SafepointSynchronize::_not_synchronized);
 666 
 667     __ jcc(Assembler::notEqual, slow_path);
 668     // ASM/C++ Interpreter
 669     // Code: _aload_0, _(i|a)getfield, _(i|a)return or any rewrites thereof; parameter size = 1
 670     // Note: We can only use this code if the getfield has been resolved
 671     //       and if we don't have a null-pointer exception => check for
 672     //       these conditions first and use slow path if necessary.
 673     // rbx,: method
 674     // rcx: receiver
 675     __ movptr(rax, Address(rsp, wordSize));
 676 
 677     // check if local 0 != NULL and read field
 678     __ testptr(rax, rax);
 679     __ jcc(Assembler::zero, slow_path);
 680 
 681     __ movptr(rdi, Address(rbx, methodOopDesc::constants_offset()));
 682     // read first instruction word and extract bytecode @ 1 and index @ 2
 683     __ movptr(rdx, Address(rbx, methodOopDesc::const_offset()));
 684     __ movl(rdx, Address(rdx, constMethodOopDesc::codes_offset()));
 685     // Shift codes right to get the index on the right.
 686     // The bytecode fetched looks like <index><0xb4><0x2a>
 687     __ shrl(rdx, 2*BitsPerByte);
 688     __ shll(rdx, exact_log2(in_words(ConstantPoolCacheEntry::size())));
 689     __ movptr(rdi, Address(rdi, constantPoolOopDesc::cache_offset_in_bytes()));
 690 
 691     // rax,: local 0
 692     // rbx,: method
 693     // rcx: receiver - do not destroy since it is needed for slow path!
 694     // rcx: scratch
 695     // rdx: constant pool cache index
 696     // rdi: constant pool cache
 697     // rsi: sender sp
 698 
 699     // check if getfield has been resolved and read constant pool cache entry
 700     // check the validity of the cache entry by testing whether _indices field
 701     // contains Bytecode::_getfield in b1 byte.
 702     assert(in_words(ConstantPoolCacheEntry::size()) == 4, "adjust shift below");
 703     __ movl(rcx,
 704             Address(rdi,
 705                     rdx,
 706                     Address::times_ptr, constantPoolCacheOopDesc::base_offset() + ConstantPoolCacheEntry::indices_offset()));
 707     __ shrl(rcx, 2*BitsPerByte);
 708     __ andl(rcx, 0xFF);
 709     __ cmpl(rcx, Bytecodes::_getfield);
 710     __ jcc(Assembler::notEqual, slow_path);
 711 
 712     // Note: constant pool entry is not valid before bytecode is resolved
 713     __ movptr(rcx,
 714               Address(rdi,
 715                       rdx,
 716                       Address::times_ptr, constantPoolCacheOopDesc::base_offset() + ConstantPoolCacheEntry::f2_offset()));
 717     __ movl(rdx,
 718             Address(rdi,
 719                     rdx,
 720                     Address::times_ptr, constantPoolCacheOopDesc::base_offset() + ConstantPoolCacheEntry::flags_offset()));
 721 
 722     Label notByte, notShort, notChar;
 723     const Address field_address (rax, rcx, Address::times_1);
 724 
 725     // Need to differentiate between igetfield, agetfield, bgetfield etc.
 726     // because they are different sizes.
 727     // Use the type from the constant pool cache
 728     __ shrl(rdx, ConstantPoolCacheEntry::tosBits);
 729     // Make sure we don't need to mask rdx for tosBits after the above shift
 730     ConstantPoolCacheEntry::verify_tosBits();
 731     __ cmpl(rdx, btos);
 732     __ jcc(Assembler::notEqual, notByte);
 733     __ load_signed_byte(rax, field_address);
 734     __ jmp(xreturn_path);
 735 
 736     __ bind(notByte);
 737     __ cmpl(rdx, stos);
 738     __ jcc(Assembler::notEqual, notShort);
 739     __ load_signed_short(rax, field_address);
 740     __ jmp(xreturn_path);
 741 
 742     __ bind(notShort);
 743     __ cmpl(rdx, ctos);
 744     __ jcc(Assembler::notEqual, notChar);
 745     __ load_unsigned_short(rax, field_address);
 746     __ jmp(xreturn_path);
 747 
 748     __ bind(notChar);
 749 #ifdef ASSERT
 750     Label okay;
 751     __ cmpl(rdx, atos);
 752     __ jcc(Assembler::equal, okay);
 753     __ cmpl(rdx, itos);
 754     __ jcc(Assembler::equal, okay);
 755     __ stop("what type is this?");
 756     __ bind(okay);
 757 #endif // ASSERT
 758     // All the rest are a 32 bit wordsize
 759     // This is ok for now. Since fast accessors should be going away
 760     __ movptr(rax, field_address);
 761 
 762     __ bind(xreturn_path);
 763 
 764     // _ireturn/_areturn
 765     __ pop(rdi);                               // get return address
 766     __ mov(rsp, rsi);                          // set sp to sender sp
 767     __ jmp(rdi);
 768 
 769     // generate a vanilla interpreter entry as the slow path
 770     __ bind(slow_path);
 771 
 772     (void) generate_normal_entry(false);
 773     return entry_point;
 774   }
 775   return NULL;
 776 
 777 }
 778 
 779 //
 780 // Interpreter stub for calling a native method. (asm interpreter)
 781 // This sets up a somewhat different looking stack for calling the native method
 782 // than the typical interpreter frame setup.
 783 //
 784 
 785 address InterpreterGenerator::generate_native_entry(bool synchronized) {
 786   // determine code generation flags
 787   bool inc_counter  = UseCompiler || CountCompiledCalls;
 788 
 789   // rbx,: methodOop
 790   // rsi: sender sp
 791   // rsi: previous interpreter state (C++ interpreter) must preserve
 792   address entry_point = __ pc();
 793 
 794 
 795   const Address size_of_parameters(rbx, methodOopDesc::size_of_parameters_offset());
 796   const Address invocation_counter(rbx, methodOopDesc::invocation_counter_offset() + InvocationCounter::counter_offset());
 797   const Address access_flags      (rbx, methodOopDesc::access_flags_offset());
 798 
 799   // get parameter size (always needed)
 800   __ load_unsigned_short(rcx, size_of_parameters);
 801 
 802   // native calls don't need the stack size check since they have no expression stack
 803   // and the arguments are already on the stack and we only add a handful of words
 804   // to the stack
 805 
 806   // rbx,: methodOop
 807   // rcx: size of parameters
 808   // rsi: sender sp
 809 
 810   __ pop(rax);                                       // get return address
 811   // for natives the size of locals is zero
 812 
 813   // compute beginning of parameters (rdi)
 814   __ lea(rdi, Address(rsp, rcx, Interpreter::stackElementScale(), -wordSize));
 815 
 816 
 817   // add 2 zero-initialized slots for native calls
 818   // NULL result handler
 819   __ push((int32_t)NULL_WORD);
 820   // NULL oop temp (mirror or jni oop result)
 821   __ push((int32_t)NULL_WORD);
 822 
 823   if (inc_counter) __ movl(rcx, invocation_counter);  // (pre-)fetch invocation count
 824   // initialize fixed part of activation frame
 825 
 826   generate_fixed_frame(true);
 827 
 828   // make sure method is native & not abstract
 829 #ifdef ASSERT
 830   __ movl(rax, access_flags);
 831   {
 832     Label L;
 833     __ testl(rax, JVM_ACC_NATIVE);
 834     __ jcc(Assembler::notZero, L);
 835     __ stop("tried to execute non-native method as native");
 836     __ bind(L);
 837   }
 838   { Label L;
 839     __ testl(rax, JVM_ACC_ABSTRACT);
 840     __ jcc(Assembler::zero, L);
 841     __ stop("tried to execute abstract method in interpreter");
 842     __ bind(L);
 843   }
 844 #endif
 845 
 846   // Since at this point in the method invocation the exception handler
 847   // would try to exit the monitor of synchronized methods which hasn't
 848   // been entered yet, we set the thread local variable
 849   // _do_not_unlock_if_synchronized to true. The remove_activation will
 850   // check this flag.
 851 
 852   __ get_thread(rax);
 853   const Address do_not_unlock_if_synchronized(rax,
 854         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
 855   __ movbool(do_not_unlock_if_synchronized, true);
 856 
 857   // increment invocation count & check for overflow
 858   Label invocation_counter_overflow;
 859   if (inc_counter) {
 860     generate_counter_incr(&invocation_counter_overflow, NULL, NULL);
 861   }
 862 
 863   Label continue_after_compile;
 864   __ bind(continue_after_compile);
 865 
 866   bang_stack_shadow_pages(true);
 867 
 868   // reset the _do_not_unlock_if_synchronized flag
 869   __ get_thread(rax);
 870   __ movbool(do_not_unlock_if_synchronized, false);
 871 
 872   // check for synchronized methods
 873   // Must happen AFTER invocation_counter check and stack overflow check,
 874   // so method is not locked if overflows.
 875   //
 876   if (synchronized) {
 877     lock_method();
 878   } else {
 879     // no synchronization necessary
 880 #ifdef ASSERT
 881       { Label L;
 882         __ movl(rax, access_flags);
 883         __ testl(rax, JVM_ACC_SYNCHRONIZED);
 884         __ jcc(Assembler::zero, L);
 885         __ stop("method needs synchronization");
 886         __ bind(L);
 887       }
 888 #endif
 889   }
 890 
 891   // start execution
 892 #ifdef ASSERT
 893   { Label L;
 894     const Address monitor_block_top (rbp,
 895                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
 896     __ movptr(rax, monitor_block_top);
 897     __ cmpptr(rax, rsp);
 898     __ jcc(Assembler::equal, L);
 899     __ stop("broken stack frame setup in interpreter");
 900     __ bind(L);
 901   }
 902 #endif
 903 
 904   // jvmti/dtrace support
 905   __ notify_method_entry();
 906 
 907   // work registers
 908   const Register method = rbx;
 909   const Register thread = rdi;
 910   const Register t      = rcx;
 911 
 912   // allocate space for parameters
 913   __ get_method(method);
 914   __ verify_oop(method);
 915   __ load_unsigned_short(t, Address(method, methodOopDesc::size_of_parameters_offset()));
 916   __ shlptr(t, Interpreter::logStackElementSize);
 917   __ addptr(t, 2*wordSize);     // allocate two more slots for JNIEnv and possible mirror
 918   __ subptr(rsp, t);
 919   __ andptr(rsp, -(StackAlignmentInBytes)); // gcc needs 16 byte aligned stacks to do XMM intrinsics
 920 
 921   // get signature handler
 922   { Label L;
 923     __ movptr(t, Address(method, methodOopDesc::signature_handler_offset()));
 924     __ testptr(t, t);
 925     __ jcc(Assembler::notZero, L);
 926     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), method);
 927     __ get_method(method);
 928     __ movptr(t, Address(method, methodOopDesc::signature_handler_offset()));
 929     __ bind(L);
 930   }
 931 
 932   // call signature handler
 933   assert(InterpreterRuntime::SignatureHandlerGenerator::from() == rdi, "adjust this code");
 934   assert(InterpreterRuntime::SignatureHandlerGenerator::to  () == rsp, "adjust this code");
 935   assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == t  , "adjust this code");
 936   // The generated handlers do not touch RBX (the method oop).
 937   // However, large signatures cannot be cached and are generated
 938   // each time here.  The slow-path generator will blow RBX
 939   // sometime, so we must reload it after the call.
 940   __ call(t);
 941   __ get_method(method);        // slow path call blows RBX on DevStudio 5.0
 942 
 943   // result handler is in rax,
 944   // set result handler
 945   __ movptr(Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize), rax);
 946 
 947   // pass mirror handle if static call
 948   { Label L;
 949     const int mirror_offset = klassOopDesc::klass_part_offset_in_bytes() + Klass::java_mirror_offset_in_bytes();
 950     __ movl(t, Address(method, methodOopDesc::access_flags_offset()));
 951     __ testl(t, JVM_ACC_STATIC);
 952     __ jcc(Assembler::zero, L);
 953     // get mirror
 954     __ movptr(t, Address(method, methodOopDesc:: constants_offset()));
 955     __ movptr(t, Address(t, constantPoolOopDesc::pool_holder_offset_in_bytes()));
 956     __ movptr(t, Address(t, mirror_offset));
 957     // copy mirror into activation frame
 958     __ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize), t);
 959     // pass handle to mirror
 960     __ lea(t, Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize));
 961     __ movptr(Address(rsp, wordSize), t);
 962     __ bind(L);
 963   }
 964 
 965   // get native function entry point
 966   { Label L;
 967     __ movptr(rax, Address(method, methodOopDesc::native_function_offset()));
 968     ExternalAddress unsatisfied(SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
 969     __ cmpptr(rax, unsatisfied.addr());
 970     __ jcc(Assembler::notEqual, L);
 971     __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), method);
 972     __ get_method(method);
 973     __ verify_oop(method);
 974     __ movptr(rax, Address(method, methodOopDesc::native_function_offset()));
 975     __ bind(L);
 976   }
 977 
 978   // pass JNIEnv
 979   __ get_thread(thread);
 980   __ lea(t, Address(thread, JavaThread::jni_environment_offset()));
 981   __ movptr(Address(rsp, 0), t);
 982 
 983   // set_last_Java_frame_before_call
 984   // It is enough that the pc()
 985   // points into the right code segment. It does not have to be the correct return pc.
 986   __ set_last_Java_frame(thread, noreg, rbp, __ pc());
 987 
 988   // change thread state
 989 #ifdef ASSERT
 990   { Label L;
 991     __ movl(t, Address(thread, JavaThread::thread_state_offset()));
 992     __ cmpl(t, _thread_in_Java);
 993     __ jcc(Assembler::equal, L);
 994     __ stop("Wrong thread state in native stub");
 995     __ bind(L);
 996   }
 997 #endif
 998 
 999   // Change state to native
1000   __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_native);
1001   __ call(rax);
1002 
1003   // result potentially in rdx:rax or ST0
1004 
1005   // Either restore the MXCSR register after returning from the JNI Call
1006   // or verify that it wasn't changed.
1007   if (VM_Version::supports_sse()) {
1008     if (RestoreMXCSROnJNICalls) {
1009       __ ldmxcsr(ExternalAddress(StubRoutines::addr_mxcsr_std()));
1010     }
1011     else if (CheckJNICalls ) {
1012       __ call(RuntimeAddress(StubRoutines::x86::verify_mxcsr_entry()));
1013     }
1014   }
1015 
1016   // Either restore the x87 floating pointer control word after returning
1017   // from the JNI call or verify that it wasn't changed.
1018   if (CheckJNICalls) {
1019     __ call(RuntimeAddress(StubRoutines::x86::verify_fpu_cntrl_wrd_entry()));
1020   }
1021 
1022   // save potential result in ST(0) & rdx:rax
1023   // (if result handler is the T_FLOAT or T_DOUBLE handler, result must be in ST0 -
1024   // the check is necessary to avoid potential Intel FPU overflow problems by saving/restoring 'empty' FPU registers)
1025   // It is safe to do this push because state is _thread_in_native and return address will be found
1026   // via _last_native_pc and not via _last_jave_sp
1027 
1028   // NOTE: the order of theses push(es) is known to frame::interpreter_frame_result.
1029   // If the order changes or anything else is added to the stack the code in
1030   // interpreter_frame_result will have to be changed.
1031 
1032   { Label L;
1033     Label push_double;
1034     ExternalAddress float_handler(AbstractInterpreter::result_handler(T_FLOAT));
1035     ExternalAddress double_handler(AbstractInterpreter::result_handler(T_DOUBLE));
1036     __ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize),
1037               float_handler.addr());
1038     __ jcc(Assembler::equal, push_double);
1039     __ cmpptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset + 1)*wordSize),
1040               double_handler.addr());
1041     __ jcc(Assembler::notEqual, L);
1042     __ bind(push_double);
1043     __ push(dtos);
1044     __ bind(L);
1045   }
1046   __ push(ltos);
1047 
1048   // change thread state
1049   __ get_thread(thread);
1050   __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
1051   if(os::is_MP()) {
1052     if (UseMembar) {
1053       // Force this write out before the read below
1054       __ membar(Assembler::Membar_mask_bits(
1055            Assembler::LoadLoad | Assembler::LoadStore |
1056            Assembler::StoreLoad | Assembler::StoreStore));
1057     } else {
1058       // Write serialization page so VM thread can do a pseudo remote membar.
1059       // We use the current thread pointer to calculate a thread specific
1060       // offset to write to within the page. This minimizes bus traffic
1061       // due to cache line collision.
1062       __ serialize_memory(thread, rcx);
1063     }
1064   }
1065 
1066   if (AlwaysRestoreFPU) {
1067     //  Make sure the control word is correct.
1068     __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1069   }
1070 
1071   // check for safepoint operation in progress and/or pending suspend requests
1072   { Label Continue;
1073 
1074     __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
1075              SafepointSynchronize::_not_synchronized);
1076 
1077     Label L;
1078     __ jcc(Assembler::notEqual, L);
1079     __ cmpl(Address(thread, JavaThread::suspend_flags_offset()), 0);
1080     __ jcc(Assembler::equal, Continue);
1081     __ bind(L);
1082 
1083     // Don't use call_VM as it will see a possible pending exception and forward it
1084     // and never return here preventing us from clearing _last_native_pc down below.
1085     // Also can't use call_VM_leaf either as it will check to see if rsi & rdi are
1086     // preserved and correspond to the bcp/locals pointers. So we do a runtime call
1087     // by hand.
1088     //
1089     __ push(thread);
1090     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address,
1091                                             JavaThread::check_special_condition_for_native_trans)));
1092     __ increment(rsp, wordSize);
1093     __ get_thread(thread);
1094 
1095     __ bind(Continue);
1096   }
1097 
1098   // change thread state
1099   __ movl(Address(thread, JavaThread::thread_state_offset()), _thread_in_Java);
1100 
1101   __ reset_last_Java_frame(thread, true, true);
1102 
1103   // reset handle block
1104   __ movptr(t, Address(thread, JavaThread::active_handles_offset()));
1105   __ movptr(Address(t, JNIHandleBlock::top_offset_in_bytes()), NULL_WORD);
1106 
1107   // If result was an oop then unbox and save it in the frame
1108   { Label L;
1109     Label no_oop, store_result;
1110     ExternalAddress handler(AbstractInterpreter::result_handler(T_OBJECT));
1111     __ cmpptr(Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize),
1112               handler.addr());
1113     __ jcc(Assembler::notEqual, no_oop);
1114     __ cmpptr(Address(rsp, 0), (int32_t)NULL_WORD);
1115     __ pop(ltos);
1116     __ testptr(rax, rax);
1117     __ jcc(Assembler::zero, store_result);
1118     // unbox
1119     __ movptr(rax, Address(rax, 0));
1120     __ bind(store_result);
1121     __ movptr(Address(rbp, (frame::interpreter_frame_oop_temp_offset)*wordSize), rax);
1122     // keep stack depth as expected by pushing oop which will eventually be discarded
1123     __ push(ltos);
1124     __ bind(no_oop);
1125   }
1126 
1127   {
1128      Label no_reguard;
1129      __ cmpl(Address(thread, JavaThread::stack_guard_state_offset()), JavaThread::stack_guard_yellow_disabled);
1130      __ jcc(Assembler::notEqual, no_reguard);
1131 
1132      __ pusha();
1133      __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
1134      __ popa();
1135 
1136      __ bind(no_reguard);
1137    }
1138 
1139   // restore rsi to have legal interpreter frame,
1140   // i.e., bci == 0 <=> rsi == code_base()
1141   // Can't call_VM until bcp is within reasonable.
1142   __ get_method(method);      // method is junk from thread_in_native to now.
1143   __ verify_oop(method);
1144   __ movptr(rsi, Address(method,methodOopDesc::const_offset()));   // get constMethodOop
1145   __ lea(rsi, Address(rsi,constMethodOopDesc::codes_offset()));    // get codebase
1146 
1147   // handle exceptions (exception handling will handle unlocking!)
1148   { Label L;
1149     __ cmpptr(Address(thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
1150     __ jcc(Assembler::zero, L);
1151     // Note: At some point we may want to unify this with the code used in call_VM_base();
1152     //       i.e., we should use the StubRoutines::forward_exception code. For now this
1153     //       doesn't work here because the rsp is not correctly set at this point.
1154     __ MacroAssembler::call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));
1155     __ should_not_reach_here();
1156     __ bind(L);
1157   }
1158 
1159   // do unlocking if necessary
1160   { Label L;
1161     __ movl(t, Address(method, methodOopDesc::access_flags_offset()));
1162     __ testl(t, JVM_ACC_SYNCHRONIZED);
1163     __ jcc(Assembler::zero, L);
1164     // the code below should be shared with interpreter macro assembler implementation
1165     { Label unlock;
1166       // BasicObjectLock will be first in list, since this is a synchronized method. However, need
1167       // to check that the object has not been unlocked by an explicit monitorexit bytecode.
1168       const Address monitor(rbp, frame::interpreter_frame_initial_sp_offset * wordSize - (int)sizeof(BasicObjectLock));
1169 
1170       __ lea(rdx, monitor);                   // address of first monitor
1171 
1172       __ movptr(t, Address(rdx, BasicObjectLock::obj_offset_in_bytes()));
1173       __ testptr(t, t);
1174       __ jcc(Assembler::notZero, unlock);
1175 
1176       // Entry already unlocked, need to throw exception
1177       __ MacroAssembler::call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
1178       __ should_not_reach_here();
1179 
1180       __ bind(unlock);
1181       __ unlock_object(rdx);
1182     }
1183     __ bind(L);
1184   }
1185 
1186   // jvmti/dtrace support
1187   // Note: This must happen _after_ handling/throwing any exceptions since
1188   //       the exception handler code notifies the runtime of method exits
1189   //       too. If this happens before, method entry/exit notifications are
1190   //       not properly paired (was bug - gri 11/22/99).
1191   __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI);
1192 
1193   // restore potential result in rdx:rax, call result handler to restore potential result in ST0 & handle result
1194   __ pop(ltos);
1195   __ movptr(t, Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize));
1196   __ call(t);
1197 
1198   // remove activation
1199   __ movptr(t, Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize)); // get sender sp
1200   __ leave();                                // remove frame anchor
1201   __ pop(rdi);                               // get return address
1202   __ mov(rsp, t);                            // set sp to sender sp
1203   __ jmp(rdi);
1204 
1205   if (inc_counter) {
1206     // Handle overflow of counter and compile method
1207     __ bind(invocation_counter_overflow);
1208     generate_counter_overflow(&continue_after_compile);
1209   }
1210 
1211   return entry_point;
1212 }
1213 
1214 //
1215 // Generic interpreted method entry to (asm) interpreter
1216 //
1217 address InterpreterGenerator::generate_normal_entry(bool synchronized) {
1218   // determine code generation flags
1219   bool inc_counter  = UseCompiler || CountCompiledCalls;
1220 
1221   // rbx,: methodOop
1222   // rsi: sender sp
1223   address entry_point = __ pc();
1224 
1225 
1226   const Address size_of_parameters(rbx, methodOopDesc::size_of_parameters_offset());
1227   const Address size_of_locals    (rbx, methodOopDesc::size_of_locals_offset());
1228   const Address invocation_counter(rbx, methodOopDesc::invocation_counter_offset() + InvocationCounter::counter_offset());
1229   const Address access_flags      (rbx, methodOopDesc::access_flags_offset());
1230 
1231   // get parameter size (always needed)
1232   __ load_unsigned_short(rcx, size_of_parameters);
1233 
1234   // rbx,: methodOop
1235   // rcx: size of parameters
1236 
1237   // rsi: sender_sp (could differ from sp+wordSize if we were called via c2i )
1238 
1239   __ load_unsigned_short(rdx, size_of_locals);       // get size of locals in words
1240   __ subl(rdx, rcx);                                // rdx = no. of additional locals
1241 
1242   // see if we've got enough room on the stack for locals plus overhead.
1243   generate_stack_overflow_check();
1244 
1245   // get return address
1246   __ pop(rax);
1247 
1248   // compute beginning of parameters (rdi)
1249   __ lea(rdi, Address(rsp, rcx, Interpreter::stackElementScale(), -wordSize));
1250 
1251   // rdx - # of additional locals
1252   // allocate space for locals
1253   // explicitly initialize locals
1254   {
1255     Label exit, loop;
1256     __ testl(rdx, rdx);
1257     __ jcc(Assembler::lessEqual, exit);               // do nothing if rdx <= 0
1258     __ bind(loop);
1259     __ push((int32_t)NULL_WORD);                      // initialize local variables
1260     __ decrement(rdx);                                // until everything initialized
1261     __ jcc(Assembler::greater, loop);
1262     __ bind(exit);
1263   }
1264 
1265   if (inc_counter) __ movl(rcx, invocation_counter);  // (pre-)fetch invocation count
1266   // initialize fixed part of activation frame
1267   generate_fixed_frame(false);
1268 
1269   // make sure method is not native & not abstract
1270 #ifdef ASSERT
1271   __ movl(rax, access_flags);
1272   {
1273     Label L;
1274     __ testl(rax, JVM_ACC_NATIVE);
1275     __ jcc(Assembler::zero, L);
1276     __ stop("tried to execute native method as non-native");
1277     __ bind(L);
1278   }
1279   { Label L;
1280     __ testl(rax, JVM_ACC_ABSTRACT);
1281     __ jcc(Assembler::zero, L);
1282     __ stop("tried to execute abstract method in interpreter");
1283     __ bind(L);
1284   }
1285 #endif
1286 
1287   // Since at this point in the method invocation the exception handler
1288   // would try to exit the monitor of synchronized methods which hasn't
1289   // been entered yet, we set the thread local variable
1290   // _do_not_unlock_if_synchronized to true. The remove_activation will
1291   // check this flag.
1292 
1293   __ get_thread(rax);
1294   const Address do_not_unlock_if_synchronized(rax,
1295         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1296   __ movbool(do_not_unlock_if_synchronized, true);
1297 
1298   // increment invocation count & check for overflow
1299   Label invocation_counter_overflow;
1300   Label profile_method;
1301   Label profile_method_continue;
1302   if (inc_counter) {
1303     generate_counter_incr(&invocation_counter_overflow, &profile_method, &profile_method_continue);
1304     if (ProfileInterpreter) {
1305       __ bind(profile_method_continue);
1306     }
1307   }
1308   Label continue_after_compile;
1309   __ bind(continue_after_compile);
1310 
1311   bang_stack_shadow_pages(false);
1312 
1313   // reset the _do_not_unlock_if_synchronized flag
1314   __ get_thread(rax);
1315   __ movbool(do_not_unlock_if_synchronized, false);
1316 
1317   // check for synchronized methods
1318   // Must happen AFTER invocation_counter check and stack overflow check,
1319   // so method is not locked if overflows.
1320   //
1321   if (synchronized) {
1322     // Allocate monitor and lock method
1323     lock_method();
1324   } else {
1325     // no synchronization necessary
1326 #ifdef ASSERT
1327       { Label L;
1328         __ movl(rax, access_flags);
1329         __ testl(rax, JVM_ACC_SYNCHRONIZED);
1330         __ jcc(Assembler::zero, L);
1331         __ stop("method needs synchronization");
1332         __ bind(L);
1333       }
1334 #endif
1335   }
1336 
1337   // start execution
1338 #ifdef ASSERT
1339   { Label L;
1340      const Address monitor_block_top (rbp,
1341                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
1342     __ movptr(rax, monitor_block_top);
1343     __ cmpptr(rax, rsp);
1344     __ jcc(Assembler::equal, L);
1345     __ stop("broken stack frame setup in interpreter");
1346     __ bind(L);
1347   }
1348 #endif
1349 
1350   // jvmti support
1351   __ notify_method_entry();
1352 
1353   __ dispatch_next(vtos);
1354 
1355   // invocation counter overflow
1356   if (inc_counter) {
1357     if (ProfileInterpreter) {
1358       // We have decided to profile this method in the interpreter
1359       __ bind(profile_method);
1360       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1361       __ set_method_data_pointer_for_bcp();
1362       __ get_method(rbx);
1363       __ jmp(profile_method_continue);
1364     }
1365     // Handle overflow of counter and compile method
1366     __ bind(invocation_counter_overflow);
1367     generate_counter_overflow(&continue_after_compile);
1368   }
1369 
1370   return entry_point;
1371 }
1372 
1373 //------------------------------------------------------------------------------------------------------------------------
1374 // Entry points
1375 //
1376 // Here we generate the various kind of entries into the interpreter.
1377 // The two main entry type are generic bytecode methods and native call method.
1378 // These both come in synchronized and non-synchronized versions but the
1379 // frame layout they create is very similar. The other method entry
1380 // types are really just special purpose entries that are really entry
1381 // and interpretation all in one. These are for trivial methods like
1382 // accessor, empty, or special math methods.
1383 //
1384 // When control flow reaches any of the entry types for the interpreter
1385 // the following holds ->
1386 //
1387 // Arguments:
1388 //
1389 // rbx,: methodOop
1390 // rcx: receiver
1391 //
1392 //
1393 // Stack layout immediately at entry
1394 //
1395 // [ return address     ] <--- rsp
1396 // [ parameter n        ]
1397 //   ...
1398 // [ parameter 1        ]
1399 // [ expression stack   ] (caller's java expression stack)
1400 
1401 // Assuming that we don't go to one of the trivial specialized
1402 // entries the stack will look like below when we are ready to execute
1403 // the first bytecode (or call the native routine). The register usage
1404 // will be as the template based interpreter expects (see interpreter_x86.hpp).
1405 //
1406 // local variables follow incoming parameters immediately; i.e.
1407 // the return address is moved to the end of the locals).
1408 //
1409 // [ monitor entry      ] <--- rsp
1410 //   ...
1411 // [ monitor entry      ]
1412 // [ expr. stack bottom ]
1413 // [ saved rsi          ]
1414 // [ current rdi        ]
1415 // [ methodOop          ]
1416 // [ saved rbp,          ] <--- rbp,
1417 // [ return address     ]
1418 // [ local variable m   ]
1419 //   ...
1420 // [ local variable 1   ]
1421 // [ parameter n        ]
1422 //   ...
1423 // [ parameter 1        ] <--- rdi
1424 
1425 address AbstractInterpreterGenerator::generate_method_entry(AbstractInterpreter::MethodKind kind) {
1426   // determine code generation flags
1427   bool synchronized = false;
1428   address entry_point = NULL;
1429 
1430   switch (kind) {
1431     case Interpreter::zerolocals             :                                                                             break;
1432     case Interpreter::zerolocals_synchronized: synchronized = true;                                                        break;
1433     case Interpreter::native                 : entry_point = ((InterpreterGenerator*)this)->generate_native_entry(false);  break;
1434     case Interpreter::native_synchronized    : entry_point = ((InterpreterGenerator*)this)->generate_native_entry(true);   break;
1435     case Interpreter::empty                  : entry_point = ((InterpreterGenerator*)this)->generate_empty_entry();        break;
1436     case Interpreter::accessor               : entry_point = ((InterpreterGenerator*)this)->generate_accessor_entry();     break;
1437     case Interpreter::abstract               : entry_point = ((InterpreterGenerator*)this)->generate_abstract_entry();     break;
1438     case Interpreter::method_handle          : entry_point = ((InterpreterGenerator*)this)->generate_method_handle_entry(); break;
1439 
1440     case Interpreter::java_lang_math_sin     : // fall thru
1441     case Interpreter::java_lang_math_cos     : // fall thru
1442     case Interpreter::java_lang_math_tan     : // fall thru
1443     case Interpreter::java_lang_math_abs     : // fall thru
1444     case Interpreter::java_lang_math_log     : // fall thru
1445     case Interpreter::java_lang_math_log10   : // fall thru
1446     case Interpreter::java_lang_math_sqrt    : entry_point = ((InterpreterGenerator*)this)->generate_math_entry(kind);     break;
1447     default                                  : ShouldNotReachHere();                                                       break;
1448   }
1449 
1450   if (entry_point) return entry_point;
1451 
1452   return ((InterpreterGenerator*)this)->generate_normal_entry(synchronized);
1453 
1454 }
1455 
1456 // These should never be compiled since the interpreter will prefer
1457 // the compiled version to the intrinsic version.
1458 bool AbstractInterpreter::can_be_compiled(methodHandle m) {
1459   switch (method_kind(m)) {
1460     case Interpreter::java_lang_math_sin     : // fall thru
1461     case Interpreter::java_lang_math_cos     : // fall thru
1462     case Interpreter::java_lang_math_tan     : // fall thru
1463     case Interpreter::java_lang_math_abs     : // fall thru
1464     case Interpreter::java_lang_math_log     : // fall thru
1465     case Interpreter::java_lang_math_log10   : // fall thru
1466     case Interpreter::java_lang_math_sqrt    :
1467       return false;
1468     default:
1469       return true;
1470   }
1471 }
1472 
1473 // How much stack a method activation needs in words.
1474 int AbstractInterpreter::size_top_interpreter_activation(methodOop method) {
1475 
1476   const int stub_code = 4;  // see generate_call_stub
1477   // Save space for one monitor to get into the interpreted method in case
1478   // the method is synchronized
1479   int monitor_size    = method->is_synchronized() ?
1480                                 1*frame::interpreter_frame_monitor_size() : 0;
1481 
1482   // total overhead size: entry_size + (saved rbp, thru expr stack bottom).
1483   // be sure to change this if you add/subtract anything to/from the overhead area
1484   const int overhead_size = -frame::interpreter_frame_initial_sp_offset;
1485 
1486   const int extra_stack = methodOopDesc::extra_stack_entries();
1487   const int method_stack = (method->max_locals() + method->max_stack() + extra_stack) *
1488                            Interpreter::stackElementWords;
1489   return overhead_size + method_stack + stub_code;
1490 }
1491 
1492 // asm based interpreter deoptimization helpers
1493 
1494 int AbstractInterpreter::layout_activation(methodOop method,
1495                                            int tempcount,
1496                                            int popframe_extra_args,
1497                                            int moncount,
1498                                            int callee_param_count,
1499                                            int callee_locals,
1500                                            frame* caller,
1501                                            frame* interpreter_frame,
1502                                            bool is_top_frame) {
1503   // Note: This calculation must exactly parallel the frame setup
1504   // in AbstractInterpreterGenerator::generate_method_entry.
1505   // If interpreter_frame!=NULL, set up the method, locals, and monitors.
1506   // The frame interpreter_frame, if not NULL, is guaranteed to be the right size,
1507   // as determined by a previous call to this method.
1508   // It is also guaranteed to be walkable even though it is in a skeletal state
1509   // NOTE: return size is in words not bytes
1510 
1511   // fixed size of an interpreter frame:
1512   int max_locals = method->max_locals() * Interpreter::stackElementWords;
1513   int extra_locals = (method->max_locals() - method->size_of_parameters()) *
1514                      Interpreter::stackElementWords;
1515 
1516   int overhead = frame::sender_sp_offset - frame::interpreter_frame_initial_sp_offset;
1517 
1518   // Our locals were accounted for by the caller (or last_frame_adjust on the transistion)
1519   // Since the callee parameters already account for the callee's params we only need to account for
1520   // the extra locals.
1521 
1522 
1523   int size = overhead +
1524          ((callee_locals - callee_param_count)*Interpreter::stackElementWords) +
1525          (moncount*frame::interpreter_frame_monitor_size()) +
1526          tempcount*Interpreter::stackElementWords + popframe_extra_args;
1527 
1528   if (interpreter_frame != NULL) {
1529 #ifdef ASSERT
1530     if (!EnableMethodHandles)
1531       // @@@ FIXME: Should we correct interpreter_frame_sender_sp in the calling sequences?
1532       // Probably, since deoptimization doesn't work yet.
1533       assert(caller->unextended_sp() == interpreter_frame->interpreter_frame_sender_sp(), "Frame not properly walkable");
1534     assert(caller->sp() == interpreter_frame->sender_sp(), "Frame not properly walkable(2)");
1535 #endif
1536 
1537     interpreter_frame->interpreter_frame_set_method(method);
1538     // NOTE the difference in using sender_sp and interpreter_frame_sender_sp
1539     // interpreter_frame_sender_sp is the original sp of the caller (the unextended_sp)
1540     // and sender_sp is fp+8
1541     intptr_t* locals = interpreter_frame->sender_sp() + max_locals - 1;
1542 
1543     interpreter_frame->interpreter_frame_set_locals(locals);
1544     BasicObjectLock* montop = interpreter_frame->interpreter_frame_monitor_begin();
1545     BasicObjectLock* monbot = montop - moncount;
1546     interpreter_frame->interpreter_frame_set_monitor_end(monbot);
1547 
1548     // Set last_sp
1549     intptr_t*  rsp = (intptr_t*) monbot  -
1550                      tempcount*Interpreter::stackElementWords -
1551                      popframe_extra_args;
1552     interpreter_frame->interpreter_frame_set_last_sp(rsp);
1553 
1554     // All frames but the initial (oldest) interpreter frame we fill in have a
1555     // value for sender_sp that allows walking the stack but isn't
1556     // truly correct. Correct the value here.
1557 
1558     if (extra_locals != 0 &&
1559         interpreter_frame->sender_sp() == interpreter_frame->interpreter_frame_sender_sp() ) {
1560       interpreter_frame->set_interpreter_frame_sender_sp(caller->sp() + extra_locals);
1561     }
1562     *interpreter_frame->interpreter_frame_cache_addr() =
1563       method->constants()->cache();
1564   }
1565   return size;
1566 }
1567 
1568 
1569 //------------------------------------------------------------------------------------------------------------------------
1570 // Exceptions
1571 
1572 void TemplateInterpreterGenerator::generate_throw_exception() {
1573   // Entry point in previous activation (i.e., if the caller was interpreted)
1574   Interpreter::_rethrow_exception_entry = __ pc();
1575   const Register thread = rcx;
1576 
1577   // Restore sp to interpreter_frame_last_sp even though we are going
1578   // to empty the expression stack for the exception processing.
1579   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
1580   // rax,: exception
1581   // rdx: return address/pc that threw exception
1582   __ restore_bcp();                              // rsi points to call/send
1583   __ restore_locals();
1584 
1585   // Entry point for exceptions thrown within interpreter code
1586   Interpreter::_throw_exception_entry = __ pc();
1587   // expression stack is undefined here
1588   // rax,: exception
1589   // rsi: exception bcp
1590   __ verify_oop(rax);
1591 
1592   // expression stack must be empty before entering the VM in case of an exception
1593   __ empty_expression_stack();
1594   __ empty_FPU_stack();
1595   // find exception handler address and preserve exception oop
1596   __ call_VM(rdx, CAST_FROM_FN_PTR(address, InterpreterRuntime::exception_handler_for_exception), rax);
1597   // rax,: exception handler entry point
1598   // rdx: preserved exception oop
1599   // rsi: bcp for exception handler
1600   __ push_ptr(rdx);                              // push exception which is now the only value on the stack
1601   __ jmp(rax);                                   // jump to exception handler (may be _remove_activation_entry!)
1602 
1603   // If the exception is not handled in the current frame the frame is removed and
1604   // the exception is rethrown (i.e. exception continuation is _rethrow_exception).
1605   //
1606   // Note: At this point the bci is still the bxi for the instruction which caused
1607   //       the exception and the expression stack is empty. Thus, for any VM calls
1608   //       at this point, GC will find a legal oop map (with empty expression stack).
1609 
1610   // In current activation
1611   // tos: exception
1612   // rsi: exception bcp
1613 
1614   //
1615   // JVMTI PopFrame support
1616   //
1617 
1618    Interpreter::_remove_activation_preserving_args_entry = __ pc();
1619   __ empty_expression_stack();
1620   __ empty_FPU_stack();
1621   // Set the popframe_processing bit in pending_popframe_condition indicating that we are
1622   // currently handling popframe, so that call_VMs that may happen later do not trigger new
1623   // popframe handling cycles.
1624   __ get_thread(thread);
1625   __ movl(rdx, Address(thread, JavaThread::popframe_condition_offset()));
1626   __ orl(rdx, JavaThread::popframe_processing_bit);
1627   __ movl(Address(thread, JavaThread::popframe_condition_offset()), rdx);
1628 
1629   {
1630     // Check to see whether we are returning to a deoptimized frame.
1631     // (The PopFrame call ensures that the caller of the popped frame is
1632     // either interpreted or compiled and deoptimizes it if compiled.)
1633     // In this case, we can't call dispatch_next() after the frame is
1634     // popped, but instead must save the incoming arguments and restore
1635     // them after deoptimization has occurred.
1636     //
1637     // Note that we don't compare the return PC against the
1638     // deoptimization blob's unpack entry because of the presence of
1639     // adapter frames in C2.
1640     Label caller_not_deoptimized;
1641     __ movptr(rdx, Address(rbp, frame::return_addr_offset * wordSize));
1642     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::interpreter_contains), rdx);
1643     __ testl(rax, rax);
1644     __ jcc(Assembler::notZero, caller_not_deoptimized);
1645 
1646     // Compute size of arguments for saving when returning to deoptimized caller
1647     __ get_method(rax);
1648     __ verify_oop(rax);
1649     __ load_unsigned_short(rax, Address(rax, in_bytes(methodOopDesc::size_of_parameters_offset())));
1650     __ shlptr(rax, Interpreter::logStackElementSize);
1651     __ restore_locals();
1652     __ subptr(rdi, rax);
1653     __ addptr(rdi, wordSize);
1654     // Save these arguments
1655     __ get_thread(thread);
1656     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::popframe_preserve_args), thread, rax, rdi);
1657 
1658     __ remove_activation(vtos, rdx,
1659                          /* throw_monitor_exception */ false,
1660                          /* install_monitor_exception */ false,
1661                          /* notify_jvmdi */ false);
1662 
1663     // Inform deoptimization that it is responsible for restoring these arguments
1664     __ get_thread(thread);
1665     __ movl(Address(thread, JavaThread::popframe_condition_offset()), JavaThread::popframe_force_deopt_reexecution_bit);
1666 
1667     // Continue in deoptimization handler
1668     __ jmp(rdx);
1669 
1670     __ bind(caller_not_deoptimized);
1671   }
1672 
1673   __ remove_activation(vtos, rdx,
1674                        /* throw_monitor_exception */ false,
1675                        /* install_monitor_exception */ false,
1676                        /* notify_jvmdi */ false);
1677 
1678   // Finish with popframe handling
1679   // A previous I2C followed by a deoptimization might have moved the
1680   // outgoing arguments further up the stack. PopFrame expects the
1681   // mutations to those outgoing arguments to be preserved and other
1682   // constraints basically require this frame to look exactly as
1683   // though it had previously invoked an interpreted activation with
1684   // no space between the top of the expression stack (current
1685   // last_sp) and the top of stack. Rather than force deopt to
1686   // maintain this kind of invariant all the time we call a small
1687   // fixup routine to move the mutated arguments onto the top of our
1688   // expression stack if necessary.
1689   __ mov(rax, rsp);
1690   __ movptr(rbx, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1691   __ get_thread(thread);
1692   // PC must point into interpreter here
1693   __ set_last_Java_frame(thread, noreg, rbp, __ pc());
1694   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), thread, rax, rbx);
1695   __ get_thread(thread);
1696   __ reset_last_Java_frame(thread, true, true);
1697   // Restore the last_sp and null it out
1698   __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1699   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
1700 
1701   __ restore_bcp();
1702   __ restore_locals();
1703   // The method data pointer was incremented already during
1704   // call profiling. We have to restore the mdp for the current bcp.
1705   if (ProfileInterpreter) {
1706     __ set_method_data_pointer_for_bcp();
1707   }
1708 
1709   // Clear the popframe condition flag
1710   __ get_thread(thread);
1711   __ movl(Address(thread, JavaThread::popframe_condition_offset()), JavaThread::popframe_inactive);
1712 
1713   __ dispatch_next(vtos);
1714   // end of PopFrame support
1715 
1716   Interpreter::_remove_activation_entry = __ pc();
1717 
1718   // preserve exception over this code sequence
1719   __ pop_ptr(rax);
1720   __ get_thread(thread);
1721   __ movptr(Address(thread, JavaThread::vm_result_offset()), rax);
1722   // remove the activation (without doing throws on illegalMonitorExceptions)
1723   __ remove_activation(vtos, rdx, false, true, false);
1724   // restore exception
1725   __ get_thread(thread);
1726   __ movptr(rax, Address(thread, JavaThread::vm_result_offset()));
1727   __ movptr(Address(thread, JavaThread::vm_result_offset()), NULL_WORD);
1728   __ verify_oop(rax);
1729 
1730   // Inbetween activations - previous activation type unknown yet
1731   // compute continuation point - the continuation point expects
1732   // the following registers set up:
1733   //
1734   // rax: exception
1735   // rdx: return address/pc that threw exception
1736   // rsp: expression stack of caller
1737   // rbp: rbp, of caller
1738   __ push(rax);                                  // save exception
1739   __ push(rdx);                                  // save return address
1740   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), thread, rdx);
1741   __ mov(rbx, rax);                              // save exception handler
1742   __ pop(rdx);                                   // restore return address
1743   __ pop(rax);                                   // restore exception
1744   // Note that an "issuing PC" is actually the next PC after the call
1745   __ jmp(rbx);                                   // jump to exception handler of caller
1746 }
1747 
1748 
1749 //
1750 // JVMTI ForceEarlyReturn support
1751 //
1752 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1753   address entry = __ pc();
1754   const Register thread = rcx;
1755 
1756   __ restore_bcp();
1757   __ restore_locals();
1758   __ empty_expression_stack();
1759   __ empty_FPU_stack();
1760   __ load_earlyret_value(state);
1761 
1762   __ get_thread(thread);
1763   __ movptr(rcx, Address(thread, JavaThread::jvmti_thread_state_offset()));
1764   const Address cond_addr(rcx, JvmtiThreadState::earlyret_state_offset());
1765 
1766   // Clear the earlyret state
1767   __ movl(cond_addr, JvmtiThreadState::earlyret_inactive);
1768 
1769   __ remove_activation(state, rsi,
1770                        false, /* throw_monitor_exception */
1771                        false, /* install_monitor_exception */
1772                        true); /* notify_jvmdi */
1773   __ jmp(rsi);
1774   return entry;
1775 } // end of ForceEarlyReturn support
1776 
1777 
1778 //------------------------------------------------------------------------------------------------------------------------
1779 // Helper for vtos entry point generation
1780 
1781 void TemplateInterpreterGenerator::set_vtos_entry_points (Template* t, address& bep, address& cep, address& sep, address& aep, address& iep, address& lep, address& fep, address& dep, address& vep) {
1782   assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1783   Label L;
1784   fep = __ pc(); __ push(ftos); __ jmp(L);
1785   dep = __ pc(); __ push(dtos); __ jmp(L);
1786   lep = __ pc(); __ push(ltos); __ jmp(L);
1787   aep = __ pc(); __ push(atos); __ jmp(L);
1788   bep = cep = sep =             // fall through
1789   iep = __ pc(); __ push(itos); // fall through
1790   vep = __ pc(); __ bind(L);    // fall through
1791   generate_and_dispatch(t);
1792 }
1793 
1794 //------------------------------------------------------------------------------------------------------------------------
1795 // Generation of individual instructions
1796 
1797 // helpers for generate_and_dispatch
1798 
1799 
1800 
1801 InterpreterGenerator::InterpreterGenerator(StubQueue* code)
1802  : TemplateInterpreterGenerator(code) {
1803    generate_all(); // down here so it can be "virtual"
1804 }
1805 
1806 //------------------------------------------------------------------------------------------------------------------------
1807 
1808 // Non-product code
1809 #ifndef PRODUCT
1810 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1811   address entry = __ pc();
1812 
1813   // prepare expression stack
1814   __ pop(rcx);          // pop return address so expression stack is 'pure'
1815   __ push(state);       // save tosca
1816 
1817   // pass tosca registers as arguments & call tracer
1818   __ call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::trace_bytecode), rcx, rax, rdx);
1819   __ mov(rcx, rax);     // make sure return address is not destroyed by pop(state)
1820   __ pop(state);        // restore tosca
1821 
1822   // return
1823   __ jmp(rcx);
1824 
1825   return entry;
1826 }
1827 
1828 
1829 void TemplateInterpreterGenerator::count_bytecode() {
1830   __ incrementl(ExternalAddress((address) &BytecodeCounter::_counter_value));
1831 }
1832 
1833 
1834 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
1835   __ incrementl(ExternalAddress((address) &BytecodeHistogram::_counters[t->bytecode()]));
1836 }
1837 
1838 
1839 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
1840   __ mov32(ExternalAddress((address) &BytecodePairHistogram::_index), rbx);
1841   __ shrl(rbx, BytecodePairHistogram::log2_number_of_codes);
1842   __ orl(rbx, ((int)t->bytecode()) << BytecodePairHistogram::log2_number_of_codes);
1843   ExternalAddress table((address) BytecodePairHistogram::_counters);
1844   Address index(noreg, rbx, Address::times_4);
1845   __ incrementl(ArrayAddress(table, index));
1846 }
1847 
1848 
1849 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
1850   // Call a little run-time stub to avoid blow-up for each bytecode.
1851   // The run-time runtime saves the right registers, depending on
1852   // the tosca in-state for the given template.
1853   assert(Interpreter::trace_code(t->tos_in()) != NULL,
1854          "entry must have been generated");
1855   __ call(RuntimeAddress(Interpreter::trace_code(t->tos_in())));
1856 }
1857 
1858 
1859 void TemplateInterpreterGenerator::stop_interpreter_at() {
1860   Label L;
1861   __ cmp32(ExternalAddress((address) &BytecodeCounter::_counter_value),
1862            StopInterpreterAt);
1863   __ jcc(Assembler::notEqual, L);
1864   __ int3();
1865   __ bind(L);
1866 }
1867 #endif // !PRODUCT
1868 #endif // CC_INTERP