1 /*
   2  * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  20  * CA 95054 USA or visit www.sun.com if you need additional information or
  21  * have any questions.
  22  *
  23  */
  24 
  25 #include "incls/_precompiled.incl"
  26 #include "incls/_interpreter_x86_64.cpp.incl"
  27 
  28 #define __ _masm->
  29 
  30 #ifndef CC_INTERP
  31 
  32 const int method_offset = frame::interpreter_frame_method_offset * wordSize;
  33 const int bci_offset    = frame::interpreter_frame_bcx_offset    * wordSize;
  34 const int locals_offset = frame::interpreter_frame_locals_offset * wordSize;
  35 
  36 //-----------------------------------------------------------------------------
  37 
  38 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
  39   address entry = __ pc();
  40 
  41 #ifdef ASSERT
  42   {
  43     Label L;
  44     __ lea(rax, Address(rbp,
  45                         frame::interpreter_frame_monitor_block_top_offset *
  46                         wordSize));
  47     __ cmpptr(rax, rsp); // rax = maximal rsp for current rbp (stack
  48                          // grows negative)
  49     __ jcc(Assembler::aboveEqual, L); // check if frame is complete
  50     __ stop ("interpreter frame not set up");
  51     __ bind(L);
  52   }
  53 #endif // ASSERT
  54   // Restore bcp under the assumption that the current frame is still
  55   // interpreted
  56   __ restore_bcp();
  57 
  58   // expression stack must be empty before entering the VM if an
  59   // exception happened
  60   __ empty_expression_stack();
  61   // throw exception
  62   __ call_VM(noreg,
  63              CAST_FROM_FN_PTR(address,
  64                               InterpreterRuntime::throw_StackOverflowError));
  65   return entry;
  66 }
  67 
  68 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler(
  69         const char* name) {
  70   address entry = __ pc();
  71   // expression stack must be empty before entering the VM if an
  72   // exception happened
  73   __ empty_expression_stack();
  74   // setup parameters
  75   // ??? convention: expect aberrant index in register ebx
  76   __ lea(c_rarg1, ExternalAddress((address)name));
  77   __ call_VM(noreg,
  78              CAST_FROM_FN_PTR(address,
  79                               InterpreterRuntime::
  80                               throw_ArrayIndexOutOfBoundsException),
  81              c_rarg1, rbx);
  82   return entry;
  83 }
  84 
  85 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
  86   address entry = __ pc();
  87 
  88   // object is at TOS
  89   __ pop(c_rarg1);
  90 
  91   // expression stack must be empty before entering the VM if an
  92   // exception happened
  93   __ empty_expression_stack();
  94 
  95   __ call_VM(noreg,
  96              CAST_FROM_FN_PTR(address,
  97                               InterpreterRuntime::
  98                               throw_ClassCastException),
  99              c_rarg1);
 100   return entry;
 101 }
 102 
 103 // Arguments are: required type at TOS+8, failing object (or NULL) at TOS+4.
 104 address TemplateInterpreterGenerator::generate_WrongMethodType_handler() {
 105   address entry = __ pc();
 106 
 107   __ pop(c_rarg2);              // failing object is at TOS
 108   __ pop(c_rarg1);              // required type is at TOS+8
 109 
 110   __ verify_oop(c_rarg1);
 111   __ verify_oop(c_rarg2);
 112 
 113   // Various method handle types use interpreter registers as temps.
 114   __ restore_bcp();
 115   __ restore_locals();
 116 
 117   // Expression stack must be empty before entering the VM for an exception.
 118   __ empty_expression_stack();
 119 
 120   __ call_VM(noreg,
 121              CAST_FROM_FN_PTR(address,
 122                               InterpreterRuntime::throw_WrongMethodTypeException),
 123              // pass required type, failing object (or NULL)
 124              c_rarg1, c_rarg2);
 125   return entry;
 126 }
 127 
 128 address TemplateInterpreterGenerator::generate_exception_handler_common(
 129         const char* name, const char* message, bool pass_oop) {
 130   assert(!pass_oop || message == NULL, "either oop or message but not both");
 131   address entry = __ pc();
 132   if (pass_oop) {
 133     // object is at TOS
 134     __ pop(c_rarg2);
 135   }
 136   // expression stack must be empty before entering the VM if an
 137   // exception happened
 138   __ empty_expression_stack();
 139   // setup parameters
 140   __ lea(c_rarg1, ExternalAddress((address)name));
 141   if (pass_oop) {
 142     __ call_VM(rax, CAST_FROM_FN_PTR(address,
 143                                      InterpreterRuntime::
 144                                      create_klass_exception),
 145                c_rarg1, c_rarg2);
 146   } else {
 147     // kind of lame ExternalAddress can't take NULL because
 148     // external_word_Relocation will assert.
 149     if (message != NULL) {
 150       __ lea(c_rarg2, ExternalAddress((address)message));
 151     } else {
 152       __ movptr(c_rarg2, NULL_WORD);
 153     }
 154     __ call_VM(rax,
 155                CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception),
 156                c_rarg1, c_rarg2);
 157   }
 158   // throw exception
 159   __ jump(ExternalAddress(Interpreter::throw_exception_entry()));
 160   return entry;
 161 }
 162 
 163 
 164 address TemplateInterpreterGenerator::generate_continuation_for(TosState state) {
 165   address entry = __ pc();
 166   // NULL last_sp until next java call
 167   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
 168   __ dispatch_next(state);
 169   return entry;
 170 }
 171 
 172 
 173 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state,
 174                                                                 int step) {
 175 
 176   // amd64 doesn't need to do anything special about compiled returns
 177   // to the interpreter so the code that exists on x86 to place a sentinel
 178   // here and the specialized cleanup code is not needed here.
 179 
 180   address entry = __ pc();
 181 
 182   // Restore stack bottom in case i2c adjusted stack
 183   __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
 184   // and NULL it as marker that esp is now tos until next java call
 185   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
 186 
 187   __ restore_bcp();
 188   __ restore_locals();
 189 
 190   Label L_got_cache, L_giant_index;
 191   if (EnableInvokeDynamic) {
 192     __ cmpb(Address(r13, 0), Bytecodes::_invokedynamic);
 193     __ jcc(Assembler::equal, L_giant_index);
 194   }
 195   __ get_cache_and_index_at_bcp(rbx, rcx, 1, false);
 196   __ bind(L_got_cache);
 197   __ movl(rbx, Address(rbx, rcx,
 198                        Address::times_ptr,
 199                        in_bytes(constantPoolCacheOopDesc::base_offset()) +
 200                        3 * wordSize));
 201   __ andl(rbx, 0xFF);
 202   if (TaggedStackInterpreter) __ shll(rbx, 1); // 2 slots per parameter.
 203   __ lea(rsp, Address(rsp, rbx, Address::times_8));
 204   __ dispatch_next(state, step);
 205 
 206   // out of the main line of code...
 207   if (EnableInvokeDynamic) {
 208     __ bind(L_giant_index);
 209     __ get_cache_and_index_at_bcp(rbx, rcx, 1, true);
 210     __ jmp(L_got_cache);
 211   }
 212 
 213   return entry;
 214 }
 215 
 216 
 217 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state,
 218                                                                int step) {
 219   address entry = __ pc();
 220   // NULL last_sp until next java call
 221   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
 222   __ restore_bcp();
 223   __ restore_locals();
 224   // handle exceptions
 225   {
 226     Label L;
 227     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
 228     __ jcc(Assembler::zero, L);
 229     __ call_VM(noreg,
 230                CAST_FROM_FN_PTR(address,
 231                                 InterpreterRuntime::throw_pending_exception));
 232     __ should_not_reach_here();
 233     __ bind(L);
 234   }
 235   __ dispatch_next(state, step);
 236   return entry;
 237 }
 238 
 239 int AbstractInterpreter::BasicType_as_index(BasicType type) {
 240   int i = 0;
 241   switch (type) {
 242     case T_BOOLEAN: i = 0; break;
 243     case T_CHAR   : i = 1; break;
 244     case T_BYTE   : i = 2; break;
 245     case T_SHORT  : i = 3; break;
 246     case T_INT    : i = 4; break;
 247     case T_LONG   : i = 5; break;
 248     case T_VOID   : i = 6; break;
 249     case T_FLOAT  : i = 7; break;
 250     case T_DOUBLE : i = 8; break;
 251     case T_OBJECT : i = 9; break;
 252     case T_ARRAY  : i = 9; break;
 253     default       : ShouldNotReachHere();
 254   }
 255   assert(0 <= i && i < AbstractInterpreter::number_of_result_handlers,
 256          "index out of bounds");
 257   return i;
 258 }
 259 
 260 
 261 address TemplateInterpreterGenerator::generate_result_handler_for(
 262         BasicType type) {
 263   address entry = __ pc();
 264   switch (type) {
 265   case T_BOOLEAN: __ c2bool(rax);            break;
 266   case T_CHAR   : __ movzwl(rax, rax);       break;
 267   case T_BYTE   : __ sign_extend_byte(rax);  break;
 268   case T_SHORT  : __ sign_extend_short(rax); break;
 269   case T_INT    : /* nothing to do */        break;
 270   case T_LONG   : /* nothing to do */        break;
 271   case T_VOID   : /* nothing to do */        break;
 272   case T_FLOAT  : /* nothing to do */        break;
 273   case T_DOUBLE : /* nothing to do */        break;
 274   case T_OBJECT :
 275     // retrieve result from frame
 276     __ movptr(rax, Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize));
 277     // and verify it
 278     __ verify_oop(rax);
 279     break;
 280   default       : ShouldNotReachHere();
 281   }
 282   __ ret(0);                                   // return from result handler
 283   return entry;
 284 }
 285 
 286 address TemplateInterpreterGenerator::generate_safept_entry_for(
 287         TosState state,
 288         address runtime_entry) {
 289   address entry = __ pc();
 290   __ push(state);
 291   __ call_VM(noreg, runtime_entry);
 292   __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));
 293   return entry;
 294 }
 295 
 296 
 297 
 298 // Helpers for commoning out cases in the various type of method entries.
 299 //
 300 
 301 
 302 // increment invocation count & check for overflow
 303 //
 304 // Note: checking for negative value instead of overflow
 305 //       so we have a 'sticky' overflow test
 306 //
 307 // rbx: method
 308 // ecx: invocation counter
 309 //
 310 void InterpreterGenerator::generate_counter_incr(
 311         Label* overflow,
 312         Label* profile_method,
 313         Label* profile_method_continue) {
 314 
 315   const Address invocation_counter(rbx,
 316                                    methodOopDesc::invocation_counter_offset() +
 317                                    InvocationCounter::counter_offset());
 318   const Address backedge_counter(rbx,
 319                                  methodOopDesc::backedge_counter_offset() +
 320                                  InvocationCounter::counter_offset());
 321 
 322   if (ProfileInterpreter) { // %%% Merge this into methodDataOop
 323     __ incrementl(Address(rbx,
 324                     methodOopDesc::interpreter_invocation_counter_offset()));
 325   }
 326   // Update standard invocation counters
 327   __ movl(rax, backedge_counter); // load backedge counter
 328 
 329   __ incrementl(rcx, InvocationCounter::count_increment);
 330   __ andl(rax, InvocationCounter::count_mask_value); // mask out the
 331                                                      // status bits
 332 
 333   __ movl(invocation_counter, rcx); // save invocation count
 334   __ addl(rcx, rax); // add both counters
 335 
 336   // profile_method is non-null only for interpreted method so
 337   // profile_method != NULL == !native_call
 338 
 339   if (ProfileInterpreter && profile_method != NULL) {
 340     // Test to see if we should create a method data oop
 341     __ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterProfileLimit));
 342     __ jcc(Assembler::less, *profile_method_continue);
 343 
 344     // if no method data exists, go to profile_method
 345     __ test_method_data_pointer(rax, *profile_method);
 346   }
 347 
 348   __ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterInvocationLimit));
 349   __ jcc(Assembler::aboveEqual, *overflow);
 350 }
 351 
 352 void InterpreterGenerator::generate_counter_overflow(Label* do_continue) {
 353 
 354   // Asm interpreter on entry
 355   // r14 - locals
 356   // r13 - bcp
 357   // rbx - method
 358   // edx - cpool --- DOES NOT APPEAR TO BE TRUE
 359   // rbp - interpreter frame
 360 
 361   // On return (i.e. jump to entry_point) [ back to invocation of interpreter ]
 362   // Everything as it was on entry
 363   // rdx is not restored. Doesn't appear to really be set.
 364 
 365   const Address size_of_parameters(rbx,
 366                                    methodOopDesc::size_of_parameters_offset());
 367 
 368   // InterpreterRuntime::frequency_counter_overflow takes two
 369   // arguments, the first (thread) is passed by call_VM, the second
 370   // indicates if the counter overflow occurs at a backwards branch
 371   // (NULL bcp).  We pass zero for it.  The call returns the address
 372   // of the verified entry point for the method or NULL if the
 373   // compilation did not complete (either went background or bailed
 374   // out).
 375   __ movl(c_rarg1, 0);
 376   __ call_VM(noreg,
 377              CAST_FROM_FN_PTR(address,
 378                               InterpreterRuntime::frequency_counter_overflow),
 379              c_rarg1);
 380 
 381   __ movptr(rbx, Address(rbp, method_offset));   // restore methodOop
 382   // Preserve invariant that r13/r14 contain bcp/locals of sender frame
 383   // and jump to the interpreted entry.
 384   __ jmp(*do_continue, relocInfo::none);
 385 }
 386 
 387 // See if we've got enough room on the stack for locals plus overhead.
 388 // The expression stack grows down incrementally, so the normal guard
 389 // page mechanism will work for that.
 390 //
 391 // NOTE: Since the additional locals are also always pushed (wasn't
 392 // obvious in generate_method_entry) so the guard should work for them
 393 // too.
 394 //
 395 // Args:
 396 //      rdx: number of additional locals this frame needs (what we must check)
 397 //      rbx: methodOop
 398 //
 399 // Kills:
 400 //      rax
 401 void InterpreterGenerator::generate_stack_overflow_check(void) {
 402 
 403   // monitor entry size: see picture of stack set
 404   // (generate_method_entry) and frame_amd64.hpp
 405   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 406 
 407   // total overhead size: entry_size + (saved rbp through expr stack
 408   // bottom).  be sure to change this if you add/subtract anything
 409   // to/from the overhead area
 410   const int overhead_size =
 411     -(frame::interpreter_frame_initial_sp_offset * wordSize) + entry_size;
 412 
 413   const int page_size = os::vm_page_size();
 414 
 415   Label after_frame_check;
 416 
 417   // see if the frame is greater than one page in size. If so,
 418   // then we need to verify there is enough stack space remaining
 419   // for the additional locals.
 420   __ cmpl(rdx, (page_size - overhead_size) / Interpreter::stackElementSize());
 421   __ jcc(Assembler::belowEqual, after_frame_check);
 422 
 423   // compute rsp as if this were going to be the last frame on
 424   // the stack before the red zone
 425 
 426   const Address stack_base(r15_thread, Thread::stack_base_offset());
 427   const Address stack_size(r15_thread, Thread::stack_size_offset());
 428 
 429   // locals + overhead, in bytes
 430   __ mov(rax, rdx);
 431   __ shlptr(rax, Interpreter::logStackElementSize()); // 2 slots per parameter.
 432   __ addptr(rax, overhead_size);
 433 
 434 #ifdef ASSERT
 435   Label stack_base_okay, stack_size_okay;
 436   // verify that thread stack base is non-zero
 437   __ cmpptr(stack_base, (int32_t)NULL_WORD);
 438   __ jcc(Assembler::notEqual, stack_base_okay);
 439   __ stop("stack base is zero");
 440   __ bind(stack_base_okay);
 441   // verify that thread stack size is non-zero
 442   __ cmpptr(stack_size, 0);
 443   __ jcc(Assembler::notEqual, stack_size_okay);
 444   __ stop("stack size is zero");
 445   __ bind(stack_size_okay);
 446 #endif
 447 
 448   // Add stack base to locals and subtract stack size
 449   __ addptr(rax, stack_base);
 450   __ subptr(rax, stack_size);
 451 
 452   // Use the maximum number of pages we might bang.
 453   const int max_pages = StackShadowPages > (StackRedPages+StackYellowPages) ? StackShadowPages :
 454                                                                               (StackRedPages+StackYellowPages);
 455 
 456   // add in the red and yellow zone sizes
 457   __ addptr(rax, max_pages * page_size);
 458 
 459   // check against the current stack bottom
 460   __ cmpptr(rsp, rax);
 461   __ jcc(Assembler::above, after_frame_check);
 462 
 463   __ pop(rax); // get return address
 464   __ jump(ExternalAddress(Interpreter::throw_StackOverflowError_entry()));
 465 
 466   // all done with frame size check
 467   __ bind(after_frame_check);
 468 }
 469 
 470 // Allocate monitor and lock method (asm interpreter)
 471 //
 472 // Args:
 473 //      rbx: methodOop
 474 //      r14: locals
 475 //
 476 // Kills:
 477 //      rax
 478 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, ...(param regs)
 479 //      rscratch1, rscratch2 (scratch regs)
 480 void InterpreterGenerator::lock_method(void) {
 481   // synchronize method
 482   const Address access_flags(rbx, methodOopDesc::access_flags_offset());
 483   const Address monitor_block_top(
 484         rbp,
 485         frame::interpreter_frame_monitor_block_top_offset * wordSize);
 486   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 487 
 488 #ifdef ASSERT
 489   {
 490     Label L;
 491     __ movl(rax, access_flags);
 492     __ testl(rax, JVM_ACC_SYNCHRONIZED);
 493     __ jcc(Assembler::notZero, L);
 494     __ stop("method doesn't need synchronization");
 495     __ bind(L);
 496   }
 497 #endif // ASSERT
 498 
 499   // get synchronization object
 500   {
 501     const int mirror_offset = klassOopDesc::klass_part_offset_in_bytes() +
 502                               Klass::java_mirror_offset_in_bytes();
 503     Label done;
 504     __ movl(rax, access_flags);
 505     __ testl(rax, JVM_ACC_STATIC);
 506     // get receiver (assume this is frequent case)
 507     __ movptr(rax, Address(r14, Interpreter::local_offset_in_bytes(0)));
 508     __ jcc(Assembler::zero, done);
 509     __ movptr(rax, Address(rbx, methodOopDesc::constants_offset()));
 510     __ movptr(rax, Address(rax,
 511                            constantPoolOopDesc::pool_holder_offset_in_bytes()));
 512     __ movptr(rax, Address(rax, mirror_offset));
 513 
 514 #ifdef ASSERT
 515     {
 516       Label L;
 517       __ testptr(rax, rax);
 518       __ jcc(Assembler::notZero, L);
 519       __ stop("synchronization object is NULL");
 520       __ bind(L);
 521     }
 522 #endif // ASSERT
 523 
 524     __ bind(done);
 525   }
 526 
 527   // add space for monitor & lock
 528   __ subptr(rsp, entry_size); // add space for a monitor entry
 529   __ movptr(monitor_block_top, rsp);  // set new monitor block top
 530   // store object
 531   __ movptr(Address(rsp, BasicObjectLock::obj_offset_in_bytes()), rax);
 532   __ movptr(c_rarg1, rsp); // object address
 533   __ lock_object(c_rarg1);
 534 }
 535 
 536 // Generate a fixed interpreter frame. This is identical setup for
 537 // interpreted methods and for native methods hence the shared code.
 538 //
 539 // Args:
 540 //      rax: return address
 541 //      rbx: methodOop
 542 //      r14: pointer to locals
 543 //      r13: sender sp
 544 //      rdx: cp cache
 545 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
 546   // initialize fixed part of activation frame
 547   __ push(rax);        // save return address
 548   __ enter();          // save old & set new rbp
 549   __ push(r13);        // set sender sp
 550   __ push((int)NULL_WORD); // leave last_sp as null
 551   __ movptr(r13, Address(rbx, methodOopDesc::const_offset()));      // get constMethodOop
 552   __ lea(r13, Address(r13, constMethodOopDesc::codes_offset())); // get codebase
 553   __ push(rbx);        // save methodOop
 554   if (ProfileInterpreter) {
 555     Label method_data_continue;
 556     __ movptr(rdx, Address(rbx, in_bytes(methodOopDesc::method_data_offset())));
 557     __ testptr(rdx, rdx);
 558     __ jcc(Assembler::zero, method_data_continue);
 559     __ addptr(rdx, in_bytes(methodDataOopDesc::data_offset()));
 560     __ bind(method_data_continue);
 561     __ push(rdx);      // set the mdp (method data pointer)
 562   } else {
 563     __ push(0);
 564   }
 565 
 566   __ movptr(rdx, Address(rbx, methodOopDesc::constants_offset()));
 567   __ movptr(rdx, Address(rdx, constantPoolOopDesc::cache_offset_in_bytes()));
 568   __ push(rdx); // set constant pool cache
 569   __ push(r14); // set locals pointer
 570   if (native_call) {
 571     __ push(0); // no bcp
 572   } else {
 573     __ push(r13); // set bcp
 574   }
 575   __ push(0); // reserve word for pointer to expression stack bottom
 576   __ movptr(Address(rsp, 0), rsp); // set expression stack bottom
 577 }
 578 
 579 // End of helpers
 580 
 581 // Various method entries
 582 //------------------------------------------------------------------------------------------------------------------------
 583 //
 584 //
 585 
 586 // Call an accessor method (assuming it is resolved, otherwise drop
 587 // into vanilla (slow path) entry
 588 address InterpreterGenerator::generate_accessor_entry(void) {
 589   // rbx: methodOop
 590 
 591   // r13: senderSP must preserver for slow path, set SP to it on fast path
 592 
 593   address entry_point = __ pc();
 594   Label xreturn_path;
 595 
 596   // do fastpath for resolved accessor methods
 597   if (UseFastAccessorMethods) {
 598     // Code: _aload_0, _(i|a)getfield, _(i|a)return or any rewrites
 599     //       thereof; parameter size = 1
 600     // Note: We can only use this code if the getfield has been resolved
 601     //       and if we don't have a null-pointer exception => check for
 602     //       these conditions first and use slow path if necessary.
 603     Label slow_path;
 604     // If we need a safepoint check, generate full interpreter entry.
 605     __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
 606              SafepointSynchronize::_not_synchronized);
 607 
 608     __ jcc(Assembler::notEqual, slow_path);
 609     // rbx: method
 610     __ movptr(rax, Address(rsp, wordSize));
 611 
 612     // check if local 0 != NULL and read field
 613     __ testptr(rax, rax);
 614     __ jcc(Assembler::zero, slow_path);
 615 
 616     __ movptr(rdi, Address(rbx, methodOopDesc::constants_offset()));
 617     // read first instruction word and extract bytecode @ 1 and index @ 2
 618     __ movptr(rdx, Address(rbx, methodOopDesc::const_offset()));
 619     __ movl(rdx, Address(rdx, constMethodOopDesc::codes_offset()));
 620     // Shift codes right to get the index on the right.
 621     // The bytecode fetched looks like <index><0xb4><0x2a>
 622     __ shrl(rdx, 2 * BitsPerByte);
 623     __ shll(rdx, exact_log2(in_words(ConstantPoolCacheEntry::size())));
 624     __ movptr(rdi, Address(rdi, constantPoolOopDesc::cache_offset_in_bytes()));
 625 
 626     // rax: local 0
 627     // rbx: method
 628     // rdx: constant pool cache index
 629     // rdi: constant pool cache
 630 
 631     // check if getfield has been resolved and read constant pool cache entry
 632     // check the validity of the cache entry by testing whether _indices field
 633     // contains Bytecode::_getfield in b1 byte.
 634     assert(in_words(ConstantPoolCacheEntry::size()) == 4,
 635            "adjust shift below");
 636     __ movl(rcx,
 637             Address(rdi,
 638                     rdx,
 639                     Address::times_8,
 640                     constantPoolCacheOopDesc::base_offset() +
 641                     ConstantPoolCacheEntry::indices_offset()));
 642     __ shrl(rcx, 2 * BitsPerByte);
 643     __ andl(rcx, 0xFF);
 644     __ cmpl(rcx, Bytecodes::_getfield);
 645     __ jcc(Assembler::notEqual, slow_path);
 646 
 647     // Note: constant pool entry is not valid before bytecode is resolved
 648     __ movptr(rcx,
 649               Address(rdi,
 650                       rdx,
 651                       Address::times_8,
 652                       constantPoolCacheOopDesc::base_offset() +
 653                       ConstantPoolCacheEntry::f2_offset()));
 654     // edx: flags
 655     __ movl(rdx,
 656             Address(rdi,
 657                     rdx,
 658                     Address::times_8,
 659                     constantPoolCacheOopDesc::base_offset() +
 660                     ConstantPoolCacheEntry::flags_offset()));
 661 
 662     Label notObj, notInt, notByte, notShort;
 663     const Address field_address(rax, rcx, Address::times_1);
 664 
 665     // Need to differentiate between igetfield, agetfield, bgetfield etc.
 666     // because they are different sizes.
 667     // Use the type from the constant pool cache
 668     __ shrl(rdx, ConstantPoolCacheEntry::tosBits);
 669     // Make sure we don't need to mask edx for tosBits after the above shift
 670     ConstantPoolCacheEntry::verify_tosBits();
 671 
 672     __ cmpl(rdx, atos);
 673     __ jcc(Assembler::notEqual, notObj);
 674     // atos
 675     __ load_heap_oop(rax, field_address);
 676     __ jmp(xreturn_path);
 677 
 678     __ bind(notObj);
 679     __ cmpl(rdx, itos);
 680     __ jcc(Assembler::notEqual, notInt);
 681     // itos
 682     __ movl(rax, field_address);
 683     __ jmp(xreturn_path);
 684 
 685     __ bind(notInt);
 686     __ cmpl(rdx, btos);
 687     __ jcc(Assembler::notEqual, notByte);
 688     // btos
 689     __ load_signed_byte(rax, field_address);
 690     __ jmp(xreturn_path);
 691 
 692     __ bind(notByte);
 693     __ cmpl(rdx, stos);
 694     __ jcc(Assembler::notEqual, notShort);
 695     // stos
 696     __ load_signed_short(rax, field_address);
 697     __ jmp(xreturn_path);
 698 
 699     __ bind(notShort);
 700 #ifdef ASSERT
 701     Label okay;
 702     __ cmpl(rdx, ctos);
 703     __ jcc(Assembler::equal, okay);
 704     __ stop("what type is this?");
 705     __ bind(okay);
 706 #endif
 707     // ctos
 708     __ load_unsigned_short(rax, field_address);
 709 
 710     __ bind(xreturn_path);
 711 
 712     // _ireturn/_areturn
 713     __ pop(rdi);
 714     __ mov(rsp, r13);
 715     __ jmp(rdi);
 716     __ ret(0);
 717 
 718     // generate a vanilla interpreter entry as the slow path
 719     __ bind(slow_path);
 720     (void) generate_normal_entry(false);
 721   } else {
 722     (void) generate_normal_entry(false);
 723   }
 724 
 725   return entry_point;
 726 }
 727 
 728 // Interpreter stub for calling a native method. (asm interpreter)
 729 // This sets up a somewhat different looking stack for calling the
 730 // native method than the typical interpreter frame setup.
 731 address InterpreterGenerator::generate_native_entry(bool synchronized) {
 732   // determine code generation flags
 733   bool inc_counter  = UseCompiler || CountCompiledCalls;
 734 
 735   // rbx: methodOop
 736   // r13: sender sp
 737 
 738   address entry_point = __ pc();
 739 
 740   const Address size_of_parameters(rbx, methodOopDesc::
 741                                         size_of_parameters_offset());
 742   const Address invocation_counter(rbx, methodOopDesc::
 743                                         invocation_counter_offset() +
 744                                         InvocationCounter::counter_offset());
 745   const Address access_flags      (rbx, methodOopDesc::access_flags_offset());
 746 
 747   // get parameter size (always needed)
 748   __ load_unsigned_short(rcx, size_of_parameters);
 749 
 750   // native calls don't need the stack size check since they have no
 751   // expression stack and the arguments are already on the stack and
 752   // we only add a handful of words to the stack
 753 
 754   // rbx: methodOop
 755   // rcx: size of parameters
 756   // r13: sender sp
 757   __ pop(rax);                                       // get return address
 758 
 759   // for natives the size of locals is zero
 760 
 761   // compute beginning of parameters (r14)
 762   if (TaggedStackInterpreter) __ shll(rcx, 1); // 2 slots per parameter.
 763   __ lea(r14, Address(rsp, rcx, Address::times_8, -wordSize));
 764 
 765   // add 2 zero-initialized slots for native calls
 766   // initialize result_handler slot
 767   __ push((int) NULL_WORD);
 768   // slot for oop temp
 769   // (static native method holder mirror/jni oop result)
 770   __ push((int) NULL_WORD);
 771 
 772   if (inc_counter) {
 773     __ movl(rcx, invocation_counter);  // (pre-)fetch invocation count
 774   }
 775 
 776   // initialize fixed part of activation frame
 777   generate_fixed_frame(true);
 778 
 779   // make sure method is native & not abstract
 780 #ifdef ASSERT
 781   __ movl(rax, access_flags);
 782   {
 783     Label L;
 784     __ testl(rax, JVM_ACC_NATIVE);
 785     __ jcc(Assembler::notZero, L);
 786     __ stop("tried to execute non-native method as native");
 787     __ bind(L);
 788   }
 789   {
 790     Label L;
 791     __ testl(rax, JVM_ACC_ABSTRACT);
 792     __ jcc(Assembler::zero, L);
 793     __ stop("tried to execute abstract method in interpreter");
 794     __ bind(L);
 795   }
 796 #endif
 797 
 798   // Since at this point in the method invocation the exception handler
 799   // would try to exit the monitor of synchronized methods which hasn't
 800   // been entered yet, we set the thread local variable
 801   // _do_not_unlock_if_synchronized to true. The remove_activation will
 802   // check this flag.
 803 
 804   const Address do_not_unlock_if_synchronized(r15_thread,
 805         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
 806   __ movbool(do_not_unlock_if_synchronized, true);
 807 
 808   // increment invocation count & check for overflow
 809   Label invocation_counter_overflow;
 810   if (inc_counter) {
 811     generate_counter_incr(&invocation_counter_overflow, NULL, NULL);
 812   }
 813 
 814   Label continue_after_compile;
 815   __ bind(continue_after_compile);
 816 
 817   bang_stack_shadow_pages(true);
 818 
 819   // reset the _do_not_unlock_if_synchronized flag
 820   __ movbool(do_not_unlock_if_synchronized, false);
 821 
 822   // check for synchronized methods
 823   // Must happen AFTER invocation_counter check and stack overflow check,
 824   // so method is not locked if overflows.
 825   if (synchronized) {
 826     lock_method();
 827   } else {
 828     // no synchronization necessary
 829 #ifdef ASSERT
 830     {
 831       Label L;
 832       __ movl(rax, access_flags);
 833       __ testl(rax, JVM_ACC_SYNCHRONIZED);
 834       __ jcc(Assembler::zero, L);
 835       __ stop("method needs synchronization");
 836       __ bind(L);
 837     }
 838 #endif
 839   }
 840 
 841   // start execution
 842 #ifdef ASSERT
 843   {
 844     Label L;
 845     const Address monitor_block_top(rbp,
 846                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
 847     __ movptr(rax, monitor_block_top);
 848     __ cmpptr(rax, rsp);
 849     __ jcc(Assembler::equal, L);
 850     __ stop("broken stack frame setup in interpreter");
 851     __ bind(L);
 852   }
 853 #endif
 854 
 855   // jvmti support
 856   __ notify_method_entry();
 857 
 858   // work registers
 859   const Register method = rbx;
 860   const Register t      = r11;
 861 
 862   // allocate space for parameters
 863   __ get_method(method);
 864   __ verify_oop(method);
 865   __ load_unsigned_short(t,
 866                          Address(method,
 867                                  methodOopDesc::size_of_parameters_offset()));
 868   __ shll(t, Interpreter::logStackElementSize());
 869 
 870   __ subptr(rsp, t);
 871   __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
 872   __ andptr(rsp, -16); // must be 16 byte boundary (see amd64 ABI)
 873 
 874   // get signature handler
 875   {
 876     Label L;
 877     __ movptr(t, Address(method, methodOopDesc::signature_handler_offset()));
 878     __ testptr(t, t);
 879     __ jcc(Assembler::notZero, L);
 880     __ call_VM(noreg,
 881                CAST_FROM_FN_PTR(address,
 882                                 InterpreterRuntime::prepare_native_call),
 883                method);
 884     __ get_method(method);
 885     __ movptr(t, Address(method, methodOopDesc::signature_handler_offset()));
 886     __ bind(L);
 887   }
 888 
 889   // call signature handler
 890   assert(InterpreterRuntime::SignatureHandlerGenerator::from() == r14,
 891          "adjust this code");
 892   assert(InterpreterRuntime::SignatureHandlerGenerator::to() == rsp,
 893          "adjust this code");
 894   assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == rscratch1,
 895           "adjust this code");
 896 
 897   // The generated handlers do not touch RBX (the method oop).
 898   // However, large signatures cannot be cached and are generated
 899   // each time here.  The slow-path generator can do a GC on return,
 900   // so we must reload it after the call.
 901   __ call(t);
 902   __ get_method(method);        // slow path can do a GC, reload RBX
 903 
 904 
 905   // result handler is in rax
 906   // set result handler
 907   __ movptr(Address(rbp,
 908                     (frame::interpreter_frame_result_handler_offset) * wordSize),
 909             rax);
 910 
 911   // pass mirror handle if static call
 912   {
 913     Label L;
 914     const int mirror_offset = klassOopDesc::klass_part_offset_in_bytes() +
 915                               Klass::java_mirror_offset_in_bytes();
 916     __ movl(t, Address(method, methodOopDesc::access_flags_offset()));
 917     __ testl(t, JVM_ACC_STATIC);
 918     __ jcc(Assembler::zero, L);
 919     // get mirror
 920     __ movptr(t, Address(method, methodOopDesc::constants_offset()));
 921     __ movptr(t, Address(t, constantPoolOopDesc::pool_holder_offset_in_bytes()));
 922     __ movptr(t, Address(t, mirror_offset));
 923     // copy mirror into activation frame
 924     __ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize),
 925             t);
 926     // pass handle to mirror
 927     __ lea(c_rarg1,
 928            Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize));
 929     __ bind(L);
 930   }
 931 
 932   // get native function entry point
 933   {
 934     Label L;
 935     __ movptr(rax, Address(method, methodOopDesc::native_function_offset()));
 936     ExternalAddress unsatisfied(SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
 937     __ movptr(rscratch2, unsatisfied.addr());
 938     __ cmpptr(rax, rscratch2);
 939     __ jcc(Assembler::notEqual, L);
 940     __ call_VM(noreg,
 941                CAST_FROM_FN_PTR(address,
 942                                 InterpreterRuntime::prepare_native_call),
 943                method);
 944     __ get_method(method);
 945     __ verify_oop(method);
 946     __ movptr(rax, Address(method, methodOopDesc::native_function_offset()));
 947     __ bind(L);
 948   }
 949 
 950   // pass JNIEnv
 951   __ lea(c_rarg0, Address(r15_thread, JavaThread::jni_environment_offset()));
 952 
 953   // It is enough that the pc() points into the right code
 954   // segment. It does not have to be the correct return pc.
 955   __ set_last_Java_frame(rsp, rbp, (address) __ pc());
 956 
 957   // change thread state
 958 #ifdef ASSERT
 959   {
 960     Label L;
 961     __ movl(t, Address(r15_thread, JavaThread::thread_state_offset()));
 962     __ cmpl(t, _thread_in_Java);
 963     __ jcc(Assembler::equal, L);
 964     __ stop("Wrong thread state in native stub");
 965     __ bind(L);
 966   }
 967 #endif
 968 
 969   // Change state to native
 970 
 971   __ movl(Address(r15_thread, JavaThread::thread_state_offset()),
 972           _thread_in_native);
 973 
 974   // Call the native method.
 975   __ call(rax);
 976   // result potentially in rax or xmm0
 977 
 978   // Depending on runtime options, either restore the MXCSR
 979   // register after returning from the JNI Call or verify that
 980   // it wasn't changed during -Xcheck:jni.
 981   if (RestoreMXCSROnJNICalls) {
 982     __ ldmxcsr(ExternalAddress(StubRoutines::x86::mxcsr_std()));
 983   }
 984   else if (CheckJNICalls) {
 985     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::verify_mxcsr_entry())));
 986   }
 987 
 988   // NOTE: The order of these pushes is known to frame::interpreter_frame_result
 989   // in order to extract the result of a method call. If the order of these
 990   // pushes change or anything else is added to the stack then the code in
 991   // interpreter_frame_result must also change.
 992 
 993   __ push(dtos);
 994   __ push(ltos);
 995 
 996   // change thread state
 997   __ movl(Address(r15_thread, JavaThread::thread_state_offset()),
 998           _thread_in_native_trans);
 999 
1000   if (os::is_MP()) {
1001     if (UseMembar) {
1002       // Force this write out before the read below
1003       __ membar(Assembler::Membar_mask_bits(
1004            Assembler::LoadLoad | Assembler::LoadStore |
1005            Assembler::StoreLoad | Assembler::StoreStore));
1006     } else {
1007       // Write serialization page so VM thread can do a pseudo remote membar.
1008       // We use the current thread pointer to calculate a thread specific
1009       // offset to write to within the page. This minimizes bus traffic
1010       // due to cache line collision.
1011       __ serialize_memory(r15_thread, rscratch2);
1012     }
1013   }
1014 
1015   // check for safepoint operation in progress and/or pending suspend requests
1016   {
1017     Label Continue;
1018     __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
1019              SafepointSynchronize::_not_synchronized);
1020 
1021     Label L;
1022     __ jcc(Assembler::notEqual, L);
1023     __ cmpl(Address(r15_thread, JavaThread::suspend_flags_offset()), 0);
1024     __ jcc(Assembler::equal, Continue);
1025     __ bind(L);
1026 
1027     // Don't use call_VM as it will see a possible pending exception
1028     // and forward it and never return here preventing us from
1029     // clearing _last_native_pc down below.  Also can't use
1030     // call_VM_leaf either as it will check to see if r13 & r14 are
1031     // preserved and correspond to the bcp/locals pointers. So we do a
1032     // runtime call by hand.
1033     //
1034     __ mov(c_rarg0, r15_thread);
1035     __ mov(r12, rsp); // remember sp
1036     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1037     __ andptr(rsp, -16); // align stack as required by ABI
1038     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
1039     __ mov(rsp, r12); // restore sp
1040     __ reinit_heapbase();
1041     __ bind(Continue);
1042   }
1043 
1044   // change thread state
1045   __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
1046 
1047   // reset_last_Java_frame
1048   __ reset_last_Java_frame(true, true);
1049 
1050   // reset handle block
1051   __ movptr(t, Address(r15_thread, JavaThread::active_handles_offset()));
1052   __ movptr(Address(t, JNIHandleBlock::top_offset_in_bytes()), (int32_t)NULL_WORD);
1053 
1054   // If result is an oop unbox and store it in frame where gc will see it
1055   // and result handler will pick it up
1056 
1057   {
1058     Label no_oop, store_result;
1059     __ lea(t, ExternalAddress(AbstractInterpreter::result_handler(T_OBJECT)));
1060     __ cmpptr(t, Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize));
1061     __ jcc(Assembler::notEqual, no_oop);
1062     // retrieve result
1063     __ pop(ltos);
1064     __ testptr(rax, rax);
1065     __ jcc(Assembler::zero, store_result);
1066     __ movptr(rax, Address(rax, 0));
1067     __ bind(store_result);
1068     __ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize), rax);
1069     // keep stack depth as expected by pushing oop which will eventually be discarde
1070     __ push(ltos);
1071     __ bind(no_oop);
1072   }
1073 
1074 
1075   {
1076     Label no_reguard;
1077     __ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()),
1078             JavaThread::stack_guard_yellow_disabled);
1079     __ jcc(Assembler::notEqual, no_reguard);
1080 
1081     __ pusha(); // XXX only save smashed registers
1082     __ mov(r12, rsp); // remember sp
1083     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1084     __ andptr(rsp, -16); // align stack as required by ABI
1085     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
1086     __ mov(rsp, r12); // restore sp
1087     __ popa(); // XXX only restore smashed registers
1088     __ reinit_heapbase();
1089 
1090     __ bind(no_reguard);
1091   }
1092 
1093 
1094   // The method register is junk from after the thread_in_native transition
1095   // until here.  Also can't call_VM until the bcp has been
1096   // restored.  Need bcp for throwing exception below so get it now.
1097   __ get_method(method);
1098   __ verify_oop(method);
1099 
1100   // restore r13 to have legal interpreter frame, i.e., bci == 0 <=>
1101   // r13 == code_base()
1102   __ movptr(r13, Address(method, methodOopDesc::const_offset()));   // get constMethodOop
1103   __ lea(r13, Address(r13, constMethodOopDesc::codes_offset()));    // get codebase
1104   // handle exceptions (exception handling will handle unlocking!)
1105   {
1106     Label L;
1107     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
1108     __ jcc(Assembler::zero, L);
1109     // Note: At some point we may want to unify this with the code
1110     // used in call_VM_base(); i.e., we should use the
1111     // StubRoutines::forward_exception code. For now this doesn't work
1112     // here because the rsp is not correctly set at this point.
1113     __ MacroAssembler::call_VM(noreg,
1114                                CAST_FROM_FN_PTR(address,
1115                                InterpreterRuntime::throw_pending_exception));
1116     __ should_not_reach_here();
1117     __ bind(L);
1118   }
1119 
1120   // do unlocking if necessary
1121   {
1122     Label L;
1123     __ movl(t, Address(method, methodOopDesc::access_flags_offset()));
1124     __ testl(t, JVM_ACC_SYNCHRONIZED);
1125     __ jcc(Assembler::zero, L);
1126     // the code below should be shared with interpreter macro
1127     // assembler implementation
1128     {
1129       Label unlock;
1130       // BasicObjectLock will be first in list, since this is a
1131       // synchronized method. However, need to check that the object
1132       // has not been unlocked by an explicit monitorexit bytecode.
1133       const Address monitor(rbp,
1134                             (intptr_t)(frame::interpreter_frame_initial_sp_offset *
1135                                        wordSize - sizeof(BasicObjectLock)));
1136 
1137       // monitor expect in c_rarg1 for slow unlock path
1138       __ lea(c_rarg1, monitor); // address of first monitor
1139 
1140       __ movptr(t, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
1141       __ testptr(t, t);
1142       __ jcc(Assembler::notZero, unlock);
1143 
1144       // Entry already unlocked, need to throw exception
1145       __ MacroAssembler::call_VM(noreg,
1146                                  CAST_FROM_FN_PTR(address,
1147                    InterpreterRuntime::throw_illegal_monitor_state_exception));
1148       __ should_not_reach_here();
1149 
1150       __ bind(unlock);
1151       __ unlock_object(c_rarg1);
1152     }
1153     __ bind(L);
1154   }
1155 
1156   // jvmti support
1157   // Note: This must happen _after_ handling/throwing any exceptions since
1158   //       the exception handler code notifies the runtime of method exits
1159   //       too. If this happens before, method entry/exit notifications are
1160   //       not properly paired (was bug - gri 11/22/99).
1161   __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI);
1162 
1163   // restore potential result in edx:eax, call result handler to
1164   // restore potential result in ST0 & handle result
1165 
1166   __ pop(ltos);
1167   __ pop(dtos);
1168 
1169   __ movptr(t, Address(rbp,
1170                        (frame::interpreter_frame_result_handler_offset) * wordSize));
1171   __ call(t);
1172 
1173   // remove activation
1174   __ movptr(t, Address(rbp,
1175                        frame::interpreter_frame_sender_sp_offset *
1176                        wordSize)); // get sender sp
1177   __ leave();                                // remove frame anchor
1178   __ pop(rdi);                               // get return address
1179   __ mov(rsp, t);                            // set sp to sender sp
1180   __ jmp(rdi);
1181 
1182   if (inc_counter) {
1183     // Handle overflow of counter and compile method
1184     __ bind(invocation_counter_overflow);
1185     generate_counter_overflow(&continue_after_compile);
1186   }
1187 
1188   return entry_point;
1189 }
1190 
1191 //
1192 // Generic interpreted method entry to (asm) interpreter
1193 //
1194 address InterpreterGenerator::generate_normal_entry(bool synchronized) {
1195   // determine code generation flags
1196   bool inc_counter  = UseCompiler || CountCompiledCalls;
1197 
1198   // ebx: methodOop
1199   // r13: sender sp
1200   address entry_point = __ pc();
1201 
1202   const Address size_of_parameters(rbx,
1203                                    methodOopDesc::size_of_parameters_offset());
1204   const Address size_of_locals(rbx, methodOopDesc::size_of_locals_offset());
1205   const Address invocation_counter(rbx,
1206                                    methodOopDesc::invocation_counter_offset() +
1207                                    InvocationCounter::counter_offset());
1208   const Address access_flags(rbx, methodOopDesc::access_flags_offset());
1209 
1210   // get parameter size (always needed)
1211   __ load_unsigned_short(rcx, size_of_parameters);
1212 
1213   // rbx: methodOop
1214   // rcx: size of parameters
1215   // r13: sender_sp (could differ from sp+wordSize if we were called via c2i )
1216 
1217   __ load_unsigned_short(rdx, size_of_locals); // get size of locals in words
1218   __ subl(rdx, rcx); // rdx = no. of additional locals
1219 
1220   // YYY
1221 //   __ incrementl(rdx);
1222 //   __ andl(rdx, -2);
1223 
1224   // see if we've got enough room on the stack for locals plus overhead.
1225   generate_stack_overflow_check();
1226 
1227   // get return address
1228   __ pop(rax);
1229 
1230   // compute beginning of parameters (r14)
1231   if (TaggedStackInterpreter) __ shll(rcx, 1); // 2 slots per parameter.
1232   __ lea(r14, Address(rsp, rcx, Address::times_8, -wordSize));
1233 
1234   // rdx - # of additional locals
1235   // allocate space for locals
1236   // explicitly initialize locals
1237   {
1238     Label exit, loop;
1239     __ testl(rdx, rdx);
1240     __ jcc(Assembler::lessEqual, exit); // do nothing if rdx <= 0
1241     __ bind(loop);
1242     if (TaggedStackInterpreter) __ push((int) NULL_WORD);  // push tag
1243     __ push((int) NULL_WORD); // initialize local variables
1244     __ decrementl(rdx); // until everything initialized
1245     __ jcc(Assembler::greater, loop);
1246     __ bind(exit);
1247   }
1248 
1249   // (pre-)fetch invocation count
1250   if (inc_counter) {
1251     __ movl(rcx, invocation_counter);
1252   }
1253   // initialize fixed part of activation frame
1254   generate_fixed_frame(false);
1255 
1256   // make sure method is not native & not abstract
1257 #ifdef ASSERT
1258   __ movl(rax, access_flags);
1259   {
1260     Label L;
1261     __ testl(rax, JVM_ACC_NATIVE);
1262     __ jcc(Assembler::zero, L);
1263     __ stop("tried to execute native method as non-native");
1264     __ bind(L);
1265   }
1266   {
1267     Label L;
1268     __ testl(rax, JVM_ACC_ABSTRACT);
1269     __ jcc(Assembler::zero, L);
1270     __ stop("tried to execute abstract method in interpreter");
1271     __ bind(L);
1272   }
1273 #endif
1274 
1275   // Since at this point in the method invocation the exception
1276   // handler would try to exit the monitor of synchronized methods
1277   // which hasn't been entered yet, we set the thread local variable
1278   // _do_not_unlock_if_synchronized to true. The remove_activation
1279   // will check this flag.
1280 
1281   const Address do_not_unlock_if_synchronized(r15_thread,
1282         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1283   __ movbool(do_not_unlock_if_synchronized, true);
1284 
1285   // increment invocation count & check for overflow
1286   Label invocation_counter_overflow;
1287   Label profile_method;
1288   Label profile_method_continue;
1289   if (inc_counter) {
1290     generate_counter_incr(&invocation_counter_overflow,
1291                           &profile_method,
1292                           &profile_method_continue);
1293     if (ProfileInterpreter) {
1294       __ bind(profile_method_continue);
1295     }
1296   }
1297 
1298   Label continue_after_compile;
1299   __ bind(continue_after_compile);
1300 
1301   // check for synchronized interpreted methods
1302   bang_stack_shadow_pages(false);
1303 
1304   // reset the _do_not_unlock_if_synchronized flag
1305   __ movbool(do_not_unlock_if_synchronized, false);
1306 
1307   // check for synchronized methods
1308   // Must happen AFTER invocation_counter check and stack overflow check,
1309   // so method is not locked if overflows.
1310   if (synchronized) {
1311     // Allocate monitor and lock method
1312     lock_method();
1313   } else {
1314     // no synchronization necessary
1315 #ifdef ASSERT
1316     {
1317       Label L;
1318       __ movl(rax, access_flags);
1319       __ testl(rax, JVM_ACC_SYNCHRONIZED);
1320       __ jcc(Assembler::zero, L);
1321       __ stop("method needs synchronization");
1322       __ bind(L);
1323     }
1324 #endif
1325   }
1326 
1327   // start execution
1328 #ifdef ASSERT
1329   {
1330     Label L;
1331      const Address monitor_block_top (rbp,
1332                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
1333     __ movptr(rax, monitor_block_top);
1334     __ cmpptr(rax, rsp);
1335     __ jcc(Assembler::equal, L);
1336     __ stop("broken stack frame setup in interpreter");
1337     __ bind(L);
1338   }
1339 #endif
1340 
1341   // jvmti support
1342   __ notify_method_entry();
1343 
1344   __ dispatch_next(vtos);
1345 
1346   // invocation counter overflow
1347   if (inc_counter) {
1348     if (ProfileInterpreter) {
1349       // We have decided to profile this method in the interpreter
1350       __ bind(profile_method);
1351 
1352       __ call_VM(noreg,
1353                  CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method),
1354                  r13, true);
1355 
1356       __ movptr(rbx, Address(rbp, method_offset)); // restore methodOop
1357       __ movptr(rax, Address(rbx,
1358                              in_bytes(methodOopDesc::method_data_offset())));
1359       __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize),
1360                 rax);
1361       __ test_method_data_pointer(rax, profile_method_continue);
1362       __ addptr(rax, in_bytes(methodDataOopDesc::data_offset()));
1363       __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize),
1364               rax);
1365       __ jmp(profile_method_continue);
1366     }
1367     // Handle overflow of counter and compile method
1368     __ bind(invocation_counter_overflow);
1369     generate_counter_overflow(&continue_after_compile);
1370   }
1371 
1372   return entry_point;
1373 }
1374 
1375 // Entry points
1376 //
1377 // Here we generate the various kind of entries into the interpreter.
1378 // The two main entry type are generic bytecode methods and native
1379 // call method.  These both come in synchronized and non-synchronized
1380 // versions but the frame layout they create is very similar. The
1381 // other method entry types are really just special purpose entries
1382 // that are really entry and interpretation all in one. These are for
1383 // trivial methods like accessor, empty, or special math methods.
1384 //
1385 // When control flow reaches any of the entry types for the interpreter
1386 // the following holds ->
1387 //
1388 // Arguments:
1389 //
1390 // rbx: methodOop
1391 //
1392 // Stack layout immediately at entry
1393 //
1394 // [ return address     ] <--- rsp
1395 // [ parameter n        ]
1396 //   ...
1397 // [ parameter 1        ]
1398 // [ expression stack   ] (caller's java expression stack)
1399 
1400 // Assuming that we don't go to one of the trivial specialized entries
1401 // the stack will look like below when we are ready to execute the
1402 // first bytecode (or call the native routine). The register usage
1403 // will be as the template based interpreter expects (see
1404 // interpreter_amd64.hpp).
1405 //
1406 // local variables follow incoming parameters immediately; i.e.
1407 // the return address is moved to the end of the locals).
1408 //
1409 // [ monitor entry      ] <--- rsp
1410 //   ...
1411 // [ monitor entry      ]
1412 // [ expr. stack bottom ]
1413 // [ saved r13          ]
1414 // [ current r14        ]
1415 // [ methodOop          ]
1416 // [ saved ebp          ] <--- rbp
1417 // [ return address     ]
1418 // [ local variable m   ]
1419 //   ...
1420 // [ local variable 1   ]
1421 // [ parameter n        ]
1422 //   ...
1423 // [ parameter 1        ] <--- r14
1424 
1425 address AbstractInterpreterGenerator::generate_method_entry(
1426                                         AbstractInterpreter::MethodKind kind) {
1427   // determine code generation flags
1428   bool synchronized = false;
1429   address entry_point = NULL;
1430 
1431   switch (kind) {
1432   case Interpreter::zerolocals             :                                                                             break;
1433   case Interpreter::zerolocals_synchronized: synchronized = true;                                                        break;
1434   case Interpreter::native                 : entry_point = ((InterpreterGenerator*) this)->generate_native_entry(false); break;
1435   case Interpreter::native_synchronized    : entry_point = ((InterpreterGenerator*) this)->generate_native_entry(true);  break;
1436   case Interpreter::empty                  : entry_point = ((InterpreterGenerator*) this)->generate_empty_entry();       break;
1437   case Interpreter::accessor               : entry_point = ((InterpreterGenerator*) this)->generate_accessor_entry();    break;
1438   case Interpreter::abstract               : entry_point = ((InterpreterGenerator*) this)->generate_abstract_entry();    break;
1439   case Interpreter::method_handle          : entry_point = ((InterpreterGenerator*) this)->generate_method_handle_entry();break;
1440 
1441   case Interpreter::java_lang_math_sin     : // fall thru
1442   case Interpreter::java_lang_math_cos     : // fall thru
1443   case Interpreter::java_lang_math_tan     : // fall thru
1444   case Interpreter::java_lang_math_abs     : // fall thru
1445   case Interpreter::java_lang_math_log     : // fall thru
1446   case Interpreter::java_lang_math_log10   : // fall thru
1447   case Interpreter::java_lang_math_sqrt    : entry_point = ((InterpreterGenerator*) this)->generate_math_entry(kind);    break;
1448   default                                  : ShouldNotReachHere();                                                       break;
1449   }
1450 
1451   if (entry_point) {
1452     return entry_point;
1453   }
1454 
1455   return ((InterpreterGenerator*) this)->
1456                                 generate_normal_entry(synchronized);
1457 }
1458 
1459 // How much stack a method activation needs in words.
1460 int AbstractInterpreter::size_top_interpreter_activation(methodOop method) {
1461   const int entry_size = frame::interpreter_frame_monitor_size();
1462 
1463   // total overhead size: entry_size + (saved rbp thru expr stack
1464   // bottom).  be sure to change this if you add/subtract anything
1465   // to/from the overhead area
1466   const int overhead_size =
1467     -(frame::interpreter_frame_initial_sp_offset) + entry_size;
1468 
1469   const int stub_code = frame::entry_frame_after_call_words;
1470   const int extra_stack = methodOopDesc::extra_stack_entries();
1471   const int method_stack = (method->max_locals() + method->max_stack() + extra_stack) *
1472                            Interpreter::stackElementWords();
1473   return (overhead_size + method_stack + stub_code);
1474 }
1475 
1476 int AbstractInterpreter::layout_activation(methodOop method,
1477                                            int tempcount,
1478                                            int popframe_extra_args,
1479                                            int moncount,
1480                                            int callee_param_count,
1481                                            int callee_locals,
1482                                            frame* caller,
1483                                            frame* interpreter_frame,
1484                                            bool is_top_frame) {
1485   // Note: This calculation must exactly parallel the frame setup
1486   // in AbstractInterpreterGenerator::generate_method_entry.
1487   // If interpreter_frame!=NULL, set up the method, locals, and monitors.
1488   // The frame interpreter_frame, if not NULL, is guaranteed to be the
1489   // right size, as determined by a previous call to this method.
1490   // It is also guaranteed to be walkable even though it is in a skeletal state
1491 
1492   // fixed size of an interpreter frame:
1493   int max_locals = method->max_locals() * Interpreter::stackElementWords();
1494   int extra_locals = (method->max_locals() - method->size_of_parameters()) *
1495                      Interpreter::stackElementWords();
1496 
1497   int overhead = frame::sender_sp_offset -
1498                  frame::interpreter_frame_initial_sp_offset;
1499   // Our locals were accounted for by the caller (or last_frame_adjust
1500   // on the transistion) Since the callee parameters already account
1501   // for the callee's params we only need to account for the extra
1502   // locals.
1503   int size = overhead +
1504          (callee_locals - callee_param_count)*Interpreter::stackElementWords() +
1505          moncount * frame::interpreter_frame_monitor_size() +
1506          tempcount* Interpreter::stackElementWords() + popframe_extra_args;
1507   if (interpreter_frame != NULL) {
1508 #ifdef ASSERT
1509     if (!EnableMethodHandles)
1510       // @@@ FIXME: Should we correct interpreter_frame_sender_sp in the calling sequences?
1511       // Probably, since deoptimization doesn't work yet.
1512       assert(caller->unextended_sp() == interpreter_frame->interpreter_frame_sender_sp(), "Frame not properly walkable");
1513     assert(caller->sp() == interpreter_frame->sender_sp(), "Frame not properly walkable(2)");
1514 #endif
1515 
1516     interpreter_frame->interpreter_frame_set_method(method);
1517     // NOTE the difference in using sender_sp and
1518     // interpreter_frame_sender_sp interpreter_frame_sender_sp is
1519     // the original sp of the caller (the unextended_sp) and
1520     // sender_sp is fp+16 XXX
1521     intptr_t* locals = interpreter_frame->sender_sp() + max_locals - 1;
1522 
1523     interpreter_frame->interpreter_frame_set_locals(locals);
1524     BasicObjectLock* montop = interpreter_frame->interpreter_frame_monitor_begin();
1525     BasicObjectLock* monbot = montop - moncount;
1526     interpreter_frame->interpreter_frame_set_monitor_end(monbot);
1527 
1528     // Set last_sp
1529     intptr_t*  esp = (intptr_t*) monbot -
1530                      tempcount*Interpreter::stackElementWords() -
1531                      popframe_extra_args;
1532     interpreter_frame->interpreter_frame_set_last_sp(esp);
1533 
1534     // All frames but the initial (oldest) interpreter frame we fill in have
1535     // a value for sender_sp that allows walking the stack but isn't
1536     // truly correct. Correct the value here.
1537     if (extra_locals != 0 &&
1538         interpreter_frame->sender_sp() ==
1539         interpreter_frame->interpreter_frame_sender_sp()) {
1540       interpreter_frame->set_interpreter_frame_sender_sp(caller->sp() +
1541                                                          extra_locals);
1542     }
1543     *interpreter_frame->interpreter_frame_cache_addr() =
1544       method->constants()->cache();
1545   }
1546   return size;
1547 }
1548 
1549 //-----------------------------------------------------------------------------
1550 // Exceptions
1551 
1552 void TemplateInterpreterGenerator::generate_throw_exception() {
1553   // Entry point in previous activation (i.e., if the caller was
1554   // interpreted)
1555   Interpreter::_rethrow_exception_entry = __ pc();
1556   // Restore sp to interpreter_frame_last_sp even though we are going
1557   // to empty the expression stack for the exception processing.
1558   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
1559   // rax: exception
1560   // rdx: return address/pc that threw exception
1561   __ restore_bcp();    // r13 points to call/send
1562   __ restore_locals();
1563   __ reinit_heapbase();  // restore r12 as heapbase.
1564   // Entry point for exceptions thrown within interpreter code
1565   Interpreter::_throw_exception_entry = __ pc();
1566   // expression stack is undefined here
1567   // rax: exception
1568   // r13: exception bcp
1569   __ verify_oop(rax);
1570   __ mov(c_rarg1, rax);
1571 
1572   // expression stack must be empty before entering the VM in case of
1573   // an exception
1574   __ empty_expression_stack();
1575   // find exception handler address and preserve exception oop
1576   __ call_VM(rdx,
1577              CAST_FROM_FN_PTR(address,
1578                           InterpreterRuntime::exception_handler_for_exception),
1579              c_rarg1);
1580   // rax: exception handler entry point
1581   // rdx: preserved exception oop
1582   // r13: bcp for exception handler
1583   __ push_ptr(rdx); // push exception which is now the only value on the stack
1584   __ jmp(rax); // jump to exception handler (may be _remove_activation_entry!)
1585 
1586   // If the exception is not handled in the current frame the frame is
1587   // removed and the exception is rethrown (i.e. exception
1588   // continuation is _rethrow_exception).
1589   //
1590   // Note: At this point the bci is still the bxi for the instruction
1591   // which caused the exception and the expression stack is
1592   // empty. Thus, for any VM calls at this point, GC will find a legal
1593   // oop map (with empty expression stack).
1594 
1595   // In current activation
1596   // tos: exception
1597   // esi: exception bcp
1598 
1599   //
1600   // JVMTI PopFrame support
1601   //
1602 
1603   Interpreter::_remove_activation_preserving_args_entry = __ pc();
1604   __ empty_expression_stack();
1605   // Set the popframe_processing bit in pending_popframe_condition
1606   // indicating that we are currently handling popframe, so that
1607   // call_VMs that may happen later do not trigger new popframe
1608   // handling cycles.
1609   __ movl(rdx, Address(r15_thread, JavaThread::popframe_condition_offset()));
1610   __ orl(rdx, JavaThread::popframe_processing_bit);
1611   __ movl(Address(r15_thread, JavaThread::popframe_condition_offset()), rdx);
1612 
1613   {
1614     // Check to see whether we are returning to a deoptimized frame.
1615     // (The PopFrame call ensures that the caller of the popped frame is
1616     // either interpreted or compiled and deoptimizes it if compiled.)
1617     // In this case, we can't call dispatch_next() after the frame is
1618     // popped, but instead must save the incoming arguments and restore
1619     // them after deoptimization has occurred.
1620     //
1621     // Note that we don't compare the return PC against the
1622     // deoptimization blob's unpack entry because of the presence of
1623     // adapter frames in C2.
1624     Label caller_not_deoptimized;
1625     __ movptr(c_rarg1, Address(rbp, frame::return_addr_offset * wordSize));
1626     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1627                                InterpreterRuntime::interpreter_contains), c_rarg1);
1628     __ testl(rax, rax);
1629     __ jcc(Assembler::notZero, caller_not_deoptimized);
1630 
1631     // Compute size of arguments for saving when returning to
1632     // deoptimized caller
1633     __ get_method(rax);
1634     __ load_unsigned_short(rax, Address(rax, in_bytes(methodOopDesc::
1635                                                 size_of_parameters_offset())));
1636     __ shll(rax, Interpreter::logStackElementSize());
1637     __ restore_locals(); // XXX do we need this?
1638     __ subptr(r14, rax);
1639     __ addptr(r14, wordSize);
1640     // Save these arguments
1641     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1642                                            Deoptimization::
1643                                            popframe_preserve_args),
1644                           r15_thread, rax, r14);
1645 
1646     __ remove_activation(vtos, rdx,
1647                          /* throw_monitor_exception */ false,
1648                          /* install_monitor_exception */ false,
1649                          /* notify_jvmdi */ false);
1650 
1651     // Inform deoptimization that it is responsible for restoring
1652     // these arguments
1653     __ movl(Address(r15_thread, JavaThread::popframe_condition_offset()),
1654             JavaThread::popframe_force_deopt_reexecution_bit);
1655 
1656     // Continue in deoptimization handler
1657     __ jmp(rdx);
1658 
1659     __ bind(caller_not_deoptimized);
1660   }
1661 
1662   __ remove_activation(vtos, rdx, /* rdx result (retaddr) is not used */
1663                        /* throw_monitor_exception */ false,
1664                        /* install_monitor_exception */ false,
1665                        /* notify_jvmdi */ false);
1666 
1667   // Finish with popframe handling
1668   // A previous I2C followed by a deoptimization might have moved the
1669   // outgoing arguments further up the stack. PopFrame expects the
1670   // mutations to those outgoing arguments to be preserved and other
1671   // constraints basically require this frame to look exactly as
1672   // though it had previously invoked an interpreted activation with
1673   // no space between the top of the expression stack (current
1674   // last_sp) and the top of stack. Rather than force deopt to
1675   // maintain this kind of invariant all the time we call a small
1676   // fixup routine to move the mutated arguments onto the top of our
1677   // expression stack if necessary.
1678   __ mov(c_rarg1, rsp);
1679   __ movptr(c_rarg2, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1680   // PC must point into interpreter here
1681   __ set_last_Java_frame(noreg, rbp, __ pc());
1682   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), r15_thread, c_rarg1, c_rarg2);
1683   __ reset_last_Java_frame(true, true);
1684   // Restore the last_sp and null it out
1685   __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1686   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
1687 
1688   __ restore_bcp();  // XXX do we need this?
1689   __ restore_locals(); // XXX do we need this?
1690   // The method data pointer was incremented already during
1691   // call profiling. We have to restore the mdp for the current bcp.
1692   if (ProfileInterpreter) {
1693     __ set_method_data_pointer_for_bcp();
1694   }
1695 
1696   // Clear the popframe condition flag
1697   __ movl(Address(r15_thread, JavaThread::popframe_condition_offset()),
1698           JavaThread::popframe_inactive);
1699 
1700   __ dispatch_next(vtos);
1701   // end of PopFrame support
1702 
1703   Interpreter::_remove_activation_entry = __ pc();
1704 
1705   // preserve exception over this code sequence
1706   __ pop_ptr(rax);
1707   __ movptr(Address(r15_thread, JavaThread::vm_result_offset()), rax);
1708   // remove the activation (without doing throws on illegalMonitorExceptions)
1709   __ remove_activation(vtos, rdx, false, true, false);
1710   // restore exception
1711   __ movptr(rax, Address(r15_thread, JavaThread::vm_result_offset()));
1712   __ movptr(Address(r15_thread, JavaThread::vm_result_offset()), (int32_t)NULL_WORD);
1713   __ verify_oop(rax);
1714 
1715   // In between activations - previous activation type unknown yet
1716   // compute continuation point - the continuation point expects the
1717   // following registers set up:
1718   //
1719   // rax: exception
1720   // rdx: return address/pc that threw exception
1721   // rsp: expression stack of caller
1722   // rbp: ebp of caller
1723   __ push(rax);                                  // save exception
1724   __ push(rdx);                                  // save return address
1725   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1726                           SharedRuntime::exception_handler_for_return_address),
1727                         rdx);
1728   __ mov(rbx, rax);                              // save exception handler
1729   __ pop(rdx);                                   // restore return address
1730   __ pop(rax);                                   // restore exception
1731   // Note that an "issuing PC" is actually the next PC after the call
1732   __ jmp(rbx);                                   // jump to exception
1733                                                  // handler of caller
1734 }
1735 
1736 
1737 //
1738 // JVMTI ForceEarlyReturn support
1739 //
1740 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1741   address entry = __ pc();
1742 
1743   __ restore_bcp();
1744   __ restore_locals();
1745   __ empty_expression_stack();
1746   __ load_earlyret_value(state);
1747 
1748   __ movptr(rdx, Address(r15_thread, JavaThread::jvmti_thread_state_offset()));
1749   Address cond_addr(rdx, JvmtiThreadState::earlyret_state_offset());
1750 
1751   // Clear the earlyret state
1752   __ movl(cond_addr, JvmtiThreadState::earlyret_inactive);
1753 
1754   __ remove_activation(state, rsi,
1755                        false, /* throw_monitor_exception */
1756                        false, /* install_monitor_exception */
1757                        true); /* notify_jvmdi */
1758   __ jmp(rsi);
1759 
1760   return entry;
1761 } // end of ForceEarlyReturn support
1762 
1763 
1764 //-----------------------------------------------------------------------------
1765 // Helper for vtos entry point generation
1766 
1767 void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t,
1768                                                          address& bep,
1769                                                          address& cep,
1770                                                          address& sep,
1771                                                          address& aep,
1772                                                          address& iep,
1773                                                          address& lep,
1774                                                          address& fep,
1775                                                          address& dep,
1776                                                          address& vep) {
1777   assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1778   Label L;
1779   aep = __ pc();  __ push_ptr();  __ jmp(L);
1780   fep = __ pc();  __ push_f();    __ jmp(L);
1781   dep = __ pc();  __ push_d();    __ jmp(L);
1782   lep = __ pc();  __ push_l();    __ jmp(L);
1783   bep = cep = sep =
1784   iep = __ pc();  __ push_i();
1785   vep = __ pc();
1786   __ bind(L);
1787   generate_and_dispatch(t);
1788 }
1789 
1790 
1791 //-----------------------------------------------------------------------------
1792 // Generation of individual instructions
1793 
1794 // helpers for generate_and_dispatch
1795 
1796 
1797 InterpreterGenerator::InterpreterGenerator(StubQueue* code)
1798   : TemplateInterpreterGenerator(code) {
1799    generate_all(); // down here so it can be "virtual"
1800 }
1801 
1802 //-----------------------------------------------------------------------------
1803 
1804 // Non-product code
1805 #ifndef PRODUCT
1806 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1807   address entry = __ pc();
1808 
1809   __ push(state);
1810   __ push(c_rarg0);
1811   __ push(c_rarg1);
1812   __ push(c_rarg2);
1813   __ push(c_rarg3);
1814   __ mov(c_rarg2, rax);  // Pass itos
1815 #ifdef _WIN64
1816   __ movflt(xmm3, xmm0); // Pass ftos
1817 #endif
1818   __ call_VM(noreg,
1819              CAST_FROM_FN_PTR(address, SharedRuntime::trace_bytecode),
1820              c_rarg1, c_rarg2, c_rarg3);
1821   __ pop(c_rarg3);
1822   __ pop(c_rarg2);
1823   __ pop(c_rarg1);
1824   __ pop(c_rarg0);
1825   __ pop(state);
1826   __ ret(0);                                   // return from result handler
1827 
1828   return entry;
1829 }
1830 
1831 void TemplateInterpreterGenerator::count_bytecode() {
1832   __ incrementl(ExternalAddress((address) &BytecodeCounter::_counter_value));
1833 }
1834 
1835 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
1836   __ incrementl(ExternalAddress((address) &BytecodeHistogram::_counters[t->bytecode()]));
1837 }
1838 
1839 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
1840   __ mov32(rbx, ExternalAddress((address) &BytecodePairHistogram::_index));
1841   __ shrl(rbx, BytecodePairHistogram::log2_number_of_codes);
1842   __ orl(rbx,
1843          ((int) t->bytecode()) <<
1844          BytecodePairHistogram::log2_number_of_codes);
1845   __ mov32(ExternalAddress((address) &BytecodePairHistogram::_index), rbx);
1846   __ lea(rscratch1, ExternalAddress((address) BytecodePairHistogram::_counters));
1847   __ incrementl(Address(rscratch1, rbx, Address::times_4));
1848 }
1849 
1850 
1851 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
1852   // Call a little run-time stub to avoid blow-up for each bytecode.
1853   // The run-time runtime saves the right registers, depending on
1854   // the tosca in-state for the given template.
1855 
1856   assert(Interpreter::trace_code(t->tos_in()) != NULL,
1857          "entry must have been generated");
1858   __ mov(r12, rsp); // remember sp
1859   __ andptr(rsp, -16); // align stack as required by ABI
1860   __ call(RuntimeAddress(Interpreter::trace_code(t->tos_in())));
1861   __ mov(rsp, r12); // restore sp
1862   __ reinit_heapbase();
1863 }
1864 
1865 
1866 void TemplateInterpreterGenerator::stop_interpreter_at() {
1867   Label L;
1868   __ cmp32(ExternalAddress((address) &BytecodeCounter::_counter_value),
1869            StopInterpreterAt);
1870   __ jcc(Assembler::notEqual, L);
1871   __ int3();
1872   __ bind(L);
1873 }
1874 #endif // !PRODUCT
1875 #endif // ! CC_INTERP