1 /*
   2  * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  20  * CA 95054 USA or visit www.sun.com if you need additional information or
  21  * have any questions.
  22  *
  23  */
  24 
  25 #include "incls/_precompiled.incl"
  26 #include "incls/_interpreter_x86_64.cpp.incl"
  27 
  28 #define __ _masm->
  29 
  30 #ifndef CC_INTERP
  31 
  32 const int method_offset = frame::interpreter_frame_method_offset * wordSize;
  33 const int bci_offset    = frame::interpreter_frame_bcx_offset    * wordSize;
  34 const int locals_offset = frame::interpreter_frame_locals_offset * wordSize;
  35 
  36 //-----------------------------------------------------------------------------
  37 
  38 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
  39   address entry = __ pc();
  40 
  41 #ifdef ASSERT
  42   {
  43     Label L;
  44     __ lea(rax, Address(rbp,
  45                         frame::interpreter_frame_monitor_block_top_offset *
  46                         wordSize));
  47     __ cmpptr(rax, rsp); // rax = maximal rsp for current rbp (stack
  48                          // grows negative)
  49     __ jcc(Assembler::aboveEqual, L); // check if frame is complete
  50     __ stop ("interpreter frame not set up");
  51     __ bind(L);
  52   }
  53 #endif // ASSERT
  54   // Restore bcp under the assumption that the current frame is still
  55   // interpreted
  56   __ restore_bcp();
  57 
  58   // expression stack must be empty before entering the VM if an
  59   // exception happened
  60   __ empty_expression_stack();
  61   // throw exception
  62   __ call_VM(noreg,
  63              CAST_FROM_FN_PTR(address,
  64                               InterpreterRuntime::throw_StackOverflowError));
  65   return entry;
  66 }
  67 
  68 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler(
  69         const char* name) {
  70   address entry = __ pc();
  71   // expression stack must be empty before entering the VM if an
  72   // exception happened
  73   __ empty_expression_stack();
  74   // setup parameters
  75   // ??? convention: expect aberrant index in register ebx
  76   __ lea(c_rarg1, ExternalAddress((address)name));
  77   __ call_VM(noreg,
  78              CAST_FROM_FN_PTR(address,
  79                               InterpreterRuntime::
  80                               throw_ArrayIndexOutOfBoundsException),
  81              c_rarg1, rbx);
  82   return entry;
  83 }
  84 
  85 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
  86   address entry = __ pc();
  87 
  88   // object is at TOS
  89   __ pop(c_rarg1);
  90 
  91   // expression stack must be empty before entering the VM if an
  92   // exception happened
  93   __ empty_expression_stack();
  94 
  95   __ call_VM(noreg,
  96              CAST_FROM_FN_PTR(address,
  97                               InterpreterRuntime::
  98                               throw_ClassCastException),
  99              c_rarg1);
 100   return entry;
 101 }
 102 
 103 // Arguments are: required type in rarg1, failing object (or NULL) in rarg2
 104 address TemplateInterpreterGenerator::generate_WrongMethodType_handler() {
 105   address entry = __ pc();
 106 
 107   __ pop(c_rarg2);              // failing object is at TOS
 108   __ pop(c_rarg1);              // required type is at TOS+8
 109 
 110   // expression stack must be empty before entering the VM if an
 111   // exception happened
 112   __ empty_expression_stack();
 113 
 114   __ call_VM(noreg,
 115              CAST_FROM_FN_PTR(address,
 116                               InterpreterRuntime::
 117                               throw_WrongMethodTypeException),
 118              // pass required type, failing object (or NULL)
 119              c_rarg1, c_rarg2);
 120   return entry;
 121 }
 122 
 123 address TemplateInterpreterGenerator::generate_exception_handler_common(
 124         const char* name, const char* message, bool pass_oop) {
 125   assert(!pass_oop || message == NULL, "either oop or message but not both");
 126   address entry = __ pc();
 127   if (pass_oop) {
 128     // object is at TOS
 129     __ pop(c_rarg2);
 130   }
 131   // expression stack must be empty before entering the VM if an
 132   // exception happened
 133   __ empty_expression_stack();
 134   // setup parameters
 135   __ lea(c_rarg1, ExternalAddress((address)name));
 136   if (pass_oop) {
 137     __ call_VM(rax, CAST_FROM_FN_PTR(address,
 138                                      InterpreterRuntime::
 139                                      create_klass_exception),
 140                c_rarg1, c_rarg2);
 141   } else {
 142     // kind of lame ExternalAddress can't take NULL because
 143     // external_word_Relocation will assert.
 144     if (message != NULL) {
 145       __ lea(c_rarg2, ExternalAddress((address)message));
 146     } else {
 147       __ movptr(c_rarg2, NULL_WORD);
 148     }
 149     __ call_VM(rax,
 150                CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception),
 151                c_rarg1, c_rarg2);
 152   }
 153   // throw exception
 154   __ jump(ExternalAddress(Interpreter::throw_exception_entry()));
 155   return entry;
 156 }
 157 
 158 
 159 address TemplateInterpreterGenerator::generate_continuation_for(TosState state) {
 160   address entry = __ pc();
 161   // NULL last_sp until next java call
 162   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
 163   __ dispatch_next(state);
 164   return entry;
 165 }
 166 
 167 
 168 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state,
 169                                                                 int step) {
 170 
 171   // amd64 doesn't need to do anything special about compiled returns
 172   // to the interpreter so the code that exists on x86 to place a sentinel
 173   // here and the specialized cleanup code is not needed here.
 174 
 175   address entry = __ pc();
 176 
 177   // Restore stack bottom in case i2c adjusted stack
 178   __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
 179   // and NULL it as marker that esp is now tos until next java call
 180   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
 181 
 182   __ restore_bcp();
 183   __ restore_locals();
 184 
 185   __ get_cache_and_index_at_bcp(rbx, rcx, 1);
 186   __ movl(rbx, Address(rbx, rcx,
 187                        Address::times_8,
 188                        in_bytes(constantPoolCacheOopDesc::base_offset()) +
 189                        3 * wordSize));
 190   __ andl(rbx, 0xFF);
 191   if (TaggedStackInterpreter) __ shll(rbx, 1); // 2 slots per parameter.
 192   __ lea(rsp, Address(rsp, rbx, Address::times_8));
 193   __ dispatch_next(state, step);
 194   return entry;
 195 }
 196 
 197 
 198 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state,
 199                                                                int step) {
 200   address entry = __ pc();
 201   // NULL last_sp until next java call
 202   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
 203   __ restore_bcp();
 204   __ restore_locals();
 205   // handle exceptions
 206   {
 207     Label L;
 208     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
 209     __ jcc(Assembler::zero, L);
 210     __ call_VM(noreg,
 211                CAST_FROM_FN_PTR(address,
 212                                 InterpreterRuntime::throw_pending_exception));
 213     __ should_not_reach_here();
 214     __ bind(L);
 215   }
 216   __ dispatch_next(state, step);
 217   return entry;
 218 }
 219 
 220 int AbstractInterpreter::BasicType_as_index(BasicType type) {
 221   int i = 0;
 222   switch (type) {
 223     case T_BOOLEAN: i = 0; break;
 224     case T_CHAR   : i = 1; break;
 225     case T_BYTE   : i = 2; break;
 226     case T_SHORT  : i = 3; break;
 227     case T_INT    : i = 4; break;
 228     case T_LONG   : i = 5; break;
 229     case T_VOID   : i = 6; break;
 230     case T_FLOAT  : i = 7; break;
 231     case T_DOUBLE : i = 8; break;
 232     case T_OBJECT : i = 9; break;
 233     case T_ARRAY  : i = 9; break;
 234     default       : ShouldNotReachHere();
 235   }
 236   assert(0 <= i && i < AbstractInterpreter::number_of_result_handlers,
 237          "index out of bounds");
 238   return i;
 239 }
 240 
 241 
 242 address TemplateInterpreterGenerator::generate_result_handler_for(
 243         BasicType type) {
 244   address entry = __ pc();
 245   switch (type) {
 246   case T_BOOLEAN: __ c2bool(rax);            break;
 247   case T_CHAR   : __ movzwl(rax, rax);       break;
 248   case T_BYTE   : __ sign_extend_byte(rax);  break;
 249   case T_SHORT  : __ sign_extend_short(rax); break;
 250   case T_INT    : /* nothing to do */        break;
 251   case T_LONG   : /* nothing to do */        break;
 252   case T_VOID   : /* nothing to do */        break;
 253   case T_FLOAT  : /* nothing to do */        break;
 254   case T_DOUBLE : /* nothing to do */        break;
 255   case T_OBJECT :
 256     // retrieve result from frame
 257     __ movptr(rax, Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize));
 258     // and verify it
 259     __ verify_oop(rax);
 260     break;
 261   default       : ShouldNotReachHere();
 262   }
 263   __ ret(0);                                   // return from result handler
 264   return entry;
 265 }
 266 
 267 address TemplateInterpreterGenerator::generate_safept_entry_for(
 268         TosState state,
 269         address runtime_entry) {
 270   address entry = __ pc();
 271   __ push(state);
 272   __ call_VM(noreg, runtime_entry);
 273   __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));
 274   return entry;
 275 }
 276 
 277 
 278 
 279 // Helpers for commoning out cases in the various type of method entries.
 280 //
 281 
 282 
 283 // increment invocation count & check for overflow
 284 //
 285 // Note: checking for negative value instead of overflow
 286 //       so we have a 'sticky' overflow test
 287 //
 288 // rbx: method
 289 // ecx: invocation counter
 290 //
 291 void InterpreterGenerator::generate_counter_incr(
 292         Label* overflow,
 293         Label* profile_method,
 294         Label* profile_method_continue) {
 295 
 296   const Address invocation_counter(rbx,
 297                                    methodOopDesc::invocation_counter_offset() +
 298                                    InvocationCounter::counter_offset());
 299   const Address backedge_counter(rbx,
 300                                  methodOopDesc::backedge_counter_offset() +
 301                                  InvocationCounter::counter_offset());
 302 
 303   if (ProfileInterpreter) { // %%% Merge this into methodDataOop
 304     __ incrementl(Address(rbx,
 305                     methodOopDesc::interpreter_invocation_counter_offset()));
 306   }
 307   // Update standard invocation counters
 308   __ movl(rax, backedge_counter); // load backedge counter
 309 
 310   __ incrementl(rcx, InvocationCounter::count_increment);
 311   __ andl(rax, InvocationCounter::count_mask_value); // mask out the
 312                                                      // status bits
 313 
 314   __ movl(invocation_counter, rcx); // save invocation count
 315   __ addl(rcx, rax); // add both counters
 316 
 317   // profile_method is non-null only for interpreted method so
 318   // profile_method != NULL == !native_call
 319 
 320   if (ProfileInterpreter && profile_method != NULL) {
 321     // Test to see if we should create a method data oop
 322     __ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterProfileLimit));
 323     __ jcc(Assembler::less, *profile_method_continue);
 324 
 325     // if no method data exists, go to profile_method
 326     __ test_method_data_pointer(rax, *profile_method);
 327   }
 328 
 329   __ cmp32(rcx, ExternalAddress((address)&InvocationCounter::InterpreterInvocationLimit));
 330   __ jcc(Assembler::aboveEqual, *overflow);
 331 }
 332 
 333 void InterpreterGenerator::generate_counter_overflow(Label* do_continue) {
 334 
 335   // Asm interpreter on entry
 336   // r14 - locals
 337   // r13 - bcp
 338   // rbx - method
 339   // edx - cpool --- DOES NOT APPEAR TO BE TRUE
 340   // rbp - interpreter frame
 341 
 342   // On return (i.e. jump to entry_point) [ back to invocation of interpreter ]
 343   // Everything as it was on entry
 344   // rdx is not restored. Doesn't appear to really be set.
 345 
 346   const Address size_of_parameters(rbx,
 347                                    methodOopDesc::size_of_parameters_offset());
 348 
 349   // InterpreterRuntime::frequency_counter_overflow takes two
 350   // arguments, the first (thread) is passed by call_VM, the second
 351   // indicates if the counter overflow occurs at a backwards branch
 352   // (NULL bcp).  We pass zero for it.  The call returns the address
 353   // of the verified entry point for the method or NULL if the
 354   // compilation did not complete (either went background or bailed
 355   // out).
 356   __ movl(c_rarg1, 0);
 357   __ call_VM(noreg,
 358              CAST_FROM_FN_PTR(address,
 359                               InterpreterRuntime::frequency_counter_overflow),
 360              c_rarg1);
 361 
 362   __ movptr(rbx, Address(rbp, method_offset));   // restore methodOop
 363   // Preserve invariant that r13/r14 contain bcp/locals of sender frame
 364   // and jump to the interpreted entry.
 365   __ jmp(*do_continue, relocInfo::none);
 366 }
 367 
 368 // See if we've got enough room on the stack for locals plus overhead.
 369 // The expression stack grows down incrementally, so the normal guard
 370 // page mechanism will work for that.
 371 //
 372 // NOTE: Since the additional locals are also always pushed (wasn't
 373 // obvious in generate_method_entry) so the guard should work for them
 374 // too.
 375 //
 376 // Args:
 377 //      rdx: number of additional locals this frame needs (what we must check)
 378 //      rbx: methodOop
 379 //
 380 // Kills:
 381 //      rax
 382 void InterpreterGenerator::generate_stack_overflow_check(void) {
 383 
 384   // monitor entry size: see picture of stack set
 385   // (generate_method_entry) and frame_amd64.hpp
 386   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 387 
 388   // total overhead size: entry_size + (saved rbp through expr stack
 389   // bottom).  be sure to change this if you add/subtract anything
 390   // to/from the overhead area
 391   const int overhead_size =
 392     -(frame::interpreter_frame_initial_sp_offset * wordSize) + entry_size;
 393 
 394   const int page_size = os::vm_page_size();
 395 
 396   Label after_frame_check;
 397 
 398   // see if the frame is greater than one page in size. If so,
 399   // then we need to verify there is enough stack space remaining
 400   // for the additional locals.
 401   __ cmpl(rdx, (page_size - overhead_size) / Interpreter::stackElementSize());
 402   __ jcc(Assembler::belowEqual, after_frame_check);
 403 
 404   // compute rsp as if this were going to be the last frame on
 405   // the stack before the red zone
 406 
 407   const Address stack_base(r15_thread, Thread::stack_base_offset());
 408   const Address stack_size(r15_thread, Thread::stack_size_offset());
 409 
 410   // locals + overhead, in bytes
 411   __ mov(rax, rdx);
 412   __ shlptr(rax, Interpreter::logStackElementSize()); // 2 slots per parameter.
 413   __ addptr(rax, overhead_size);
 414 
 415 #ifdef ASSERT
 416   Label stack_base_okay, stack_size_okay;
 417   // verify that thread stack base is non-zero
 418   __ cmpptr(stack_base, (int32_t)NULL_WORD);
 419   __ jcc(Assembler::notEqual, stack_base_okay);
 420   __ stop("stack base is zero");
 421   __ bind(stack_base_okay);
 422   // verify that thread stack size is non-zero
 423   __ cmpptr(stack_size, 0);
 424   __ jcc(Assembler::notEqual, stack_size_okay);
 425   __ stop("stack size is zero");
 426   __ bind(stack_size_okay);
 427 #endif
 428 
 429   // Add stack base to locals and subtract stack size
 430   __ addptr(rax, stack_base);
 431   __ subptr(rax, stack_size);
 432 
 433   // add in the red and yellow zone sizes
 434   __ addptr(rax, (StackRedPages + StackYellowPages) * page_size);
 435 
 436   // check against the current stack bottom
 437   __ cmpptr(rsp, rax);
 438   __ jcc(Assembler::above, after_frame_check);
 439 
 440   __ pop(rax); // get return address
 441   __ jump(ExternalAddress(Interpreter::throw_StackOverflowError_entry()));
 442 
 443   // all done with frame size check
 444   __ bind(after_frame_check);
 445 }
 446 
 447 // Allocate monitor and lock method (asm interpreter)
 448 //
 449 // Args:
 450 //      rbx: methodOop
 451 //      r14: locals
 452 //
 453 // Kills:
 454 //      rax
 455 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, ...(param regs)
 456 //      rscratch1, rscratch2 (scratch regs)
 457 void InterpreterGenerator::lock_method(void) {
 458   // synchronize method
 459   const Address access_flags(rbx, methodOopDesc::access_flags_offset());
 460   const Address monitor_block_top(
 461         rbp,
 462         frame::interpreter_frame_monitor_block_top_offset * wordSize);
 463   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 464 
 465 #ifdef ASSERT
 466   {
 467     Label L;
 468     __ movl(rax, access_flags);
 469     __ testl(rax, JVM_ACC_SYNCHRONIZED);
 470     __ jcc(Assembler::notZero, L);
 471     __ stop("method doesn't need synchronization");
 472     __ bind(L);
 473   }
 474 #endif // ASSERT
 475 
 476   // get synchronization object
 477   {
 478     const int mirror_offset = klassOopDesc::klass_part_offset_in_bytes() +
 479                               Klass::java_mirror_offset_in_bytes();
 480     Label done;
 481     __ movl(rax, access_flags);
 482     __ testl(rax, JVM_ACC_STATIC);
 483     // get receiver (assume this is frequent case)
 484     __ movptr(rax, Address(r14, Interpreter::local_offset_in_bytes(0)));
 485     __ jcc(Assembler::zero, done);
 486     __ movptr(rax, Address(rbx, methodOopDesc::constants_offset()));
 487     __ movptr(rax, Address(rax,
 488                            constantPoolOopDesc::pool_holder_offset_in_bytes()));
 489     __ movptr(rax, Address(rax, mirror_offset));
 490 
 491 #ifdef ASSERT
 492     {
 493       Label L;
 494       __ testptr(rax, rax);
 495       __ jcc(Assembler::notZero, L);
 496       __ stop("synchronization object is NULL");
 497       __ bind(L);
 498     }
 499 #endif // ASSERT
 500 
 501     __ bind(done);
 502   }
 503 
 504   // add space for monitor & lock
 505   __ subptr(rsp, entry_size); // add space for a monitor entry
 506   __ movptr(monitor_block_top, rsp);  // set new monitor block top
 507   // store object
 508   __ movptr(Address(rsp, BasicObjectLock::obj_offset_in_bytes()), rax);
 509   __ movptr(c_rarg1, rsp); // object address
 510   __ lock_object(c_rarg1);
 511 }
 512 
 513 // Generate a fixed interpreter frame. This is identical setup for
 514 // interpreted methods and for native methods hence the shared code.
 515 //
 516 // Args:
 517 //      rax: return address
 518 //      rbx: methodOop
 519 //      r14: pointer to locals
 520 //      r13: sender sp
 521 //      rdx: cp cache
 522 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
 523   // initialize fixed part of activation frame
 524   __ push(rax);        // save return address
 525   __ enter();          // save old & set new rbp
 526   __ push(r13);        // set sender sp
 527   __ push((int)NULL_WORD); // leave last_sp as null
 528   __ movptr(r13, Address(rbx, methodOopDesc::const_offset()));      // get constMethodOop
 529   __ lea(r13, Address(r13, constMethodOopDesc::codes_offset())); // get codebase
 530   __ push(rbx);        // save methodOop
 531   if (ProfileInterpreter) {
 532     Label method_data_continue;
 533     __ movptr(rdx, Address(rbx, in_bytes(methodOopDesc::method_data_offset())));
 534     __ testptr(rdx, rdx);
 535     __ jcc(Assembler::zero, method_data_continue);
 536     __ addptr(rdx, in_bytes(methodDataOopDesc::data_offset()));
 537     __ bind(method_data_continue);
 538     __ push(rdx);      // set the mdp (method data pointer)
 539   } else {
 540     __ push(0);
 541   }
 542 
 543   __ movptr(rdx, Address(rbx, methodOopDesc::constants_offset()));
 544   __ movptr(rdx, Address(rdx, constantPoolOopDesc::cache_offset_in_bytes()));
 545   __ push(rdx); // set constant pool cache
 546   __ push(r14); // set locals pointer
 547   if (native_call) {
 548     __ push(0); // no bcp
 549   } else {
 550     __ push(r13); // set bcp
 551   }
 552   __ push(0); // reserve word for pointer to expression stack bottom
 553   __ movptr(Address(rsp, 0), rsp); // set expression stack bottom
 554 }
 555 
 556 // End of helpers
 557 
 558 // Various method entries
 559 //------------------------------------------------------------------------------------------------------------------------
 560 //
 561 //
 562 
 563 // Call an accessor method (assuming it is resolved, otherwise drop
 564 // into vanilla (slow path) entry
 565 address InterpreterGenerator::generate_accessor_entry(void) {
 566   // rbx: methodOop
 567 
 568   // r13: senderSP must preserver for slow path, set SP to it on fast path
 569 
 570   address entry_point = __ pc();
 571   Label xreturn_path;
 572 
 573   // do fastpath for resolved accessor methods
 574   if (UseFastAccessorMethods) {
 575     // Code: _aload_0, _(i|a)getfield, _(i|a)return or any rewrites
 576     //       thereof; parameter size = 1
 577     // Note: We can only use this code if the getfield has been resolved
 578     //       and if we don't have a null-pointer exception => check for
 579     //       these conditions first and use slow path if necessary.
 580     Label slow_path;
 581     // If we need a safepoint check, generate full interpreter entry.
 582     __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
 583              SafepointSynchronize::_not_synchronized);
 584 
 585     __ jcc(Assembler::notEqual, slow_path);
 586     // rbx: method
 587     __ movptr(rax, Address(rsp, wordSize));
 588 
 589     // check if local 0 != NULL and read field
 590     __ testptr(rax, rax);
 591     __ jcc(Assembler::zero, slow_path);
 592 
 593     __ movptr(rdi, Address(rbx, methodOopDesc::constants_offset()));
 594     // read first instruction word and extract bytecode @ 1 and index @ 2
 595     __ movptr(rdx, Address(rbx, methodOopDesc::const_offset()));
 596     __ movl(rdx, Address(rdx, constMethodOopDesc::codes_offset()));
 597     // Shift codes right to get the index on the right.
 598     // The bytecode fetched looks like <index><0xb4><0x2a>
 599     __ shrl(rdx, 2 * BitsPerByte);
 600     __ shll(rdx, exact_log2(in_words(ConstantPoolCacheEntry::size())));
 601     __ movptr(rdi, Address(rdi, constantPoolOopDesc::cache_offset_in_bytes()));
 602 
 603     // rax: local 0
 604     // rbx: method
 605     // rdx: constant pool cache index
 606     // rdi: constant pool cache
 607 
 608     // check if getfield has been resolved and read constant pool cache entry
 609     // check the validity of the cache entry by testing whether _indices field
 610     // contains Bytecode::_getfield in b1 byte.
 611     assert(in_words(ConstantPoolCacheEntry::size()) == 4,
 612            "adjust shift below");
 613     __ movl(rcx,
 614             Address(rdi,
 615                     rdx,
 616                     Address::times_8,
 617                     constantPoolCacheOopDesc::base_offset() +
 618                     ConstantPoolCacheEntry::indices_offset()));
 619     __ shrl(rcx, 2 * BitsPerByte);
 620     __ andl(rcx, 0xFF);
 621     __ cmpl(rcx, Bytecodes::_getfield);
 622     __ jcc(Assembler::notEqual, slow_path);
 623 
 624     // Note: constant pool entry is not valid before bytecode is resolved
 625     __ movptr(rcx,
 626               Address(rdi,
 627                       rdx,
 628                       Address::times_8,
 629                       constantPoolCacheOopDesc::base_offset() +
 630                       ConstantPoolCacheEntry::f2_offset()));
 631     // edx: flags
 632     __ movl(rdx,
 633             Address(rdi,
 634                     rdx,
 635                     Address::times_8,
 636                     constantPoolCacheOopDesc::base_offset() +
 637                     ConstantPoolCacheEntry::flags_offset()));
 638 
 639     Label notObj, notInt, notByte, notShort;
 640     const Address field_address(rax, rcx, Address::times_1);
 641 
 642     // Need to differentiate between igetfield, agetfield, bgetfield etc.
 643     // because they are different sizes.
 644     // Use the type from the constant pool cache
 645     __ shrl(rdx, ConstantPoolCacheEntry::tosBits);
 646     // Make sure we don't need to mask edx for tosBits after the above shift
 647     ConstantPoolCacheEntry::verify_tosBits();
 648 
 649     __ cmpl(rdx, atos);
 650     __ jcc(Assembler::notEqual, notObj);
 651     // atos
 652     __ load_heap_oop(rax, field_address);
 653     __ jmp(xreturn_path);
 654 
 655     __ bind(notObj);
 656     __ cmpl(rdx, itos);
 657     __ jcc(Assembler::notEqual, notInt);
 658     // itos
 659     __ movl(rax, field_address);
 660     __ jmp(xreturn_path);
 661 
 662     __ bind(notInt);
 663     __ cmpl(rdx, btos);
 664     __ jcc(Assembler::notEqual, notByte);
 665     // btos
 666     __ load_signed_byte(rax, field_address);
 667     __ jmp(xreturn_path);
 668 
 669     __ bind(notByte);
 670     __ cmpl(rdx, stos);
 671     __ jcc(Assembler::notEqual, notShort);
 672     // stos
 673     __ load_signed_short(rax, field_address);
 674     __ jmp(xreturn_path);
 675 
 676     __ bind(notShort);
 677 #ifdef ASSERT
 678     Label okay;
 679     __ cmpl(rdx, ctos);
 680     __ jcc(Assembler::equal, okay);
 681     __ stop("what type is this?");
 682     __ bind(okay);
 683 #endif
 684     // ctos
 685     __ load_unsigned_short(rax, field_address);
 686 
 687     __ bind(xreturn_path);
 688 
 689     // _ireturn/_areturn
 690     __ pop(rdi);
 691     __ mov(rsp, r13);
 692     __ jmp(rdi);
 693     __ ret(0);
 694 
 695     // generate a vanilla interpreter entry as the slow path
 696     __ bind(slow_path);
 697     (void) generate_normal_entry(false);
 698   } else {
 699     (void) generate_normal_entry(false);
 700   }
 701 
 702   return entry_point;
 703 }
 704 
 705 // Interpreter stub for calling a native method. (asm interpreter)
 706 // This sets up a somewhat different looking stack for calling the
 707 // native method than the typical interpreter frame setup.
 708 address InterpreterGenerator::generate_native_entry(bool synchronized) {
 709   // determine code generation flags
 710   bool inc_counter  = UseCompiler || CountCompiledCalls;
 711 
 712   // rbx: methodOop
 713   // r13: sender sp
 714 
 715   address entry_point = __ pc();
 716 
 717   const Address size_of_parameters(rbx, methodOopDesc::
 718                                         size_of_parameters_offset());
 719   const Address invocation_counter(rbx, methodOopDesc::
 720                                         invocation_counter_offset() +
 721                                         InvocationCounter::counter_offset());
 722   const Address access_flags      (rbx, methodOopDesc::access_flags_offset());
 723 
 724   // get parameter size (always needed)
 725   __ load_unsigned_short(rcx, size_of_parameters);
 726 
 727   // native calls don't need the stack size check since they have no
 728   // expression stack and the arguments are already on the stack and
 729   // we only add a handful of words to the stack
 730 
 731   // rbx: methodOop
 732   // rcx: size of parameters
 733   // r13: sender sp
 734   __ pop(rax);                                       // get return address
 735 
 736   // for natives the size of locals is zero
 737 
 738   // compute beginning of parameters (r14)
 739   if (TaggedStackInterpreter) __ shll(rcx, 1); // 2 slots per parameter.
 740   __ lea(r14, Address(rsp, rcx, Address::times_8, -wordSize));
 741 
 742   // add 2 zero-initialized slots for native calls
 743   // initialize result_handler slot
 744   __ push((int) NULL_WORD);
 745   // slot for oop temp
 746   // (static native method holder mirror/jni oop result)
 747   __ push((int) NULL_WORD);
 748 
 749   if (inc_counter) {
 750     __ movl(rcx, invocation_counter);  // (pre-)fetch invocation count
 751   }
 752 
 753   // initialize fixed part of activation frame
 754   generate_fixed_frame(true);
 755 
 756   // make sure method is native & not abstract
 757 #ifdef ASSERT
 758   __ movl(rax, access_flags);
 759   {
 760     Label L;
 761     __ testl(rax, JVM_ACC_NATIVE);
 762     __ jcc(Assembler::notZero, L);
 763     __ stop("tried to execute non-native method as native");
 764     __ bind(L);
 765   }
 766   {
 767     Label L;
 768     __ testl(rax, JVM_ACC_ABSTRACT);
 769     __ jcc(Assembler::zero, L);
 770     __ stop("tried to execute abstract method in interpreter");
 771     __ bind(L);
 772   }
 773 #endif
 774 
 775   // Since at this point in the method invocation the exception handler
 776   // would try to exit the monitor of synchronized methods which hasn't
 777   // been entered yet, we set the thread local variable
 778   // _do_not_unlock_if_synchronized to true. The remove_activation will
 779   // check this flag.
 780 
 781   const Address do_not_unlock_if_synchronized(r15_thread,
 782         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
 783   __ movbool(do_not_unlock_if_synchronized, true);
 784 
 785   // increment invocation count & check for overflow
 786   Label invocation_counter_overflow;
 787   if (inc_counter) {
 788     generate_counter_incr(&invocation_counter_overflow, NULL, NULL);
 789   }
 790 
 791   Label continue_after_compile;
 792   __ bind(continue_after_compile);
 793 
 794   bang_stack_shadow_pages(true);
 795 
 796   // reset the _do_not_unlock_if_synchronized flag
 797   __ movbool(do_not_unlock_if_synchronized, false);
 798 
 799   // check for synchronized methods
 800   // Must happen AFTER invocation_counter check and stack overflow check,
 801   // so method is not locked if overflows.
 802   if (synchronized) {
 803     lock_method();
 804   } else {
 805     // no synchronization necessary
 806 #ifdef ASSERT
 807     {
 808       Label L;
 809       __ movl(rax, access_flags);
 810       __ testl(rax, JVM_ACC_SYNCHRONIZED);
 811       __ jcc(Assembler::zero, L);
 812       __ stop("method needs synchronization");
 813       __ bind(L);
 814     }
 815 #endif
 816   }
 817 
 818   // start execution
 819 #ifdef ASSERT
 820   {
 821     Label L;
 822     const Address monitor_block_top(rbp,
 823                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
 824     __ movptr(rax, monitor_block_top);
 825     __ cmpptr(rax, rsp);
 826     __ jcc(Assembler::equal, L);
 827     __ stop("broken stack frame setup in interpreter");
 828     __ bind(L);
 829   }
 830 #endif
 831 
 832   // jvmti support
 833   __ notify_method_entry();
 834 
 835   // work registers
 836   const Register method = rbx;
 837   const Register t      = r11;
 838 
 839   // allocate space for parameters
 840   __ get_method(method);
 841   __ verify_oop(method);
 842   __ load_unsigned_short(t,
 843                          Address(method,
 844                                  methodOopDesc::size_of_parameters_offset()));
 845   __ shll(t, Interpreter::logStackElementSize());
 846 
 847   __ subptr(rsp, t);
 848   __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
 849   __ andptr(rsp, -16); // must be 16 byte boundary (see amd64 ABI)
 850 
 851   // get signature handler
 852   {
 853     Label L;
 854     __ movptr(t, Address(method, methodOopDesc::signature_handler_offset()));
 855     __ testptr(t, t);
 856     __ jcc(Assembler::notZero, L);
 857     __ call_VM(noreg,
 858                CAST_FROM_FN_PTR(address,
 859                                 InterpreterRuntime::prepare_native_call),
 860                method);
 861     __ get_method(method);
 862     __ movptr(t, Address(method, methodOopDesc::signature_handler_offset()));
 863     __ bind(L);
 864   }
 865 
 866   // call signature handler
 867   assert(InterpreterRuntime::SignatureHandlerGenerator::from() == r14,
 868          "adjust this code");
 869   assert(InterpreterRuntime::SignatureHandlerGenerator::to() == rsp,
 870          "adjust this code");
 871   assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == rscratch1,
 872           "adjust this code");
 873 
 874   // The generated handlers do not touch RBX (the method oop).
 875   // However, large signatures cannot be cached and are generated
 876   // each time here.  The slow-path generator can do a GC on return,
 877   // so we must reload it after the call.
 878   __ call(t);
 879   __ get_method(method);        // slow path can do a GC, reload RBX
 880 
 881 
 882   // result handler is in rax
 883   // set result handler
 884   __ movptr(Address(rbp,
 885                     (frame::interpreter_frame_result_handler_offset) * wordSize),
 886             rax);
 887 
 888   // pass mirror handle if static call
 889   {
 890     Label L;
 891     const int mirror_offset = klassOopDesc::klass_part_offset_in_bytes() +
 892                               Klass::java_mirror_offset_in_bytes();
 893     __ movl(t, Address(method, methodOopDesc::access_flags_offset()));
 894     __ testl(t, JVM_ACC_STATIC);
 895     __ jcc(Assembler::zero, L);
 896     // get mirror
 897     __ movptr(t, Address(method, methodOopDesc::constants_offset()));
 898     __ movptr(t, Address(t, constantPoolOopDesc::pool_holder_offset_in_bytes()));
 899     __ movptr(t, Address(t, mirror_offset));
 900     // copy mirror into activation frame
 901     __ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize),
 902             t);
 903     // pass handle to mirror
 904     __ lea(c_rarg1,
 905            Address(rbp, frame::interpreter_frame_oop_temp_offset * wordSize));
 906     __ bind(L);
 907   }
 908 
 909   // get native function entry point
 910   {
 911     Label L;
 912     __ movptr(rax, Address(method, methodOopDesc::native_function_offset()));
 913     ExternalAddress unsatisfied(SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
 914     __ movptr(rscratch2, unsatisfied.addr());
 915     __ cmpptr(rax, rscratch2);
 916     __ jcc(Assembler::notEqual, L);
 917     __ call_VM(noreg,
 918                CAST_FROM_FN_PTR(address,
 919                                 InterpreterRuntime::prepare_native_call),
 920                method);
 921     __ get_method(method);
 922     __ verify_oop(method);
 923     __ movptr(rax, Address(method, methodOopDesc::native_function_offset()));
 924     __ bind(L);
 925   }
 926 
 927   // pass JNIEnv
 928   __ lea(c_rarg0, Address(r15_thread, JavaThread::jni_environment_offset()));
 929 
 930   // It is enough that the pc() points into the right code
 931   // segment. It does not have to be the correct return pc.
 932   __ set_last_Java_frame(rsp, rbp, (address) __ pc());
 933 
 934   // change thread state
 935 #ifdef ASSERT
 936   {
 937     Label L;
 938     __ movl(t, Address(r15_thread, JavaThread::thread_state_offset()));
 939     __ cmpl(t, _thread_in_Java);
 940     __ jcc(Assembler::equal, L);
 941     __ stop("Wrong thread state in native stub");
 942     __ bind(L);
 943   }
 944 #endif
 945 
 946   // Change state to native
 947 
 948   __ movl(Address(r15_thread, JavaThread::thread_state_offset()),
 949           _thread_in_native);
 950 
 951   // Call the native method.
 952   __ call(rax);
 953   // result potentially in rax or xmm0
 954 
 955   // Depending on runtime options, either restore the MXCSR
 956   // register after returning from the JNI Call or verify that
 957   // it wasn't changed during -Xcheck:jni.
 958   if (RestoreMXCSROnJNICalls) {
 959     __ ldmxcsr(ExternalAddress(StubRoutines::x86::mxcsr_std()));
 960   }
 961   else if (CheckJNICalls) {
 962     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::verify_mxcsr_entry())));
 963   }
 964 
 965   // NOTE: The order of these pushes is known to frame::interpreter_frame_result
 966   // in order to extract the result of a method call. If the order of these
 967   // pushes change or anything else is added to the stack then the code in
 968   // interpreter_frame_result must also change.
 969 
 970   __ push(dtos);
 971   __ push(ltos);
 972 
 973   // change thread state
 974   __ movl(Address(r15_thread, JavaThread::thread_state_offset()),
 975           _thread_in_native_trans);
 976 
 977   if (os::is_MP()) {
 978     if (UseMembar) {
 979       // Force this write out before the read below
 980       __ membar(Assembler::Membar_mask_bits(
 981            Assembler::LoadLoad | Assembler::LoadStore |
 982            Assembler::StoreLoad | Assembler::StoreStore));
 983     } else {
 984       // Write serialization page so VM thread can do a pseudo remote membar.
 985       // We use the current thread pointer to calculate a thread specific
 986       // offset to write to within the page. This minimizes bus traffic
 987       // due to cache line collision.
 988       __ serialize_memory(r15_thread, rscratch2);
 989     }
 990   }
 991 
 992   // check for safepoint operation in progress and/or pending suspend requests
 993   {
 994     Label Continue;
 995     __ cmp32(ExternalAddress(SafepointSynchronize::address_of_state()),
 996              SafepointSynchronize::_not_synchronized);
 997 
 998     Label L;
 999     __ jcc(Assembler::notEqual, L);
1000     __ cmpl(Address(r15_thread, JavaThread::suspend_flags_offset()), 0);
1001     __ jcc(Assembler::equal, Continue);
1002     __ bind(L);
1003 
1004     // Don't use call_VM as it will see a possible pending exception
1005     // and forward it and never return here preventing us from
1006     // clearing _last_native_pc down below.  Also can't use
1007     // call_VM_leaf either as it will check to see if r13 & r14 are
1008     // preserved and correspond to the bcp/locals pointers. So we do a
1009     // runtime call by hand.
1010     //
1011     __ mov(c_rarg0, r15_thread);
1012     __ mov(r12, rsp); // remember sp
1013     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1014     __ andptr(rsp, -16); // align stack as required by ABI
1015     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
1016     __ mov(rsp, r12); // restore sp
1017     __ reinit_heapbase();
1018     __ bind(Continue);
1019   }
1020 
1021   // change thread state
1022   __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
1023 
1024   // reset_last_Java_frame
1025   __ reset_last_Java_frame(true, true);
1026 
1027   // reset handle block
1028   __ movptr(t, Address(r15_thread, JavaThread::active_handles_offset()));
1029   __ movptr(Address(t, JNIHandleBlock::top_offset_in_bytes()), (int32_t)NULL_WORD);
1030 
1031   // If result is an oop unbox and store it in frame where gc will see it
1032   // and result handler will pick it up
1033 
1034   {
1035     Label no_oop, store_result;
1036     __ lea(t, ExternalAddress(AbstractInterpreter::result_handler(T_OBJECT)));
1037     __ cmpptr(t, Address(rbp, frame::interpreter_frame_result_handler_offset*wordSize));
1038     __ jcc(Assembler::notEqual, no_oop);
1039     // retrieve result
1040     __ pop(ltos);
1041     __ testptr(rax, rax);
1042     __ jcc(Assembler::zero, store_result);
1043     __ movptr(rax, Address(rax, 0));
1044     __ bind(store_result);
1045     __ movptr(Address(rbp, frame::interpreter_frame_oop_temp_offset*wordSize), rax);
1046     // keep stack depth as expected by pushing oop which will eventually be discarde
1047     __ push(ltos);
1048     __ bind(no_oop);
1049   }
1050 
1051 
1052   {
1053     Label no_reguard;
1054     __ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()),
1055             JavaThread::stack_guard_yellow_disabled);
1056     __ jcc(Assembler::notEqual, no_reguard);
1057 
1058     __ pusha(); // XXX only save smashed registers
1059     __ mov(r12, rsp); // remember sp
1060     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1061     __ andptr(rsp, -16); // align stack as required by ABI
1062     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
1063     __ mov(rsp, r12); // restore sp
1064     __ popa(); // XXX only restore smashed registers
1065     __ reinit_heapbase();
1066 
1067     __ bind(no_reguard);
1068   }
1069 
1070 
1071   // The method register is junk from after the thread_in_native transition
1072   // until here.  Also can't call_VM until the bcp has been
1073   // restored.  Need bcp for throwing exception below so get it now.
1074   __ get_method(method);
1075   __ verify_oop(method);
1076 
1077   // restore r13 to have legal interpreter frame, i.e., bci == 0 <=>
1078   // r13 == code_base()
1079   __ movptr(r13, Address(method, methodOopDesc::const_offset()));   // get constMethodOop
1080   __ lea(r13, Address(r13, constMethodOopDesc::codes_offset()));    // get codebase
1081   // handle exceptions (exception handling will handle unlocking!)
1082   {
1083     Label L;
1084     __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t) NULL_WORD);
1085     __ jcc(Assembler::zero, L);
1086     // Note: At some point we may want to unify this with the code
1087     // used in call_VM_base(); i.e., we should use the
1088     // StubRoutines::forward_exception code. For now this doesn't work
1089     // here because the rsp is not correctly set at this point.
1090     __ MacroAssembler::call_VM(noreg,
1091                                CAST_FROM_FN_PTR(address,
1092                                InterpreterRuntime::throw_pending_exception));
1093     __ should_not_reach_here();
1094     __ bind(L);
1095   }
1096 
1097   // do unlocking if necessary
1098   {
1099     Label L;
1100     __ movl(t, Address(method, methodOopDesc::access_flags_offset()));
1101     __ testl(t, JVM_ACC_SYNCHRONIZED);
1102     __ jcc(Assembler::zero, L);
1103     // the code below should be shared with interpreter macro
1104     // assembler implementation
1105     {
1106       Label unlock;
1107       // BasicObjectLock will be first in list, since this is a
1108       // synchronized method. However, need to check that the object
1109       // has not been unlocked by an explicit monitorexit bytecode.
1110       const Address monitor(rbp,
1111                             (intptr_t)(frame::interpreter_frame_initial_sp_offset *
1112                                        wordSize - sizeof(BasicObjectLock)));
1113 
1114       // monitor expect in c_rarg1 for slow unlock path
1115       __ lea(c_rarg1, monitor); // address of first monitor
1116 
1117       __ movptr(t, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
1118       __ testptr(t, t);
1119       __ jcc(Assembler::notZero, unlock);
1120 
1121       // Entry already unlocked, need to throw exception
1122       __ MacroAssembler::call_VM(noreg,
1123                                  CAST_FROM_FN_PTR(address,
1124                    InterpreterRuntime::throw_illegal_monitor_state_exception));
1125       __ should_not_reach_here();
1126 
1127       __ bind(unlock);
1128       __ unlock_object(c_rarg1);
1129     }
1130     __ bind(L);
1131   }
1132 
1133   // jvmti support
1134   // Note: This must happen _after_ handling/throwing any exceptions since
1135   //       the exception handler code notifies the runtime of method exits
1136   //       too. If this happens before, method entry/exit notifications are
1137   //       not properly paired (was bug - gri 11/22/99).
1138   __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI);
1139 
1140   // restore potential result in edx:eax, call result handler to
1141   // restore potential result in ST0 & handle result
1142 
1143   __ pop(ltos);
1144   __ pop(dtos);
1145 
1146   __ movptr(t, Address(rbp,
1147                        (frame::interpreter_frame_result_handler_offset) * wordSize));
1148   __ call(t);
1149 
1150   // remove activation
1151   __ movptr(t, Address(rbp,
1152                        frame::interpreter_frame_sender_sp_offset *
1153                        wordSize)); // get sender sp
1154   __ leave();                                // remove frame anchor
1155   __ pop(rdi);                               // get return address
1156   __ mov(rsp, t);                            // set sp to sender sp
1157   __ jmp(rdi);
1158 
1159   if (inc_counter) {
1160     // Handle overflow of counter and compile method
1161     __ bind(invocation_counter_overflow);
1162     generate_counter_overflow(&continue_after_compile);
1163   }
1164 
1165   return entry_point;
1166 }
1167 
1168 //
1169 // Generic interpreted method entry to (asm) interpreter
1170 //
1171 address InterpreterGenerator::generate_normal_entry(bool synchronized) {
1172   // determine code generation flags
1173   bool inc_counter  = UseCompiler || CountCompiledCalls;
1174 
1175   // ebx: methodOop
1176   // r13: sender sp
1177   address entry_point = __ pc();
1178 
1179   const Address size_of_parameters(rbx,
1180                                    methodOopDesc::size_of_parameters_offset());
1181   const Address size_of_locals(rbx, methodOopDesc::size_of_locals_offset());
1182   const Address invocation_counter(rbx,
1183                                    methodOopDesc::invocation_counter_offset() +
1184                                    InvocationCounter::counter_offset());
1185   const Address access_flags(rbx, methodOopDesc::access_flags_offset());
1186 
1187   // get parameter size (always needed)
1188   __ load_unsigned_short(rcx, size_of_parameters);
1189 
1190   // rbx: methodOop
1191   // rcx: size of parameters
1192   // r13: sender_sp (could differ from sp+wordSize if we were called via c2i )
1193 
1194   __ load_unsigned_short(rdx, size_of_locals); // get size of locals in words
1195   __ subl(rdx, rcx); // rdx = no. of additional locals
1196 
1197   // YYY
1198 //   __ incrementl(rdx);
1199 //   __ andl(rdx, -2);
1200 
1201   // see if we've got enough room on the stack for locals plus overhead.
1202   generate_stack_overflow_check();
1203 
1204   // get return address
1205   __ pop(rax);
1206 
1207   // compute beginning of parameters (r14)
1208   if (TaggedStackInterpreter) __ shll(rcx, 1); // 2 slots per parameter.
1209   __ lea(r14, Address(rsp, rcx, Address::times_8, -wordSize));
1210 
1211   // rdx - # of additional locals
1212   // allocate space for locals
1213   // explicitly initialize locals
1214   {
1215     Label exit, loop;
1216     __ testl(rdx, rdx);
1217     __ jcc(Assembler::lessEqual, exit); // do nothing if rdx <= 0
1218     __ bind(loop);
1219     if (TaggedStackInterpreter) __ push((int) NULL_WORD);  // push tag
1220     __ push((int) NULL_WORD); // initialize local variables
1221     __ decrementl(rdx); // until everything initialized
1222     __ jcc(Assembler::greater, loop);
1223     __ bind(exit);
1224   }
1225 
1226   // (pre-)fetch invocation count
1227   if (inc_counter) {
1228     __ movl(rcx, invocation_counter);
1229   }
1230   // initialize fixed part of activation frame
1231   generate_fixed_frame(false);
1232 
1233   // make sure method is not native & not abstract
1234 #ifdef ASSERT
1235   __ movl(rax, access_flags);
1236   {
1237     Label L;
1238     __ testl(rax, JVM_ACC_NATIVE);
1239     __ jcc(Assembler::zero, L);
1240     __ stop("tried to execute native method as non-native");
1241     __ bind(L);
1242   }
1243   {
1244     Label L;
1245     __ testl(rax, JVM_ACC_ABSTRACT);
1246     __ jcc(Assembler::zero, L);
1247     __ stop("tried to execute abstract method in interpreter");
1248     __ bind(L);
1249   }
1250 #endif
1251 
1252   // Since at this point in the method invocation the exception
1253   // handler would try to exit the monitor of synchronized methods
1254   // which hasn't been entered yet, we set the thread local variable
1255   // _do_not_unlock_if_synchronized to true. The remove_activation
1256   // will check this flag.
1257 
1258   const Address do_not_unlock_if_synchronized(r15_thread,
1259         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1260   __ movbool(do_not_unlock_if_synchronized, true);
1261 
1262   // increment invocation count & check for overflow
1263   Label invocation_counter_overflow;
1264   Label profile_method;
1265   Label profile_method_continue;
1266   if (inc_counter) {
1267     generate_counter_incr(&invocation_counter_overflow,
1268                           &profile_method,
1269                           &profile_method_continue);
1270     if (ProfileInterpreter) {
1271       __ bind(profile_method_continue);
1272     }
1273   }
1274 
1275   Label continue_after_compile;
1276   __ bind(continue_after_compile);
1277 
1278   // check for synchronized interpreted methods
1279   bang_stack_shadow_pages(false);
1280 
1281   // reset the _do_not_unlock_if_synchronized flag
1282   __ movbool(do_not_unlock_if_synchronized, false);
1283 
1284   // check for synchronized methods
1285   // Must happen AFTER invocation_counter check and stack overflow check,
1286   // so method is not locked if overflows.
1287   if (synchronized) {
1288     // Allocate monitor and lock method
1289     lock_method();
1290   } else {
1291     // no synchronization necessary
1292 #ifdef ASSERT
1293     {
1294       Label L;
1295       __ movl(rax, access_flags);
1296       __ testl(rax, JVM_ACC_SYNCHRONIZED);
1297       __ jcc(Assembler::zero, L);
1298       __ stop("method needs synchronization");
1299       __ bind(L);
1300     }
1301 #endif
1302   }
1303 
1304   // start execution
1305 #ifdef ASSERT
1306   {
1307     Label L;
1308      const Address monitor_block_top (rbp,
1309                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
1310     __ movptr(rax, monitor_block_top);
1311     __ cmpptr(rax, rsp);
1312     __ jcc(Assembler::equal, L);
1313     __ stop("broken stack frame setup in interpreter");
1314     __ bind(L);
1315   }
1316 #endif
1317 
1318   // jvmti support
1319   __ notify_method_entry();
1320 
1321   __ dispatch_next(vtos);
1322 
1323   // invocation counter overflow
1324   if (inc_counter) {
1325     if (ProfileInterpreter) {
1326       // We have decided to profile this method in the interpreter
1327       __ bind(profile_method);
1328 
1329       __ call_VM(noreg,
1330                  CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method),
1331                  r13, true);
1332 
1333       __ movptr(rbx, Address(rbp, method_offset)); // restore methodOop
1334       __ movptr(rax, Address(rbx,
1335                              in_bytes(methodOopDesc::method_data_offset())));
1336       __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize),
1337                 rax);
1338       __ test_method_data_pointer(rax, profile_method_continue);
1339       __ addptr(rax, in_bytes(methodDataOopDesc::data_offset()));
1340       __ movptr(Address(rbp, frame::interpreter_frame_mdx_offset * wordSize),
1341               rax);
1342       __ jmp(profile_method_continue);
1343     }
1344     // Handle overflow of counter and compile method
1345     __ bind(invocation_counter_overflow);
1346     generate_counter_overflow(&continue_after_compile);
1347   }
1348 
1349   return entry_point;
1350 }
1351 
1352 // Entry points
1353 //
1354 // Here we generate the various kind of entries into the interpreter.
1355 // The two main entry type are generic bytecode methods and native
1356 // call method.  These both come in synchronized and non-synchronized
1357 // versions but the frame layout they create is very similar. The
1358 // other method entry types are really just special purpose entries
1359 // that are really entry and interpretation all in one. These are for
1360 // trivial methods like accessor, empty, or special math methods.
1361 //
1362 // When control flow reaches any of the entry types for the interpreter
1363 // the following holds ->
1364 //
1365 // Arguments:
1366 //
1367 // rbx: methodOop
1368 //
1369 // Stack layout immediately at entry
1370 //
1371 // [ return address     ] <--- rsp
1372 // [ parameter n        ]
1373 //   ...
1374 // [ parameter 1        ]
1375 // [ expression stack   ] (caller's java expression stack)
1376 
1377 // Assuming that we don't go to one of the trivial specialized entries
1378 // the stack will look like below when we are ready to execute the
1379 // first bytecode (or call the native routine). The register usage
1380 // will be as the template based interpreter expects (see
1381 // interpreter_amd64.hpp).
1382 //
1383 // local variables follow incoming parameters immediately; i.e.
1384 // the return address is moved to the end of the locals).
1385 //
1386 // [ monitor entry      ] <--- rsp
1387 //   ...
1388 // [ monitor entry      ]
1389 // [ expr. stack bottom ]
1390 // [ saved r13          ]
1391 // [ current r14        ]
1392 // [ methodOop          ]
1393 // [ saved ebp          ] <--- rbp
1394 // [ return address     ]
1395 // [ local variable m   ]
1396 //   ...
1397 // [ local variable 1   ]
1398 // [ parameter n        ]
1399 //   ...
1400 // [ parameter 1        ] <--- r14
1401 
1402 address AbstractInterpreterGenerator::generate_method_entry(
1403                                         AbstractInterpreter::MethodKind kind) {
1404   // determine code generation flags
1405   bool synchronized = false;
1406   address entry_point = NULL;
1407 
1408   switch (kind) {
1409   case Interpreter::zerolocals             :                                                                             break;
1410   case Interpreter::zerolocals_synchronized: synchronized = true;                                                        break;
1411   case Interpreter::native                 : entry_point = ((InterpreterGenerator*) this)->generate_native_entry(false); break;
1412   case Interpreter::native_synchronized    : entry_point = ((InterpreterGenerator*) this)->generate_native_entry(true);  break;
1413   case Interpreter::empty                  : entry_point = ((InterpreterGenerator*) this)->generate_empty_entry();       break;
1414   case Interpreter::accessor               : entry_point = ((InterpreterGenerator*) this)->generate_accessor_entry();    break;
1415   case Interpreter::abstract               : entry_point = ((InterpreterGenerator*) this)->generate_abstract_entry();    break;
1416   case Interpreter::method_handle          : entry_point = ((InterpreterGenerator*) this)->generate_method_handle_entry();break;
1417 
1418   case Interpreter::java_lang_math_sin     : // fall thru
1419   case Interpreter::java_lang_math_cos     : // fall thru
1420   case Interpreter::java_lang_math_tan     : // fall thru
1421   case Interpreter::java_lang_math_abs     : // fall thru
1422   case Interpreter::java_lang_math_log     : // fall thru
1423   case Interpreter::java_lang_math_log10   : // fall thru
1424   case Interpreter::java_lang_math_sqrt    : entry_point = ((InterpreterGenerator*) this)->generate_math_entry(kind);    break;
1425   default                                  : ShouldNotReachHere();                                                       break;
1426   }
1427 
1428   if (entry_point) {
1429     return entry_point;
1430   }
1431 
1432   return ((InterpreterGenerator*) this)->
1433                                 generate_normal_entry(synchronized);
1434 }
1435 
1436 // How much stack a method activation needs in words.
1437 int AbstractInterpreter::size_top_interpreter_activation(methodOop method) {
1438   const int entry_size = frame::interpreter_frame_monitor_size();
1439 
1440   // total overhead size: entry_size + (saved rbp thru expr stack
1441   // bottom).  be sure to change this if you add/subtract anything
1442   // to/from the overhead area
1443   const int overhead_size =
1444     -(frame::interpreter_frame_initial_sp_offset) + entry_size;
1445 
1446   const int stub_code = frame::entry_frame_after_call_words;
1447   const int extra_stack = methodOopDesc::extra_stack_entries();
1448   const int method_stack = (method->max_locals() + method->max_stack() + extra_stack) *
1449                            Interpreter::stackElementWords();
1450   return (overhead_size + method_stack + stub_code);
1451 }
1452 
1453 int AbstractInterpreter::layout_activation(methodOop method,
1454                                            int tempcount,
1455                                            int popframe_extra_args,
1456                                            int moncount,
1457                                            int callee_param_count,
1458                                            int callee_locals,
1459                                            frame* caller,
1460                                            frame* interpreter_frame,
1461                                            bool is_top_frame) {
1462   // Note: This calculation must exactly parallel the frame setup
1463   // in AbstractInterpreterGenerator::generate_method_entry.
1464   // If interpreter_frame!=NULL, set up the method, locals, and monitors.
1465   // The frame interpreter_frame, if not NULL, is guaranteed to be the
1466   // right size, as determined by a previous call to this method.
1467   // It is also guaranteed to be walkable even though it is in a skeletal state
1468 
1469   // fixed size of an interpreter frame:
1470   int max_locals = method->max_locals() * Interpreter::stackElementWords();
1471   int extra_locals = (method->max_locals() - method->size_of_parameters()) *
1472                      Interpreter::stackElementWords();
1473 
1474   int overhead = frame::sender_sp_offset -
1475                  frame::interpreter_frame_initial_sp_offset;
1476   // Our locals were accounted for by the caller (or last_frame_adjust
1477   // on the transistion) Since the callee parameters already account
1478   // for the callee's params we only need to account for the extra
1479   // locals.
1480   int size = overhead +
1481          (callee_locals - callee_param_count)*Interpreter::stackElementWords() +
1482          moncount * frame::interpreter_frame_monitor_size() +
1483          tempcount* Interpreter::stackElementWords() + popframe_extra_args;
1484   if (interpreter_frame != NULL) {
1485 #ifdef ASSERT
1486     assert(caller->unextended_sp() == interpreter_frame->interpreter_frame_sender_sp(),
1487            "Frame not properly walkable");
1488     assert(caller->sp() == interpreter_frame->sender_sp(), "Frame not properly walkable(2)");
1489 #endif
1490 
1491     interpreter_frame->interpreter_frame_set_method(method);
1492     // NOTE the difference in using sender_sp and
1493     // interpreter_frame_sender_sp interpreter_frame_sender_sp is
1494     // the original sp of the caller (the unextended_sp) and
1495     // sender_sp is fp+16 XXX
1496     intptr_t* locals = interpreter_frame->sender_sp() + max_locals - 1;
1497 
1498     interpreter_frame->interpreter_frame_set_locals(locals);
1499     BasicObjectLock* montop = interpreter_frame->interpreter_frame_monitor_begin();
1500     BasicObjectLock* monbot = montop - moncount;
1501     interpreter_frame->interpreter_frame_set_monitor_end(monbot);
1502 
1503     // Set last_sp
1504     intptr_t*  esp = (intptr_t*) monbot -
1505                      tempcount*Interpreter::stackElementWords() -
1506                      popframe_extra_args;
1507     interpreter_frame->interpreter_frame_set_last_sp(esp);
1508 
1509     // All frames but the initial (oldest) interpreter frame we fill in have
1510     // a value for sender_sp that allows walking the stack but isn't
1511     // truly correct. Correct the value here.
1512     if (extra_locals != 0 &&
1513         interpreter_frame->sender_sp() ==
1514         interpreter_frame->interpreter_frame_sender_sp()) {
1515       interpreter_frame->set_interpreter_frame_sender_sp(caller->sp() +
1516                                                          extra_locals);
1517     }
1518     *interpreter_frame->interpreter_frame_cache_addr() =
1519       method->constants()->cache();
1520   }
1521   return size;
1522 }
1523 
1524 //-----------------------------------------------------------------------------
1525 // Exceptions
1526 
1527 void TemplateInterpreterGenerator::generate_throw_exception() {
1528   // Entry point in previous activation (i.e., if the caller was
1529   // interpreted)
1530   Interpreter::_rethrow_exception_entry = __ pc();
1531   // Restore sp to interpreter_frame_last_sp even though we are going
1532   // to empty the expression stack for the exception processing.
1533   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
1534   // rax: exception
1535   // rdx: return address/pc that threw exception
1536   __ restore_bcp();    // r13 points to call/send
1537   __ restore_locals();
1538   __ reinit_heapbase();  // restore r12 as heapbase.
1539   // Entry point for exceptions thrown within interpreter code
1540   Interpreter::_throw_exception_entry = __ pc();
1541   // expression stack is undefined here
1542   // rax: exception
1543   // r13: exception bcp
1544   __ verify_oop(rax);
1545   __ mov(c_rarg1, rax);
1546 
1547   // expression stack must be empty before entering the VM in case of
1548   // an exception
1549   __ empty_expression_stack();
1550   // find exception handler address and preserve exception oop
1551   __ call_VM(rdx,
1552              CAST_FROM_FN_PTR(address,
1553                           InterpreterRuntime::exception_handler_for_exception),
1554              c_rarg1);
1555   // rax: exception handler entry point
1556   // rdx: preserved exception oop
1557   // r13: bcp for exception handler
1558   __ push_ptr(rdx); // push exception which is now the only value on the stack
1559   __ jmp(rax); // jump to exception handler (may be _remove_activation_entry!)
1560 
1561   // If the exception is not handled in the current frame the frame is
1562   // removed and the exception is rethrown (i.e. exception
1563   // continuation is _rethrow_exception).
1564   //
1565   // Note: At this point the bci is still the bxi for the instruction
1566   // which caused the exception and the expression stack is
1567   // empty. Thus, for any VM calls at this point, GC will find a legal
1568   // oop map (with empty expression stack).
1569 
1570   // In current activation
1571   // tos: exception
1572   // esi: exception bcp
1573 
1574   //
1575   // JVMTI PopFrame support
1576   //
1577 
1578   Interpreter::_remove_activation_preserving_args_entry = __ pc();
1579   __ empty_expression_stack();
1580   // Set the popframe_processing bit in pending_popframe_condition
1581   // indicating that we are currently handling popframe, so that
1582   // call_VMs that may happen later do not trigger new popframe
1583   // handling cycles.
1584   __ movl(rdx, Address(r15_thread, JavaThread::popframe_condition_offset()));
1585   __ orl(rdx, JavaThread::popframe_processing_bit);
1586   __ movl(Address(r15_thread, JavaThread::popframe_condition_offset()), rdx);
1587 
1588   {
1589     // Check to see whether we are returning to a deoptimized frame.
1590     // (The PopFrame call ensures that the caller of the popped frame is
1591     // either interpreted or compiled and deoptimizes it if compiled.)
1592     // In this case, we can't call dispatch_next() after the frame is
1593     // popped, but instead must save the incoming arguments and restore
1594     // them after deoptimization has occurred.
1595     //
1596     // Note that we don't compare the return PC against the
1597     // deoptimization blob's unpack entry because of the presence of
1598     // adapter frames in C2.
1599     Label caller_not_deoptimized;
1600     __ movptr(c_rarg1, Address(rbp, frame::return_addr_offset * wordSize));
1601     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1602                                InterpreterRuntime::interpreter_contains), c_rarg1);
1603     __ testl(rax, rax);
1604     __ jcc(Assembler::notZero, caller_not_deoptimized);
1605 
1606     // Compute size of arguments for saving when returning to
1607     // deoptimized caller
1608     __ get_method(rax);
1609     __ load_unsigned_short(rax, Address(rax, in_bytes(methodOopDesc::
1610                                                 size_of_parameters_offset())));
1611     __ shll(rax, Interpreter::logStackElementSize());
1612     __ restore_locals(); // XXX do we need this?
1613     __ subptr(r14, rax);
1614     __ addptr(r14, wordSize);
1615     // Save these arguments
1616     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1617                                            Deoptimization::
1618                                            popframe_preserve_args),
1619                           r15_thread, rax, r14);
1620 
1621     __ remove_activation(vtos, rdx,
1622                          /* throw_monitor_exception */ false,
1623                          /* install_monitor_exception */ false,
1624                          /* notify_jvmdi */ false);
1625 
1626     // Inform deoptimization that it is responsible for restoring
1627     // these arguments
1628     __ movl(Address(r15_thread, JavaThread::popframe_condition_offset()),
1629             JavaThread::popframe_force_deopt_reexecution_bit);
1630 
1631     // Continue in deoptimization handler
1632     __ jmp(rdx);
1633 
1634     __ bind(caller_not_deoptimized);
1635   }
1636 
1637   __ remove_activation(vtos, rdx, /* rdx result (retaddr) is not used */
1638                        /* throw_monitor_exception */ false,
1639                        /* install_monitor_exception */ false,
1640                        /* notify_jvmdi */ false);
1641 
1642   // Finish with popframe handling
1643   // A previous I2C followed by a deoptimization might have moved the
1644   // outgoing arguments further up the stack. PopFrame expects the
1645   // mutations to those outgoing arguments to be preserved and other
1646   // constraints basically require this frame to look exactly as
1647   // though it had previously invoked an interpreted activation with
1648   // no space between the top of the expression stack (current
1649   // last_sp) and the top of stack. Rather than force deopt to
1650   // maintain this kind of invariant all the time we call a small
1651   // fixup routine to move the mutated arguments onto the top of our
1652   // expression stack if necessary.
1653   __ mov(c_rarg1, rsp);
1654   __ movptr(c_rarg2, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1655   // PC must point into interpreter here
1656   __ set_last_Java_frame(noreg, rbp, __ pc());
1657   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), r15_thread, c_rarg1, c_rarg2);
1658   __ reset_last_Java_frame(true, true);
1659   // Restore the last_sp and null it out
1660   __ movptr(rsp, Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize));
1661   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD);
1662 
1663   __ restore_bcp();  // XXX do we need this?
1664   __ restore_locals(); // XXX do we need this?
1665   // The method data pointer was incremented already during
1666   // call profiling. We have to restore the mdp for the current bcp.
1667   if (ProfileInterpreter) {
1668     __ set_method_data_pointer_for_bcp();
1669   }
1670 
1671   // Clear the popframe condition flag
1672   __ movl(Address(r15_thread, JavaThread::popframe_condition_offset()),
1673           JavaThread::popframe_inactive);
1674 
1675   __ dispatch_next(vtos);
1676   // end of PopFrame support
1677 
1678   Interpreter::_remove_activation_entry = __ pc();
1679 
1680   // preserve exception over this code sequence
1681   __ pop_ptr(rax);
1682   __ movptr(Address(r15_thread, JavaThread::vm_result_offset()), rax);
1683   // remove the activation (without doing throws on illegalMonitorExceptions)
1684   __ remove_activation(vtos, rdx, false, true, false);
1685   // restore exception
1686   __ movptr(rax, Address(r15_thread, JavaThread::vm_result_offset()));
1687   __ movptr(Address(r15_thread, JavaThread::vm_result_offset()), (int32_t)NULL_WORD);
1688   __ verify_oop(rax);
1689 
1690   // In between activations - previous activation type unknown yet
1691   // compute continuation point - the continuation point expects the
1692   // following registers set up:
1693   //
1694   // rax: exception
1695   // rdx: return address/pc that threw exception
1696   // rsp: expression stack of caller
1697   // rbp: ebp of caller
1698   __ push(rax);                                  // save exception
1699   __ push(rdx);                                  // save return address
1700   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1701                           SharedRuntime::exception_handler_for_return_address),
1702                         rdx);
1703   __ mov(rbx, rax);                              // save exception handler
1704   __ pop(rdx);                                   // restore return address
1705   __ pop(rax);                                   // restore exception
1706   // Note that an "issuing PC" is actually the next PC after the call
1707   __ jmp(rbx);                                   // jump to exception
1708                                                  // handler of caller
1709 }
1710 
1711 
1712 //
1713 // JVMTI ForceEarlyReturn support
1714 //
1715 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1716   address entry = __ pc();
1717 
1718   __ restore_bcp();
1719   __ restore_locals();
1720   __ empty_expression_stack();
1721   __ load_earlyret_value(state);
1722 
1723   __ movptr(rdx, Address(r15_thread, JavaThread::jvmti_thread_state_offset()));
1724   Address cond_addr(rdx, JvmtiThreadState::earlyret_state_offset());
1725 
1726   // Clear the earlyret state
1727   __ movl(cond_addr, JvmtiThreadState::earlyret_inactive);
1728 
1729   __ remove_activation(state, rsi,
1730                        false, /* throw_monitor_exception */
1731                        false, /* install_monitor_exception */
1732                        true); /* notify_jvmdi */
1733   __ jmp(rsi);
1734 
1735   return entry;
1736 } // end of ForceEarlyReturn support
1737 
1738 
1739 //-----------------------------------------------------------------------------
1740 // Helper for vtos entry point generation
1741 
1742 void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t,
1743                                                          address& bep,
1744                                                          address& cep,
1745                                                          address& sep,
1746                                                          address& aep,
1747                                                          address& iep,
1748                                                          address& lep,
1749                                                          address& fep,
1750                                                          address& dep,
1751                                                          address& vep) {
1752   assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1753   Label L;
1754   aep = __ pc();  __ push_ptr();  __ jmp(L);
1755   fep = __ pc();  __ push_f();    __ jmp(L);
1756   dep = __ pc();  __ push_d();    __ jmp(L);
1757   lep = __ pc();  __ push_l();    __ jmp(L);
1758   bep = cep = sep =
1759   iep = __ pc();  __ push_i();
1760   vep = __ pc();
1761   __ bind(L);
1762   generate_and_dispatch(t);
1763 }
1764 
1765 
1766 //-----------------------------------------------------------------------------
1767 // Generation of individual instructions
1768 
1769 // helpers for generate_and_dispatch
1770 
1771 
1772 InterpreterGenerator::InterpreterGenerator(StubQueue* code)
1773   : TemplateInterpreterGenerator(code) {
1774    generate_all(); // down here so it can be "virtual"
1775 }
1776 
1777 //-----------------------------------------------------------------------------
1778 
1779 // Non-product code
1780 #ifndef PRODUCT
1781 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1782   address entry = __ pc();
1783 
1784   __ push(state);
1785   __ push(c_rarg0);
1786   __ push(c_rarg1);
1787   __ push(c_rarg2);
1788   __ push(c_rarg3);
1789   __ mov(c_rarg2, rax);  // Pass itos
1790 #ifdef _WIN64
1791   __ movflt(xmm3, xmm0); // Pass ftos
1792 #endif
1793   __ call_VM(noreg,
1794              CAST_FROM_FN_PTR(address, SharedRuntime::trace_bytecode),
1795              c_rarg1, c_rarg2, c_rarg3);
1796   __ pop(c_rarg3);
1797   __ pop(c_rarg2);
1798   __ pop(c_rarg1);
1799   __ pop(c_rarg0);
1800   __ pop(state);
1801   __ ret(0);                                   // return from result handler
1802 
1803   return entry;
1804 }
1805 
1806 void TemplateInterpreterGenerator::count_bytecode() {
1807   __ incrementl(ExternalAddress((address) &BytecodeCounter::_counter_value));
1808 }
1809 
1810 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
1811   __ incrementl(ExternalAddress((address) &BytecodeHistogram::_counters[t->bytecode()]));
1812 }
1813 
1814 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
1815   __ mov32(rbx, ExternalAddress((address) &BytecodePairHistogram::_index));
1816   __ shrl(rbx, BytecodePairHistogram::log2_number_of_codes);
1817   __ orl(rbx,
1818          ((int) t->bytecode()) <<
1819          BytecodePairHistogram::log2_number_of_codes);
1820   __ mov32(ExternalAddress((address) &BytecodePairHistogram::_index), rbx);
1821   __ lea(rscratch1, ExternalAddress((address) BytecodePairHistogram::_counters));
1822   __ incrementl(Address(rscratch1, rbx, Address::times_4));
1823 }
1824 
1825 
1826 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
1827   // Call a little run-time stub to avoid blow-up for each bytecode.
1828   // The run-time runtime saves the right registers, depending on
1829   // the tosca in-state for the given template.
1830 
1831   assert(Interpreter::trace_code(t->tos_in()) != NULL,
1832          "entry must have been generated");
1833   __ mov(r12, rsp); // remember sp
1834   __ andptr(rsp, -16); // align stack as required by ABI
1835   __ call(RuntimeAddress(Interpreter::trace_code(t->tos_in())));
1836   __ mov(rsp, r12); // restore sp
1837   __ reinit_heapbase();
1838 }
1839 
1840 
1841 void TemplateInterpreterGenerator::stop_interpreter_at() {
1842   Label L;
1843   __ cmp32(ExternalAddress((address) &BytecodeCounter::_counter_value),
1844            StopInterpreterAt);
1845   __ jcc(Assembler::notEqual, L);
1846   __ int3();
1847   __ bind(L);
1848 }
1849 #endif // !PRODUCT
1850 #endif // ! CC_INTERP