1 /*
   2  * Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.hpp"
  28 #include "interpreter/bytecodeHistogram.hpp"
  29 #include "interpreter/interpreter.hpp"
  30 #include "interpreter/interpreterRuntime.hpp"
  31 #include "interpreter/interp_masm.hpp"
  32 #include "interpreter/templateInterpreterGenerator.hpp"
  33 #include "interpreter/templateTable.hpp"
  34 #include "interpreter/bytecodeTracer.hpp"
  35 #include "memory/resourceArea.hpp"
  36 #include "oops/arrayOop.hpp"
  37 #include "oops/methodData.hpp"
  38 #include "oops/method.hpp"
  39 #include "oops/oop.inline.hpp"
  40 #include "prims/jvmtiExport.hpp"
  41 #include "prims/jvmtiThreadState.hpp"
  42 #include "runtime/arguments.hpp"
  43 #include "runtime/deoptimization.hpp"
  44 #include "runtime/frame.inline.hpp"
  45 #include "runtime/sharedRuntime.hpp"
  46 #include "runtime/stubRoutines.hpp"
  47 #include "runtime/synchronizer.hpp"
  48 #include "runtime/timer.hpp"
  49 #include "runtime/vframeArray.hpp"
  50 #include "utilities/debug.hpp"
  51 #include <sys/types.h>
  52 
  53 #ifndef PRODUCT
  54 #include "oops/method.hpp"
  55 #endif // !PRODUCT
  56 
  57 #ifdef BUILTIN_SIM
  58 #include "../../../../../../simulator/simulator.hpp"
  59 #endif
  60 
  61 // Size of interpreter code.  Increase if too small.  Interpreter will
  62 // fail with a guarantee ("not enough space for interpreter generation");
  63 // if too small.
  64 // Run with +PrintInterpreter to get the VM to print out the size.
  65 // Max size with JVMTI
  66 int TemplateInterpreter::InterpreterCodeSize = 200 * 1024;
  67 
  68 #define __ _masm->
  69 
  70 //-----------------------------------------------------------------------------
  71 
  72 extern "C" void entry(CodeBuffer*);
  73 
  74 //-----------------------------------------------------------------------------
  75 
  76 address TemplateInterpreterGenerator::generate_slow_signature_handler() {
  77   address entry = __ pc();
  78 
  79   __ andr(esp, esp, -16);
  80   __ mov(c_rarg3, esp);
  81   // rmethod
  82   // rlocals
  83   // c_rarg3: first stack arg - wordSize
  84 
  85   // adjust sp
  86   __ sub(sp, c_rarg3, 18 * wordSize);
  87   __ str(lr, Address(__ pre(sp, -2 * wordSize)));
  88   __ call_VM(noreg,
  89              CAST_FROM_FN_PTR(address,
  90                               InterpreterRuntime::slow_signature_handler),
  91              rmethod, rlocals, c_rarg3);
  92 
  93   // r0: result handler
  94 
  95   // Stack layout:
  96   // rsp: return address           <- sp
  97   //      1 garbage
  98   //      8 integer args (if static first is unused)
  99   //      1 float/double identifiers
 100   //      8 double args
 101   //        stack args              <- esp
 102   //        garbage
 103   //        expression stack bottom
 104   //        bcp (NULL)
 105   //        ...
 106 
 107   // Restore LR
 108   __ ldr(lr, Address(__ post(sp, 2 * wordSize)));
 109 
 110   // Do FP first so we can use c_rarg3 as temp
 111   __ ldrw(c_rarg3, Address(sp, 9 * wordSize)); // float/double identifiers
 112 
 113   for (int i = 0; i < Argument::n_float_register_parameters_c; i++) {
 114     const FloatRegister r = as_FloatRegister(i);
 115 
 116     Label d, done;
 117 
 118     __ tbnz(c_rarg3, i, d);
 119     __ ldrs(r, Address(sp, (10 + i) * wordSize));
 120     __ b(done);
 121     __ bind(d);
 122     __ ldrd(r, Address(sp, (10 + i) * wordSize));
 123     __ bind(done);
 124   }
 125 
 126   // c_rarg0 contains the result from the call of
 127   // InterpreterRuntime::slow_signature_handler so we don't touch it
 128   // here.  It will be loaded with the JNIEnv* later.
 129   __ ldr(c_rarg1, Address(sp, 1 * wordSize));
 130   for (int i = c_rarg2->encoding(); i <= c_rarg7->encoding(); i += 2) {
 131     Register rm = as_Register(i), rn = as_Register(i+1);
 132     __ ldp(rm, rn, Address(sp, i * wordSize));
 133   }
 134 
 135   __ add(sp, sp, 18 * wordSize);
 136   __ ret(lr);
 137 
 138   return entry;
 139 }
 140 
 141 
 142 //
 143 // Various method entries
 144 //
 145 
 146 address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) {
 147   // rmethod: Method*
 148   // r13: sender sp
 149   // esp: args
 150 
 151   if (!InlineIntrinsics) return NULL; // Generate a vanilla entry
 152 
 153   // These don't need a safepoint check because they aren't virtually
 154   // callable. We won't enter these intrinsics from compiled code.
 155   // If in the future we added an intrinsic which was virtually callable
 156   // we'd have to worry about how to safepoint so that this code is used.
 157 
 158   // mathematical functions inlined by compiler
 159   // (interpreter must provide identical implementation
 160   // in order to avoid monotonicity bugs when switching
 161   // from interpreter to compiler in the middle of some
 162   // computation)
 163   //
 164   // stack:
 165   //        [ arg ] <-- esp
 166   //        [ arg ]
 167   // retaddr in lr
 168 
 169   address entry_point = NULL;
 170   Register continuation = lr;
 171   switch (kind) {
 172   case Interpreter::java_lang_math_abs:
 173     entry_point = __ pc();
 174     __ ldrd(v0, Address(esp));
 175     __ fabsd(v0, v0);
 176     __ mov(sp, r13); // Restore caller's SP
 177     break;
 178   case Interpreter::java_lang_math_sqrt:
 179     entry_point = __ pc();
 180     __ ldrd(v0, Address(esp));
 181     __ fsqrtd(v0, v0);
 182     __ mov(sp, r13);
 183     break;
 184   case Interpreter::java_lang_math_sin :
 185   case Interpreter::java_lang_math_cos :
 186   case Interpreter::java_lang_math_tan :
 187   case Interpreter::java_lang_math_log :
 188   case Interpreter::java_lang_math_log10 :
 189   case Interpreter::java_lang_math_exp :
 190     entry_point = __ pc();
 191     __ ldrd(v0, Address(esp));
 192     __ mov(sp, r13);
 193     __ mov(r19, lr);
 194     continuation = r19;  // The first callee-saved register
 195     generate_transcendental_entry(kind, 1);
 196     break;
 197   case Interpreter::java_lang_math_pow :
 198     entry_point = __ pc();
 199     __ mov(r19, lr);
 200     continuation = r19;
 201     __ ldrd(v0, Address(esp, 2 * Interpreter::stackElementSize));
 202     __ ldrd(v1, Address(esp));
 203     __ mov(sp, r13);
 204     generate_transcendental_entry(kind, 2);
 205     break;
 206   default:
 207     ;
 208   }
 209   if (entry_point) {
 210     __ br(continuation);
 211   }
 212 
 213   return entry_point;
 214 }
 215 
 216   // double trigonometrics and transcendentals
 217   // static jdouble dsin(jdouble x);
 218   // static jdouble dcos(jdouble x);
 219   // static jdouble dtan(jdouble x);
 220   // static jdouble dlog(jdouble x);
 221   // static jdouble dlog10(jdouble x);
 222   // static jdouble dexp(jdouble x);
 223   // static jdouble dpow(jdouble x, jdouble y);
 224 
 225 void TemplateInterpreterGenerator::generate_transcendental_entry(AbstractInterpreter::MethodKind kind, int fpargs) {
 226   address fn;
 227   switch (kind) {
 228   case Interpreter::java_lang_math_sin :
 229     fn = CAST_FROM_FN_PTR(address, SharedRuntime::dsin);
 230     break;
 231   case Interpreter::java_lang_math_cos :
 232     fn = CAST_FROM_FN_PTR(address, SharedRuntime::dcos);
 233     break;
 234   case Interpreter::java_lang_math_tan :
 235     fn = CAST_FROM_FN_PTR(address, SharedRuntime::dtan);
 236     break;
 237   case Interpreter::java_lang_math_log :
 238     fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog);
 239     break;
 240   case Interpreter::java_lang_math_log10 :
 241     fn = CAST_FROM_FN_PTR(address, SharedRuntime::dlog10);
 242     break;
 243   case Interpreter::java_lang_math_exp :
 244     fn = CAST_FROM_FN_PTR(address, SharedRuntime::dexp);
 245     break;
 246   case Interpreter::java_lang_math_pow :
 247     fpargs = 2;
 248     fn = CAST_FROM_FN_PTR(address, SharedRuntime::dpow);
 249     break;
 250   default:
 251     ShouldNotReachHere();
 252     fn = NULL;  // unreachable
 253   }
 254   const int gpargs = 0, rtype = 3;
 255   __ mov(rscratch1, fn);
 256   __ blrt(rscratch1, gpargs, fpargs, rtype);
 257 }
 258 
 259 // Abstract method entry
 260 // Attempt to execute abstract method. Throw exception
 261 address TemplateInterpreterGenerator::generate_abstract_entry(void) {
 262   // rmethod: Method*
 263   // r13: sender SP
 264 
 265   address entry_point = __ pc();
 266 
 267   // abstract method entry
 268 
 269   //  pop return address, reset last_sp to NULL
 270   __ empty_expression_stack();
 271   __ restore_bcp();      // bcp must be correct for exception handler   (was destroyed)
 272   __ restore_locals();   // make sure locals pointer is correct as well (was destroyed)
 273 
 274   // throw exception
 275   __ call_VM(noreg, CAST_FROM_FN_PTR(address,
 276                              InterpreterRuntime::throw_AbstractMethodError));
 277   // the call_VM checks for exception, so we should never return here.
 278   __ should_not_reach_here();
 279 
 280   return entry_point;
 281 }
 282 
 283 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
 284   address entry = __ pc();
 285 
 286 #ifdef ASSERT
 287   {
 288     Label L;
 289     __ ldr(rscratch1, Address(rfp,
 290                        frame::interpreter_frame_monitor_block_top_offset *
 291                        wordSize));
 292     __ mov(rscratch2, sp);
 293     __ cmp(rscratch1, rscratch2); // maximal rsp for current rfp (stack
 294                            // grows negative)
 295     __ br(Assembler::HS, L); // check if frame is complete
 296     __ stop ("interpreter frame not set up");
 297     __ bind(L);
 298   }
 299 #endif // ASSERT
 300   // Restore bcp under the assumption that the current frame is still
 301   // interpreted
 302   __ restore_bcp();
 303 
 304   // expression stack must be empty before entering the VM if an
 305   // exception happened
 306   __ empty_expression_stack();
 307   // throw exception
 308   __ call_VM(noreg,
 309              CAST_FROM_FN_PTR(address,
 310                               InterpreterRuntime::throw_StackOverflowError));
 311   return entry;
 312 }
 313 
 314 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler(
 315         const char* name) {
 316   address entry = __ pc();
 317   // expression stack must be empty before entering the VM if an
 318   // exception happened
 319   __ empty_expression_stack();
 320   // setup parameters
 321   // ??? convention: expect aberrant index in register r1
 322   __ movw(c_rarg2, r1);
 323   __ mov(c_rarg1, (address)name);
 324   __ call_VM(noreg,
 325              CAST_FROM_FN_PTR(address,
 326                               InterpreterRuntime::
 327                               throw_ArrayIndexOutOfBoundsException),
 328              c_rarg1, c_rarg2);
 329   return entry;
 330 }
 331 
 332 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
 333   address entry = __ pc();
 334 
 335   // object is at TOS
 336   __ pop(c_rarg1);
 337 
 338   // expression stack must be empty before entering the VM if an
 339   // exception happened
 340   __ empty_expression_stack();
 341 
 342   __ call_VM(noreg,
 343              CAST_FROM_FN_PTR(address,
 344                               InterpreterRuntime::
 345                               throw_ClassCastException),
 346              c_rarg1);
 347   return entry;
 348 }
 349 
 350 address TemplateInterpreterGenerator::generate_exception_handler_common(
 351         const char* name, const char* message, bool pass_oop) {
 352   assert(!pass_oop || message == NULL, "either oop or message but not both");
 353   address entry = __ pc();
 354   if (pass_oop) {
 355     // object is at TOS
 356     __ pop(c_rarg2);
 357   }
 358   // expression stack must be empty before entering the VM if an
 359   // exception happened
 360   __ empty_expression_stack();
 361   // setup parameters
 362   __ lea(c_rarg1, Address((address)name));
 363   if (pass_oop) {
 364     __ call_VM(r0, CAST_FROM_FN_PTR(address,
 365                                     InterpreterRuntime::
 366                                     create_klass_exception),
 367                c_rarg1, c_rarg2);
 368   } else {
 369     // kind of lame ExternalAddress can't take NULL because
 370     // external_word_Relocation will assert.
 371     if (message != NULL) {
 372       __ lea(c_rarg2, Address((address)message));
 373     } else {
 374       __ mov(c_rarg2, NULL_WORD);
 375     }
 376     __ call_VM(r0,
 377                CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception),
 378                c_rarg1, c_rarg2);
 379   }
 380   // throw exception
 381   __ b(address(Interpreter::throw_exception_entry()));
 382   return entry;
 383 }
 384 
 385 address TemplateInterpreterGenerator::generate_continuation_for(TosState state) {
 386   address entry = __ pc();
 387   // NULL last_sp until next java call
 388   __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
 389   __ dispatch_next(state);
 390   return entry;
 391 }
 392 
 393 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {
 394   address entry = __ pc();
 395 
 396   // Restore stack bottom in case i2c adjusted stack
 397   __ ldr(esp, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
 398   // and NULL it as marker that esp is now tos until next java call
 399   __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
 400   __ restore_bcp();
 401   __ restore_locals();
 402   __ restore_constant_pool_cache();
 403   __ get_method(rmethod);
 404 
 405   // Pop N words from the stack
 406   __ get_cache_and_index_at_bcp(r1, r2, 1, index_size);
 407   __ ldr(r1, Address(r1, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));
 408   __ andr(r1, r1, ConstantPoolCacheEntry::parameter_size_mask);
 409 
 410   __ add(esp, esp, r1, Assembler::LSL, 3);
 411 
 412   // Restore machine SP
 413   __ ldr(rscratch1, Address(rmethod, Method::const_offset()));
 414   __ ldrh(rscratch1, Address(rscratch1, ConstMethod::max_stack_offset()));
 415   __ add(rscratch1, rscratch1, frame::interpreter_frame_monitor_size() + 2);
 416   __ ldr(rscratch2,
 417          Address(rfp, frame::interpreter_frame_initial_sp_offset * wordSize));
 418   __ sub(rscratch1, rscratch2, rscratch1, ext::uxtw, 3);
 419   __ andr(sp, rscratch1, -16);
 420 
 421 #ifndef PRODUCT
 422   // tell the simulator that the method has been reentered
 423   if (NotifySimulator) {
 424     __ notify(Assembler::method_reentry);
 425   }
 426 #endif
 427   __ get_dispatch();
 428   __ dispatch_next(state, step);
 429 
 430   return entry;
 431 }
 432 
 433 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state,
 434                                                                int step) {
 435   address entry = __ pc();
 436   __ restore_bcp();
 437   __ restore_locals();
 438   __ restore_constant_pool_cache();
 439   __ get_method(rmethod);
 440 
 441 #if INCLUDE_JVMCI
 442   // Check if we need to take lock at entry of synchronized method.
 443   if (UseJVMCICompiler) {
 444     Label L;
 445     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 446     __ cbz(rscratch1, L);
 447     // Clear flag.
 448     __ strb(zr, Address(rthread, JavaThread::pending_monitorenter_offset()));
 449     // Take lock.
 450     lock_method();
 451     __ bind(L);
 452   }
 453 #endif
 454   // handle exceptions
 455   {
 456     Label L;
 457     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 458     __ cbz(rscratch1, L);
 459     __ call_VM(noreg,
 460                CAST_FROM_FN_PTR(address,
 461                                 InterpreterRuntime::throw_pending_exception));
 462     __ should_not_reach_here();
 463     __ bind(L);
 464   }
 465 
 466   __ get_dispatch();
 467 
 468   // Calculate stack limit
 469   __ ldr(rscratch1, Address(rmethod, Method::const_offset()));
 470   __ ldrh(rscratch1, Address(rscratch1, ConstMethod::max_stack_offset()));
 471   __ add(rscratch1, rscratch1, frame::interpreter_frame_monitor_size() + 2);
 472   __ ldr(rscratch2,
 473          Address(rfp, frame::interpreter_frame_initial_sp_offset * wordSize));
 474   __ sub(rscratch1, rscratch2, rscratch1, ext::uxtx, 3);
 475   __ andr(sp, rscratch1, -16);
 476 
 477   // Restore expression stack pointer
 478   __ ldr(esp, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
 479   // NULL last_sp until next java call
 480   __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
 481 
 482   __ dispatch_next(state, step);
 483   return entry;
 484 }
 485 
 486 address TemplateInterpreterGenerator::generate_result_handler_for(
 487         BasicType type) {
 488     address entry = __ pc();
 489   switch (type) {
 490   case T_BOOLEAN: __ uxtb(r0, r0);        break;
 491   case T_CHAR   : __ uxth(r0, r0);       break;
 492   case T_BYTE   : __ sxtb(r0, r0);        break;
 493   case T_SHORT  : __ sxth(r0, r0);        break;
 494   case T_INT    : __ uxtw(r0, r0);        break;  // FIXME: We almost certainly don't need this
 495   case T_LONG   : /* nothing to do */        break;
 496   case T_VOID   : /* nothing to do */        break;
 497   case T_FLOAT  : /* nothing to do */        break;
 498   case T_DOUBLE : /* nothing to do */        break;
 499   case T_OBJECT :
 500     // retrieve result from frame
 501     __ ldr(r0, Address(rfp, frame::interpreter_frame_oop_temp_offset*wordSize));
 502     // and verify it
 503     __ verify_oop(r0);
 504     break;
 505   default       : ShouldNotReachHere();
 506   }
 507   __ ret(lr);                                  // return from result handler
 508   return entry;
 509 }
 510 
 511 address TemplateInterpreterGenerator::generate_safept_entry_for(
 512         TosState state,
 513         address runtime_entry) {
 514   address entry = __ pc();
 515   __ push(state);
 516   __ call_VM(noreg, runtime_entry);
 517   __ membar(Assembler::AnyAny);
 518   __ dispatch_via(vtos, Interpreter::_normal_table.table_for(vtos));
 519   return entry;
 520 }
 521 
 522 // Helpers for commoning out cases in the various type of method entries.
 523 //
 524 
 525 
 526 // increment invocation count & check for overflow
 527 //
 528 // Note: checking for negative value instead of overflow
 529 //       so we have a 'sticky' overflow test
 530 //
 531 // rmethod: method
 532 //
 533 void TemplateInterpreterGenerator::generate_counter_incr(
 534         Label* overflow,
 535         Label* profile_method,
 536         Label* profile_method_continue) {
 537   Label done;
 538   // Note: In tiered we increment either counters in Method* or in MDO depending if we're profiling or not.
 539   if (TieredCompilation) {
 540     int increment = InvocationCounter::count_increment;
 541     Label no_mdo;
 542     if (ProfileInterpreter) {
 543       // Are we profiling?
 544       __ ldr(r0, Address(rmethod, Method::method_data_offset()));
 545       __ cbz(r0, no_mdo);
 546       // Increment counter in the MDO
 547       const Address mdo_invocation_counter(r0, in_bytes(MethodData::invocation_counter_offset()) +
 548                                                 in_bytes(InvocationCounter::counter_offset()));
 549       const Address mask(r0, in_bytes(MethodData::invoke_mask_offset()));
 550       __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, rscratch1, rscratch2, false, Assembler::EQ, overflow);
 551       __ b(done);
 552     }
 553     __ bind(no_mdo);
 554     // Increment counter in MethodCounters
 555     const Address invocation_counter(rscratch2,
 556                   MethodCounters::invocation_counter_offset() +
 557                   InvocationCounter::counter_offset());
 558     __ get_method_counters(rmethod, rscratch2, done);
 559     const Address mask(rscratch2, in_bytes(MethodCounters::invoke_mask_offset()));
 560     __ increment_mask_and_jump(invocation_counter, increment, mask, rscratch1, r1, false, Assembler::EQ, overflow);
 561     __ bind(done);
 562   } else { // not TieredCompilation
 563     const Address backedge_counter(rscratch2,
 564                   MethodCounters::backedge_counter_offset() +
 565                   InvocationCounter::counter_offset());
 566     const Address invocation_counter(rscratch2,
 567                   MethodCounters::invocation_counter_offset() +
 568                   InvocationCounter::counter_offset());
 569 
 570     __ get_method_counters(rmethod, rscratch2, done);
 571 
 572     if (ProfileInterpreter) { // %%% Merge this into MethodData*
 573       __ ldrw(r1, Address(rscratch2, MethodCounters::interpreter_invocation_counter_offset()));
 574       __ addw(r1, r1, 1);
 575       __ strw(r1, Address(rscratch2, MethodCounters::interpreter_invocation_counter_offset()));
 576     }
 577     // Update standard invocation counters
 578     __ ldrw(r1, invocation_counter);
 579     __ ldrw(r0, backedge_counter);
 580 
 581     __ addw(r1, r1, InvocationCounter::count_increment);
 582     __ andw(r0, r0, InvocationCounter::count_mask_value);
 583 
 584     __ strw(r1, invocation_counter);
 585     __ addw(r0, r0, r1);                // add both counters
 586 
 587     // profile_method is non-null only for interpreted method so
 588     // profile_method != NULL == !native_call
 589 
 590     if (ProfileInterpreter && profile_method != NULL) {
 591       // Test to see if we should create a method data oop
 592       __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
 593       __ ldrw(rscratch2, Address(rscratch2, in_bytes(MethodCounters::interpreter_profile_limit_offset())));
 594       __ cmpw(r0, rscratch2);
 595       __ br(Assembler::LT, *profile_method_continue);
 596 
 597       // if no method data exists, go to profile_method
 598       __ test_method_data_pointer(rscratch2, *profile_method);
 599     }
 600 
 601     {
 602       __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
 603       __ ldrw(rscratch2, Address(rscratch2, in_bytes(MethodCounters::interpreter_invocation_limit_offset())));
 604       __ cmpw(r0, rscratch2);
 605       __ br(Assembler::HS, *overflow);
 606     }
 607     __ bind(done);
 608   }
 609 }
 610 
 611 void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) {
 612 
 613   // Asm interpreter on entry
 614   // On return (i.e. jump to entry_point) [ back to invocation of interpreter ]
 615   // Everything as it was on entry
 616 
 617   // InterpreterRuntime::frequency_counter_overflow takes two
 618   // arguments, the first (thread) is passed by call_VM, the second
 619   // indicates if the counter overflow occurs at a backwards branch
 620   // (NULL bcp).  We pass zero for it.  The call returns the address
 621   // of the verified entry point for the method or NULL if the
 622   // compilation did not complete (either went background or bailed
 623   // out).
 624   __ mov(c_rarg1, 0);
 625   __ call_VM(noreg,
 626              CAST_FROM_FN_PTR(address,
 627                               InterpreterRuntime::frequency_counter_overflow),
 628              c_rarg1);
 629 
 630   __ b(do_continue);
 631 }
 632 
 633 // See if we've got enough room on the stack for locals plus overhead.
 634 // The expression stack grows down incrementally, so the normal guard
 635 // page mechanism will work for that.
 636 //
 637 // NOTE: Since the additional locals are also always pushed (wasn't
 638 // obvious in generate_method_entry) so the guard should work for them
 639 // too.
 640 //
 641 // Args:
 642 //      r3: number of additional locals this frame needs (what we must check)
 643 //      rmethod: Method*
 644 //
 645 // Kills:
 646 //      r0
 647 void TemplateInterpreterGenerator::generate_stack_overflow_check(void) {
 648 
 649   // monitor entry size: see picture of stack set
 650   // (generate_method_entry) and frame_amd64.hpp
 651   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 652 
 653   // total overhead size: entry_size + (saved rbp through expr stack
 654   // bottom).  be sure to change this if you add/subtract anything
 655   // to/from the overhead area
 656   const int overhead_size =
 657     -(frame::interpreter_frame_initial_sp_offset * wordSize) + entry_size;
 658 
 659   const int page_size = os::vm_page_size();
 660 
 661   Label after_frame_check;
 662 
 663   // see if the frame is greater than one page in size. If so,
 664   // then we need to verify there is enough stack space remaining
 665   // for the additional locals.
 666   //
 667   // Note that we use SUBS rather than CMP here because the immediate
 668   // field of this instruction may overflow.  SUBS can cope with this
 669   // because it is a macro that will expand to some number of MOV
 670   // instructions and a register operation.
 671   __ subs(rscratch1, r3, (page_size - overhead_size) / Interpreter::stackElementSize);
 672   __ br(Assembler::LS, after_frame_check);
 673 
 674   // compute rsp as if this were going to be the last frame on
 675   // the stack before the red zone
 676 
 677   const Address stack_base(rthread, Thread::stack_base_offset());
 678   const Address stack_size(rthread, Thread::stack_size_offset());
 679 
 680   // locals + overhead, in bytes
 681   __ mov(r0, overhead_size);
 682   __ add(r0, r0, r3, Assembler::LSL, Interpreter::logStackElementSize);  // 2 slots per parameter.
 683 
 684   __ ldr(rscratch1, stack_base);
 685   __ ldr(rscratch2, stack_size);
 686 
 687 #ifdef ASSERT
 688   Label stack_base_okay, stack_size_okay;
 689   // verify that thread stack base is non-zero
 690   __ cbnz(rscratch1, stack_base_okay);
 691   __ stop("stack base is zero");
 692   __ bind(stack_base_okay);
 693   // verify that thread stack size is non-zero
 694   __ cbnz(rscratch2, stack_size_okay);
 695   __ stop("stack size is zero");
 696   __ bind(stack_size_okay);
 697 #endif
 698 
 699   // Add stack base to locals and subtract stack size
 700   __ sub(rscratch1, rscratch1, rscratch2); // Stack limit
 701   __ add(r0, r0, rscratch1);
 702 
 703   // Use the bigger size for banging.
 704   const int max_bang_size = MAX2(JavaThread::stack_shadow_zone_size(),
 705                                  JavaThread::stack_red_zone_size() + JavaThread::stack_yellow_zone_size());
 706 
 707   // add in the red and yellow zone sizes
 708   __ add(r0, r0, max_bang_size * 2);
 709 
 710   // check against the current stack bottom
 711   __ cmp(sp, r0);
 712   __ br(Assembler::HI, after_frame_check);
 713 
 714   // Remove the incoming args, peeling the machine SP back to where it
 715   // was in the caller.  This is not strictly necessary, but unless we
 716   // do so the stack frame may have a garbage FP; this ensures a
 717   // correct call stack that we can always unwind.  The ANDR should be
 718   // unnecessary because the sender SP in r13 is always aligned, but
 719   // it doesn't hurt.
 720   __ andr(sp, r13, -16);
 721 
 722   // Note: the restored frame is not necessarily interpreted.
 723   // Use the shared runtime version of the StackOverflowError.
 724   assert(StubRoutines::throw_StackOverflowError_entry() != NULL, "stub not yet generated");
 725   __ far_jump(RuntimeAddress(StubRoutines::throw_StackOverflowError_entry()));
 726 
 727   // all done with frame size check
 728   __ bind(after_frame_check);
 729 }
 730 
 731 // Allocate monitor and lock method (asm interpreter)
 732 //
 733 // Args:
 734 //      rmethod: Method*
 735 //      rlocals: locals
 736 //
 737 // Kills:
 738 //      r0
 739 //      c_rarg0, c_rarg1, c_rarg2, c_rarg3, ...(param regs)
 740 //      rscratch1, rscratch2 (scratch regs)
 741 void TemplateInterpreterGenerator::lock_method() {
 742   // synchronize method
 743   const Address access_flags(rmethod, Method::access_flags_offset());
 744   const Address monitor_block_top(
 745         rfp,
 746         frame::interpreter_frame_monitor_block_top_offset * wordSize);
 747   const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
 748 
 749 #ifdef ASSERT
 750   {
 751     Label L;
 752     __ ldrw(r0, access_flags);
 753     __ tst(r0, JVM_ACC_SYNCHRONIZED);
 754     __ br(Assembler::NE, L);
 755     __ stop("method doesn't need synchronization");
 756     __ bind(L);
 757   }
 758 #endif // ASSERT
 759 
 760   // get synchronization object
 761   {
 762     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 763     Label done;
 764     __ ldrw(r0, access_flags);
 765     __ tst(r0, JVM_ACC_STATIC);
 766     // get receiver (assume this is frequent case)
 767     __ ldr(r0, Address(rlocals, Interpreter::local_offset_in_bytes(0)));
 768     __ br(Assembler::EQ, done);
 769     __ ldr(r0, Address(rmethod, Method::const_offset()));
 770     __ ldr(r0, Address(r0, ConstMethod::constants_offset()));
 771     __ ldr(r0, Address(r0,
 772                            ConstantPool::pool_holder_offset_in_bytes()));
 773     __ ldr(r0, Address(r0, mirror_offset));
 774 
 775 #ifdef ASSERT
 776     {
 777       Label L;
 778       __ cbnz(r0, L);
 779       __ stop("synchronization object is NULL");
 780       __ bind(L);
 781     }
 782 #endif // ASSERT
 783 
 784     __ bind(done);
 785   }
 786 
 787   // add space for monitor & lock
 788   __ sub(sp, sp, entry_size); // add space for a monitor entry
 789   __ sub(esp, esp, entry_size);
 790   __ mov(rscratch1, esp);
 791   __ str(rscratch1, monitor_block_top);  // set new monitor block top
 792   // store object
 793   __ str(r0, Address(esp, BasicObjectLock::obj_offset_in_bytes()));
 794   __ mov(c_rarg1, esp); // object address
 795   __ lock_object(c_rarg1);
 796 }
 797 
 798 // Generate a fixed interpreter frame. This is identical setup for
 799 // interpreted methods and for native methods hence the shared code.
 800 //
 801 // Args:
 802 //      lr: return address
 803 //      rmethod: Method*
 804 //      rlocals: pointer to locals
 805 //      rcpool: cp cache
 806 //      stack_pointer: previous sp
 807 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
 808   // initialize fixed part of activation frame
 809   if (native_call) {
 810     __ sub(esp, sp, 12 *  wordSize);
 811     __ mov(rbcp, zr);
 812     __ stp(esp, zr, Address(__ pre(sp, -12 * wordSize)));
 813     // add 2 zero-initialized slots for native calls
 814     __ stp(zr, zr, Address(sp, 10 * wordSize));
 815   } else {
 816     __ sub(esp, sp, 10 *  wordSize);
 817     __ ldr(rscratch1, Address(rmethod, Method::const_offset()));      // get ConstMethod
 818     __ add(rbcp, rscratch1, in_bytes(ConstMethod::codes_offset())); // get codebase
 819     __ stp(esp, rbcp, Address(__ pre(sp, -10 * wordSize)));
 820   }
 821 
 822   if (ProfileInterpreter) {
 823     Label method_data_continue;
 824     __ ldr(rscratch1, Address(rmethod, Method::method_data_offset()));
 825     __ cbz(rscratch1, method_data_continue);
 826     __ lea(rscratch1, Address(rscratch1, in_bytes(MethodData::data_offset())));
 827     __ bind(method_data_continue);
 828     __ stp(rscratch1, rmethod, Address(sp, 4 * wordSize));  // save Method* and mdp (method data pointer)
 829   } else {
 830     __ stp(zr, rmethod, Address(sp, 4 * wordSize));        // save Method* (no mdp)
 831   }
 832 
 833   __ ldr(rcpool, Address(rmethod, Method::const_offset()));
 834   __ ldr(rcpool, Address(rcpool, ConstMethod::constants_offset()));
 835   __ ldr(rcpool, Address(rcpool, ConstantPool::cache_offset_in_bytes()));
 836   __ stp(rlocals, rcpool, Address(sp, 2 * wordSize));
 837 
 838   __ stp(rfp, lr, Address(sp, 8 * wordSize));
 839   __ lea(rfp, Address(sp, 8 * wordSize));
 840 
 841   // set sender sp
 842   // leave last_sp as null
 843   __ stp(zr, r13, Address(sp, 6 * wordSize));
 844 
 845   // Move SP out of the way
 846   if (! native_call) {
 847     __ ldr(rscratch1, Address(rmethod, Method::const_offset()));
 848     __ ldrh(rscratch1, Address(rscratch1, ConstMethod::max_stack_offset()));
 849     __ add(rscratch1, rscratch1, frame::interpreter_frame_monitor_size() + 2);
 850     __ sub(rscratch1, sp, rscratch1, ext::uxtw, 3);
 851     __ andr(sp, rscratch1, -16);
 852   }
 853 }
 854 
 855 // End of helpers
 856 
 857 // Various method entries
 858 //------------------------------------------------------------------------------------------------------------------------
 859 //
 860 //
 861 
 862 // Method entry for java.lang.ref.Reference.get.
 863 address TemplateInterpreterGenerator::generate_Reference_get_entry(void) {
 864 #if INCLUDE_ALL_GCS
 865   // Code: _aload_0, _getfield, _areturn
 866   // parameter size = 1
 867   //
 868   // The code that gets generated by this routine is split into 2 parts:
 869   //    1. The "intrinsified" code for G1 (or any SATB based GC),
 870   //    2. The slow path - which is an expansion of the regular method entry.
 871   //
 872   // Notes:-
 873   // * In the G1 code we do not check whether we need to block for
 874   //   a safepoint. If G1 is enabled then we must execute the specialized
 875   //   code for Reference.get (except when the Reference object is null)
 876   //   so that we can log the value in the referent field with an SATB
 877   //   update buffer.
 878   //   If the code for the getfield template is modified so that the
 879   //   G1 pre-barrier code is executed when the current method is
 880   //   Reference.get() then going through the normal method entry
 881   //   will be fine.
 882   // * The G1 code can, however, check the receiver object (the instance
 883   //   of java.lang.Reference) and jump to the slow path if null. If the
 884   //   Reference object is null then we obviously cannot fetch the referent
 885   //   and so we don't need to call the G1 pre-barrier. Thus we can use the
 886   //   regular method entry code to generate the NPE.
 887   //
 888   // This code is based on generate_accessor_enty.
 889   //
 890   // rmethod: Method*
 891   // r13: senderSP must preserve for slow path, set SP to it on fast path
 892 
 893   address entry = __ pc();
 894 
 895   const int referent_offset = java_lang_ref_Reference::referent_offset;
 896   guarantee(referent_offset > 0, "referent offset not initialized");
 897 
 898   if (UseG1GC) {
 899     Label slow_path;
 900     const Register local_0 = c_rarg0;
 901     // Check if local 0 != NULL
 902     // If the receiver is null then it is OK to jump to the slow path.
 903     __ ldr(local_0, Address(esp, 0));
 904     __ cbz(local_0, slow_path);
 905 
 906 
 907     // Load the value of the referent field.
 908     const Address field_address(local_0, referent_offset);
 909     __ load_heap_oop(local_0, field_address);
 910 
 911     // Generate the G1 pre-barrier code to log the value of
 912     // the referent field in an SATB buffer.
 913     __ enter(); // g1_write may call runtime
 914     __ g1_write_barrier_pre(noreg /* obj */,
 915                             local_0 /* pre_val */,
 916                             rthread /* thread */,
 917                             rscratch2 /* tmp */,
 918                             true /* tosca_live */,
 919                             true /* expand_call */);
 920     __ leave();
 921     // areturn
 922     __ andr(sp, r13, -16);  // done with stack
 923     __ ret(lr);
 924 
 925     // generate a vanilla interpreter entry as the slow path
 926     __ bind(slow_path);
 927     __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals));
 928     return entry;
 929   }
 930 #endif // INCLUDE_ALL_GCS
 931 
 932   // If G1 is not enabled then attempt to go through the accessor entry point
 933   // Reference.get is an accessor
 934   return NULL;
 935 }
 936 
 937 /**
 938  * Method entry for static native methods:
 939  *   int java.util.zip.CRC32.update(int crc, int b)
 940  */
 941 address TemplateInterpreterGenerator::generate_CRC32_update_entry() {
 942   if (UseCRC32Intrinsics) {
 943     address entry = __ pc();
 944 
 945     // rmethod: Method*
 946     // r13: senderSP must preserved for slow path
 947     // esp: args
 948 
 949     Label slow_path;
 950     // If we need a safepoint check, generate full interpreter entry.
 951     ExternalAddress state(SafepointSynchronize::address_of_state());
 952     unsigned long offset;
 953     __ adrp(rscratch1, ExternalAddress(SafepointSynchronize::address_of_state()), offset);
 954     __ ldrw(rscratch1, Address(rscratch1, offset));
 955     assert(SafepointSynchronize::_not_synchronized == 0, "rewrite this code");
 956     __ cbnz(rscratch1, slow_path);
 957 
 958     // We don't generate local frame and don't align stack because
 959     // we call stub code and there is no safepoint on this path.
 960 
 961     // Load parameters
 962     const Register crc = c_rarg0;  // crc
 963     const Register val = c_rarg1;  // source java byte value
 964     const Register tbl = c_rarg2;  // scratch
 965 
 966     // Arguments are reversed on java expression stack
 967     __ ldrw(val, Address(esp, 0));              // byte value
 968     __ ldrw(crc, Address(esp, wordSize));       // Initial CRC
 969 
 970     __ adrp(tbl, ExternalAddress(StubRoutines::crc_table_addr()), offset);
 971     __ add(tbl, tbl, offset);
 972 
 973     __ ornw(crc, zr, crc); // ~crc
 974     __ update_byte_crc32(crc, val, tbl);
 975     __ ornw(crc, zr, crc); // ~crc
 976 
 977     // result in c_rarg0
 978 
 979     __ andr(sp, r13, -16);
 980     __ ret(lr);
 981 
 982     // generate a vanilla native entry as the slow path
 983     __ bind(slow_path);
 984     __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::native));
 985     return entry;
 986   }
 987   return NULL;
 988 }
 989 
 990 /**
 991  * Method entry for static native methods:
 992  *   int java.util.zip.CRC32.updateBytes(int crc, byte[] b, int off, int len)
 993  *   int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len)
 994  */
 995 address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) {
 996   if (UseCRC32Intrinsics) {
 997     address entry = __ pc();
 998 
 999     // rmethod,: Method*
1000     // r13: senderSP must preserved for slow path
1001 
1002     Label slow_path;
1003     // If we need a safepoint check, generate full interpreter entry.
1004     ExternalAddress state(SafepointSynchronize::address_of_state());
1005     unsigned long offset;
1006     __ adrp(rscratch1, ExternalAddress(SafepointSynchronize::address_of_state()), offset);
1007     __ ldrw(rscratch1, Address(rscratch1, offset));
1008     assert(SafepointSynchronize::_not_synchronized == 0, "rewrite this code");
1009     __ cbnz(rscratch1, slow_path);
1010 
1011     // We don't generate local frame and don't align stack because
1012     // we call stub code and there is no safepoint on this path.
1013 
1014     // Load parameters
1015     const Register crc = c_rarg0;  // crc
1016     const Register buf = c_rarg1;  // source java byte array address
1017     const Register len = c_rarg2;  // length
1018     const Register off = len;      // offset (never overlaps with 'len')
1019 
1020     // Arguments are reversed on java expression stack
1021     // Calculate address of start element
1022     if (kind == Interpreter::java_util_zip_CRC32_updateByteBuffer) {
1023       __ ldr(buf, Address(esp, 2*wordSize)); // long buf
1024       __ ldrw(off, Address(esp, wordSize)); // offset
1025       __ add(buf, buf, off); // + offset
1026       __ ldrw(crc,   Address(esp, 4*wordSize)); // Initial CRC
1027     } else {
1028       __ ldr(buf, Address(esp, 2*wordSize)); // byte[] array
1029       __ add(buf, buf, arrayOopDesc::base_offset_in_bytes(T_BYTE)); // + header size
1030       __ ldrw(off, Address(esp, wordSize)); // offset
1031       __ add(buf, buf, off); // + offset
1032       __ ldrw(crc,   Address(esp, 3*wordSize)); // Initial CRC
1033     }
1034     // Can now load 'len' since we're finished with 'off'
1035     __ ldrw(len, Address(esp, 0x0)); // Length
1036 
1037     __ andr(sp, r13, -16); // Restore the caller's SP
1038 
1039     // We are frameless so we can just jump to the stub.
1040     __ b(CAST_FROM_FN_PTR(address, StubRoutines::updateBytesCRC32()));
1041 
1042     // generate a vanilla native entry as the slow path
1043     __ bind(slow_path);
1044     __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::native));
1045     return entry;
1046   }
1047   return NULL;
1048 }
1049 
1050 // Not supported
1051 address TemplateInterpreterGenerator::generate_CRC32C_updateBytes_entry(AbstractInterpreter::MethodKind kind) {
1052   return NULL;
1053 }
1054 
1055 void TemplateInterpreterGenerator::bang_stack_shadow_pages(bool native_call) {
1056   // Bang each page in the shadow zone. We can't assume it's been done for
1057   // an interpreter frame with greater than a page of locals, so each page
1058   // needs to be checked.  Only true for non-native.
1059   if (UseStackBanging) {
1060     const int n_shadow_pages = JavaThread::stack_shadow_zone_size() / os::vm_page_size();
1061     const int start_page = native_call ? n_shadow_pages : 1;
1062     const int page_size = os::vm_page_size();
1063     for (int pages = start_page; pages <= n_shadow_pages ; pages++) {
1064       __ sub(rscratch2, sp, pages*page_size);
1065       __ str(zr, Address(rscratch2));
1066     }
1067   }
1068 }
1069 
1070 
1071 // Interpreter stub for calling a native method. (asm interpreter)
1072 // This sets up a somewhat different looking stack for calling the
1073 // native method than the typical interpreter frame setup.
1074 address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
1075   // determine code generation flags
1076   bool inc_counter  = UseCompiler || CountCompiledCalls || LogTouchedMethods;
1077 
1078   // r1: Method*
1079   // rscratch1: sender sp
1080 
1081   address entry_point = __ pc();
1082 
1083   const Address constMethod       (rmethod, Method::const_offset());
1084   const Address access_flags      (rmethod, Method::access_flags_offset());
1085   const Address size_of_parameters(r2, ConstMethod::
1086                                        size_of_parameters_offset());
1087 
1088   // get parameter size (always needed)
1089   __ ldr(r2, constMethod);
1090   __ load_unsigned_short(r2, size_of_parameters);
1091 
1092   // native calls don't need the stack size check since they have no
1093   // expression stack and the arguments are already on the stack and
1094   // we only add a handful of words to the stack
1095 
1096   // rmethod: Method*
1097   // r2: size of parameters
1098   // rscratch1: sender sp
1099 
1100   // for natives the size of locals is zero
1101 
1102   // compute beginning of parameters (rlocals)
1103   __ add(rlocals, esp, r2, ext::uxtx, 3);
1104   __ add(rlocals, rlocals, -wordSize);
1105 
1106   // Pull SP back to minimum size: this avoids holes in the stack
1107   __ andr(sp, esp, -16);
1108 
1109   // initialize fixed part of activation frame
1110   generate_fixed_frame(true);
1111 #ifndef PRODUCT
1112   // tell the simulator that a method has been entered
1113   if (NotifySimulator) {
1114     __ notify(Assembler::method_entry);
1115   }
1116 #endif
1117 
1118   // make sure method is native & not abstract
1119 #ifdef ASSERT
1120   __ ldrw(r0, access_flags);
1121   {
1122     Label L;
1123     __ tst(r0, JVM_ACC_NATIVE);
1124     __ br(Assembler::NE, L);
1125     __ stop("tried to execute non-native method as native");
1126     __ bind(L);
1127   }
1128   {
1129     Label L;
1130     __ tst(r0, JVM_ACC_ABSTRACT);
1131     __ br(Assembler::EQ, L);
1132     __ stop("tried to execute abstract method in interpreter");
1133     __ bind(L);
1134   }
1135 #endif
1136 
1137   // Since at this point in the method invocation the exception
1138   // handler would try to exit the monitor of synchronized methods
1139   // which hasn't been entered yet, we set the thread local variable
1140   // _do_not_unlock_if_synchronized to true. The remove_activation
1141   // will check this flag.
1142 
1143    const Address do_not_unlock_if_synchronized(rthread,
1144         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1145   __ mov(rscratch2, true);
1146   __ strb(rscratch2, do_not_unlock_if_synchronized);
1147 
1148   // increment invocation count & check for overflow
1149   Label invocation_counter_overflow;
1150   if (inc_counter) {
1151     generate_counter_incr(&invocation_counter_overflow, NULL, NULL);
1152   }
1153 
1154   Label continue_after_compile;
1155   __ bind(continue_after_compile);
1156 
1157   bang_stack_shadow_pages(true);
1158 
1159   // reset the _do_not_unlock_if_synchronized flag
1160   __ strb(zr, do_not_unlock_if_synchronized);
1161 
1162   // check for synchronized methods
1163   // Must happen AFTER invocation_counter check and stack overflow check,
1164   // so method is not locked if overflows.
1165   if (synchronized) {
1166     lock_method();
1167   } else {
1168     // no synchronization necessary
1169 #ifdef ASSERT
1170     {
1171       Label L;
1172       __ ldrw(r0, access_flags);
1173       __ tst(r0, JVM_ACC_SYNCHRONIZED);
1174       __ br(Assembler::EQ, L);
1175       __ stop("method needs synchronization");
1176       __ bind(L);
1177     }
1178 #endif
1179   }
1180 
1181   // start execution
1182 #ifdef ASSERT
1183   {
1184     Label L;
1185     const Address monitor_block_top(rfp,
1186                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
1187     __ ldr(rscratch1, monitor_block_top);
1188     __ cmp(esp, rscratch1);
1189     __ br(Assembler::EQ, L);
1190     __ stop("broken stack frame setup in interpreter");
1191     __ bind(L);
1192   }
1193 #endif
1194 
1195   // jvmti support
1196   __ notify_method_entry();
1197 
1198   // work registers
1199   const Register t = r17;
1200   const Register result_handler = r19;
1201 
1202   // allocate space for parameters
1203   __ ldr(t, Address(rmethod, Method::const_offset()));
1204   __ load_unsigned_short(t, Address(t, ConstMethod::size_of_parameters_offset()));
1205 
1206   __ sub(rscratch1, esp, t, ext::uxtx, Interpreter::logStackElementSize);
1207   __ andr(sp, rscratch1, -16);
1208   __ mov(esp, rscratch1);
1209 
1210   // get signature handler
1211   {
1212     Label L;
1213     __ ldr(t, Address(rmethod, Method::signature_handler_offset()));
1214     __ cbnz(t, L);
1215     __ call_VM(noreg,
1216                CAST_FROM_FN_PTR(address,
1217                                 InterpreterRuntime::prepare_native_call),
1218                rmethod);
1219     __ ldr(t, Address(rmethod, Method::signature_handler_offset()));
1220     __ bind(L);
1221   }
1222 
1223   // call signature handler
1224   assert(InterpreterRuntime::SignatureHandlerGenerator::from() == rlocals,
1225          "adjust this code");
1226   assert(InterpreterRuntime::SignatureHandlerGenerator::to() == sp,
1227          "adjust this code");
1228   assert(InterpreterRuntime::SignatureHandlerGenerator::temp() == rscratch1,
1229           "adjust this code");
1230 
1231   // The generated handlers do not touch rmethod (the method).
1232   // However, large signatures cannot be cached and are generated
1233   // each time here.  The slow-path generator can do a GC on return,
1234   // so we must reload it after the call.
1235   __ blr(t);
1236   __ get_method(rmethod);        // slow path can do a GC, reload rmethod
1237 
1238 
1239   // result handler is in r0
1240   // set result handler
1241   __ mov(result_handler, r0);
1242   // pass mirror handle if static call
1243   {
1244     Label L;
1245     const int mirror_offset = in_bytes(Klass::java_mirror_offset());
1246     __ ldrw(t, Address(rmethod, Method::access_flags_offset()));
1247     __ tst(t, JVM_ACC_STATIC);
1248     __ br(Assembler::EQ, L);
1249     // get mirror
1250     __ ldr(t, Address(rmethod, Method::const_offset()));
1251     __ ldr(t, Address(t, ConstMethod::constants_offset()));
1252     __ ldr(t, Address(t, ConstantPool::pool_holder_offset_in_bytes()));
1253     __ ldr(t, Address(t, mirror_offset));
1254     // copy mirror into activation frame
1255     __ str(t, Address(rfp, frame::interpreter_frame_oop_temp_offset * wordSize));
1256     // pass handle to mirror
1257     __ add(c_rarg1, rfp, frame::interpreter_frame_oop_temp_offset * wordSize);
1258     __ bind(L);
1259   }
1260 
1261   // get native function entry point in r10
1262   {
1263     Label L;
1264     __ ldr(r10, Address(rmethod, Method::native_function_offset()));
1265     address unsatisfied = (SharedRuntime::native_method_throw_unsatisfied_link_error_entry());
1266     __ mov(rscratch2, unsatisfied);
1267     __ ldr(rscratch2, rscratch2);
1268     __ cmp(r10, rscratch2);
1269     __ br(Assembler::NE, L);
1270     __ call_VM(noreg,
1271                CAST_FROM_FN_PTR(address,
1272                                 InterpreterRuntime::prepare_native_call),
1273                rmethod);
1274     __ get_method(rmethod);
1275     __ ldr(r10, Address(rmethod, Method::native_function_offset()));
1276     __ bind(L);
1277   }
1278 
1279   // pass JNIEnv
1280   __ add(c_rarg0, rthread, in_bytes(JavaThread::jni_environment_offset()));
1281 
1282   // It is enough that the pc() points into the right code
1283   // segment. It does not have to be the correct return pc.
1284   __ set_last_Java_frame(esp, rfp, (address)NULL, rscratch1);
1285 
1286   // change thread state
1287 #ifdef ASSERT
1288   {
1289     Label L;
1290     __ ldrw(t, Address(rthread, JavaThread::thread_state_offset()));
1291     __ cmp(t, _thread_in_Java);
1292     __ br(Assembler::EQ, L);
1293     __ stop("Wrong thread state in native stub");
1294     __ bind(L);
1295   }
1296 #endif
1297 
1298   // Change state to native
1299   __ mov(rscratch1, _thread_in_native);
1300   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
1301   __ stlrw(rscratch1, rscratch2);
1302 
1303   // Call the native method.
1304   __ blrt(r10, rscratch1);
1305   __ maybe_isb();
1306   __ get_method(rmethod);
1307   // result potentially in r0 or v0
1308 
1309   // make room for the pushes we're about to do
1310   __ sub(rscratch1, esp, 4 * wordSize);
1311   __ andr(sp, rscratch1, -16);
1312 
1313   // NOTE: The order of these pushes is known to frame::interpreter_frame_result
1314   // in order to extract the result of a method call. If the order of these
1315   // pushes change or anything else is added to the stack then the code in
1316   // interpreter_frame_result must also change.
1317   __ push(dtos);
1318   __ push(ltos);
1319 
1320   // change thread state
1321   __ mov(rscratch1, _thread_in_native_trans);
1322   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
1323   __ stlrw(rscratch1, rscratch2);
1324 
1325   if (os::is_MP()) {
1326     if (UseMembar) {
1327       // Force this write out before the read below
1328       __ dsb(Assembler::SY);
1329     } else {
1330       // Write serialization page so VM thread can do a pseudo remote membar.
1331       // We use the current thread pointer to calculate a thread specific
1332       // offset to write to within the page. This minimizes bus traffic
1333       // due to cache line collision.
1334       __ serialize_memory(rthread, rscratch2);
1335     }
1336   }
1337 
1338   // check for safepoint operation in progress and/or pending suspend requests
1339   {
1340     Label Continue;
1341     {
1342       unsigned long offset;
1343       __ adrp(rscratch2, SafepointSynchronize::address_of_state(), offset);
1344       __ ldrw(rscratch2, Address(rscratch2, offset));
1345     }
1346     assert(SafepointSynchronize::_not_synchronized == 0,
1347            "SafepointSynchronize::_not_synchronized");
1348     Label L;
1349     __ cbnz(rscratch2, L);
1350     __ ldrw(rscratch2, Address(rthread, JavaThread::suspend_flags_offset()));
1351     __ cbz(rscratch2, Continue);
1352     __ bind(L);
1353 
1354     // Don't use call_VM as it will see a possible pending exception
1355     // and forward it and never return here preventing us from
1356     // clearing _last_native_pc down below. So we do a runtime call by
1357     // hand.
1358     //
1359     __ mov(c_rarg0, rthread);
1360     __ mov(rscratch2, CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
1361     __ blrt(rscratch2, 1, 0, 0);
1362     __ maybe_isb();
1363     __ get_method(rmethod);
1364     __ reinit_heapbase();
1365     __ bind(Continue);
1366   }
1367 
1368   // change thread state
1369   __ mov(rscratch1, _thread_in_Java);
1370   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
1371   __ stlrw(rscratch1, rscratch2);
1372 
1373   // reset_last_Java_frame
1374   __ reset_last_Java_frame(true, true);
1375 
1376   // reset handle block
1377   __ ldr(t, Address(rthread, JavaThread::active_handles_offset()));
1378   __ str(zr, Address(t, JNIHandleBlock::top_offset_in_bytes()));
1379 
1380   // If result is an oop unbox and store it in frame where gc will see it
1381   // and result handler will pick it up
1382 
1383   {
1384     Label no_oop, store_result;
1385     __ adr(t, ExternalAddress(AbstractInterpreter::result_handler(T_OBJECT)));
1386     __ cmp(t, result_handler);
1387     __ br(Assembler::NE, no_oop);
1388     // retrieve result
1389     __ pop(ltos);
1390     __ cbz(r0, store_result);
1391     __ ldr(r0, Address(r0, 0));
1392     __ bind(store_result);
1393     __ str(r0, Address(rfp, frame::interpreter_frame_oop_temp_offset*wordSize));
1394     // keep stack depth as expected by pushing oop which will eventually be discarded
1395     __ push(ltos);
1396     __ bind(no_oop);
1397   }
1398 
1399   {
1400     Label no_reguard;
1401     __ lea(rscratch1, Address(rthread, in_bytes(JavaThread::stack_guard_state_offset())));
1402     __ ldrw(rscratch1, Address(rscratch1));
1403     __ cmp(rscratch1, JavaThread::stack_guard_yellow_reserved_disabled);
1404     __ br(Assembler::NE, no_reguard);
1405 
1406     __ pusha(); // XXX only save smashed registers
1407     __ mov(c_rarg0, rthread);
1408     __ mov(rscratch2, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
1409     __ blrt(rscratch2, 0, 0, 0);
1410     __ popa(); // XXX only restore smashed registers
1411     __ bind(no_reguard);
1412   }
1413 
1414   // The method register is junk from after the thread_in_native transition
1415   // until here.  Also can't call_VM until the bcp has been
1416   // restored.  Need bcp for throwing exception below so get it now.
1417   __ get_method(rmethod);
1418 
1419   // restore bcp to have legal interpreter frame, i.e., bci == 0 <=>
1420   // rbcp == code_base()
1421   __ ldr(rbcp, Address(rmethod, Method::const_offset()));   // get ConstMethod*
1422   __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));          // get codebase
1423   // handle exceptions (exception handling will handle unlocking!)
1424   {
1425     Label L;
1426     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
1427     __ cbz(rscratch1, L);
1428     // Note: At some point we may want to unify this with the code
1429     // used in call_VM_base(); i.e., we should use the
1430     // StubRoutines::forward_exception code. For now this doesn't work
1431     // here because the rsp is not correctly set at this point.
1432     __ MacroAssembler::call_VM(noreg,
1433                                CAST_FROM_FN_PTR(address,
1434                                InterpreterRuntime::throw_pending_exception));
1435     __ should_not_reach_here();
1436     __ bind(L);
1437   }
1438 
1439   // do unlocking if necessary
1440   {
1441     Label L;
1442     __ ldrw(t, Address(rmethod, Method::access_flags_offset()));
1443     __ tst(t, JVM_ACC_SYNCHRONIZED);
1444     __ br(Assembler::EQ, L);
1445     // the code below should be shared with interpreter macro
1446     // assembler implementation
1447     {
1448       Label unlock;
1449       // BasicObjectLock will be first in list, since this is a
1450       // synchronized method. However, need to check that the object
1451       // has not been unlocked by an explicit monitorexit bytecode.
1452 
1453       // monitor expect in c_rarg1 for slow unlock path
1454       __ lea (c_rarg1, Address(rfp,   // address of first monitor
1455                                (intptr_t)(frame::interpreter_frame_initial_sp_offset *
1456                                           wordSize - sizeof(BasicObjectLock))));
1457 
1458       __ ldr(t, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
1459       __ cbnz(t, unlock);
1460 
1461       // Entry already unlocked, need to throw exception
1462       __ MacroAssembler::call_VM(noreg,
1463                                  CAST_FROM_FN_PTR(address,
1464                    InterpreterRuntime::throw_illegal_monitor_state_exception));
1465       __ should_not_reach_here();
1466 
1467       __ bind(unlock);
1468       __ unlock_object(c_rarg1);
1469     }
1470     __ bind(L);
1471   }
1472 
1473   // jvmti support
1474   // Note: This must happen _after_ handling/throwing any exceptions since
1475   //       the exception handler code notifies the runtime of method exits
1476   //       too. If this happens before, method entry/exit notifications are
1477   //       not properly paired (was bug - gri 11/22/99).
1478   __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI);
1479 
1480   // restore potential result in r0:d0, call result handler to
1481   // restore potential result in ST0 & handle result
1482 
1483   __ pop(ltos);
1484   __ pop(dtos);
1485 
1486   __ blr(result_handler);
1487 
1488   // remove activation
1489   __ ldr(esp, Address(rfp,
1490                     frame::interpreter_frame_sender_sp_offset *
1491                     wordSize)); // get sender sp
1492   // remove frame anchor
1493   __ leave();
1494 
1495   // resture sender sp
1496   __ mov(sp, esp);
1497 
1498   __ ret(lr);
1499 
1500   if (inc_counter) {
1501     // Handle overflow of counter and compile method
1502     __ bind(invocation_counter_overflow);
1503     generate_counter_overflow(continue_after_compile);
1504   }
1505 
1506   return entry_point;
1507 }
1508 
1509 //
1510 // Generic interpreted method entry to (asm) interpreter
1511 //
1512 address TemplateInterpreterGenerator::generate_normal_entry(bool synchronized) {
1513   // determine code generation flags
1514   bool inc_counter  = UseCompiler || CountCompiledCalls || LogTouchedMethods;
1515 
1516   // rscratch1: sender sp
1517   address entry_point = __ pc();
1518 
1519   const Address constMethod(rmethod, Method::const_offset());
1520   const Address access_flags(rmethod, Method::access_flags_offset());
1521   const Address size_of_parameters(r3,
1522                                    ConstMethod::size_of_parameters_offset());
1523   const Address size_of_locals(r3, ConstMethod::size_of_locals_offset());
1524 
1525   // get parameter size (always needed)
1526   // need to load the const method first
1527   __ ldr(r3, constMethod);
1528   __ load_unsigned_short(r2, size_of_parameters);
1529 
1530   // r2: size of parameters
1531 
1532   __ load_unsigned_short(r3, size_of_locals); // get size of locals in words
1533   __ sub(r3, r3, r2); // r3 = no. of additional locals
1534 
1535   // see if we've got enough room on the stack for locals plus overhead.
1536   generate_stack_overflow_check();
1537 
1538   // compute beginning of parameters (rlocals)
1539   __ add(rlocals, esp, r2, ext::uxtx, 3);
1540   __ sub(rlocals, rlocals, wordSize);
1541 
1542   // Make room for locals
1543   __ sub(rscratch1, esp, r3, ext::uxtx, 3);
1544   __ andr(sp, rscratch1, -16);
1545 
1546   // r3 - # of additional locals
1547   // allocate space for locals
1548   // explicitly initialize locals
1549   {
1550     Label exit, loop;
1551     __ ands(zr, r3, r3);
1552     __ br(Assembler::LE, exit); // do nothing if r3 <= 0
1553     __ bind(loop);
1554     __ str(zr, Address(__ post(rscratch1, wordSize)));
1555     __ sub(r3, r3, 1); // until everything initialized
1556     __ cbnz(r3, loop);
1557     __ bind(exit);
1558   }
1559 
1560   // And the base dispatch table
1561   __ get_dispatch();
1562 
1563   // initialize fixed part of activation frame
1564   generate_fixed_frame(false);
1565 #ifndef PRODUCT
1566   // tell the simulator that a method has been entered
1567   if (NotifySimulator) {
1568     __ notify(Assembler::method_entry);
1569   }
1570 #endif
1571   // make sure method is not native & not abstract
1572 #ifdef ASSERT
1573   __ ldrw(r0, access_flags);
1574   {
1575     Label L;
1576     __ tst(r0, JVM_ACC_NATIVE);
1577     __ br(Assembler::EQ, L);
1578     __ stop("tried to execute native method as non-native");
1579     __ bind(L);
1580   }
1581  {
1582     Label L;
1583     __ tst(r0, JVM_ACC_ABSTRACT);
1584     __ br(Assembler::EQ, L);
1585     __ stop("tried to execute abstract method in interpreter");
1586     __ bind(L);
1587   }
1588 #endif
1589 
1590   // Since at this point in the method invocation the exception
1591   // handler would try to exit the monitor of synchronized methods
1592   // which hasn't been entered yet, we set the thread local variable
1593   // _do_not_unlock_if_synchronized to true. The remove_activation
1594   // will check this flag.
1595 
1596    const Address do_not_unlock_if_synchronized(rthread,
1597         in_bytes(JavaThread::do_not_unlock_if_synchronized_offset()));
1598   __ mov(rscratch2, true);
1599   __ strb(rscratch2, do_not_unlock_if_synchronized);
1600 
1601   // increment invocation count & check for overflow
1602   Label invocation_counter_overflow;
1603   Label profile_method;
1604   Label profile_method_continue;
1605   if (inc_counter) {
1606     generate_counter_incr(&invocation_counter_overflow,
1607                           &profile_method,
1608                           &profile_method_continue);
1609     if (ProfileInterpreter) {
1610       __ bind(profile_method_continue);
1611     }
1612   }
1613 
1614   Label continue_after_compile;
1615   __ bind(continue_after_compile);
1616 
1617   bang_stack_shadow_pages(false);
1618 
1619   // reset the _do_not_unlock_if_synchronized flag
1620   __ strb(zr, do_not_unlock_if_synchronized);
1621 
1622   // check for synchronized methods
1623   // Must happen AFTER invocation_counter check and stack overflow check,
1624   // so method is not locked if overflows.
1625   if (synchronized) {
1626     // Allocate monitor and lock method
1627     lock_method();
1628   } else {
1629     // no synchronization necessary
1630 #ifdef ASSERT
1631     {
1632       Label L;
1633       __ ldrw(r0, access_flags);
1634       __ tst(r0, JVM_ACC_SYNCHRONIZED);
1635       __ br(Assembler::EQ, L);
1636       __ stop("method needs synchronization");
1637       __ bind(L);
1638     }
1639 #endif
1640   }
1641 
1642   // start execution
1643 #ifdef ASSERT
1644   {
1645     Label L;
1646      const Address monitor_block_top (rfp,
1647                  frame::interpreter_frame_monitor_block_top_offset * wordSize);
1648     __ ldr(rscratch1, monitor_block_top);
1649     __ cmp(esp, rscratch1);
1650     __ br(Assembler::EQ, L);
1651     __ stop("broken stack frame setup in interpreter");
1652     __ bind(L);
1653   }
1654 #endif
1655 
1656   // jvmti support
1657   __ notify_method_entry();
1658 
1659   __ dispatch_next(vtos);
1660 
1661   // invocation counter overflow
1662   if (inc_counter) {
1663     if (ProfileInterpreter) {
1664       // We have decided to profile this method in the interpreter
1665       __ bind(profile_method);
1666       __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1667       __ set_method_data_pointer_for_bcp();
1668       // don't think we need this
1669       __ get_method(r1);
1670       __ b(profile_method_continue);
1671     }
1672     // Handle overflow of counter and compile method
1673     __ bind(invocation_counter_overflow);
1674     generate_counter_overflow(continue_after_compile);
1675   }
1676 
1677   return entry_point;
1678 }
1679 
1680 //-----------------------------------------------------------------------------
1681 // Exceptions
1682 
1683 void TemplateInterpreterGenerator::generate_throw_exception() {
1684   // Entry point in previous activation (i.e., if the caller was
1685   // interpreted)
1686   Interpreter::_rethrow_exception_entry = __ pc();
1687   // Restore sp to interpreter_frame_last_sp even though we are going
1688   // to empty the expression stack for the exception processing.
1689   __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1690   // r0: exception
1691   // r3: return address/pc that threw exception
1692   __ restore_bcp();    // rbcp points to call/send
1693   __ restore_locals();
1694   __ restore_constant_pool_cache();
1695   __ reinit_heapbase();  // restore rheapbase as heapbase.
1696   __ get_dispatch();
1697 
1698 #ifndef PRODUCT
1699   // tell the simulator that the caller method has been reentered
1700   if (NotifySimulator) {
1701     __ get_method(rmethod);
1702     __ notify(Assembler::method_reentry);
1703   }
1704 #endif
1705   // Entry point for exceptions thrown within interpreter code
1706   Interpreter::_throw_exception_entry = __ pc();
1707   // If we came here via a NullPointerException on the receiver of a
1708   // method, rmethod may be corrupt.
1709   __ get_method(rmethod);
1710   // expression stack is undefined here
1711   // r0: exception
1712   // rbcp: exception bcp
1713   __ verify_oop(r0);
1714   __ mov(c_rarg1, r0);
1715 
1716   // expression stack must be empty before entering the VM in case of
1717   // an exception
1718   __ empty_expression_stack();
1719   // find exception handler address and preserve exception oop
1720   __ call_VM(r3,
1721              CAST_FROM_FN_PTR(address,
1722                           InterpreterRuntime::exception_handler_for_exception),
1723              c_rarg1);
1724 
1725   // Calculate stack limit
1726   __ ldr(rscratch1, Address(rmethod, Method::const_offset()));
1727   __ ldrh(rscratch1, Address(rscratch1, ConstMethod::max_stack_offset()));
1728   __ add(rscratch1, rscratch1, frame::interpreter_frame_monitor_size() + 4);
1729   __ ldr(rscratch2,
1730          Address(rfp, frame::interpreter_frame_initial_sp_offset * wordSize));
1731   __ sub(rscratch1, rscratch2, rscratch1, ext::uxtx, 3);
1732   __ andr(sp, rscratch1, -16);
1733 
1734   // r0: exception handler entry point
1735   // r3: preserved exception oop
1736   // rbcp: bcp for exception handler
1737   __ push_ptr(r3); // push exception which is now the only value on the stack
1738   __ br(r0); // jump to exception handler (may be _remove_activation_entry!)
1739 
1740   // If the exception is not handled in the current frame the frame is
1741   // removed and the exception is rethrown (i.e. exception
1742   // continuation is _rethrow_exception).
1743   //
1744   // Note: At this point the bci is still the bxi for the instruction
1745   // which caused the exception and the expression stack is
1746   // empty. Thus, for any VM calls at this point, GC will find a legal
1747   // oop map (with empty expression stack).
1748 
1749   //
1750   // JVMTI PopFrame support
1751   //
1752 
1753   Interpreter::_remove_activation_preserving_args_entry = __ pc();
1754   __ empty_expression_stack();
1755   // Set the popframe_processing bit in pending_popframe_condition
1756   // indicating that we are currently handling popframe, so that
1757   // call_VMs that may happen later do not trigger new popframe
1758   // handling cycles.
1759   __ ldrw(r3, Address(rthread, JavaThread::popframe_condition_offset()));
1760   __ orr(r3, r3, JavaThread::popframe_processing_bit);
1761   __ strw(r3, Address(rthread, JavaThread::popframe_condition_offset()));
1762 
1763   {
1764     // Check to see whether we are returning to a deoptimized frame.
1765     // (The PopFrame call ensures that the caller of the popped frame is
1766     // either interpreted or compiled and deoptimizes it if compiled.)
1767     // In this case, we can't call dispatch_next() after the frame is
1768     // popped, but instead must save the incoming arguments and restore
1769     // them after deoptimization has occurred.
1770     //
1771     // Note that we don't compare the return PC against the
1772     // deoptimization blob's unpack entry because of the presence of
1773     // adapter frames in C2.
1774     Label caller_not_deoptimized;
1775     __ ldr(c_rarg1, Address(rfp, frame::return_addr_offset * wordSize));
1776     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1777                                InterpreterRuntime::interpreter_contains), c_rarg1);
1778     __ cbnz(r0, caller_not_deoptimized);
1779 
1780     // Compute size of arguments for saving when returning to
1781     // deoptimized caller
1782     __ get_method(r0);
1783     __ ldr(r0, Address(r0, Method::const_offset()));
1784     __ load_unsigned_short(r0, Address(r0, in_bytes(ConstMethod::
1785                                                     size_of_parameters_offset())));
1786     __ lsl(r0, r0, Interpreter::logStackElementSize);
1787     __ restore_locals(); // XXX do we need this?
1788     __ sub(rlocals, rlocals, r0);
1789     __ add(rlocals, rlocals, wordSize);
1790     // Save these arguments
1791     __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1792                                            Deoptimization::
1793                                            popframe_preserve_args),
1794                           rthread, r0, rlocals);
1795 
1796     __ remove_activation(vtos,
1797                          /* throw_monitor_exception */ false,
1798                          /* install_monitor_exception */ false,
1799                          /* notify_jvmdi */ false);
1800 
1801     // Inform deoptimization that it is responsible for restoring
1802     // these arguments
1803     __ mov(rscratch1, JavaThread::popframe_force_deopt_reexecution_bit);
1804     __ strw(rscratch1, Address(rthread, JavaThread::popframe_condition_offset()));
1805 
1806     // Continue in deoptimization handler
1807     __ ret(lr);
1808 
1809     __ bind(caller_not_deoptimized);
1810   }
1811 
1812   __ remove_activation(vtos,
1813                        /* throw_monitor_exception */ false,
1814                        /* install_monitor_exception */ false,
1815                        /* notify_jvmdi */ false);
1816 
1817   // Restore the last_sp and null it out
1818   __ ldr(esp, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1819   __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
1820 
1821   __ restore_bcp();
1822   __ restore_locals();
1823   __ restore_constant_pool_cache();
1824   __ get_method(rmethod);
1825 
1826   // The method data pointer was incremented already during
1827   // call profiling. We have to restore the mdp for the current bcp.
1828   if (ProfileInterpreter) {
1829     __ set_method_data_pointer_for_bcp();
1830   }
1831 
1832   // Clear the popframe condition flag
1833   __ strw(zr, Address(rthread, JavaThread::popframe_condition_offset()));
1834   assert(JavaThread::popframe_inactive == 0, "fix popframe_inactive");
1835 
1836 #if INCLUDE_JVMTI
1837   {
1838     Label L_done;
1839 
1840     __ ldrb(rscratch1, Address(rbcp, 0));
1841     __ cmpw(r1, Bytecodes::_invokestatic);
1842     __ br(Assembler::EQ, L_done);
1843 
1844     // The member name argument must be restored if _invokestatic is re-executed after a PopFrame call.
1845     // Detect such a case in the InterpreterRuntime function and return the member name argument, or NULL.
1846 
1847     __ ldr(c_rarg0, Address(rlocals, 0));
1848     __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null), c_rarg0, rmethod, rbcp);
1849 
1850     __ cbz(r0, L_done);
1851 
1852     __ str(r0, Address(esp, 0));
1853     __ bind(L_done);
1854   }
1855 #endif // INCLUDE_JVMTI
1856 
1857   // Restore machine SP
1858   __ ldr(rscratch1, Address(rmethod, Method::const_offset()));
1859   __ ldrh(rscratch1, Address(rscratch1, ConstMethod::max_stack_offset()));
1860   __ add(rscratch1, rscratch1, frame::interpreter_frame_monitor_size() + 4);
1861   __ ldr(rscratch2,
1862          Address(rfp, frame::interpreter_frame_initial_sp_offset * wordSize));
1863   __ sub(rscratch1, rscratch2, rscratch1, ext::uxtw, 3);
1864   __ andr(sp, rscratch1, -16);
1865 
1866   __ dispatch_next(vtos);
1867   // end of PopFrame support
1868 
1869   Interpreter::_remove_activation_entry = __ pc();
1870 
1871   // preserve exception over this code sequence
1872   __ pop_ptr(r0);
1873   __ str(r0, Address(rthread, JavaThread::vm_result_offset()));
1874   // remove the activation (without doing throws on illegalMonitorExceptions)
1875   __ remove_activation(vtos, false, true, false);
1876   // restore exception
1877   // restore exception
1878   __ get_vm_result(r0, rthread);
1879 
1880   // In between activations - previous activation type unknown yet
1881   // compute continuation point - the continuation point expects the
1882   // following registers set up:
1883   //
1884   // r0: exception
1885   // lr: return address/pc that threw exception
1886   // rsp: expression stack of caller
1887   // rfp: fp of caller
1888   // FIXME: There's no point saving LR here because VM calls don't trash it
1889   __ stp(r0, lr, Address(__ pre(sp, -2 * wordSize)));  // save exception & return address
1890   __ super_call_VM_leaf(CAST_FROM_FN_PTR(address,
1891                           SharedRuntime::exception_handler_for_return_address),
1892                         rthread, lr);
1893   __ mov(r1, r0);                               // save exception handler
1894   __ ldp(r0, lr, Address(__ post(sp, 2 * wordSize)));  // restore exception & return address
1895   // We might be returning to a deopt handler that expects r3 to
1896   // contain the exception pc
1897   __ mov(r3, lr);
1898   // Note that an "issuing PC" is actually the next PC after the call
1899   __ br(r1);                                    // jump to exception
1900                                                 // handler of caller
1901 }
1902 
1903 
1904 //
1905 // JVMTI ForceEarlyReturn support
1906 //
1907 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1908   address entry = __ pc();
1909 
1910   __ restore_bcp();
1911   __ restore_locals();
1912   __ empty_expression_stack();
1913   __ load_earlyret_value(state);
1914 
1915   __ ldr(rscratch1, Address(rthread, JavaThread::jvmti_thread_state_offset()));
1916   Address cond_addr(rscratch1, JvmtiThreadState::earlyret_state_offset());
1917 
1918   // Clear the earlyret state
1919   assert(JvmtiThreadState::earlyret_inactive == 0, "should be");
1920   __ str(zr, cond_addr);
1921 
1922   __ remove_activation(state,
1923                        false, /* throw_monitor_exception */
1924                        false, /* install_monitor_exception */
1925                        true); /* notify_jvmdi */
1926   __ ret(lr);
1927 
1928   return entry;
1929 } // end of ForceEarlyReturn support
1930 
1931 
1932 
1933 //-----------------------------------------------------------------------------
1934 // Helper for vtos entry point generation
1935 
1936 void TemplateInterpreterGenerator::set_vtos_entry_points(Template* t,
1937                                                          address& bep,
1938                                                          address& cep,
1939                                                          address& sep,
1940                                                          address& aep,
1941                                                          address& iep,
1942                                                          address& lep,
1943                                                          address& fep,
1944                                                          address& dep,
1945                                                          address& vep) {
1946   assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1947   Label L;
1948   aep = __ pc();  __ push_ptr();  __ b(L);
1949   fep = __ pc();  __ push_f();    __ b(L);
1950   dep = __ pc();  __ push_d();    __ b(L);
1951   lep = __ pc();  __ push_l();    __ b(L);
1952   bep = cep = sep =
1953   iep = __ pc();  __ push_i();
1954   vep = __ pc();
1955   __ bind(L);
1956   generate_and_dispatch(t);
1957 }
1958 
1959 //-----------------------------------------------------------------------------
1960 
1961 // Non-product code
1962 #ifndef PRODUCT
1963 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1964   address entry = __ pc();
1965 
1966   __ push(lr);
1967   __ push(state);
1968   __ push(RegSet::range(r0, r15), sp);
1969   __ mov(c_rarg2, r0);  // Pass itos
1970   __ call_VM(noreg,
1971              CAST_FROM_FN_PTR(address, SharedRuntime::trace_bytecode),
1972              c_rarg1, c_rarg2, c_rarg3);
1973   __ pop(RegSet::range(r0, r15), sp);
1974   __ pop(state);
1975   __ pop(lr);
1976   __ ret(lr);                                   // return from result handler
1977 
1978   return entry;
1979 }
1980 
1981 void TemplateInterpreterGenerator::count_bytecode() {
1982   Register rscratch3 = r0;
1983   __ push(rscratch1);
1984   __ push(rscratch2);
1985   __ push(rscratch3);
1986   Label L;
1987   __ mov(rscratch2, (address) &BytecodeCounter::_counter_value);
1988   __ bind(L);
1989   __ ldxr(rscratch1, rscratch2);
1990   __ add(rscratch1, rscratch1, 1);
1991   __ stxr(rscratch3, rscratch1, rscratch2);
1992   __ cbnzw(rscratch3, L);
1993   __ pop(rscratch3);
1994   __ pop(rscratch2);
1995   __ pop(rscratch1);
1996 }
1997 
1998 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) { ; }
1999 
2000 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) { ; }
2001 
2002 
2003 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
2004   // Call a little run-time stub to avoid blow-up for each bytecode.
2005   // The run-time runtime saves the right registers, depending on
2006   // the tosca in-state for the given template.
2007 
2008   assert(Interpreter::trace_code(t->tos_in()) != NULL,
2009          "entry must have been generated");
2010   __ bl(Interpreter::trace_code(t->tos_in()));
2011   __ reinit_heapbase();
2012 }
2013 
2014 
2015 void TemplateInterpreterGenerator::stop_interpreter_at() {
2016   Label L;
2017   __ push(rscratch1);
2018   __ mov(rscratch1, (address) &BytecodeCounter::_counter_value);
2019   __ ldr(rscratch1, Address(rscratch1));
2020   __ mov(rscratch2, StopInterpreterAt);
2021   __ cmpw(rscratch1, rscratch2);
2022   __ br(Assembler::NE, L);
2023   __ brk(0);
2024   __ bind(L);
2025   __ pop(rscratch1);
2026 }
2027 
2028 #ifdef BUILTIN_SIM
2029 
2030 #include <sys/mman.h>
2031 #include <unistd.h>
2032 
2033 extern "C" {
2034   static int PAGESIZE = getpagesize();
2035   int is_mapped_address(u_int64_t address)
2036   {
2037     address = (address & ~((u_int64_t)PAGESIZE - 1));
2038     if (msync((void *)address, PAGESIZE, MS_ASYNC) == 0) {
2039       return true;
2040     }
2041     if (errno != ENOMEM) {
2042       return true;
2043     }
2044     return false;
2045   }
2046 
2047   void bccheck1(u_int64_t pc, u_int64_t fp, char *method, int *bcidx, int *framesize, char *decode)
2048   {
2049     if (method != 0) {
2050       method[0] = '\0';
2051     }
2052     if (bcidx != 0) {
2053       *bcidx = -2;
2054     }
2055     if (decode != 0) {
2056       decode[0] = 0;
2057     }
2058 
2059     if (framesize != 0) {
2060       *framesize = -1;
2061     }
2062 
2063     if (Interpreter::contains((address)pc)) {
2064       AArch64Simulator *sim = AArch64Simulator::get_current(UseSimulatorCache, DisableBCCheck);
2065       Method* meth;
2066       address bcp;
2067       if (fp) {
2068 #define FRAME_SLOT_METHOD 3
2069 #define FRAME_SLOT_BCP 7
2070         meth = (Method*)sim->getMemory()->loadU64(fp - (FRAME_SLOT_METHOD << 3));
2071         bcp = (address)sim->getMemory()->loadU64(fp - (FRAME_SLOT_BCP << 3));
2072 #undef FRAME_SLOT_METHOD
2073 #undef FRAME_SLOT_BCP
2074       } else {
2075         meth = (Method*)sim->getCPUState().xreg(RMETHOD, 0);
2076         bcp = (address)sim->getCPUState().xreg(RBCP, 0);
2077       }
2078       if (meth->is_native()) {
2079         return;
2080       }
2081       if(method && meth->is_method()) {
2082         ResourceMark rm;
2083         method[0] = 'I';
2084         method[1] = ' ';
2085         meth->name_and_sig_as_C_string(method + 2, 398);
2086       }
2087       if (bcidx) {
2088         if (meth->contains(bcp)) {
2089           *bcidx = meth->bci_from(bcp);
2090         } else {
2091           *bcidx = -2;
2092         }
2093       }
2094       if (decode) {
2095         if (!BytecodeTracer::closure()) {
2096           BytecodeTracer::set_closure(BytecodeTracer::std_closure());
2097         }
2098         stringStream str(decode, 400);
2099         BytecodeTracer::trace(meth, bcp, &str);
2100       }
2101     } else {
2102       if (method) {
2103         CodeBlob *cb = CodeCache::find_blob((address)pc);
2104         if (cb != NULL) {
2105           if (cb->is_nmethod()) {
2106             ResourceMark rm;
2107             nmethod* nm = (nmethod*)cb;
2108             method[0] = 'C';
2109             method[1] = ' ';
2110             nm->method()->name_and_sig_as_C_string(method + 2, 398);
2111           } else if (cb->is_adapter_blob()) {
2112             strcpy(method, "B adapter blob");
2113           } else if (cb->is_runtime_stub()) {
2114             strcpy(method, "B runtime stub");
2115           } else if (cb->is_exception_stub()) {
2116             strcpy(method, "B exception stub");
2117           } else if (cb->is_deoptimization_stub()) {
2118             strcpy(method, "B deoptimization stub");
2119           } else if (cb->is_safepoint_stub()) {
2120             strcpy(method, "B safepoint stub");
2121           } else if (cb->is_uncommon_trap_stub()) {
2122             strcpy(method, "B uncommon trap stub");
2123           } else if (cb->contains((address)StubRoutines::call_stub())) {
2124             strcpy(method, "B call stub");
2125           } else {
2126             strcpy(method, "B unknown blob : ");
2127             strcat(method, cb->name());
2128           }
2129           if (framesize != NULL) {
2130             *framesize = cb->frame_size();
2131           }
2132         }
2133       }
2134     }
2135   }
2136 
2137 
2138   JNIEXPORT void bccheck(u_int64_t pc, u_int64_t fp, char *method, int *bcidx, int *framesize, char *decode)
2139   {
2140     bccheck1(pc, fp, method, bcidx, framesize, decode);
2141   }
2142 }
2143 
2144 #endif // BUILTIN_SIM
2145 #endif // !PRODUCT