1 /*
2 * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "interpreter/bytecodeHistogram.hpp"
28 #include "interpreter/interp_masm.hpp"
29 #include "interpreter/interpreter.hpp"
30 #include "interpreter/interpreterRuntime.hpp"
31 #include "interpreter/templateInterpreterGenerator.hpp"
32 #include "interpreter/templateTable.hpp"
33 #include "oops/arrayOop.hpp"
34 #include "oops/methodData.hpp"
35 #include "oops/method.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "prims/jvmtiExport.hpp"
38 #include "prims/jvmtiThreadState.hpp"
39 #include "prims/methodHandles.hpp"
40 #include "runtime/arguments.hpp"
41 #include "runtime/deoptimization.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "runtime/synchronizer.hpp"
46 #include "runtime/timer.hpp"
47 #include "runtime/vframeArray.hpp"
48 #include "utilities/align.hpp"
49 #include "utilities/debug.hpp"
50 #include "utilities/macros.hpp"
51
52 // Size of interpreter code. Increase if too small. Interpreter will
53 // fail with a guarantee ("not enough space for interpreter generation");
54 // if too small.
55 // Run with +PrintInterpreter to get the VM to print out the size.
56 // Max size with JVMTI
57 int TemplateInterpreter::InterpreterCodeSize = 180 * 1024;
58
59 #define __ _masm->
60
61 //------------------------------------------------------------------------------------------------------------------------
62
63 address TemplateInterpreterGenerator::generate_slow_signature_handler() {
64 address entry = __ pc();
65
66 // callee-save register for saving LR, shared with generate_native_entry
67 const Register Rsaved_ret_addr = AARCH64_ONLY(R21) NOT_AARCH64(Rtmp_save0);
68
69 __ mov(Rsaved_ret_addr, LR);
70
71 __ mov(R1, Rmethod);
72 __ mov(R2, Rlocals);
73 __ mov(R3, SP);
74
75 #ifdef AARCH64
76 // expand expr. stack and extended SP to avoid cutting SP in call_VM
77 __ mov(Rstack_top, SP);
78 __ str(Rstack_top, Address(FP, frame::interpreter_frame_extended_sp_offset * wordSize));
79 __ check_stack_top();
80
81 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::slow_signature_handler), R1, R2, R3, false);
82
83 __ ldp(ZR, c_rarg1, Address(SP, 2*wordSize, post_indexed));
84 __ ldp(c_rarg2, c_rarg3, Address(SP, 2*wordSize, post_indexed));
85 __ ldp(c_rarg4, c_rarg5, Address(SP, 2*wordSize, post_indexed));
86 __ ldp(c_rarg6, c_rarg7, Address(SP, 2*wordSize, post_indexed));
87
88 __ ldp_d(V0, V1, Address(SP, 2*wordSize, post_indexed));
89 __ ldp_d(V2, V3, Address(SP, 2*wordSize, post_indexed));
90 __ ldp_d(V4, V5, Address(SP, 2*wordSize, post_indexed));
91 __ ldp_d(V6, V7, Address(SP, 2*wordSize, post_indexed));
92 #else
93
94 // Safer to save R9 (when scratched) since callers may have been
95 // written assuming R9 survives. This is suboptimal but
96 // probably not important for this slow case call site.
97 // Note for R9 saving: slow_signature_handler may copy register
98 // arguments above the current SP (passed as R3). It is safe for
99 // call_VM to use push and pop to protect additional values on the
100 // stack if needed.
101 __ call_VM(CAST_FROM_FN_PTR(address, InterpreterRuntime::slow_signature_handler), true /* save R9 if needed*/);
102 __ add(SP, SP, wordSize); // Skip R0
103 __ pop(RegisterSet(R1, R3)); // Load arguments passed in registers
104 #ifdef __ABI_HARD__
105 // Few alternatives to an always-load-FP-registers approach:
106 // - parse method signature to detect FP arguments
107 // - keep a counter/flag on a stack indicationg number of FP arguments in the method.
108 // The later has been originally implemented and tested but a conditional path could
109 // eliminate any gain imposed by avoiding 8 double word loads.
110 __ fldmiad(SP, FloatRegisterSet(D0, 8), writeback);
111 #endif // __ABI_HARD__
112 #endif // AARCH64
113
114 __ ret(Rsaved_ret_addr);
115
116 return entry;
117 }
118
119
120 //
121 // Various method entries (that c++ and asm interpreter agree upon)
122 //------------------------------------------------------------------------------------------------------------------------
123 //
124 //
125
126 // Abstract method entry
127 // Attempt to execute abstract method. Throw exception
128 address TemplateInterpreterGenerator::generate_abstract_entry(void) {
129 address entry_point = __ pc();
130
131 #ifdef AARCH64
132 __ restore_sp_after_call(Rtemp);
133 __ restore_stack_top();
134 #endif
135
136 __ empty_expression_stack();
137
138 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
139
140 DEBUG_ONLY(STOP("generate_abstract_entry");) // Should not reach here
141 return entry_point;
142 }
143
144 address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) {
145 if (!InlineIntrinsics) return NULL; // Generate a vanilla entry
146
147 // TODO: ARM
148 return NULL;
149
150 address entry_point = __ pc();
151 STOP("generate_math_entry");
152 return entry_point;
153 }
154
155 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
156 address entry = __ pc();
157
158 // Note: There should be a minimal interpreter frame set up when stack
159 // overflow occurs since we check explicitly for it now.
160 //
161 #ifdef ASSERT
162 { Label L;
163 __ sub(Rtemp, FP, - frame::interpreter_frame_monitor_block_top_offset * wordSize);
164 __ cmp(SP, Rtemp); // Rtemp = maximal SP for current FP,
165 // (stack grows negative)
166 __ b(L, ls); // check if frame is complete
167 __ stop ("interpreter frame not set up");
168 __ bind(L);
169 }
170 #endif // ASSERT
171
172 // Restore bcp under the assumption that the current frame is still
173 // interpreted
174 __ restore_bcp();
175
176 // expression stack must be empty before entering the VM if an exception
177 // happened
178 __ empty_expression_stack();
179
180 // throw exception
181 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_StackOverflowError));
182
183 __ should_not_reach_here();
184
185 return entry;
186 }
187
188 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler(const char* name) {
189 address entry = __ pc();
190
191 // index is in R4_ArrayIndexOutOfBounds_index
192
193 InlinedString Lname(name);
194
195 // expression stack must be empty before entering the VM if an exception happened
196 __ empty_expression_stack();
197
198 // setup parameters
199 __ ldr_literal(R1, Lname);
200 __ mov(R2, R4_ArrayIndexOutOfBounds_index);
201
202 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_ArrayIndexOutOfBoundsException), R1, R2);
203
204 __ nop(); // to avoid filling CPU pipeline with invalid instructions
205 __ nop();
206 __ should_not_reach_here();
207 __ bind_literal(Lname);
208
209 return entry;
210 }
211
212 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
213 address entry = __ pc();
214
215 // object is in R2_ClassCastException_obj
216
217 // expression stack must be empty before entering the VM if an exception
218 // happened
219 __ empty_expression_stack();
220
221 __ mov(R1, R2_ClassCastException_obj);
222 __ call_VM(noreg,
223 CAST_FROM_FN_PTR(address,
224 InterpreterRuntime::throw_ClassCastException),
225 R1);
226
227 __ should_not_reach_here();
228
229 return entry;
230 }
231
232 address TemplateInterpreterGenerator::generate_exception_handler_common(const char* name, const char* message, bool pass_oop) {
233 assert(!pass_oop || message == NULL, "either oop or message but not both");
234 address entry = __ pc();
235
236 InlinedString Lname(name);
237 InlinedString Lmessage(message);
238
239 if (pass_oop) {
240 // object is at TOS
241 __ pop_ptr(R2);
242 }
243
244 // expression stack must be empty before entering the VM if an exception happened
245 __ empty_expression_stack();
246
247 // setup parameters
248 __ ldr_literal(R1, Lname);
249
250 if (pass_oop) {
251 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_klass_exception), R1, R2);
252 } else {
253 if (message != NULL) {
254 __ ldr_literal(R2, Lmessage);
255 } else {
256 __ mov(R2, 0);
257 }
258 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception), R1, R2);
259 }
260
261 // throw exception
262 __ b(Interpreter::throw_exception_entry());
263
264 __ nop(); // to avoid filling CPU pipeline with invalid instructions
265 __ nop();
266 __ bind_literal(Lname);
267 if (!pass_oop && (message != NULL)) {
268 __ bind_literal(Lmessage);
269 }
270
271 return entry;
272 }
273
274 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {
275 address entry = __ pc();
276
277 __ interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
278
279 #ifdef AARCH64
280 __ restore_sp_after_call(Rtemp); // Restore SP to extended SP
281 __ restore_stack_top();
282 #else
283 // Restore stack bottom in case i2c adjusted stack
284 __ ldr(SP, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
285 // and NULL it as marker that SP is now tos until next java call
286 __ mov(Rtemp, (int)NULL_WORD);
287 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
288 #endif // AARCH64
289
290 __ restore_method();
291 __ restore_bcp();
292 __ restore_dispatch();
293 __ restore_locals();
294
295 const Register Rcache = R2_tmp;
296 const Register Rindex = R3_tmp;
297 __ get_cache_and_index_at_bcp(Rcache, Rindex, 1, index_size);
298
299 __ add(Rtemp, Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord));
300 __ ldrb(Rtemp, Address(Rtemp, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));
301 __ check_stack_top();
302 __ add(Rstack_top, Rstack_top, AsmOperand(Rtemp, lsl, Interpreter::logStackElementSize));
303
304 #ifndef AARCH64
305 __ convert_retval_to_tos(state);
306 #endif // !AARCH64
307
308 __ check_and_handle_popframe();
309 __ check_and_handle_earlyret();
310
311 __ dispatch_next(state, step);
312
313 return entry;
314 }
315
316
317 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, int step) {
318 address entry = __ pc();
319
320 __ interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
321
322 #ifdef AARCH64
323 __ restore_sp_after_call(Rtemp); // Restore SP to extended SP
324 __ restore_stack_top();
325 #else
326 // The stack is not extended by deopt but we must NULL last_sp as this
327 // entry is like a "return".
328 __ mov(Rtemp, 0);
329 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
330 #endif // AARCH64
331
332 __ restore_method();
333 __ restore_bcp();
334 __ restore_dispatch();
335 __ restore_locals();
336
337 // handle exceptions
338 { Label L;
339 __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
340 __ cbz(Rtemp, L);
341 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));
342 __ should_not_reach_here();
343 __ bind(L);
344 }
345
346 __ dispatch_next(state, step);
347
348 return entry;
349 }
350
351 address TemplateInterpreterGenerator::generate_result_handler_for(BasicType type) {
352 #ifdef AARCH64
353 address entry = __ pc();
354 switch (type) {
355 case T_BOOLEAN:
356 __ tst(R0, 0xff);
357 __ cset(R0, ne);
358 break;
359 case T_CHAR : __ zero_extend(R0, R0, 16); break;
360 case T_BYTE : __ sign_extend(R0, R0, 8); break;
361 case T_SHORT : __ sign_extend(R0, R0, 16); break;
362 case T_INT : // fall through
363 case T_LONG : // fall through
364 case T_VOID : // fall through
365 case T_FLOAT : // fall through
366 case T_DOUBLE : /* nothing to do */ break;
367 case T_OBJECT :
368 // retrieve result from frame
369 __ ldr(R0, Address(FP, frame::interpreter_frame_oop_temp_offset * wordSize));
370 // and verify it
371 __ verify_oop(R0);
372 break;
373 default : ShouldNotReachHere();
374 }
375 __ ret();
376 return entry;
377 #else
378 // Result handlers are not used on 32-bit ARM
379 // since the returned value is already in appropriate format.
380 __ should_not_reach_here(); // to avoid empty code block
381
382 // The result handler non-zero indicates an object is returned and this is
383 // used in the native entry code.
384 return type == T_OBJECT ? (address)(-1) : NULL;
385 #endif // AARCH64
386 }
387
388 address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, address runtime_entry) {
389 address entry = __ pc();
390 __ push(state);
391 __ call_VM(noreg, runtime_entry);
392
393 // load current bytecode
394 __ ldrb(R3_bytecode, Address(Rbcp));
395 __ dispatch_only_normal(vtos);
396 return entry;
397 }
398
399
400 // Helpers for commoning out cases in the various type of method entries.
401 //
402
403 // increment invocation count & check for overflow
404 //
405 // Note: checking for negative value instead of overflow
406 // so we have a 'sticky' overflow test
407 //
408 // In: Rmethod.
409 //
410 // Uses R0, R1, Rtemp.
411 //
412 void TemplateInterpreterGenerator::generate_counter_incr(Label* overflow,
413 Label* profile_method,
414 Label* profile_method_continue) {
415 Label done;
416 const Register Rcounters = Rtemp;
417 const Address invocation_counter(Rcounters,
418 MethodCounters::invocation_counter_offset() +
419 InvocationCounter::counter_offset());
420
421 // Note: In tiered we increment either counters in MethodCounters* or
422 // in MDO depending if we're profiling or not.
423 if (TieredCompilation) {
424 int increment = InvocationCounter::count_increment;
425 Label no_mdo;
426 if (ProfileInterpreter) {
427 // Are we profiling?
428 __ ldr(R1_tmp, Address(Rmethod, Method::method_data_offset()));
429 __ cbz(R1_tmp, no_mdo);
430 // Increment counter in the MDO
431 const Address mdo_invocation_counter(R1_tmp,
432 in_bytes(MethodData::invocation_counter_offset()) +
433 in_bytes(InvocationCounter::counter_offset()));
434 const Address mask(R1_tmp, in_bytes(MethodData::invoke_mask_offset()));
435 __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, R0_tmp, Rtemp, eq, overflow);
436 __ b(done);
437 }
438 __ bind(no_mdo);
439 __ get_method_counters(Rmethod, Rcounters, done);
440 const Address mask(Rcounters, in_bytes(MethodCounters::invoke_mask_offset()));
441 __ increment_mask_and_jump(invocation_counter, increment, mask, R0_tmp, R1_tmp, eq, overflow);
442 __ bind(done);
443 } else { // not TieredCompilation
444 const Address backedge_counter(Rcounters,
445 MethodCounters::backedge_counter_offset() +
446 InvocationCounter::counter_offset());
447
448 const Register Ricnt = R0_tmp; // invocation counter
449 const Register Rbcnt = R1_tmp; // backedge counter
450
451 __ get_method_counters(Rmethod, Rcounters, done);
452
453 if (ProfileInterpreter) {
454 const Register Riic = R1_tmp;
455 __ ldr_s32(Riic, Address(Rcounters, MethodCounters::interpreter_invocation_counter_offset()));
456 __ add(Riic, Riic, 1);
457 __ str_32(Riic, Address(Rcounters, MethodCounters::interpreter_invocation_counter_offset()));
458 }
459
460 // Update standard invocation counters
461
462 __ ldr_u32(Ricnt, invocation_counter);
463 __ ldr_u32(Rbcnt, backedge_counter);
464
465 __ add(Ricnt, Ricnt, InvocationCounter::count_increment);
466
467 #ifdef AARCH64
468 __ andr(Rbcnt, Rbcnt, (unsigned int)InvocationCounter::count_mask_value); // mask out the status bits
469 #else
470 __ bic(Rbcnt, Rbcnt, ~InvocationCounter::count_mask_value); // mask out the status bits
471 #endif // AARCH64
472
473 __ str_32(Ricnt, invocation_counter); // save invocation count
474 __ add(Ricnt, Ricnt, Rbcnt); // add both counters
475
476 // profile_method is non-null only for interpreted method so
477 // profile_method != NULL == !native_call
478 // BytecodeInterpreter only calls for native so code is elided.
479
480 if (ProfileInterpreter && profile_method != NULL) {
481 assert(profile_method_continue != NULL, "should be non-null");
482
483 // Test to see if we should create a method data oop
484 // Reuse R1_tmp as we don't need backedge counters anymore.
485 Address profile_limit(Rcounters, in_bytes(MethodCounters::interpreter_profile_limit_offset()));
486 __ ldr_s32(R1_tmp, profile_limit);
487 __ cmp_32(Ricnt, R1_tmp);
488 __ b(*profile_method_continue, lt);
489
490 // if no method data exists, go to profile_method
491 __ test_method_data_pointer(R1_tmp, *profile_method);
492 }
493
494 Address invoke_limit(Rcounters, in_bytes(MethodCounters::interpreter_invocation_limit_offset()));
495 __ ldr_s32(R1_tmp, invoke_limit);
496 __ cmp_32(Ricnt, R1_tmp);
497 __ b(*overflow, hs);
498 __ bind(done);
499 }
500 }
501
502 void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) {
503 // InterpreterRuntime::frequency_counter_overflow takes one argument
504 // indicating if the counter overflow occurs at a backwards branch (non-NULL bcp).
505 // The call returns the address of the verified entry point for the method or NULL
506 // if the compilation did not complete (either went background or bailed out).
507 __ mov(R1, (int)false);
508 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), R1);
509
510 // jump to the interpreted entry.
511 __ b(do_continue);
512 }
513
514 void TemplateInterpreterGenerator::generate_stack_overflow_check(void) {
515 // Check if we've got enough room on the stack for
516 // - overhead;
517 // - locals;
518 // - expression stack.
519 //
520 // Registers on entry:
521 //
522 // R3 = number of additional locals
523 // R11 = max expression stack slots (AArch64 only)
524 // Rthread
525 // Rmethod
526 // Registers used: R0, R1, R2, Rtemp.
527
528 const Register Radditional_locals = R3;
529 const Register RmaxStack = AARCH64_ONLY(R11) NOT_AARCH64(R2);
530
531 // monitor entry size
532 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
533
534 // total overhead size: entry_size + (saved registers, thru expr stack bottom).
535 // be sure to change this if you add/subtract anything to/from the overhead area
536 const int overhead_size = (frame::sender_sp_offset - frame::interpreter_frame_initial_sp_offset)*wordSize + entry_size;
537
538 // Pages reserved for VM runtime calls and subsequent Java calls.
539 const int reserved_pages = JavaThread::stack_shadow_zone_size();
540
541 // Thread::stack_size() includes guard pages, and they should not be touched.
542 const int guard_pages = JavaThread::stack_guard_zone_size();
543
544 __ ldr(R0, Address(Rthread, Thread::stack_base_offset()));
545 __ ldr(R1, Address(Rthread, Thread::stack_size_offset()));
546 #ifndef AARCH64
547 __ ldr(Rtemp, Address(Rmethod, Method::const_offset()));
548 __ ldrh(RmaxStack, Address(Rtemp, ConstMethod::max_stack_offset()));
549 #endif // !AARCH64
550 __ sub_slow(Rtemp, SP, overhead_size + reserved_pages + guard_pages + Method::extra_stack_words());
551
552 // reserve space for additional locals
553 __ sub(Rtemp, Rtemp, AsmOperand(Radditional_locals, lsl, Interpreter::logStackElementSize));
554
555 // stack size
556 __ sub(R0, R0, R1);
557
558 // reserve space for expression stack
559 __ sub(Rtemp, Rtemp, AsmOperand(RmaxStack, lsl, Interpreter::logStackElementSize));
560
561 __ cmp(Rtemp, R0);
562
563 #ifdef AARCH64
564 Label L;
565 __ b(L, hi);
566 __ mov(SP, Rsender_sp); // restore SP
567 __ b(StubRoutines::throw_StackOverflowError_entry());
568 __ bind(L);
569 #else
570 __ mov(SP, Rsender_sp, ls); // restore SP
571 __ b(StubRoutines::throw_StackOverflowError_entry(), ls);
572 #endif // AARCH64
573 }
574
575
576 // Allocate monitor and lock method (asm interpreter)
577 //
578 void TemplateInterpreterGenerator::lock_method() {
579 // synchronize method
580
581 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
582 assert ((entry_size % StackAlignmentInBytes) == 0, "should keep stack alignment");
583
584 #ifdef ASSERT
585 { Label L;
586 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
587 __ tbnz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
588 __ stop("method doesn't need synchronization");
589 __ bind(L);
590 }
591 #endif // ASSERT
592
593 // get synchronization object
594 { Label done;
595 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
596 #ifdef AARCH64
597 __ ldr(R0, Address(Rlocals, Interpreter::local_offset_in_bytes(0))); // get receiver (assume this is frequent case)
598 __ tbz(Rtemp, JVM_ACC_STATIC_BIT, done);
599 #else
600 __ tst(Rtemp, JVM_ACC_STATIC);
601 __ ldr(R0, Address(Rlocals, Interpreter::local_offset_in_bytes(0)), eq); // get receiver (assume this is frequent case)
602 __ b(done, eq);
603 #endif // AARCH64
604 __ load_mirror(R0, Rmethod, Rtemp);
605 __ bind(done);
606 }
607
608 // add space for monitor & lock
609
610 #ifdef AARCH64
611 __ check_extended_sp(Rtemp);
612 __ sub(SP, SP, entry_size); // adjust extended SP
613 __ mov(Rtemp, SP);
614 __ str(Rtemp, Address(FP, frame::interpreter_frame_extended_sp_offset * wordSize));
615 #endif // AARCH64
616
617 __ sub(Rstack_top, Rstack_top, entry_size);
618 __ check_stack_top_on_expansion();
619 // add space for a monitor entry
620 __ str(Rstack_top, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
621 // set new monitor block top
622 __ str(R0, Address(Rstack_top, BasicObjectLock::obj_offset_in_bytes()));
623 // store object
624 __ mov(R1, Rstack_top); // monitor entry address
625 __ lock_object(R1);
626 }
627
628 #ifdef AARCH64
629
630 //
631 // Generate a fixed interpreter frame. This is identical setup for interpreted methods
632 // and for native methods hence the shared code.
633 //
634 // On entry:
635 // R10 = ConstMethod
636 // R11 = max expr. stack (in slots), if !native_call
637 //
638 // On exit:
639 // Rbcp, Rstack_top are initialized, SP is extended
640 //
641 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
642 // Incoming registers
643 const Register RconstMethod = R10;
644 const Register RmaxStack = R11;
645 // Temporary registers
646 const Register RextendedSP = R0;
647 const Register Rcache = R1;
648 const Register Rmdp = ProfileInterpreter ? R2 : ZR;
649
650 // Generates the following stack layout (stack grows up in this picture):
651 //
652 // [ expr. stack bottom ]
653 // [ saved Rbcp ]
654 // [ current Rlocals ]
655 // [ cache ]
656 // [ mdx ]
657 // [ mirror ]
658 // [ Method* ]
659 // [ extended SP ]
660 // [ expr. stack top ]
661 // [ sender_sp ]
662 // [ saved FP ] <--- FP
663 // [ saved LR ]
664
665 // initialize fixed part of activation frame
666 __ stp(FP, LR, Address(SP, -2*wordSize, pre_indexed));
667 __ mov(FP, SP); // establish new FP
668
669 // setup Rbcp
670 if (native_call) {
671 __ mov(Rbcp, ZR); // bcp = 0 for native calls
672 } else {
673 __ add(Rbcp, RconstMethod, in_bytes(ConstMethod::codes_offset())); // get codebase
674 }
675
676 // Rstack_top & RextendedSP
677 __ sub(Rstack_top, SP, 10*wordSize);
678 if (native_call) {
679 __ sub(RextendedSP, Rstack_top, align_up(wordSize, StackAlignmentInBytes)); // reserve 1 slot for exception handling
680 } else {
681 __ sub(RextendedSP, Rstack_top, AsmOperand(RmaxStack, lsl, Interpreter::logStackElementSize));
682 __ align_reg(RextendedSP, RextendedSP, StackAlignmentInBytes);
683 }
684 __ mov(SP, RextendedSP);
685 __ check_stack_top();
686
687 // Load Rmdp
688 if (ProfileInterpreter) {
689 __ ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
690 __ tst(Rtemp, Rtemp);
691 __ add(Rtemp, Rtemp, in_bytes(MethodData::data_offset()));
692 __ csel(Rmdp, ZR, Rtemp, eq);
693 }
694
695 // Load Rcache
696 __ ldr(Rtemp, Address(RconstMethod, ConstMethod::constants_offset()));
697 __ ldr(Rcache, Address(Rtemp, ConstantPool::cache_offset_in_bytes()));
698 // Get mirror and store it in the frame as GC root for this Method*
699 __ load_mirror(Rtemp, Rmethod, Rtemp);
700
701 // Build fixed frame
702 __ stp(Rstack_top, Rbcp, Address(FP, -10*wordSize));
703 __ stp(Rlocals, Rcache, Address(FP, -8*wordSize));
704 __ stp(Rmdp, Rtemp, Address(FP, -6*wordSize));
705 __ stp(Rmethod, RextendedSP, Address(FP, -4*wordSize));
706 __ stp(ZR, Rsender_sp, Address(FP, -2*wordSize));
707 assert(frame::interpreter_frame_initial_sp_offset == -10, "interpreter frame broken");
708 assert(frame::interpreter_frame_stack_top_offset == -2, "stack top broken");
709 }
710
711 #else // AARCH64
712
713 //
714 // Generate a fixed interpreter frame. This is identical setup for interpreted methods
715 // and for native methods hence the shared code.
716
717 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
718 // Generates the following stack layout:
719 //
720 // [ expr. stack bottom ]
721 // [ saved Rbcp ]
722 // [ current Rlocals ]
723 // [ cache ]
724 // [ mdx ]
725 // [ Method* ]
726 // [ last_sp ]
727 // [ sender_sp ]
728 // [ saved FP ] <--- FP
729 // [ saved LR ]
730
731 // initialize fixed part of activation frame
732 __ push(LR); // save return address
733 __ push(FP); // save FP
734 __ mov(FP, SP); // establish new FP
735
736 __ push(Rsender_sp);
737
738 __ mov(R0, 0);
739 __ push(R0); // leave last_sp as null
740
741 // setup Rbcp
742 if (native_call) {
743 __ mov(Rbcp, 0); // bcp = 0 for native calls
744 } else {
745 __ ldr(Rtemp, Address(Rmethod, Method::const_offset())); // get ConstMethod*
746 __ add(Rbcp, Rtemp, ConstMethod::codes_offset()); // get codebase
747 }
748
749 __ push(Rmethod); // save Method*
750 // Get mirror and store it in the frame as GC root for this Method*
751 __ load_mirror(Rtemp, Rmethod, Rtemp);
752 __ push(Rtemp);
753
754 if (ProfileInterpreter) {
755 __ ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
756 __ tst(Rtemp, Rtemp);
757 __ add(Rtemp, Rtemp, in_bytes(MethodData::data_offset()), ne);
758 __ push(Rtemp); // set the mdp (method data pointer)
759 } else {
760 __ push(R0);
761 }
762
763 __ ldr(Rtemp, Address(Rmethod, Method::const_offset()));
764 __ ldr(Rtemp, Address(Rtemp, ConstMethod::constants_offset()));
765 __ ldr(Rtemp, Address(Rtemp, ConstantPool::cache_offset_in_bytes()));
766 __ push(Rtemp); // set constant pool cache
767 __ push(Rlocals); // set locals pointer
768 __ push(Rbcp); // set bcp
769 __ push(R0); // reserve word for pointer to expression stack bottom
770 __ str(SP, Address(SP, 0)); // set expression stack bottom
771 }
772
773 #endif // AARCH64
774
775 // End of helpers
776
777 //------------------------------------------------------------------------------------------------------------------------
778 // Entry points
779 //
780 // Here we generate the various kind of entries into the interpreter.
781 // The two main entry type are generic bytecode methods and native call method.
782 // These both come in synchronized and non-synchronized versions but the
783 // frame layout they create is very similar. The other method entry
784 // types are really just special purpose entries that are really entry
785 // and interpretation all in one. These are for trivial methods like
786 // accessor, empty, or special math methods.
787 //
788 // When control flow reaches any of the entry types for the interpreter
789 // the following holds ->
790 //
791 // Arguments:
792 //
793 // Rmethod: Method*
794 // Rthread: thread
795 // Rsender_sp: sender sp
796 // Rparams (SP on 32-bit ARM): pointer to method parameters
797 //
798 // LR: return address
799 //
800 // Stack layout immediately at entry
801 //
802 // [ optional padding(*)] <--- SP (AArch64)
803 // [ parameter n ] <--- Rparams (SP on 32-bit ARM)
804 // ...
805 // [ parameter 1 ]
806 // [ expression stack ] (caller's java expression stack)
807
808 // Assuming that we don't go to one of the trivial specialized
809 // entries the stack will look like below when we are ready to execute
810 // the first bytecode (or call the native routine). The register usage
811 // will be as the template based interpreter expects.
812 //
813 // local variables follow incoming parameters immediately; i.e.
814 // the return address is saved at the end of the locals.
815 //
816 // [ reserved stack (*) ] <--- SP (AArch64)
817 // [ expr. stack ] <--- Rstack_top (SP on 32-bit ARM)
818 // [ monitor entry ]
819 // ...
820 // [ monitor entry ]
821 // [ expr. stack bottom ]
822 // [ saved Rbcp ]
823 // [ current Rlocals ]
824 // [ cache ]
825 // [ mdx ]
826 // [ mirror ]
827 // [ Method* ]
828 //
829 // 32-bit ARM:
830 // [ last_sp ]
831 //
832 // AArch64:
833 // [ extended SP (*) ]
834 // [ stack top (*) ]
835 //
836 // [ sender_sp ]
837 // [ saved FP ] <--- FP
838 // [ saved LR ]
839 // [ optional padding(*)]
840 // [ local variable m ]
841 // ...
842 // [ local variable 1 ]
843 // [ parameter n ]
844 // ...
845 // [ parameter 1 ] <--- Rlocals
846 //
847 // (*) - AArch64 only
848 //
849
850 address TemplateInterpreterGenerator::generate_Reference_get_entry(void) {
851 #if INCLUDE_ALL_GCS
852 if (UseG1GC) {
853 // Code: _aload_0, _getfield, _areturn
854 // parameter size = 1
855 //
856 // The code that gets generated by this routine is split into 2 parts:
857 // 1. The "intrinsified" code for G1 (or any SATB based GC),
858 // 2. The slow path - which is an expansion of the regular method entry.
859 //
860 // Notes:-
861 // * In the G1 code we do not check whether we need to block for
862 // a safepoint. If G1 is enabled then we must execute the specialized
863 // code for Reference.get (except when the Reference object is null)
864 // so that we can log the value in the referent field with an SATB
865 // update buffer.
866 // If the code for the getfield template is modified so that the
867 // G1 pre-barrier code is executed when the current method is
868 // Reference.get() then going through the normal method entry
869 // will be fine.
870 // * The G1 code can, however, check the receiver object (the instance
871 // of java.lang.Reference) and jump to the slow path if null. If the
872 // Reference object is null then we obviously cannot fetch the referent
873 // and so we don't need to call the G1 pre-barrier. Thus we can use the
874 // regular method entry code to generate the NPE.
875 //
876 // This code is based on generate_accessor_enty.
877 //
878 // Rmethod: Method*
879 // Rthread: thread
880 // Rsender_sp: sender sp, must be preserved for slow path, set SP to it on fast path
881 // Rparams: parameters
882
883 address entry = __ pc();
884 Label slow_path;
885 const Register Rthis = R0;
886 const Register Rret_addr = Rtmp_save1;
887 assert_different_registers(Rthis, Rret_addr, Rsender_sp);
888
889 const int referent_offset = java_lang_ref_Reference::referent_offset;
890 guarantee(referent_offset > 0, "referent offset not initialized");
891
892 // Check if local 0 != NULL
893 // If the receiver is null then it is OK to jump to the slow path.
894 __ ldr(Rthis, Address(Rparams));
895 __ cbz(Rthis, slow_path);
896
897 // Generate the G1 pre-barrier code to log the value of
898 // the referent field in an SATB buffer.
899
900 // Load the value of the referent field.
901 __ load_heap_oop(R0, Address(Rthis, referent_offset));
902
903 // Preserve LR
904 __ mov(Rret_addr, LR);
905
906 __ g1_write_barrier_pre(noreg, // store_addr
907 noreg, // new_val
908 R0, // pre_val
909 Rtemp, // tmp1
910 R1_tmp); // tmp2
911
912 // _areturn
913 __ mov(SP, Rsender_sp);
914 __ ret(Rret_addr);
915
916 // generate a vanilla interpreter entry as the slow path
917 __ bind(slow_path);
918 __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals));
919 return entry;
920 }
921 #endif // INCLUDE_ALL_GCS
922
923 // If G1 is not enabled then attempt to go through the normal entry point
924 return NULL;
925 }
926
927 // Not supported
928 address TemplateInterpreterGenerator::generate_CRC32_update_entry() { return NULL; }
929 address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) { return NULL; }
930 address TemplateInterpreterGenerator::generate_CRC32C_updateBytes_entry(AbstractInterpreter::MethodKind kind) { return NULL; }
931
932 //
933 // Interpreter stub for calling a native method. (asm interpreter)
934 // This sets up a somewhat different looking stack for calling the native method
935 // than the typical interpreter frame setup.
936 //
937
938 address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
939 // determine code generation flags
940 bool inc_counter = UseCompiler || CountCompiledCalls || LogTouchedMethods;
941
942 // Incoming registers:
943 //
944 // Rmethod: Method*
945 // Rthread: thread
946 // Rsender_sp: sender sp
947 // Rparams: parameters
948
949 address entry_point = __ pc();
950
951 // Register allocation
952 const Register Rsize_of_params = AARCH64_ONLY(R20) NOT_AARCH64(R6);
953 const Register Rsig_handler = AARCH64_ONLY(R21) NOT_AARCH64(Rtmp_save0 /* R4 */);
954 const Register Rnative_code = AARCH64_ONLY(R22) NOT_AARCH64(Rtmp_save1 /* R5 */);
955 const Register Rresult_handler = AARCH64_ONLY(Rsig_handler) NOT_AARCH64(R6);
956
957 #ifdef AARCH64
958 const Register RconstMethod = R10; // also used in generate_fixed_frame (should match)
959 const Register Rsaved_result = Rnative_code;
960 const FloatRegister Dsaved_result = V8;
961 #else
962 const Register Rsaved_result_lo = Rtmp_save0; // R4
963 const Register Rsaved_result_hi = Rtmp_save1; // R5
964 FloatRegister saved_result_fp;
965 #endif // AARCH64
966
967
968 #ifdef AARCH64
969 __ ldr(RconstMethod, Address(Rmethod, Method::const_offset()));
970 __ ldrh(Rsize_of_params, Address(RconstMethod, ConstMethod::size_of_parameters_offset()));
971 #else
972 __ ldr(Rsize_of_params, Address(Rmethod, Method::const_offset()));
973 __ ldrh(Rsize_of_params, Address(Rsize_of_params, ConstMethod::size_of_parameters_offset()));
974 #endif // AARCH64
975
976 // native calls don't need the stack size check since they have no expression stack
977 // and the arguments are already on the stack and we only add a handful of words
978 // to the stack
979
980 // compute beginning of parameters (Rlocals)
981 __ sub(Rlocals, Rparams, wordSize);
982 __ add(Rlocals, Rlocals, AsmOperand(Rsize_of_params, lsl, Interpreter::logStackElementSize));
983
984 #ifdef AARCH64
985 int extra_stack_reserve = 2*wordSize; // extra space for oop_temp
986 if(__ can_post_interpreter_events()) {
987 // extra space for saved results
988 extra_stack_reserve += 2*wordSize;
989 }
990 // reserve extra stack space and nullify oop_temp slot
991 __ stp(ZR, ZR, Address(SP, -extra_stack_reserve, pre_indexed));
992 #else
993 // reserve stack space for oop_temp
994 __ mov(R0, 0);
995 __ push(R0);
996 #endif // AARCH64
997
998 generate_fixed_frame(true); // Note: R9 is now saved in the frame
999
1000 // make sure method is native & not abstract
1001 #ifdef ASSERT
1002 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1003 {
1004 Label L;
1005 __ tbnz(Rtemp, JVM_ACC_NATIVE_BIT, L);
1006 __ stop("tried to execute non-native method as native");
1007 __ bind(L);
1008 }
1009 { Label L;
1010 __ tbz(Rtemp, JVM_ACC_ABSTRACT_BIT, L);
1011 __ stop("tried to execute abstract method in interpreter");
1012 __ bind(L);
1013 }
1014 #endif
1015
1016 // increment invocation count & check for overflow
1017 Label invocation_counter_overflow;
1018 if (inc_counter) {
1019 if (synchronized) {
1020 // Avoid unlocking method's monitor in case of exception, as it has not
1021 // been locked yet.
1022 __ set_do_not_unlock_if_synchronized(true, Rtemp);
1023 }
1024 generate_counter_incr(&invocation_counter_overflow, NULL, NULL);
1025 }
1026
1027 Label continue_after_compile;
1028 __ bind(continue_after_compile);
1029
1030 if (inc_counter && synchronized) {
1031 __ set_do_not_unlock_if_synchronized(false, Rtemp);
1032 }
1033
1034 // check for synchronized methods
1035 // Must happen AFTER invocation_counter check and stack overflow check,
1036 // so method is not locked if overflows.
1037 //
1038 if (synchronized) {
1039 lock_method();
1040 } else {
1041 // no synchronization necessary
1042 #ifdef ASSERT
1043 { Label L;
1044 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1045 __ tbz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
1046 __ stop("method needs synchronization");
1047 __ bind(L);
1048 }
1049 #endif
1050 }
1051
1052 // start execution
1053 #ifdef ASSERT
1054 { Label L;
1055 __ ldr(Rtemp, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
1056 __ cmp(Rtemp, Rstack_top);
1057 __ b(L, eq);
1058 __ stop("broken stack frame setup in interpreter");
1059 __ bind(L);
1060 }
1061 #endif
1062 __ check_extended_sp(Rtemp);
1063
1064 // jvmti/dtrace support
1065 __ notify_method_entry();
1066 #if R9_IS_SCRATCHED
1067 __ restore_method();
1068 #endif
1069
1070 {
1071 Label L;
1072 __ ldr(Rsig_handler, Address(Rmethod, Method::signature_handler_offset()));
1073 __ cbnz(Rsig_handler, L);
1074 __ mov(R1, Rmethod);
1075 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), R1, true);
1076 __ ldr(Rsig_handler, Address(Rmethod, Method::signature_handler_offset()));
1077 __ bind(L);
1078 }
1079
1080 {
1081 Label L;
1082 __ ldr(Rnative_code, Address(Rmethod, Method::native_function_offset()));
1083 __ cbnz(Rnative_code, L);
1084 __ mov(R1, Rmethod);
1085 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), R1);
1086 __ ldr(Rnative_code, Address(Rmethod, Method::native_function_offset()));
1087 __ bind(L);
1088 }
1089
1090 // Allocate stack space for arguments
1091
1092 #ifdef AARCH64
1093 __ sub(Rtemp, SP, Rsize_of_params, ex_uxtw, LogBytesPerWord);
1094 __ align_reg(SP, Rtemp, StackAlignmentInBytes);
1095
1096 // Allocate more stack space to accomodate all arguments passed on GP and FP registers:
1097 // 8 * wordSize for GPRs
1098 // 8 * wordSize for FPRs
1099 int reg_arguments = align_up(8*wordSize + 8*wordSize, StackAlignmentInBytes);
1100 #else
1101
1102 // C functions need aligned stack
1103 __ bic(SP, SP, StackAlignmentInBytes - 1);
1104 // Multiply by BytesPerLong instead of BytesPerWord, because calling convention
1105 // may require empty slots due to long alignment, e.g. func(int, jlong, int, jlong)
1106 __ sub(SP, SP, AsmOperand(Rsize_of_params, lsl, LogBytesPerLong));
1107
1108 #ifdef __ABI_HARD__
1109 // Allocate more stack space to accomodate all GP as well as FP registers:
1110 // 4 * wordSize
1111 // 8 * BytesPerLong
1112 int reg_arguments = align_up((4*wordSize) + (8*BytesPerLong), StackAlignmentInBytes);
1113 #else
1114 // Reserve at least 4 words on the stack for loading
1115 // of parameters passed on registers (R0-R3).
1116 // See generate_slow_signature_handler().
1117 // It is also used for JNIEnv & class additional parameters.
1118 int reg_arguments = 4 * wordSize;
1119 #endif // __ABI_HARD__
1120 #endif // AARCH64
1121
1122 __ sub(SP, SP, reg_arguments);
1123
1124
1125 // Note: signature handler blows R4 (32-bit ARM) or R21 (AArch64) besides all scratch registers.
1126 // See AbstractInterpreterGenerator::generate_slow_signature_handler().
1127 __ call(Rsig_handler);
1128 #if R9_IS_SCRATCHED
1129 __ restore_method();
1130 #endif
1131 __ mov(Rresult_handler, R0);
1132
1133 // Pass JNIEnv and mirror for static methods
1134 {
1135 Label L;
1136 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1137 __ add(R0, Rthread, in_bytes(JavaThread::jni_environment_offset()));
1138 __ tbz(Rtemp, JVM_ACC_STATIC_BIT, L);
1139 __ load_mirror(Rtemp, Rmethod, Rtemp);
1140 __ add(R1, FP, frame::interpreter_frame_oop_temp_offset * wordSize);
1141 __ str(Rtemp, Address(R1, 0));
1142 __ bind(L);
1143 }
1144
1145 __ set_last_Java_frame(SP, FP, true, Rtemp);
1146
1147 // Changing state to _thread_in_native must be the last thing to do
1148 // before the jump to native code. At this moment stack must be
1149 // safepoint-safe and completely prepared for stack walking.
1150 #ifdef ASSERT
1151 {
1152 Label L;
1153 __ ldr_u32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1154 __ cmp_32(Rtemp, _thread_in_Java);
1155 __ b(L, eq);
1156 __ stop("invalid thread state");
1157 __ bind(L);
1158 }
1159 #endif
1160
1161 #ifdef AARCH64
1162 __ mov(Rtemp, _thread_in_native);
1163 __ add(Rtemp2, Rthread, in_bytes(JavaThread::thread_state_offset()));
1164 // STLR is used to force all preceding writes to be observed prior to thread state change
1165 __ stlr_w(Rtemp, Rtemp2);
1166 #else
1167 // Force all preceding writes to be observed prior to thread state change
1168 __ membar(MacroAssembler::StoreStore, Rtemp);
1169
1170 __ mov(Rtemp, _thread_in_native);
1171 __ str(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1172 #endif // AARCH64
1173
1174 __ call(Rnative_code);
1175 #if R9_IS_SCRATCHED
1176 __ restore_method();
1177 #endif
1178
1179 // Set FPSCR/FPCR to a known state
1180 if (AlwaysRestoreFPU) {
1181 __ restore_default_fp_mode();
1182 }
1183
1184 // Do safepoint check
1185 __ mov(Rtemp, _thread_in_native_trans);
1186 __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1187
1188 // Force this write out before the read below
1189 __ membar(MacroAssembler::StoreLoad, Rtemp);
1190
1191 __ ldr_global_s32(Rtemp, SafepointSynchronize::address_of_state());
1192
1193 // Protect the return value in the interleaved code: save it to callee-save registers.
1194 #ifdef AARCH64
1195 __ mov(Rsaved_result, R0);
1196 __ fmov_d(Dsaved_result, D0);
1197 #else
1198 __ mov(Rsaved_result_lo, R0);
1199 __ mov(Rsaved_result_hi, R1);
1200 #ifdef __ABI_HARD__
1201 // preserve native FP result in a callee-saved register
1202 saved_result_fp = D8;
1203 __ fcpyd(saved_result_fp, D0);
1204 #else
1205 saved_result_fp = fnoreg;
1206 #endif // __ABI_HARD__
1207 #endif // AARCH64
1208
1209 {
1210 __ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
1211 __ cmp(Rtemp, SafepointSynchronize::_not_synchronized);
1212 __ cond_cmp(R3, 0, eq);
1213
1214 #ifdef AARCH64
1215 Label L;
1216 __ b(L, eq);
1217 __ mov(R0, Rthread);
1218 __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans), relocInfo::none);
1219 __ bind(L);
1220 #else
1221 __ mov(R0, Rthread, ne);
1222 __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans), relocInfo::none, ne);
1223 #if R9_IS_SCRATCHED
1224 __ restore_method();
1225 #endif
1226 #endif // AARCH64
1227 }
1228
1229 // Perform Native->Java thread transition
1230 __ mov(Rtemp, _thread_in_Java);
1231 __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1232
1233 // Zero handles and last_java_sp
1234 __ reset_last_Java_frame(Rtemp);
1235 __ ldr(R3, Address(Rthread, JavaThread::active_handles_offset()));
1236 __ str_32(__ zero_register(Rtemp), Address(R3, JNIHandleBlock::top_offset_in_bytes()));
1237 if (CheckJNICalls) {
1238 __ str(__ zero_register(Rtemp), Address(Rthread, JavaThread::pending_jni_exception_check_fn_offset()));
1239 }
1240
1241 // Unbox oop result, e.g. JNIHandles::resolve result if it's an oop.
1242 {
1243 Label Lnot_oop;
1244 #ifdef AARCH64
1245 __ mov_slow(Rtemp, AbstractInterpreter::result_handler(T_OBJECT));
1246 __ cmp(Rresult_handler, Rtemp);
1247 __ b(Lnot_oop, ne);
1248 #else // !AARCH64
1249 // For ARM32, Rresult_handler is -1 for oop result, 0 otherwise.
1250 __ cbz(Rresult_handler, Lnot_oop);
1251 #endif // !AARCH64
1252 Register value = AARCH64_ONLY(Rsaved_result) NOT_AARCH64(Rsaved_result_lo);
1253 __ resolve_jobject(value, // value
1254 Rtemp, // tmp1
1255 R1_tmp); // tmp2
1256 // Store resolved result in frame for GC visibility.
1257 __ str(value, Address(FP, frame::interpreter_frame_oop_temp_offset * wordSize));
1258 __ bind(Lnot_oop);
1259 }
1260
1261 #ifdef AARCH64
1262 // Restore SP (drop native parameters area), to keep SP in sync with extended_sp in frame
1263 __ restore_sp_after_call(Rtemp);
1264 __ check_stack_top();
1265 #endif // AARCH64
1266
1267 // reguard stack if StackOverflow exception happened while in native.
1268 {
1269 __ ldr_u32(Rtemp, Address(Rthread, JavaThread::stack_guard_state_offset()));
1270 __ cmp_32(Rtemp, JavaThread::stack_guard_yellow_reserved_disabled);
1271 #ifdef AARCH64
1272 Label L;
1273 __ b(L, ne);
1274 __ call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages), relocInfo::none);
1275 __ bind(L);
1276 #else
1277 __ call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages), relocInfo::none, eq);
1278 #if R9_IS_SCRATCHED
1279 __ restore_method();
1280 #endif
1281 #endif // AARCH64
1282 }
1283
1284 // check pending exceptions
1285 {
1286 __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
1287 #ifdef AARCH64
1288 Label L;
1289 __ cbz(Rtemp, L);
1290 __ mov_pc_to(Rexception_pc);
1291 __ b(StubRoutines::forward_exception_entry());
1292 __ bind(L);
1293 #else
1294 __ cmp(Rtemp, 0);
1295 __ mov(Rexception_pc, PC, ne);
1296 __ b(StubRoutines::forward_exception_entry(), ne);
1297 #endif // AARCH64
1298 }
1299
1300 if (synchronized) {
1301 // address of first monitor
1302 __ sub(R1, FP, - (frame::interpreter_frame_monitor_block_bottom_offset - frame::interpreter_frame_monitor_size()) * wordSize);
1303 __ unlock_object(R1);
1304 }
1305
1306 // jvmti/dtrace support
1307 // Note: This must happen _after_ handling/throwing any exceptions since
1308 // the exception handler code notifies the runtime of method exits
1309 // too. If this happens before, method entry/exit notifications are
1310 // not properly paired (was bug - gri 11/22/99).
1311 #ifdef AARCH64
1312 __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI, true, Rsaved_result, noreg, Dsaved_result);
1313 #else
1314 __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI, true, Rsaved_result_lo, Rsaved_result_hi, saved_result_fp);
1315 #endif // AARCH64
1316
1317 // Restore the result. Oop result is restored from the stack.
1318 #ifdef AARCH64
1319 __ mov(R0, Rsaved_result);
1320 __ fmov_d(D0, Dsaved_result);
1321
1322 __ blr(Rresult_handler);
1323 #else
1324 __ cmp(Rresult_handler, 0);
1325 __ ldr(R0, Address(FP, frame::interpreter_frame_oop_temp_offset * wordSize), ne);
1326 __ mov(R0, Rsaved_result_lo, eq);
1327 __ mov(R1, Rsaved_result_hi);
1328
1329 #ifdef __ABI_HARD__
1330 // reload native FP result
1331 __ fcpyd(D0, D8);
1332 #endif // __ABI_HARD__
1333
1334 #ifdef ASSERT
1335 if (VerifyOops) {
1336 Label L;
1337 __ cmp(Rresult_handler, 0);
1338 __ b(L, eq);
1339 __ verify_oop(R0);
1340 __ bind(L);
1341 }
1342 #endif // ASSERT
1343 #endif // AARCH64
1344
1345 // Restore FP/LR, sender_sp and return
1346 #ifdef AARCH64
1347 __ ldr(Rtemp, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize));
1348 __ ldp(FP, LR, Address(FP));
1349 __ mov(SP, Rtemp);
1350 #else
1351 __ mov(Rtemp, FP);
1352 __ ldmia(FP, RegisterSet(FP) | RegisterSet(LR));
1353 __ ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize));
1354 #endif // AARCH64
1355
1356 __ ret();
1357
1358 if (inc_counter) {
1359 // Handle overflow of counter and compile method
1360 __ bind(invocation_counter_overflow);
1361 generate_counter_overflow(continue_after_compile);
1362 }
1363
1364 return entry_point;
1365 }
1366
1367 //
1368 // Generic interpreted method entry to (asm) interpreter
1369 //
1370 address TemplateInterpreterGenerator::generate_normal_entry(bool synchronized) {
1371 // determine code generation flags
1372 bool inc_counter = UseCompiler || CountCompiledCalls || LogTouchedMethods;
1373
1374 // Rmethod: Method*
1375 // Rthread: thread
1376 // Rsender_sp: sender sp (could differ from SP if we were called via c2i)
1377 // Rparams: pointer to the last parameter in the stack
1378
1379 address entry_point = __ pc();
1380
1381 const Register RconstMethod = AARCH64_ONLY(R10) NOT_AARCH64(R3);
1382
1383 #ifdef AARCH64
1384 const Register RmaxStack = R11;
1385 const Register RlocalsBase = R12;
1386 #endif // AARCH64
1387
1388 __ ldr(RconstMethod, Address(Rmethod, Method::const_offset()));
1389
1390 __ ldrh(R2, Address(RconstMethod, ConstMethod::size_of_parameters_offset()));
1391 __ ldrh(R3, Address(RconstMethod, ConstMethod::size_of_locals_offset()));
1392
1393 // setup Rlocals
1394 __ sub(Rlocals, Rparams, wordSize);
1395 __ add(Rlocals, Rlocals, AsmOperand(R2, lsl, Interpreter::logStackElementSize));
1396
1397 __ sub(R3, R3, R2); // number of additional locals
1398
1399 #ifdef AARCH64
1400 // setup RmaxStack
1401 __ ldrh(RmaxStack, Address(RconstMethod, ConstMethod::max_stack_offset()));
1402 // We have to add extra reserved slots to max_stack. There are 3 users of the extra slots,
1403 // none of which are at the same time, so we just need to make sure there is enough room
1404 // for the biggest user:
1405 // -reserved slot for exception handler
1406 // -reserved slots for JSR292. Method::extra_stack_entries() is the size.
1407 // -3 reserved slots so get_method_counters() can save some registers before call_VM().
1408 __ add(RmaxStack, RmaxStack, MAX2(3, Method::extra_stack_entries()));
1409 #endif // AARCH64
1410
1411 // see if we've got enough room on the stack for locals plus overhead.
1412 generate_stack_overflow_check();
1413
1414 #ifdef AARCH64
1415
1416 // allocate space for locals
1417 {
1418 __ sub(RlocalsBase, Rparams, AsmOperand(R3, lsl, Interpreter::logStackElementSize));
1419 __ align_reg(SP, RlocalsBase, StackAlignmentInBytes);
1420 }
1421
1422 // explicitly initialize locals
1423 {
1424 Label zero_loop, done;
1425 __ cbz(R3, done);
1426
1427 __ tbz(R3, 0, zero_loop);
1428 __ subs(R3, R3, 1);
1429 __ str(ZR, Address(RlocalsBase, wordSize, post_indexed));
1430 __ b(done, eq);
1431
1432 __ bind(zero_loop);
1433 __ subs(R3, R3, 2);
1434 __ stp(ZR, ZR, Address(RlocalsBase, 2*wordSize, post_indexed));
1435 __ b(zero_loop, ne);
1436
1437 __ bind(done);
1438 }
1439
1440 #else
1441 // allocate space for locals
1442 // explicitly initialize locals
1443
1444 // Loop is unrolled 4 times
1445 Label loop;
1446 __ mov(R0, 0);
1447 __ bind(loop);
1448
1449 // #1
1450 __ subs(R3, R3, 1);
1451 __ push(R0, ge);
1452
1453 // #2
1454 __ subs(R3, R3, 1, ge);
1455 __ push(R0, ge);
1456
1457 // #3
1458 __ subs(R3, R3, 1, ge);
1459 __ push(R0, ge);
1460
1461 // #4
1462 __ subs(R3, R3, 1, ge);
1463 __ push(R0, ge);
1464
1465 __ b(loop, gt);
1466 #endif // AARCH64
1467
1468 // initialize fixed part of activation frame
1469 generate_fixed_frame(false);
1470
1471 __ restore_dispatch();
1472
1473 // make sure method is not native & not abstract
1474 #ifdef ASSERT
1475 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1476 {
1477 Label L;
1478 __ tbz(Rtemp, JVM_ACC_NATIVE_BIT, L);
1479 __ stop("tried to execute native method as non-native");
1480 __ bind(L);
1481 }
1482 { Label L;
1483 __ tbz(Rtemp, JVM_ACC_ABSTRACT_BIT, L);
1484 __ stop("tried to execute abstract method in interpreter");
1485 __ bind(L);
1486 }
1487 #endif
1488
1489 // increment invocation count & check for overflow
1490 Label invocation_counter_overflow;
1491 Label profile_method;
1492 Label profile_method_continue;
1493 if (inc_counter) {
1494 if (synchronized) {
1495 // Avoid unlocking method's monitor in case of exception, as it has not
1496 // been locked yet.
1497 __ set_do_not_unlock_if_synchronized(true, Rtemp);
1498 }
1499 generate_counter_incr(&invocation_counter_overflow, &profile_method, &profile_method_continue);
1500 if (ProfileInterpreter) {
1501 __ bind(profile_method_continue);
1502 }
1503 }
1504 Label continue_after_compile;
1505 __ bind(continue_after_compile);
1506
1507 if (inc_counter && synchronized) {
1508 __ set_do_not_unlock_if_synchronized(false, Rtemp);
1509 }
1510 #if R9_IS_SCRATCHED
1511 __ restore_method();
1512 #endif
1513
1514 // check for synchronized methods
1515 // Must happen AFTER invocation_counter check and stack overflow check,
1516 // so method is not locked if overflows.
1517 //
1518 if (synchronized) {
1519 // Allocate monitor and lock method
1520 lock_method();
1521 } else {
1522 // no synchronization necessary
1523 #ifdef ASSERT
1524 { Label L;
1525 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1526 __ tbz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
1527 __ stop("method needs synchronization");
1528 __ bind(L);
1529 }
1530 #endif
1531 }
1532
1533 // start execution
1534 #ifdef ASSERT
1535 { Label L;
1536 __ ldr(Rtemp, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
1537 __ cmp(Rtemp, Rstack_top);
1538 __ b(L, eq);
1539 __ stop("broken stack frame setup in interpreter");
1540 __ bind(L);
1541 }
1542 #endif
1543 __ check_extended_sp(Rtemp);
1544
1545 // jvmti support
1546 __ notify_method_entry();
1547 #if R9_IS_SCRATCHED
1548 __ restore_method();
1549 #endif
1550
1551 __ dispatch_next(vtos);
1552
1553 // invocation counter overflow
1554 if (inc_counter) {
1555 if (ProfileInterpreter) {
1556 // We have decided to profile this method in the interpreter
1557 __ bind(profile_method);
1558
1559 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1560 __ set_method_data_pointer_for_bcp();
1561
1562 __ b(profile_method_continue);
1563 }
1564
1565 // Handle overflow of counter and compile method
1566 __ bind(invocation_counter_overflow);
1567 generate_counter_overflow(continue_after_compile);
1568 }
1569
1570 return entry_point;
1571 }
1572
1573 //------------------------------------------------------------------------------------------------------------------------
1574 // Exceptions
1575
1576 void TemplateInterpreterGenerator::generate_throw_exception() {
1577 // Entry point in previous activation (i.e., if the caller was interpreted)
1578 Interpreter::_rethrow_exception_entry = __ pc();
1579 // Rexception_obj: exception
1580
1581 #ifndef AARCH64
1582 // Clear interpreter_frame_last_sp.
1583 __ mov(Rtemp, 0);
1584 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1585 #endif // !AARCH64
1586
1587 #if R9_IS_SCRATCHED
1588 __ restore_method();
1589 #endif
1590 __ restore_bcp();
1591 __ restore_dispatch();
1592 __ restore_locals();
1593
1594 #ifdef AARCH64
1595 __ restore_sp_after_call(Rtemp);
1596 #endif // AARCH64
1597
1598 // Entry point for exceptions thrown within interpreter code
1599 Interpreter::_throw_exception_entry = __ pc();
1600
1601 // expression stack is undefined here
1602 // Rexception_obj: exception
1603 // Rbcp: exception bcp
1604 __ verify_oop(Rexception_obj);
1605
1606 // expression stack must be empty before entering the VM in case of an exception
1607 __ empty_expression_stack();
1608 // find exception handler address and preserve exception oop
1609 __ mov(R1, Rexception_obj);
1610 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::exception_handler_for_exception), R1);
1611 // R0: exception handler entry point
1612 // Rexception_obj: preserved exception oop
1613 // Rbcp: bcp for exception handler
1614 __ push_ptr(Rexception_obj); // push exception which is now the only value on the stack
1615 __ jump(R0); // jump to exception handler (may be _remove_activation_entry!)
1616
1617 // If the exception is not handled in the current frame the frame is removed and
1618 // the exception is rethrown (i.e. exception continuation is _rethrow_exception).
1619 //
1620 // Note: At this point the bci is still the bxi for the instruction which caused
1621 // the exception and the expression stack is empty. Thus, for any VM calls
1622 // at this point, GC will find a legal oop map (with empty expression stack).
1623
1624 // In current activation
1625 // tos: exception
1626 // Rbcp: exception bcp
1627
1628 //
1629 // JVMTI PopFrame support
1630 //
1631 Interpreter::_remove_activation_preserving_args_entry = __ pc();
1632
1633 #ifdef AARCH64
1634 __ restore_sp_after_call(Rtemp); // restore SP to extended SP
1635 #endif // AARCH64
1636
1637 __ empty_expression_stack();
1638
1639 // Set the popframe_processing bit in _popframe_condition indicating that we are
1640 // currently handling popframe, so that call_VMs that may happen later do not trigger new
1641 // popframe handling cycles.
1642
1643 __ ldr_s32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1644 __ orr(Rtemp, Rtemp, (unsigned)JavaThread::popframe_processing_bit);
1645 __ str_32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1646
1647 {
1648 // Check to see whether we are returning to a deoptimized frame.
1649 // (The PopFrame call ensures that the caller of the popped frame is
1650 // either interpreted or compiled and deoptimizes it if compiled.)
1651 // In this case, we can't call dispatch_next() after the frame is
1652 // popped, but instead must save the incoming arguments and restore
1653 // them after deoptimization has occurred.
1654 //
1655 // Note that we don't compare the return PC against the
1656 // deoptimization blob's unpack entry because of the presence of
1657 // adapter frames in C2.
1658 Label caller_not_deoptimized;
1659 __ ldr(R0, Address(FP, frame::return_addr_offset * wordSize));
1660 __ call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::interpreter_contains), R0);
1661 __ cbnz_32(R0, caller_not_deoptimized);
1662 #ifdef AARCH64
1663 __ NOT_TESTED();
1664 #endif
1665
1666 // Compute size of arguments for saving when returning to deoptimized caller
1667 __ restore_method();
1668 __ ldr(R0, Address(Rmethod, Method::const_offset()));
1669 __ ldrh(R0, Address(R0, ConstMethod::size_of_parameters_offset()));
1670
1671 __ logical_shift_left(R1, R0, Interpreter::logStackElementSize);
1672 // Save these arguments
1673 __ restore_locals();
1674 __ sub(R2, Rlocals, R1);
1675 __ add(R2, R2, wordSize);
1676 __ mov(R0, Rthread);
1677 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::popframe_preserve_args), R0, R1, R2);
1678
1679 __ remove_activation(vtos, LR,
1680 /* throw_monitor_exception */ false,
1681 /* install_monitor_exception */ false,
1682 /* notify_jvmdi */ false);
1683
1684 // Inform deoptimization that it is responsible for restoring these arguments
1685 __ mov(Rtemp, JavaThread::popframe_force_deopt_reexecution_bit);
1686 __ str_32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1687
1688 // Continue in deoptimization handler
1689 __ ret();
1690
1691 __ bind(caller_not_deoptimized);
1692 }
1693
1694 __ remove_activation(vtos, R4,
1695 /* throw_monitor_exception */ false,
1696 /* install_monitor_exception */ false,
1697 /* notify_jvmdi */ false);
1698
1699 #ifndef AARCH64
1700 // Finish with popframe handling
1701 // A previous I2C followed by a deoptimization might have moved the
1702 // outgoing arguments further up the stack. PopFrame expects the
1703 // mutations to those outgoing arguments to be preserved and other
1704 // constraints basically require this frame to look exactly as
1705 // though it had previously invoked an interpreted activation with
1706 // no space between the top of the expression stack (current
1707 // last_sp) and the top of stack. Rather than force deopt to
1708 // maintain this kind of invariant all the time we call a small
1709 // fixup routine to move the mutated arguments onto the top of our
1710 // expression stack if necessary.
1711 __ mov(R1, SP);
1712 __ ldr(R2, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1713 // PC must point into interpreter here
1714 __ set_last_Java_frame(SP, FP, true, Rtemp);
1715 __ mov(R0, Rthread);
1716 __ call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), R0, R1, R2);
1717 __ reset_last_Java_frame(Rtemp);
1718 #endif // !AARCH64
1719
1720 #ifdef AARCH64
1721 __ restore_sp_after_call(Rtemp);
1722 __ restore_stack_top();
1723 #else
1724 // Restore the last_sp and null it out
1725 __ ldr(SP, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1726 __ mov(Rtemp, (int)NULL_WORD);
1727 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1728 #endif // AARCH64
1729
1730 __ restore_bcp();
1731 __ restore_dispatch();
1732 __ restore_locals();
1733 __ restore_method();
1734
1735 // The method data pointer was incremented already during
1736 // call profiling. We have to restore the mdp for the current bcp.
1737 if (ProfileInterpreter) {
1738 __ set_method_data_pointer_for_bcp();
1739 }
1740
1741 // Clear the popframe condition flag
1742 assert(JavaThread::popframe_inactive == 0, "adjust this code");
1743 __ str_32(__ zero_register(Rtemp), Address(Rthread, JavaThread::popframe_condition_offset()));
1744
1745 #if INCLUDE_JVMTI
1746 {
1747 Label L_done;
1748
1749 __ ldrb(Rtemp, Address(Rbcp, 0));
1750 __ cmp(Rtemp, Bytecodes::_invokestatic);
1751 __ b(L_done, ne);
1752
1753 // The member name argument must be restored if _invokestatic is re-executed after a PopFrame call.
1754 // Detect such a case in the InterpreterRuntime function and return the member name argument, or NULL.
1755
1756 // get local0
1757 __ ldr(R1, Address(Rlocals, 0));
1758 __ mov(R2, Rmethod);
1759 __ mov(R3, Rbcp);
1760 __ call_VM(R0, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null), R1, R2, R3);
1761
1762 __ cbz(R0, L_done);
1763
1764 __ str(R0, Address(Rstack_top));
1765 __ bind(L_done);
1766 }
1767 #endif // INCLUDE_JVMTI
1768
1769 __ dispatch_next(vtos);
1770 // end of PopFrame support
1771
1772 Interpreter::_remove_activation_entry = __ pc();
1773
1774 // preserve exception over this code sequence
1775 __ pop_ptr(R0_tos);
1776 __ str(R0_tos, Address(Rthread, JavaThread::vm_result_offset()));
1777 // remove the activation (without doing throws on illegalMonitorExceptions)
1778 __ remove_activation(vtos, Rexception_pc, false, true, false);
1779 // restore exception
1780 __ get_vm_result(Rexception_obj, Rtemp);
1781
1782 // Inbetween activations - previous activation type unknown yet
1783 // compute continuation point - the continuation point expects
1784 // the following registers set up:
1785 //
1786 // Rexception_obj: exception
1787 // Rexception_pc: return address/pc that threw exception
1788 // SP: expression stack of caller
1789 // FP: frame pointer of caller
1790 __ mov(c_rarg0, Rthread);
1791 __ mov(c_rarg1, Rexception_pc);
1792 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), c_rarg0, c_rarg1);
1793 // Note that an "issuing PC" is actually the next PC after the call
1794
1795 __ jump(R0); // jump to exception handler of caller
1796 }
1797
1798
1799 //
1800 // JVMTI ForceEarlyReturn support
1801 //
1802 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1803 address entry = __ pc();
1804
1805 #ifdef AARCH64
1806 __ restore_sp_after_call(Rtemp); // restore SP to extended SP
1807 #endif // AARCH64
1808
1809 __ restore_bcp();
1810 __ restore_dispatch();
1811 __ restore_locals();
1812
1813 __ empty_expression_stack();
1814
1815 __ load_earlyret_value(state);
1816
1817 // Clear the earlyret state
1818 __ ldr(Rtemp, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
1819
1820 assert(JvmtiThreadState::earlyret_inactive == 0, "adjust this code");
1821 __ str_32(__ zero_register(R2), Address(Rtemp, JvmtiThreadState::earlyret_state_offset()));
1822
1823 __ remove_activation(state, LR,
1824 false, /* throw_monitor_exception */
1825 false, /* install_monitor_exception */
1826 true); /* notify_jvmdi */
1827
1828 #ifndef AARCH64
1829 // According to interpreter calling conventions, result is returned in R0/R1,
1830 // so ftos (S0) and dtos (D0) are moved to R0/R1.
1831 // This conversion should be done after remove_activation, as it uses
1832 // push(state) & pop(state) to preserve return value.
1833 __ convert_tos_to_retval(state);
1834 #endif // !AARCH64
1835 __ ret();
1836
1837 return entry;
1838 } // end of ForceEarlyReturn support
1839
1840
1841 //------------------------------------------------------------------------------------------------------------------------
1842 // Helper for vtos entry point generation
1843
1844 void TemplateInterpreterGenerator::set_vtos_entry_points (Template* t, address& bep, address& cep, address& sep, address& aep, address& iep, address& lep, address& fep, address& dep, address& vep) {
1845 assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1846 Label L;
1847
1848 #ifdef __SOFTFP__
1849 dep = __ pc(); // fall through
1850 #else
1851 fep = __ pc(); __ push(ftos); __ b(L);
1852 dep = __ pc(); __ push(dtos); __ b(L);
1853 #endif // __SOFTFP__
1854
1855 lep = __ pc(); __ push(ltos); __ b(L);
1856
1857 if (AARCH64_ONLY(true) NOT_AARCH64(VerifyOops)) { // can't share atos entry with itos on AArch64 or if VerifyOops
1858 aep = __ pc(); __ push(atos); __ b(L);
1859 } else {
1860 aep = __ pc(); // fall through
1861 }
1862
1863 #ifdef __SOFTFP__
1864 fep = __ pc(); // fall through
1865 #endif // __SOFTFP__
1866
1867 bep = cep = sep = // fall through
1868 iep = __ pc(); __ push(itos); // fall through
1869 vep = __ pc(); __ bind(L); // fall through
1870 generate_and_dispatch(t);
1871 }
1872
1873 //------------------------------------------------------------------------------------------------------------------------
1874
1875 // Non-product code
1876 #ifndef PRODUCT
1877 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1878 address entry = __ pc();
1879
1880 // prepare expression stack
1881 __ push(state); // save tosca
1882
1883 // pass tosca registers as arguments
1884 __ mov(R2, R0_tos);
1885 #ifdef AARCH64
1886 __ mov(R3, ZR);
1887 #else
1888 __ mov(R3, R1_tos_hi);
1889 #endif // AARCH64
1890 __ mov(R1, LR); // save return address
1891
1892 // call tracer
1893 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::trace_bytecode), R1, R2, R3);
1894
1895 __ mov(LR, R0); // restore return address
1896 __ pop(state); // restore tosca
1897
1898 // return
1899 __ ret();
1900
1901 return entry;
1902 }
1903
1904
1905 void TemplateInterpreterGenerator::count_bytecode() {
1906 __ inc_global_counter((address) &BytecodeCounter::_counter_value, 0, Rtemp, R2_tmp, true);
1907 }
1908
1909
1910 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
1911 __ inc_global_counter((address)&BytecodeHistogram::_counters[0], sizeof(BytecodeHistogram::_counters[0]) * t->bytecode(), Rtemp, R2_tmp, true);
1912 }
1913
1914
1915 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
1916 const Register Rindex_addr = R2_tmp;
1917 Label Lcontinue;
1918 InlinedAddress Lcounters((address)BytecodePairHistogram::_counters);
1919 InlinedAddress Lindex((address)&BytecodePairHistogram::_index);
1920 const Register Rcounters_addr = R2_tmp;
1921 const Register Rindex = R4_tmp;
1922
1923 // calculate new index for counter:
1924 // index = (_index >> log2_number_of_codes) | (bytecode << log2_number_of_codes).
1925 // (_index >> log2_number_of_codes) is previous bytecode
1926
1927 __ ldr_literal(Rindex_addr, Lindex);
1928 __ ldr_s32(Rindex, Address(Rindex_addr));
1929 __ mov_slow(Rtemp, ((int)t->bytecode()) << BytecodePairHistogram::log2_number_of_codes);
1930 __ orr(Rindex, Rtemp, AsmOperand(Rindex, lsr, BytecodePairHistogram::log2_number_of_codes));
1931 __ str_32(Rindex, Address(Rindex_addr));
1932
1933 // Rindex (R4) contains index of counter
1934
1935 __ ldr_literal(Rcounters_addr, Lcounters);
1936 __ ldr_s32(Rtemp, Address::indexed_32(Rcounters_addr, Rindex));
1937 __ adds_32(Rtemp, Rtemp, 1);
1938 __ b(Lcontinue, mi); // avoid overflow
1939 __ str_32(Rtemp, Address::indexed_32(Rcounters_addr, Rindex));
1940
1941 __ b(Lcontinue);
1942
1943 __ bind_literal(Lindex);
1944 __ bind_literal(Lcounters);
1945
1946 __ bind(Lcontinue);
1947 }
1948
1949
1950 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
1951 // Call a little run-time stub to avoid blow-up for each bytecode.
1952 // The run-time runtime saves the right registers, depending on
1953 // the tosca in-state for the given template.
1954 assert(Interpreter::trace_code(t->tos_in()) != NULL,
1955 "entry must have been generated");
1956 address trace_entry = Interpreter::trace_code(t->tos_in());
1957 __ call(trace_entry, relocInfo::none);
1958 }
1959
1960
1961 void TemplateInterpreterGenerator::stop_interpreter_at() {
1962 Label Lcontinue;
1963 const Register stop_at = R2_tmp;
1964
1965 __ ldr_global_s32(Rtemp, (address) &BytecodeCounter::_counter_value);
1966 __ mov_slow(stop_at, StopInterpreterAt);
1967
1968 // test bytecode counter
1969 __ cmp(Rtemp, stop_at);
1970 __ b(Lcontinue, ne);
1971
1972 __ trace_state("stop_interpreter_at");
1973 __ breakpoint();
1974
1975 __ bind(Lcontinue);
1976 }
1977 #endif // !PRODUCT