1 /*
2 * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/assembler.hpp"
27 #include "interpreter/bytecodeHistogram.hpp"
28 #include "interpreter/interp_masm.hpp"
29 #include "interpreter/interpreter.hpp"
30 #include "interpreter/interpreterRuntime.hpp"
31 #include "interpreter/templateInterpreterGenerator.hpp"
32 #include "interpreter/templateTable.hpp"
33 #include "oops/arrayOop.hpp"
34 #include "oops/methodData.hpp"
35 #include "oops/method.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "prims/jvmtiExport.hpp"
38 #include "prims/jvmtiThreadState.hpp"
39 #include "prims/methodHandles.hpp"
40 #include "runtime/arguments.hpp"
41 #include "runtime/deoptimization.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "runtime/synchronizer.hpp"
46 #include "runtime/timer.hpp"
47 #include "runtime/vframeArray.hpp"
48 #include "utilities/align.hpp"
49 #include "utilities/debug.hpp"
50 #include "utilities/macros.hpp"
51
52 // Size of interpreter code. Increase if too small. Interpreter will
53 // fail with a guarantee ("not enough space for interpreter generation");
54 // if too small.
55 // Run with +PrintInterpreter to get the VM to print out the size.
56 // Max size with JVMTI
57 int TemplateInterpreter::InterpreterCodeSize = 180 * 1024;
58
59 #define __ _masm->
60
61 //------------------------------------------------------------------------------------------------------------------------
62
63 address TemplateInterpreterGenerator::generate_slow_signature_handler() {
64 address entry = __ pc();
65
66 // callee-save register for saving LR, shared with generate_native_entry
67 const Register Rsaved_ret_addr = AARCH64_ONLY(R21) NOT_AARCH64(Rtmp_save0);
68
69 __ mov(Rsaved_ret_addr, LR);
70
71 __ mov(R1, Rmethod);
72 __ mov(R2, Rlocals);
73 __ mov(R3, SP);
74
75 #ifdef AARCH64
76 // expand expr. stack and extended SP to avoid cutting SP in call_VM
77 __ mov(Rstack_top, SP);
78 __ str(Rstack_top, Address(FP, frame::interpreter_frame_extended_sp_offset * wordSize));
79 __ check_stack_top();
80
81 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::slow_signature_handler), R1, R2, R3, false);
82
83 __ ldp(ZR, c_rarg1, Address(SP, 2*wordSize, post_indexed));
84 __ ldp(c_rarg2, c_rarg3, Address(SP, 2*wordSize, post_indexed));
85 __ ldp(c_rarg4, c_rarg5, Address(SP, 2*wordSize, post_indexed));
86 __ ldp(c_rarg6, c_rarg7, Address(SP, 2*wordSize, post_indexed));
87
88 __ ldp_d(V0, V1, Address(SP, 2*wordSize, post_indexed));
89 __ ldp_d(V2, V3, Address(SP, 2*wordSize, post_indexed));
90 __ ldp_d(V4, V5, Address(SP, 2*wordSize, post_indexed));
91 __ ldp_d(V6, V7, Address(SP, 2*wordSize, post_indexed));
92 #else
93
94 // Safer to save R9 (when scratched) since callers may have been
95 // written assuming R9 survives. This is suboptimal but
96 // probably not important for this slow case call site.
97 // Note for R9 saving: slow_signature_handler may copy register
98 // arguments above the current SP (passed as R3). It is safe for
99 // call_VM to use push and pop to protect additional values on the
100 // stack if needed.
101 __ call_VM(CAST_FROM_FN_PTR(address, InterpreterRuntime::slow_signature_handler), true /* save R9 if needed*/);
102 __ add(SP, SP, wordSize); // Skip R0
103 __ pop(RegisterSet(R1, R3)); // Load arguments passed in registers
104 #ifdef __ABI_HARD__
105 // Few alternatives to an always-load-FP-registers approach:
106 // - parse method signature to detect FP arguments
107 // - keep a counter/flag on a stack indicationg number of FP arguments in the method.
108 // The later has been originally implemented and tested but a conditional path could
109 // eliminate any gain imposed by avoiding 8 double word loads.
110 __ fldmiad(SP, FloatRegisterSet(D0, 8), writeback);
111 #endif // __ABI_HARD__
112 #endif // AARCH64
113
114 __ ret(Rsaved_ret_addr);
115
116 return entry;
117 }
118
119
120 //
121 // Various method entries (that c++ and asm interpreter agree upon)
122 //------------------------------------------------------------------------------------------------------------------------
123 //
124 //
125
126 // Abstract method entry
127 // Attempt to execute abstract method. Throw exception
128 address TemplateInterpreterGenerator::generate_abstract_entry(void) {
129 address entry_point = __ pc();
130
131 #ifdef AARCH64
132 __ restore_sp_after_call(Rtemp);
133 __ restore_stack_top();
134 #endif
135
136 __ empty_expression_stack();
137
138 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
139
140 DEBUG_ONLY(STOP("generate_abstract_entry");) // Should not reach here
141 return entry_point;
142 }
143
144 address TemplateInterpreterGenerator::generate_math_entry(AbstractInterpreter::MethodKind kind) {
145 if (!InlineIntrinsics) return NULL; // Generate a vanilla entry
146
147 // TODO: ARM
148 return NULL;
149
150 address entry_point = __ pc();
151 STOP("generate_math_entry");
152 return entry_point;
153 }
154
155 address TemplateInterpreterGenerator::generate_StackOverflowError_handler() {
156 address entry = __ pc();
157
158 // Note: There should be a minimal interpreter frame set up when stack
159 // overflow occurs since we check explicitly for it now.
160 //
161 #ifdef ASSERT
162 { Label L;
163 __ sub(Rtemp, FP, - frame::interpreter_frame_monitor_block_top_offset * wordSize);
164 __ cmp(SP, Rtemp); // Rtemp = maximal SP for current FP,
165 // (stack grows negative)
166 __ b(L, ls); // check if frame is complete
167 __ stop ("interpreter frame not set up");
168 __ bind(L);
169 }
170 #endif // ASSERT
171
172 // Restore bcp under the assumption that the current frame is still
173 // interpreted
174 __ restore_bcp();
175
176 // expression stack must be empty before entering the VM if an exception
177 // happened
178 __ empty_expression_stack();
179
180 // throw exception
181 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_StackOverflowError));
182
183 __ should_not_reach_here();
184
185 return entry;
186 }
187
188 address TemplateInterpreterGenerator::generate_ArrayIndexOutOfBounds_handler() {
189 address entry = __ pc();
190
191 // index is in R4_ArrayIndexOutOfBounds_index
192
193 // expression stack must be empty before entering the VM if an exception happened
194 __ empty_expression_stack();
195
196 // setup parameters
197 // Array expected in R1.
198 __ mov(R2, R4_ArrayIndexOutOfBounds_index);
199
200 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_ArrayIndexOutOfBoundsException), R1, R2);
201
202 __ nop(); // to avoid filling CPU pipeline with invalid instructions
203 __ nop();
204 __ should_not_reach_here();
205 __ bind_literal(Lname);
206
207 return entry;
208 }
209
210 address TemplateInterpreterGenerator::generate_ClassCastException_handler() {
211 address entry = __ pc();
212
213 // object is in R2_ClassCastException_obj
214
215 // expression stack must be empty before entering the VM if an exception
216 // happened
217 __ empty_expression_stack();
218
219 __ mov(R1, R2_ClassCastException_obj);
220 __ call_VM(noreg,
221 CAST_FROM_FN_PTR(address,
222 InterpreterRuntime::throw_ClassCastException),
223 R1);
224
225 __ should_not_reach_here();
226
227 return entry;
228 }
229
230 address TemplateInterpreterGenerator::generate_exception_handler_common(const char* name, const char* message, bool pass_oop) {
231 assert(!pass_oop || message == NULL, "either oop or message but not both");
232 address entry = __ pc();
233
234 InlinedString Lname(name);
235 InlinedString Lmessage(message);
236
237 if (pass_oop) {
238 // object is at TOS
239 __ pop_ptr(R2);
240 }
241
242 // expression stack must be empty before entering the VM if an exception happened
243 __ empty_expression_stack();
244
245 // setup parameters
246 __ ldr_literal(R1, Lname);
247
248 if (pass_oop) {
249 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_klass_exception), R1, R2);
250 } else {
251 if (message != NULL) {
252 __ ldr_literal(R2, Lmessage);
253 } else {
254 __ mov(R2, 0);
255 }
256 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::create_exception), R1, R2);
257 }
258
259 // throw exception
260 __ b(Interpreter::throw_exception_entry());
261
262 __ nop(); // to avoid filling CPU pipeline with invalid instructions
263 __ nop();
264 __ bind_literal(Lname);
265 if (!pass_oop && (message != NULL)) {
266 __ bind_literal(Lmessage);
267 }
268
269 return entry;
270 }
271
272 address TemplateInterpreterGenerator::generate_return_entry_for(TosState state, int step, size_t index_size) {
273 address entry = __ pc();
274
275 __ interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
276
277 #ifdef AARCH64
278 __ restore_sp_after_call(Rtemp); // Restore SP to extended SP
279 __ restore_stack_top();
280 #else
281 // Restore stack bottom in case i2c adjusted stack
282 __ ldr(SP, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
283 // and NULL it as marker that SP is now tos until next java call
284 __ mov(Rtemp, (int)NULL_WORD);
285 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
286 #endif // AARCH64
287
288 __ restore_method();
289 __ restore_bcp();
290 __ restore_dispatch();
291 __ restore_locals();
292
293 const Register Rcache = R2_tmp;
294 const Register Rindex = R3_tmp;
295 __ get_cache_and_index_at_bcp(Rcache, Rindex, 1, index_size);
296
297 __ add(Rtemp, Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord));
298 __ ldrb(Rtemp, Address(Rtemp, ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::flags_offset()));
299 __ check_stack_top();
300 __ add(Rstack_top, Rstack_top, AsmOperand(Rtemp, lsl, Interpreter::logStackElementSize));
301
302 #ifndef AARCH64
303 __ convert_retval_to_tos(state);
304 #endif // !AARCH64
305
306 __ check_and_handle_popframe();
307 __ check_and_handle_earlyret();
308
309 __ dispatch_next(state, step);
310
311 return entry;
312 }
313
314
315 address TemplateInterpreterGenerator::generate_deopt_entry_for(TosState state, int step, address continuation) {
316 address entry = __ pc();
317
318 __ interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
319
320 #ifdef AARCH64
321 __ restore_sp_after_call(Rtemp); // Restore SP to extended SP
322 __ restore_stack_top();
323 #else
324 // The stack is not extended by deopt but we must NULL last_sp as this
325 // entry is like a "return".
326 __ mov(Rtemp, 0);
327 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
328 #endif // AARCH64
329
330 __ restore_method();
331 __ restore_bcp();
332 __ restore_dispatch();
333 __ restore_locals();
334
335 // handle exceptions
336 { Label L;
337 __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
338 __ cbz(Rtemp, L);
339 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_pending_exception));
340 __ should_not_reach_here();
341 __ bind(L);
342 }
343
344 if (continuation == NULL) {
345 __ dispatch_next(state, step);
346 } else {
347 __ jump_to_entry(continuation);
348 }
349
350 return entry;
351 }
352
353 address TemplateInterpreterGenerator::generate_result_handler_for(BasicType type) {
354 #ifdef AARCH64
355 address entry = __ pc();
356 switch (type) {
357 case T_BOOLEAN:
358 __ tst(R0, 0xff);
359 __ cset(R0, ne);
360 break;
361 case T_CHAR : __ zero_extend(R0, R0, 16); break;
362 case T_BYTE : __ sign_extend(R0, R0, 8); break;
363 case T_SHORT : __ sign_extend(R0, R0, 16); break;
364 case T_INT : // fall through
365 case T_LONG : // fall through
366 case T_VOID : // fall through
367 case T_FLOAT : // fall through
368 case T_DOUBLE : /* nothing to do */ break;
369 case T_OBJECT :
370 // retrieve result from frame
371 __ ldr(R0, Address(FP, frame::interpreter_frame_oop_temp_offset * wordSize));
372 // and verify it
373 __ verify_oop(R0);
374 break;
375 default : ShouldNotReachHere();
376 }
377 __ ret();
378 return entry;
379 #else
380 // Result handlers are not used on 32-bit ARM
381 // since the returned value is already in appropriate format.
382 __ should_not_reach_here(); // to avoid empty code block
383
384 // The result handler non-zero indicates an object is returned and this is
385 // used in the native entry code.
386 return type == T_OBJECT ? (address)(-1) : NULL;
387 #endif // AARCH64
388 }
389
390 address TemplateInterpreterGenerator::generate_safept_entry_for(TosState state, address runtime_entry) {
391 address entry = __ pc();
392 __ push(state);
393 __ call_VM(noreg, runtime_entry);
394
395 // load current bytecode
396 __ ldrb(R3_bytecode, Address(Rbcp));
397 __ dispatch_only_normal(vtos);
398 return entry;
399 }
400
401
402 // Helpers for commoning out cases in the various type of method entries.
403 //
404
405 // increment invocation count & check for overflow
406 //
407 // Note: checking for negative value instead of overflow
408 // so we have a 'sticky' overflow test
409 //
410 // In: Rmethod.
411 //
412 // Uses R0, R1, Rtemp.
413 //
414 void TemplateInterpreterGenerator::generate_counter_incr(Label* overflow,
415 Label* profile_method,
416 Label* profile_method_continue) {
417 Label done;
418 const Register Rcounters = Rtemp;
419 const Address invocation_counter(Rcounters,
420 MethodCounters::invocation_counter_offset() +
421 InvocationCounter::counter_offset());
422
423 // Note: In tiered we increment either counters in MethodCounters* or
424 // in MDO depending if we're profiling or not.
425 if (TieredCompilation) {
426 int increment = InvocationCounter::count_increment;
427 Label no_mdo;
428 if (ProfileInterpreter) {
429 // Are we profiling?
430 __ ldr(R1_tmp, Address(Rmethod, Method::method_data_offset()));
431 __ cbz(R1_tmp, no_mdo);
432 // Increment counter in the MDO
433 const Address mdo_invocation_counter(R1_tmp,
434 in_bytes(MethodData::invocation_counter_offset()) +
435 in_bytes(InvocationCounter::counter_offset()));
436 const Address mask(R1_tmp, in_bytes(MethodData::invoke_mask_offset()));
437 __ increment_mask_and_jump(mdo_invocation_counter, increment, mask, R0_tmp, Rtemp, eq, overflow);
438 __ b(done);
439 }
440 __ bind(no_mdo);
441 __ get_method_counters(Rmethod, Rcounters, done);
442 const Address mask(Rcounters, in_bytes(MethodCounters::invoke_mask_offset()));
443 __ increment_mask_and_jump(invocation_counter, increment, mask, R0_tmp, R1_tmp, eq, overflow);
444 __ bind(done);
445 } else { // not TieredCompilation
446 const Address backedge_counter(Rcounters,
447 MethodCounters::backedge_counter_offset() +
448 InvocationCounter::counter_offset());
449
450 const Register Ricnt = R0_tmp; // invocation counter
451 const Register Rbcnt = R1_tmp; // backedge counter
452
453 __ get_method_counters(Rmethod, Rcounters, done);
454
455 if (ProfileInterpreter) {
456 const Register Riic = R1_tmp;
457 __ ldr_s32(Riic, Address(Rcounters, MethodCounters::interpreter_invocation_counter_offset()));
458 __ add(Riic, Riic, 1);
459 __ str_32(Riic, Address(Rcounters, MethodCounters::interpreter_invocation_counter_offset()));
460 }
461
462 // Update standard invocation counters
463
464 __ ldr_u32(Ricnt, invocation_counter);
465 __ ldr_u32(Rbcnt, backedge_counter);
466
467 __ add(Ricnt, Ricnt, InvocationCounter::count_increment);
468
469 #ifdef AARCH64
470 __ andr(Rbcnt, Rbcnt, (unsigned int)InvocationCounter::count_mask_value); // mask out the status bits
471 #else
472 __ bic(Rbcnt, Rbcnt, ~InvocationCounter::count_mask_value); // mask out the status bits
473 #endif // AARCH64
474
475 __ str_32(Ricnt, invocation_counter); // save invocation count
476 __ add(Ricnt, Ricnt, Rbcnt); // add both counters
477
478 // profile_method is non-null only for interpreted method so
479 // profile_method != NULL == !native_call
480 // BytecodeInterpreter only calls for native so code is elided.
481
482 if (ProfileInterpreter && profile_method != NULL) {
483 assert(profile_method_continue != NULL, "should be non-null");
484
485 // Test to see if we should create a method data oop
486 // Reuse R1_tmp as we don't need backedge counters anymore.
487 Address profile_limit(Rcounters, in_bytes(MethodCounters::interpreter_profile_limit_offset()));
488 __ ldr_s32(R1_tmp, profile_limit);
489 __ cmp_32(Ricnt, R1_tmp);
490 __ b(*profile_method_continue, lt);
491
492 // if no method data exists, go to profile_method
493 __ test_method_data_pointer(R1_tmp, *profile_method);
494 }
495
496 Address invoke_limit(Rcounters, in_bytes(MethodCounters::interpreter_invocation_limit_offset()));
497 __ ldr_s32(R1_tmp, invoke_limit);
498 __ cmp_32(Ricnt, R1_tmp);
499 __ b(*overflow, hs);
500 __ bind(done);
501 }
502 }
503
504 void TemplateInterpreterGenerator::generate_counter_overflow(Label& do_continue) {
505 // InterpreterRuntime::frequency_counter_overflow takes one argument
506 // indicating if the counter overflow occurs at a backwards branch (non-NULL bcp).
507 // The call returns the address of the verified entry point for the method or NULL
508 // if the compilation did not complete (either went background or bailed out).
509 __ mov(R1, (int)false);
510 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::frequency_counter_overflow), R1);
511
512 // jump to the interpreted entry.
513 __ b(do_continue);
514 }
515
516 void TemplateInterpreterGenerator::generate_stack_overflow_check(void) {
517 // Check if we've got enough room on the stack for
518 // - overhead;
519 // - locals;
520 // - expression stack.
521 //
522 // Registers on entry:
523 //
524 // R3 = number of additional locals
525 // R11 = max expression stack slots (AArch64 only)
526 // Rthread
527 // Rmethod
528 // Registers used: R0, R1, R2, Rtemp.
529
530 const Register Radditional_locals = R3;
531 const Register RmaxStack = AARCH64_ONLY(R11) NOT_AARCH64(R2);
532
533 // monitor entry size
534 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
535
536 // total overhead size: entry_size + (saved registers, thru expr stack bottom).
537 // be sure to change this if you add/subtract anything to/from the overhead area
538 const int overhead_size = (frame::sender_sp_offset - frame::interpreter_frame_initial_sp_offset)*wordSize + entry_size;
539
540 // Pages reserved for VM runtime calls and subsequent Java calls.
541 const int reserved_pages = JavaThread::stack_shadow_zone_size();
542
543 // Thread::stack_size() includes guard pages, and they should not be touched.
544 const int guard_pages = JavaThread::stack_guard_zone_size();
545
546 __ ldr(R0, Address(Rthread, Thread::stack_base_offset()));
547 __ ldr(R1, Address(Rthread, Thread::stack_size_offset()));
548 #ifndef AARCH64
549 __ ldr(Rtemp, Address(Rmethod, Method::const_offset()));
550 __ ldrh(RmaxStack, Address(Rtemp, ConstMethod::max_stack_offset()));
551 #endif // !AARCH64
552 __ sub_slow(Rtemp, SP, overhead_size + reserved_pages + guard_pages + Method::extra_stack_words());
553
554 // reserve space for additional locals
555 __ sub(Rtemp, Rtemp, AsmOperand(Radditional_locals, lsl, Interpreter::logStackElementSize));
556
557 // stack size
558 __ sub(R0, R0, R1);
559
560 // reserve space for expression stack
561 __ sub(Rtemp, Rtemp, AsmOperand(RmaxStack, lsl, Interpreter::logStackElementSize));
562
563 __ cmp(Rtemp, R0);
564
565 #ifdef AARCH64
566 Label L;
567 __ b(L, hi);
568 __ mov(SP, Rsender_sp); // restore SP
569 __ b(StubRoutines::throw_StackOverflowError_entry());
570 __ bind(L);
571 #else
572 __ mov(SP, Rsender_sp, ls); // restore SP
573 __ b(StubRoutines::throw_StackOverflowError_entry(), ls);
574 #endif // AARCH64
575 }
576
577
578 // Allocate monitor and lock method (asm interpreter)
579 //
580 void TemplateInterpreterGenerator::lock_method() {
581 // synchronize method
582
583 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
584 assert ((entry_size % StackAlignmentInBytes) == 0, "should keep stack alignment");
585
586 #ifdef ASSERT
587 { Label L;
588 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
589 __ tbnz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
590 __ stop("method doesn't need synchronization");
591 __ bind(L);
592 }
593 #endif // ASSERT
594
595 // get synchronization object
596 { Label done;
597 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
598 #ifdef AARCH64
599 __ ldr(R0, Address(Rlocals, Interpreter::local_offset_in_bytes(0))); // get receiver (assume this is frequent case)
600 __ tbz(Rtemp, JVM_ACC_STATIC_BIT, done);
601 #else
602 __ tst(Rtemp, JVM_ACC_STATIC);
603 __ ldr(R0, Address(Rlocals, Interpreter::local_offset_in_bytes(0)), eq); // get receiver (assume this is frequent case)
604 __ b(done, eq);
605 #endif // AARCH64
606 __ load_mirror(R0, Rmethod, Rtemp);
607 __ bind(done);
608 }
609
610 // add space for monitor & lock
611
612 #ifdef AARCH64
613 __ check_extended_sp(Rtemp);
614 __ sub(SP, SP, entry_size); // adjust extended SP
615 __ mov(Rtemp, SP);
616 __ str(Rtemp, Address(FP, frame::interpreter_frame_extended_sp_offset * wordSize));
617 #endif // AARCH64
618
619 __ sub(Rstack_top, Rstack_top, entry_size);
620 __ check_stack_top_on_expansion();
621 // add space for a monitor entry
622 __ str(Rstack_top, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
623 // set new monitor block top
624 __ str(R0, Address(Rstack_top, BasicObjectLock::obj_offset_in_bytes()));
625 // store object
626 __ mov(R1, Rstack_top); // monitor entry address
627 __ lock_object(R1);
628 }
629
630 #ifdef AARCH64
631
632 //
633 // Generate a fixed interpreter frame. This is identical setup for interpreted methods
634 // and for native methods hence the shared code.
635 //
636 // On entry:
637 // R10 = ConstMethod
638 // R11 = max expr. stack (in slots), if !native_call
639 //
640 // On exit:
641 // Rbcp, Rstack_top are initialized, SP is extended
642 //
643 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
644 // Incoming registers
645 const Register RconstMethod = R10;
646 const Register RmaxStack = R11;
647 // Temporary registers
648 const Register RextendedSP = R0;
649 const Register Rcache = R1;
650 const Register Rmdp = ProfileInterpreter ? R2 : ZR;
651
652 // Generates the following stack layout (stack grows up in this picture):
653 //
654 // [ expr. stack bottom ]
655 // [ saved Rbcp ]
656 // [ current Rlocals ]
657 // [ cache ]
658 // [ mdx ]
659 // [ mirror ]
660 // [ Method* ]
661 // [ extended SP ]
662 // [ expr. stack top ]
663 // [ sender_sp ]
664 // [ saved FP ] <--- FP
665 // [ saved LR ]
666
667 // initialize fixed part of activation frame
668 __ stp(FP, LR, Address(SP, -2*wordSize, pre_indexed));
669 __ mov(FP, SP); // establish new FP
670
671 // setup Rbcp
672 if (native_call) {
673 __ mov(Rbcp, ZR); // bcp = 0 for native calls
674 } else {
675 __ add(Rbcp, RconstMethod, in_bytes(ConstMethod::codes_offset())); // get codebase
676 }
677
678 // Rstack_top & RextendedSP
679 __ sub(Rstack_top, SP, 10*wordSize);
680 if (native_call) {
681 __ sub(RextendedSP, Rstack_top, align_up(wordSize, StackAlignmentInBytes)); // reserve 1 slot for exception handling
682 } else {
683 __ sub(RextendedSP, Rstack_top, AsmOperand(RmaxStack, lsl, Interpreter::logStackElementSize));
684 __ align_reg(RextendedSP, RextendedSP, StackAlignmentInBytes);
685 }
686 __ mov(SP, RextendedSP);
687 __ check_stack_top();
688
689 // Load Rmdp
690 if (ProfileInterpreter) {
691 __ ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
692 __ tst(Rtemp, Rtemp);
693 __ add(Rtemp, Rtemp, in_bytes(MethodData::data_offset()));
694 __ csel(Rmdp, ZR, Rtemp, eq);
695 }
696
697 // Load Rcache
698 __ ldr(Rtemp, Address(RconstMethod, ConstMethod::constants_offset()));
699 __ ldr(Rcache, Address(Rtemp, ConstantPool::cache_offset_in_bytes()));
700 // Get mirror and store it in the frame as GC root for this Method*
701 __ load_mirror(Rtemp, Rmethod, Rtemp);
702
703 // Build fixed frame
704 __ stp(Rstack_top, Rbcp, Address(FP, -10*wordSize));
705 __ stp(Rlocals, Rcache, Address(FP, -8*wordSize));
706 __ stp(Rmdp, Rtemp, Address(FP, -6*wordSize));
707 __ stp(Rmethod, RextendedSP, Address(FP, -4*wordSize));
708 __ stp(ZR, Rsender_sp, Address(FP, -2*wordSize));
709 assert(frame::interpreter_frame_initial_sp_offset == -10, "interpreter frame broken");
710 assert(frame::interpreter_frame_stack_top_offset == -2, "stack top broken");
711 }
712
713 #else // AARCH64
714
715 //
716 // Generate a fixed interpreter frame. This is identical setup for interpreted methods
717 // and for native methods hence the shared code.
718
719 void TemplateInterpreterGenerator::generate_fixed_frame(bool native_call) {
720 // Generates the following stack layout:
721 //
722 // [ expr. stack bottom ]
723 // [ saved Rbcp ]
724 // [ current Rlocals ]
725 // [ cache ]
726 // [ mdx ]
727 // [ Method* ]
728 // [ last_sp ]
729 // [ sender_sp ]
730 // [ saved FP ] <--- FP
731 // [ saved LR ]
732
733 // initialize fixed part of activation frame
734 __ push(LR); // save return address
735 __ push(FP); // save FP
736 __ mov(FP, SP); // establish new FP
737
738 __ push(Rsender_sp);
739
740 __ mov(R0, 0);
741 __ push(R0); // leave last_sp as null
742
743 // setup Rbcp
744 if (native_call) {
745 __ mov(Rbcp, 0); // bcp = 0 for native calls
746 } else {
747 __ ldr(Rtemp, Address(Rmethod, Method::const_offset())); // get ConstMethod*
748 __ add(Rbcp, Rtemp, ConstMethod::codes_offset()); // get codebase
749 }
750
751 __ push(Rmethod); // save Method*
752 // Get mirror and store it in the frame as GC root for this Method*
753 __ load_mirror(Rtemp, Rmethod, Rtemp);
754 __ push(Rtemp);
755
756 if (ProfileInterpreter) {
757 __ ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
758 __ tst(Rtemp, Rtemp);
759 __ add(Rtemp, Rtemp, in_bytes(MethodData::data_offset()), ne);
760 __ push(Rtemp); // set the mdp (method data pointer)
761 } else {
762 __ push(R0);
763 }
764
765 __ ldr(Rtemp, Address(Rmethod, Method::const_offset()));
766 __ ldr(Rtemp, Address(Rtemp, ConstMethod::constants_offset()));
767 __ ldr(Rtemp, Address(Rtemp, ConstantPool::cache_offset_in_bytes()));
768 __ push(Rtemp); // set constant pool cache
769 __ push(Rlocals); // set locals pointer
770 __ push(Rbcp); // set bcp
771 __ push(R0); // reserve word for pointer to expression stack bottom
772 __ str(SP, Address(SP, 0)); // set expression stack bottom
773 }
774
775 #endif // AARCH64
776
777 // End of helpers
778
779 //------------------------------------------------------------------------------------------------------------------------
780 // Entry points
781 //
782 // Here we generate the various kind of entries into the interpreter.
783 // The two main entry type are generic bytecode methods and native call method.
784 // These both come in synchronized and non-synchronized versions but the
785 // frame layout they create is very similar. The other method entry
786 // types are really just special purpose entries that are really entry
787 // and interpretation all in one. These are for trivial methods like
788 // accessor, empty, or special math methods.
789 //
790 // When control flow reaches any of the entry types for the interpreter
791 // the following holds ->
792 //
793 // Arguments:
794 //
795 // Rmethod: Method*
796 // Rthread: thread
797 // Rsender_sp: sender sp
798 // Rparams (SP on 32-bit ARM): pointer to method parameters
799 //
800 // LR: return address
801 //
802 // Stack layout immediately at entry
803 //
804 // [ optional padding(*)] <--- SP (AArch64)
805 // [ parameter n ] <--- Rparams (SP on 32-bit ARM)
806 // ...
807 // [ parameter 1 ]
808 // [ expression stack ] (caller's java expression stack)
809
810 // Assuming that we don't go to one of the trivial specialized
811 // entries the stack will look like below when we are ready to execute
812 // the first bytecode (or call the native routine). The register usage
813 // will be as the template based interpreter expects.
814 //
815 // local variables follow incoming parameters immediately; i.e.
816 // the return address is saved at the end of the locals.
817 //
818 // [ reserved stack (*) ] <--- SP (AArch64)
819 // [ expr. stack ] <--- Rstack_top (SP on 32-bit ARM)
820 // [ monitor entry ]
821 // ...
822 // [ monitor entry ]
823 // [ expr. stack bottom ]
824 // [ saved Rbcp ]
825 // [ current Rlocals ]
826 // [ cache ]
827 // [ mdx ]
828 // [ mirror ]
829 // [ Method* ]
830 //
831 // 32-bit ARM:
832 // [ last_sp ]
833 //
834 // AArch64:
835 // [ extended SP (*) ]
836 // [ stack top (*) ]
837 //
838 // [ sender_sp ]
839 // [ saved FP ] <--- FP
840 // [ saved LR ]
841 // [ optional padding(*)]
842 // [ local variable m ]
843 // ...
844 // [ local variable 1 ]
845 // [ parameter n ]
846 // ...
847 // [ parameter 1 ] <--- Rlocals
848 //
849 // (*) - AArch64 only
850 //
851
852 address TemplateInterpreterGenerator::generate_Reference_get_entry(void) {
853 #if INCLUDE_ALL_GCS
854 if (UseG1GC) {
855 // Code: _aload_0, _getfield, _areturn
856 // parameter size = 1
857 //
858 // The code that gets generated by this routine is split into 2 parts:
859 // 1. The "intrinsified" code for G1 (or any SATB based GC),
860 // 2. The slow path - which is an expansion of the regular method entry.
861 //
862 // Notes:-
863 // * In the G1 code we do not check whether we need to block for
864 // a safepoint. If G1 is enabled then we must execute the specialized
865 // code for Reference.get (except when the Reference object is null)
866 // so that we can log the value in the referent field with an SATB
867 // update buffer.
868 // If the code for the getfield template is modified so that the
869 // G1 pre-barrier code is executed when the current method is
870 // Reference.get() then going through the normal method entry
871 // will be fine.
872 // * The G1 code can, however, check the receiver object (the instance
873 // of java.lang.Reference) and jump to the slow path if null. If the
874 // Reference object is null then we obviously cannot fetch the referent
875 // and so we don't need to call the G1 pre-barrier. Thus we can use the
876 // regular method entry code to generate the NPE.
877 //
878 // This code is based on generate_accessor_enty.
879 //
880 // Rmethod: Method*
881 // Rthread: thread
882 // Rsender_sp: sender sp, must be preserved for slow path, set SP to it on fast path
883 // Rparams: parameters
884
885 address entry = __ pc();
886 Label slow_path;
887 const Register Rthis = R0;
888 const Register Rret_addr = Rtmp_save1;
889 assert_different_registers(Rthis, Rret_addr, Rsender_sp);
890
891 const int referent_offset = java_lang_ref_Reference::referent_offset;
892 guarantee(referent_offset > 0, "referent offset not initialized");
893
894 // Check if local 0 != NULL
895 // If the receiver is null then it is OK to jump to the slow path.
896 __ ldr(Rthis, Address(Rparams));
897 __ cbz(Rthis, slow_path);
898
899 // Generate the G1 pre-barrier code to log the value of
900 // the referent field in an SATB buffer.
901
902 // Load the value of the referent field.
903 __ load_heap_oop(R0, Address(Rthis, referent_offset));
904
905 // Preserve LR
906 __ mov(Rret_addr, LR);
907
908 __ g1_write_barrier_pre(noreg, // store_addr
909 noreg, // new_val
910 R0, // pre_val
911 Rtemp, // tmp1
912 R1_tmp); // tmp2
913
914 // _areturn
915 __ mov(SP, Rsender_sp);
916 __ ret(Rret_addr);
917
918 // generate a vanilla interpreter entry as the slow path
919 __ bind(slow_path);
920 __ jump_to_entry(Interpreter::entry_for_kind(Interpreter::zerolocals));
921 return entry;
922 }
923 #endif // INCLUDE_ALL_GCS
924
925 // If G1 is not enabled then attempt to go through the normal entry point
926 return NULL;
927 }
928
929 // Not supported
930 address TemplateInterpreterGenerator::generate_CRC32_update_entry() { return NULL; }
931 address TemplateInterpreterGenerator::generate_CRC32_updateBytes_entry(AbstractInterpreter::MethodKind kind) { return NULL; }
932 address TemplateInterpreterGenerator::generate_CRC32C_updateBytes_entry(AbstractInterpreter::MethodKind kind) { return NULL; }
933
934 //
935 // Interpreter stub for calling a native method. (asm interpreter)
936 // This sets up a somewhat different looking stack for calling the native method
937 // than the typical interpreter frame setup.
938 //
939
940 address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
941 // determine code generation flags
942 bool inc_counter = UseCompiler || CountCompiledCalls || LogTouchedMethods;
943
944 // Incoming registers:
945 //
946 // Rmethod: Method*
947 // Rthread: thread
948 // Rsender_sp: sender sp
949 // Rparams: parameters
950
951 address entry_point = __ pc();
952
953 // Register allocation
954 const Register Rsize_of_params = AARCH64_ONLY(R20) NOT_AARCH64(R6);
955 const Register Rsig_handler = AARCH64_ONLY(R21) NOT_AARCH64(Rtmp_save0 /* R4 */);
956 const Register Rnative_code = AARCH64_ONLY(R22) NOT_AARCH64(Rtmp_save1 /* R5 */);
957 const Register Rresult_handler = AARCH64_ONLY(Rsig_handler) NOT_AARCH64(R6);
958
959 #ifdef AARCH64
960 const Register RconstMethod = R10; // also used in generate_fixed_frame (should match)
961 const Register Rsaved_result = Rnative_code;
962 const FloatRegister Dsaved_result = V8;
963 #else
964 const Register Rsaved_result_lo = Rtmp_save0; // R4
965 const Register Rsaved_result_hi = Rtmp_save1; // R5
966 FloatRegister saved_result_fp;
967 #endif // AARCH64
968
969
970 #ifdef AARCH64
971 __ ldr(RconstMethod, Address(Rmethod, Method::const_offset()));
972 __ ldrh(Rsize_of_params, Address(RconstMethod, ConstMethod::size_of_parameters_offset()));
973 #else
974 __ ldr(Rsize_of_params, Address(Rmethod, Method::const_offset()));
975 __ ldrh(Rsize_of_params, Address(Rsize_of_params, ConstMethod::size_of_parameters_offset()));
976 #endif // AARCH64
977
978 // native calls don't need the stack size check since they have no expression stack
979 // and the arguments are already on the stack and we only add a handful of words
980 // to the stack
981
982 // compute beginning of parameters (Rlocals)
983 __ sub(Rlocals, Rparams, wordSize);
984 __ add(Rlocals, Rlocals, AsmOperand(Rsize_of_params, lsl, Interpreter::logStackElementSize));
985
986 #ifdef AARCH64
987 int extra_stack_reserve = 2*wordSize; // extra space for oop_temp
988 if(__ can_post_interpreter_events()) {
989 // extra space for saved results
990 extra_stack_reserve += 2*wordSize;
991 }
992 // reserve extra stack space and nullify oop_temp slot
993 __ stp(ZR, ZR, Address(SP, -extra_stack_reserve, pre_indexed));
994 #else
995 // reserve stack space for oop_temp
996 __ mov(R0, 0);
997 __ push(R0);
998 #endif // AARCH64
999
1000 generate_fixed_frame(true); // Note: R9 is now saved in the frame
1001
1002 // make sure method is native & not abstract
1003 #ifdef ASSERT
1004 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1005 {
1006 Label L;
1007 __ tbnz(Rtemp, JVM_ACC_NATIVE_BIT, L);
1008 __ stop("tried to execute non-native method as native");
1009 __ bind(L);
1010 }
1011 { Label L;
1012 __ tbz(Rtemp, JVM_ACC_ABSTRACT_BIT, L);
1013 __ stop("tried to execute abstract method in interpreter");
1014 __ bind(L);
1015 }
1016 #endif
1017
1018 // increment invocation count & check for overflow
1019 Label invocation_counter_overflow;
1020 if (inc_counter) {
1021 if (synchronized) {
1022 // Avoid unlocking method's monitor in case of exception, as it has not
1023 // been locked yet.
1024 __ set_do_not_unlock_if_synchronized(true, Rtemp);
1025 }
1026 generate_counter_incr(&invocation_counter_overflow, NULL, NULL);
1027 }
1028
1029 Label continue_after_compile;
1030 __ bind(continue_after_compile);
1031
1032 if (inc_counter && synchronized) {
1033 __ set_do_not_unlock_if_synchronized(false, Rtemp);
1034 }
1035
1036 // check for synchronized methods
1037 // Must happen AFTER invocation_counter check and stack overflow check,
1038 // so method is not locked if overflows.
1039 //
1040 if (synchronized) {
1041 lock_method();
1042 } else {
1043 // no synchronization necessary
1044 #ifdef ASSERT
1045 { Label L;
1046 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1047 __ tbz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
1048 __ stop("method needs synchronization");
1049 __ bind(L);
1050 }
1051 #endif
1052 }
1053
1054 // start execution
1055 #ifdef ASSERT
1056 { Label L;
1057 __ ldr(Rtemp, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
1058 __ cmp(Rtemp, Rstack_top);
1059 __ b(L, eq);
1060 __ stop("broken stack frame setup in interpreter");
1061 __ bind(L);
1062 }
1063 #endif
1064 __ check_extended_sp(Rtemp);
1065
1066 // jvmti/dtrace support
1067 __ notify_method_entry();
1068 #if R9_IS_SCRATCHED
1069 __ restore_method();
1070 #endif
1071
1072 {
1073 Label L;
1074 __ ldr(Rsig_handler, Address(Rmethod, Method::signature_handler_offset()));
1075 __ cbnz(Rsig_handler, L);
1076 __ mov(R1, Rmethod);
1077 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), R1, true);
1078 __ ldr(Rsig_handler, Address(Rmethod, Method::signature_handler_offset()));
1079 __ bind(L);
1080 }
1081
1082 {
1083 Label L;
1084 __ ldr(Rnative_code, Address(Rmethod, Method::native_function_offset()));
1085 __ cbnz(Rnative_code, L);
1086 __ mov(R1, Rmethod);
1087 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::prepare_native_call), R1);
1088 __ ldr(Rnative_code, Address(Rmethod, Method::native_function_offset()));
1089 __ bind(L);
1090 }
1091
1092 // Allocate stack space for arguments
1093
1094 #ifdef AARCH64
1095 __ sub(Rtemp, SP, Rsize_of_params, ex_uxtw, LogBytesPerWord);
1096 __ align_reg(SP, Rtemp, StackAlignmentInBytes);
1097
1098 // Allocate more stack space to accomodate all arguments passed on GP and FP registers:
1099 // 8 * wordSize for GPRs
1100 // 8 * wordSize for FPRs
1101 int reg_arguments = align_up(8*wordSize + 8*wordSize, StackAlignmentInBytes);
1102 #else
1103
1104 // C functions need aligned stack
1105 __ bic(SP, SP, StackAlignmentInBytes - 1);
1106 // Multiply by BytesPerLong instead of BytesPerWord, because calling convention
1107 // may require empty slots due to long alignment, e.g. func(int, jlong, int, jlong)
1108 __ sub(SP, SP, AsmOperand(Rsize_of_params, lsl, LogBytesPerLong));
1109
1110 #ifdef __ABI_HARD__
1111 // Allocate more stack space to accomodate all GP as well as FP registers:
1112 // 4 * wordSize
1113 // 8 * BytesPerLong
1114 int reg_arguments = align_up((4*wordSize) + (8*BytesPerLong), StackAlignmentInBytes);
1115 #else
1116 // Reserve at least 4 words on the stack for loading
1117 // of parameters passed on registers (R0-R3).
1118 // See generate_slow_signature_handler().
1119 // It is also used for JNIEnv & class additional parameters.
1120 int reg_arguments = 4 * wordSize;
1121 #endif // __ABI_HARD__
1122 #endif // AARCH64
1123
1124 __ sub(SP, SP, reg_arguments);
1125
1126
1127 // Note: signature handler blows R4 (32-bit ARM) or R21 (AArch64) besides all scratch registers.
1128 // See AbstractInterpreterGenerator::generate_slow_signature_handler().
1129 __ call(Rsig_handler);
1130 #if R9_IS_SCRATCHED
1131 __ restore_method();
1132 #endif
1133 __ mov(Rresult_handler, R0);
1134
1135 // Pass JNIEnv and mirror for static methods
1136 {
1137 Label L;
1138 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1139 __ add(R0, Rthread, in_bytes(JavaThread::jni_environment_offset()));
1140 __ tbz(Rtemp, JVM_ACC_STATIC_BIT, L);
1141 __ load_mirror(Rtemp, Rmethod, Rtemp);
1142 __ add(R1, FP, frame::interpreter_frame_oop_temp_offset * wordSize);
1143 __ str(Rtemp, Address(R1, 0));
1144 __ bind(L);
1145 }
1146
1147 __ set_last_Java_frame(SP, FP, true, Rtemp);
1148
1149 // Changing state to _thread_in_native must be the last thing to do
1150 // before the jump to native code. At this moment stack must be
1151 // safepoint-safe and completely prepared for stack walking.
1152 #ifdef ASSERT
1153 {
1154 Label L;
1155 __ ldr_u32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1156 __ cmp_32(Rtemp, _thread_in_Java);
1157 __ b(L, eq);
1158 __ stop("invalid thread state");
1159 __ bind(L);
1160 }
1161 #endif
1162
1163 #ifdef AARCH64
1164 __ mov(Rtemp, _thread_in_native);
1165 __ add(Rtemp2, Rthread, in_bytes(JavaThread::thread_state_offset()));
1166 // STLR is used to force all preceding writes to be observed prior to thread state change
1167 __ stlr_w(Rtemp, Rtemp2);
1168 #else
1169 // Force all preceding writes to be observed prior to thread state change
1170 __ membar(MacroAssembler::StoreStore, Rtemp);
1171
1172 __ mov(Rtemp, _thread_in_native);
1173 __ str(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1174 #endif // AARCH64
1175
1176 __ call(Rnative_code);
1177 #if R9_IS_SCRATCHED
1178 __ restore_method();
1179 #endif
1180
1181 // Set FPSCR/FPCR to a known state
1182 if (AlwaysRestoreFPU) {
1183 __ restore_default_fp_mode();
1184 }
1185
1186 // Do safepoint check
1187 __ mov(Rtemp, _thread_in_native_trans);
1188 __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1189
1190 // Force this write out before the read below
1191 __ membar(MacroAssembler::StoreLoad, Rtemp);
1192
1193 __ ldr_global_s32(Rtemp, SafepointSynchronize::address_of_state());
1194
1195 // Protect the return value in the interleaved code: save it to callee-save registers.
1196 #ifdef AARCH64
1197 __ mov(Rsaved_result, R0);
1198 __ fmov_d(Dsaved_result, D0);
1199 #else
1200 __ mov(Rsaved_result_lo, R0);
1201 __ mov(Rsaved_result_hi, R1);
1202 #ifdef __ABI_HARD__
1203 // preserve native FP result in a callee-saved register
1204 saved_result_fp = D8;
1205 __ fcpyd(saved_result_fp, D0);
1206 #else
1207 saved_result_fp = fnoreg;
1208 #endif // __ABI_HARD__
1209 #endif // AARCH64
1210
1211 {
1212 __ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
1213 __ cmp(Rtemp, SafepointSynchronize::_not_synchronized);
1214 __ cond_cmp(R3, 0, eq);
1215
1216 #ifdef AARCH64
1217 Label L;
1218 __ b(L, eq);
1219 __ mov(R0, Rthread);
1220 __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans), relocInfo::none);
1221 __ bind(L);
1222 #else
1223 __ mov(R0, Rthread, ne);
1224 __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans), relocInfo::none, ne);
1225 #if R9_IS_SCRATCHED
1226 __ restore_method();
1227 #endif
1228 #endif // AARCH64
1229 }
1230
1231 // Perform Native->Java thread transition
1232 __ mov(Rtemp, _thread_in_Java);
1233 __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1234
1235 // Zero handles and last_java_sp
1236 __ reset_last_Java_frame(Rtemp);
1237 __ ldr(R3, Address(Rthread, JavaThread::active_handles_offset()));
1238 __ str_32(__ zero_register(Rtemp), Address(R3, JNIHandleBlock::top_offset_in_bytes()));
1239 if (CheckJNICalls) {
1240 __ str(__ zero_register(Rtemp), Address(Rthread, JavaThread::pending_jni_exception_check_fn_offset()));
1241 }
1242
1243 // Unbox oop result, e.g. JNIHandles::resolve result if it's an oop.
1244 {
1245 Label Lnot_oop;
1246 #ifdef AARCH64
1247 __ mov_slow(Rtemp, AbstractInterpreter::result_handler(T_OBJECT));
1248 __ cmp(Rresult_handler, Rtemp);
1249 __ b(Lnot_oop, ne);
1250 #else // !AARCH64
1251 // For ARM32, Rresult_handler is -1 for oop result, 0 otherwise.
1252 __ cbz(Rresult_handler, Lnot_oop);
1253 #endif // !AARCH64
1254 Register value = AARCH64_ONLY(Rsaved_result) NOT_AARCH64(Rsaved_result_lo);
1255 __ resolve_jobject(value, // value
1256 Rtemp, // tmp1
1257 R1_tmp); // tmp2
1258 // Store resolved result in frame for GC visibility.
1259 __ str(value, Address(FP, frame::interpreter_frame_oop_temp_offset * wordSize));
1260 __ bind(Lnot_oop);
1261 }
1262
1263 #ifdef AARCH64
1264 // Restore SP (drop native parameters area), to keep SP in sync with extended_sp in frame
1265 __ restore_sp_after_call(Rtemp);
1266 __ check_stack_top();
1267 #endif // AARCH64
1268
1269 // reguard stack if StackOverflow exception happened while in native.
1270 {
1271 __ ldr_u32(Rtemp, Address(Rthread, JavaThread::stack_guard_state_offset()));
1272 __ cmp_32(Rtemp, JavaThread::stack_guard_yellow_reserved_disabled);
1273 #ifdef AARCH64
1274 Label L;
1275 __ b(L, ne);
1276 __ call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages), relocInfo::none);
1277 __ bind(L);
1278 #else
1279 __ call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages), relocInfo::none, eq);
1280 #if R9_IS_SCRATCHED
1281 __ restore_method();
1282 #endif
1283 #endif // AARCH64
1284 }
1285
1286 // check pending exceptions
1287 {
1288 __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
1289 #ifdef AARCH64
1290 Label L;
1291 __ cbz(Rtemp, L);
1292 __ mov_pc_to(Rexception_pc);
1293 __ b(StubRoutines::forward_exception_entry());
1294 __ bind(L);
1295 #else
1296 __ cmp(Rtemp, 0);
1297 __ mov(Rexception_pc, PC, ne);
1298 __ b(StubRoutines::forward_exception_entry(), ne);
1299 #endif // AARCH64
1300 }
1301
1302 if (synchronized) {
1303 // address of first monitor
1304 __ sub(R1, FP, - (frame::interpreter_frame_monitor_block_bottom_offset - frame::interpreter_frame_monitor_size()) * wordSize);
1305 __ unlock_object(R1);
1306 }
1307
1308 // jvmti/dtrace support
1309 // Note: This must happen _after_ handling/throwing any exceptions since
1310 // the exception handler code notifies the runtime of method exits
1311 // too. If this happens before, method entry/exit notifications are
1312 // not properly paired (was bug - gri 11/22/99).
1313 #ifdef AARCH64
1314 __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI, true, Rsaved_result, noreg, Dsaved_result);
1315 #else
1316 __ notify_method_exit(vtos, InterpreterMacroAssembler::NotifyJVMTI, true, Rsaved_result_lo, Rsaved_result_hi, saved_result_fp);
1317 #endif // AARCH64
1318
1319 // Restore the result. Oop result is restored from the stack.
1320 #ifdef AARCH64
1321 __ mov(R0, Rsaved_result);
1322 __ fmov_d(D0, Dsaved_result);
1323
1324 __ blr(Rresult_handler);
1325 #else
1326 __ cmp(Rresult_handler, 0);
1327 __ ldr(R0, Address(FP, frame::interpreter_frame_oop_temp_offset * wordSize), ne);
1328 __ mov(R0, Rsaved_result_lo, eq);
1329 __ mov(R1, Rsaved_result_hi);
1330
1331 #ifdef __ABI_HARD__
1332 // reload native FP result
1333 __ fcpyd(D0, D8);
1334 #endif // __ABI_HARD__
1335
1336 #ifdef ASSERT
1337 if (VerifyOops) {
1338 Label L;
1339 __ cmp(Rresult_handler, 0);
1340 __ b(L, eq);
1341 __ verify_oop(R0);
1342 __ bind(L);
1343 }
1344 #endif // ASSERT
1345 #endif // AARCH64
1346
1347 // Restore FP/LR, sender_sp and return
1348 #ifdef AARCH64
1349 __ ldr(Rtemp, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize));
1350 __ ldp(FP, LR, Address(FP));
1351 __ mov(SP, Rtemp);
1352 #else
1353 __ mov(Rtemp, FP);
1354 __ ldmia(FP, RegisterSet(FP) | RegisterSet(LR));
1355 __ ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize));
1356 #endif // AARCH64
1357
1358 __ ret();
1359
1360 if (inc_counter) {
1361 // Handle overflow of counter and compile method
1362 __ bind(invocation_counter_overflow);
1363 generate_counter_overflow(continue_after_compile);
1364 }
1365
1366 return entry_point;
1367 }
1368
1369 //
1370 // Generic interpreted method entry to (asm) interpreter
1371 //
1372 address TemplateInterpreterGenerator::generate_normal_entry(bool synchronized) {
1373 // determine code generation flags
1374 bool inc_counter = UseCompiler || CountCompiledCalls || LogTouchedMethods;
1375
1376 // Rmethod: Method*
1377 // Rthread: thread
1378 // Rsender_sp: sender sp (could differ from SP if we were called via c2i)
1379 // Rparams: pointer to the last parameter in the stack
1380
1381 address entry_point = __ pc();
1382
1383 const Register RconstMethod = AARCH64_ONLY(R10) NOT_AARCH64(R3);
1384
1385 #ifdef AARCH64
1386 const Register RmaxStack = R11;
1387 const Register RlocalsBase = R12;
1388 #endif // AARCH64
1389
1390 __ ldr(RconstMethod, Address(Rmethod, Method::const_offset()));
1391
1392 __ ldrh(R2, Address(RconstMethod, ConstMethod::size_of_parameters_offset()));
1393 __ ldrh(R3, Address(RconstMethod, ConstMethod::size_of_locals_offset()));
1394
1395 // setup Rlocals
1396 __ sub(Rlocals, Rparams, wordSize);
1397 __ add(Rlocals, Rlocals, AsmOperand(R2, lsl, Interpreter::logStackElementSize));
1398
1399 __ sub(R3, R3, R2); // number of additional locals
1400
1401 #ifdef AARCH64
1402 // setup RmaxStack
1403 __ ldrh(RmaxStack, Address(RconstMethod, ConstMethod::max_stack_offset()));
1404 // We have to add extra reserved slots to max_stack. There are 3 users of the extra slots,
1405 // none of which are at the same time, so we just need to make sure there is enough room
1406 // for the biggest user:
1407 // -reserved slot for exception handler
1408 // -reserved slots for JSR292. Method::extra_stack_entries() is the size.
1409 // -3 reserved slots so get_method_counters() can save some registers before call_VM().
1410 __ add(RmaxStack, RmaxStack, MAX2(3, Method::extra_stack_entries()));
1411 #endif // AARCH64
1412
1413 // see if we've got enough room on the stack for locals plus overhead.
1414 generate_stack_overflow_check();
1415
1416 #ifdef AARCH64
1417
1418 // allocate space for locals
1419 {
1420 __ sub(RlocalsBase, Rparams, AsmOperand(R3, lsl, Interpreter::logStackElementSize));
1421 __ align_reg(SP, RlocalsBase, StackAlignmentInBytes);
1422 }
1423
1424 // explicitly initialize locals
1425 {
1426 Label zero_loop, done;
1427 __ cbz(R3, done);
1428
1429 __ tbz(R3, 0, zero_loop);
1430 __ subs(R3, R3, 1);
1431 __ str(ZR, Address(RlocalsBase, wordSize, post_indexed));
1432 __ b(done, eq);
1433
1434 __ bind(zero_loop);
1435 __ subs(R3, R3, 2);
1436 __ stp(ZR, ZR, Address(RlocalsBase, 2*wordSize, post_indexed));
1437 __ b(zero_loop, ne);
1438
1439 __ bind(done);
1440 }
1441
1442 #else
1443 // allocate space for locals
1444 // explicitly initialize locals
1445
1446 // Loop is unrolled 4 times
1447 Label loop;
1448 __ mov(R0, 0);
1449 __ bind(loop);
1450
1451 // #1
1452 __ subs(R3, R3, 1);
1453 __ push(R0, ge);
1454
1455 // #2
1456 __ subs(R3, R3, 1, ge);
1457 __ push(R0, ge);
1458
1459 // #3
1460 __ subs(R3, R3, 1, ge);
1461 __ push(R0, ge);
1462
1463 // #4
1464 __ subs(R3, R3, 1, ge);
1465 __ push(R0, ge);
1466
1467 __ b(loop, gt);
1468 #endif // AARCH64
1469
1470 // initialize fixed part of activation frame
1471 generate_fixed_frame(false);
1472
1473 __ restore_dispatch();
1474
1475 // make sure method is not native & not abstract
1476 #ifdef ASSERT
1477 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1478 {
1479 Label L;
1480 __ tbz(Rtemp, JVM_ACC_NATIVE_BIT, L);
1481 __ stop("tried to execute native method as non-native");
1482 __ bind(L);
1483 }
1484 { Label L;
1485 __ tbz(Rtemp, JVM_ACC_ABSTRACT_BIT, L);
1486 __ stop("tried to execute abstract method in interpreter");
1487 __ bind(L);
1488 }
1489 #endif
1490
1491 // increment invocation count & check for overflow
1492 Label invocation_counter_overflow;
1493 Label profile_method;
1494 Label profile_method_continue;
1495 if (inc_counter) {
1496 if (synchronized) {
1497 // Avoid unlocking method's monitor in case of exception, as it has not
1498 // been locked yet.
1499 __ set_do_not_unlock_if_synchronized(true, Rtemp);
1500 }
1501 generate_counter_incr(&invocation_counter_overflow, &profile_method, &profile_method_continue);
1502 if (ProfileInterpreter) {
1503 __ bind(profile_method_continue);
1504 }
1505 }
1506 Label continue_after_compile;
1507 __ bind(continue_after_compile);
1508
1509 if (inc_counter && synchronized) {
1510 __ set_do_not_unlock_if_synchronized(false, Rtemp);
1511 }
1512 #if R9_IS_SCRATCHED
1513 __ restore_method();
1514 #endif
1515
1516 // check for synchronized methods
1517 // Must happen AFTER invocation_counter check and stack overflow check,
1518 // so method is not locked if overflows.
1519 //
1520 if (synchronized) {
1521 // Allocate monitor and lock method
1522 lock_method();
1523 } else {
1524 // no synchronization necessary
1525 #ifdef ASSERT
1526 { Label L;
1527 __ ldr_u32(Rtemp, Address(Rmethod, Method::access_flags_offset()));
1528 __ tbz(Rtemp, JVM_ACC_SYNCHRONIZED_BIT, L);
1529 __ stop("method needs synchronization");
1530 __ bind(L);
1531 }
1532 #endif
1533 }
1534
1535 // start execution
1536 #ifdef ASSERT
1537 { Label L;
1538 __ ldr(Rtemp, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
1539 __ cmp(Rtemp, Rstack_top);
1540 __ b(L, eq);
1541 __ stop("broken stack frame setup in interpreter");
1542 __ bind(L);
1543 }
1544 #endif
1545 __ check_extended_sp(Rtemp);
1546
1547 // jvmti support
1548 __ notify_method_entry();
1549 #if R9_IS_SCRATCHED
1550 __ restore_method();
1551 #endif
1552
1553 __ dispatch_next(vtos);
1554
1555 // invocation counter overflow
1556 if (inc_counter) {
1557 if (ProfileInterpreter) {
1558 // We have decided to profile this method in the interpreter
1559 __ bind(profile_method);
1560
1561 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1562 __ set_method_data_pointer_for_bcp();
1563
1564 __ b(profile_method_continue);
1565 }
1566
1567 // Handle overflow of counter and compile method
1568 __ bind(invocation_counter_overflow);
1569 generate_counter_overflow(continue_after_compile);
1570 }
1571
1572 return entry_point;
1573 }
1574
1575 //------------------------------------------------------------------------------------------------------------------------
1576 // Exceptions
1577
1578 void TemplateInterpreterGenerator::generate_throw_exception() {
1579 // Entry point in previous activation (i.e., if the caller was interpreted)
1580 Interpreter::_rethrow_exception_entry = __ pc();
1581 // Rexception_obj: exception
1582
1583 #ifndef AARCH64
1584 // Clear interpreter_frame_last_sp.
1585 __ mov(Rtemp, 0);
1586 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1587 #endif // !AARCH64
1588
1589 #if R9_IS_SCRATCHED
1590 __ restore_method();
1591 #endif
1592 __ restore_bcp();
1593 __ restore_dispatch();
1594 __ restore_locals();
1595
1596 #ifdef AARCH64
1597 __ restore_sp_after_call(Rtemp);
1598 #endif // AARCH64
1599
1600 // Entry point for exceptions thrown within interpreter code
1601 Interpreter::_throw_exception_entry = __ pc();
1602
1603 // expression stack is undefined here
1604 // Rexception_obj: exception
1605 // Rbcp: exception bcp
1606 __ verify_oop(Rexception_obj);
1607
1608 // expression stack must be empty before entering the VM in case of an exception
1609 __ empty_expression_stack();
1610 // find exception handler address and preserve exception oop
1611 __ mov(R1, Rexception_obj);
1612 __ call_VM(Rexception_obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::exception_handler_for_exception), R1);
1613 // R0: exception handler entry point
1614 // Rexception_obj: preserved exception oop
1615 // Rbcp: bcp for exception handler
1616 __ push_ptr(Rexception_obj); // push exception which is now the only value on the stack
1617 __ jump(R0); // jump to exception handler (may be _remove_activation_entry!)
1618
1619 // If the exception is not handled in the current frame the frame is removed and
1620 // the exception is rethrown (i.e. exception continuation is _rethrow_exception).
1621 //
1622 // Note: At this point the bci is still the bxi for the instruction which caused
1623 // the exception and the expression stack is empty. Thus, for any VM calls
1624 // at this point, GC will find a legal oop map (with empty expression stack).
1625
1626 // In current activation
1627 // tos: exception
1628 // Rbcp: exception bcp
1629
1630 //
1631 // JVMTI PopFrame support
1632 //
1633 Interpreter::_remove_activation_preserving_args_entry = __ pc();
1634
1635 #ifdef AARCH64
1636 __ restore_sp_after_call(Rtemp); // restore SP to extended SP
1637 #endif // AARCH64
1638
1639 __ empty_expression_stack();
1640
1641 // Set the popframe_processing bit in _popframe_condition indicating that we are
1642 // currently handling popframe, so that call_VMs that may happen later do not trigger new
1643 // popframe handling cycles.
1644
1645 __ ldr_s32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1646 __ orr(Rtemp, Rtemp, (unsigned)JavaThread::popframe_processing_bit);
1647 __ str_32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1648
1649 {
1650 // Check to see whether we are returning to a deoptimized frame.
1651 // (The PopFrame call ensures that the caller of the popped frame is
1652 // either interpreted or compiled and deoptimizes it if compiled.)
1653 // In this case, we can't call dispatch_next() after the frame is
1654 // popped, but instead must save the incoming arguments and restore
1655 // them after deoptimization has occurred.
1656 //
1657 // Note that we don't compare the return PC against the
1658 // deoptimization blob's unpack entry because of the presence of
1659 // adapter frames in C2.
1660 Label caller_not_deoptimized;
1661 __ ldr(R0, Address(FP, frame::return_addr_offset * wordSize));
1662 __ call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::interpreter_contains), R0);
1663 __ cbnz_32(R0, caller_not_deoptimized);
1664 #ifdef AARCH64
1665 __ NOT_TESTED();
1666 #endif
1667
1668 // Compute size of arguments for saving when returning to deoptimized caller
1669 __ restore_method();
1670 __ ldr(R0, Address(Rmethod, Method::const_offset()));
1671 __ ldrh(R0, Address(R0, ConstMethod::size_of_parameters_offset()));
1672
1673 __ logical_shift_left(R1, R0, Interpreter::logStackElementSize);
1674 // Save these arguments
1675 __ restore_locals();
1676 __ sub(R2, Rlocals, R1);
1677 __ add(R2, R2, wordSize);
1678 __ mov(R0, Rthread);
1679 __ call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::popframe_preserve_args), R0, R1, R2);
1680
1681 __ remove_activation(vtos, LR,
1682 /* throw_monitor_exception */ false,
1683 /* install_monitor_exception */ false,
1684 /* notify_jvmdi */ false);
1685
1686 // Inform deoptimization that it is responsible for restoring these arguments
1687 __ mov(Rtemp, JavaThread::popframe_force_deopt_reexecution_bit);
1688 __ str_32(Rtemp, Address(Rthread, JavaThread::popframe_condition_offset()));
1689
1690 // Continue in deoptimization handler
1691 __ ret();
1692
1693 __ bind(caller_not_deoptimized);
1694 }
1695
1696 __ remove_activation(vtos, R4,
1697 /* throw_monitor_exception */ false,
1698 /* install_monitor_exception */ false,
1699 /* notify_jvmdi */ false);
1700
1701 #ifndef AARCH64
1702 // Finish with popframe handling
1703 // A previous I2C followed by a deoptimization might have moved the
1704 // outgoing arguments further up the stack. PopFrame expects the
1705 // mutations to those outgoing arguments to be preserved and other
1706 // constraints basically require this frame to look exactly as
1707 // though it had previously invoked an interpreted activation with
1708 // no space between the top of the expression stack (current
1709 // last_sp) and the top of stack. Rather than force deopt to
1710 // maintain this kind of invariant all the time we call a small
1711 // fixup routine to move the mutated arguments onto the top of our
1712 // expression stack if necessary.
1713 __ mov(R1, SP);
1714 __ ldr(R2, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1715 // PC must point into interpreter here
1716 __ set_last_Java_frame(SP, FP, true, Rtemp);
1717 __ mov(R0, Rthread);
1718 __ call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::popframe_move_outgoing_args), R0, R1, R2);
1719 __ reset_last_Java_frame(Rtemp);
1720 #endif // !AARCH64
1721
1722 #ifdef AARCH64
1723 __ restore_sp_after_call(Rtemp);
1724 __ restore_stack_top();
1725 #else
1726 // Restore the last_sp and null it out
1727 __ ldr(SP, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1728 __ mov(Rtemp, (int)NULL_WORD);
1729 __ str(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1730 #endif // AARCH64
1731
1732 __ restore_bcp();
1733 __ restore_dispatch();
1734 __ restore_locals();
1735 __ restore_method();
1736
1737 // The method data pointer was incremented already during
1738 // call profiling. We have to restore the mdp for the current bcp.
1739 if (ProfileInterpreter) {
1740 __ set_method_data_pointer_for_bcp();
1741 }
1742
1743 // Clear the popframe condition flag
1744 assert(JavaThread::popframe_inactive == 0, "adjust this code");
1745 __ str_32(__ zero_register(Rtemp), Address(Rthread, JavaThread::popframe_condition_offset()));
1746
1747 #if INCLUDE_JVMTI
1748 {
1749 Label L_done;
1750
1751 __ ldrb(Rtemp, Address(Rbcp, 0));
1752 __ cmp(Rtemp, Bytecodes::_invokestatic);
1753 __ b(L_done, ne);
1754
1755 // The member name argument must be restored if _invokestatic is re-executed after a PopFrame call.
1756 // Detect such a case in the InterpreterRuntime function and return the member name argument, or NULL.
1757
1758 // get local0
1759 __ ldr(R1, Address(Rlocals, 0));
1760 __ mov(R2, Rmethod);
1761 __ mov(R3, Rbcp);
1762 __ call_VM(R0, CAST_FROM_FN_PTR(address, InterpreterRuntime::member_name_arg_or_null), R1, R2, R3);
1763
1764 __ cbz(R0, L_done);
1765
1766 __ str(R0, Address(Rstack_top));
1767 __ bind(L_done);
1768 }
1769 #endif // INCLUDE_JVMTI
1770
1771 __ dispatch_next(vtos);
1772 // end of PopFrame support
1773
1774 Interpreter::_remove_activation_entry = __ pc();
1775
1776 // preserve exception over this code sequence
1777 __ pop_ptr(R0_tos);
1778 __ str(R0_tos, Address(Rthread, JavaThread::vm_result_offset()));
1779 // remove the activation (without doing throws on illegalMonitorExceptions)
1780 __ remove_activation(vtos, Rexception_pc, false, true, false);
1781 // restore exception
1782 __ get_vm_result(Rexception_obj, Rtemp);
1783
1784 // Inbetween activations - previous activation type unknown yet
1785 // compute continuation point - the continuation point expects
1786 // the following registers set up:
1787 //
1788 // Rexception_obj: exception
1789 // Rexception_pc: return address/pc that threw exception
1790 // SP: expression stack of caller
1791 // FP: frame pointer of caller
1792 __ mov(c_rarg0, Rthread);
1793 __ mov(c_rarg1, Rexception_pc);
1794 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), c_rarg0, c_rarg1);
1795 // Note that an "issuing PC" is actually the next PC after the call
1796
1797 __ jump(R0); // jump to exception handler of caller
1798 }
1799
1800
1801 //
1802 // JVMTI ForceEarlyReturn support
1803 //
1804 address TemplateInterpreterGenerator::generate_earlyret_entry_for(TosState state) {
1805 address entry = __ pc();
1806
1807 #ifdef AARCH64
1808 __ restore_sp_after_call(Rtemp); // restore SP to extended SP
1809 #endif // AARCH64
1810
1811 __ restore_bcp();
1812 __ restore_dispatch();
1813 __ restore_locals();
1814
1815 __ empty_expression_stack();
1816
1817 __ load_earlyret_value(state);
1818
1819 // Clear the earlyret state
1820 __ ldr(Rtemp, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
1821
1822 assert(JvmtiThreadState::earlyret_inactive == 0, "adjust this code");
1823 __ str_32(__ zero_register(R2), Address(Rtemp, JvmtiThreadState::earlyret_state_offset()));
1824
1825 __ remove_activation(state, LR,
1826 false, /* throw_monitor_exception */
1827 false, /* install_monitor_exception */
1828 true); /* notify_jvmdi */
1829
1830 #ifndef AARCH64
1831 // According to interpreter calling conventions, result is returned in R0/R1,
1832 // so ftos (S0) and dtos (D0) are moved to R0/R1.
1833 // This conversion should be done after remove_activation, as it uses
1834 // push(state) & pop(state) to preserve return value.
1835 __ convert_tos_to_retval(state);
1836 #endif // !AARCH64
1837 __ ret();
1838
1839 return entry;
1840 } // end of ForceEarlyReturn support
1841
1842
1843 //------------------------------------------------------------------------------------------------------------------------
1844 // Helper for vtos entry point generation
1845
1846 void TemplateInterpreterGenerator::set_vtos_entry_points (Template* t, address& bep, address& cep, address& sep, address& aep, address& iep, address& lep, address& fep, address& dep, address& vep) {
1847 assert(t->is_valid() && t->tos_in() == vtos, "illegal template");
1848 Label L;
1849
1850 #ifdef __SOFTFP__
1851 dep = __ pc(); // fall through
1852 #else
1853 fep = __ pc(); __ push(ftos); __ b(L);
1854 dep = __ pc(); __ push(dtos); __ b(L);
1855 #endif // __SOFTFP__
1856
1857 lep = __ pc(); __ push(ltos); __ b(L);
1858
1859 if (AARCH64_ONLY(true) NOT_AARCH64(VerifyOops)) { // can't share atos entry with itos on AArch64 or if VerifyOops
1860 aep = __ pc(); __ push(atos); __ b(L);
1861 } else {
1862 aep = __ pc(); // fall through
1863 }
1864
1865 #ifdef __SOFTFP__
1866 fep = __ pc(); // fall through
1867 #endif // __SOFTFP__
1868
1869 bep = cep = sep = // fall through
1870 iep = __ pc(); __ push(itos); // fall through
1871 vep = __ pc(); __ bind(L); // fall through
1872 generate_and_dispatch(t);
1873 }
1874
1875 //------------------------------------------------------------------------------------------------------------------------
1876
1877 // Non-product code
1878 #ifndef PRODUCT
1879 address TemplateInterpreterGenerator::generate_trace_code(TosState state) {
1880 address entry = __ pc();
1881
1882 // prepare expression stack
1883 __ push(state); // save tosca
1884
1885 // pass tosca registers as arguments
1886 __ mov(R2, R0_tos);
1887 #ifdef AARCH64
1888 __ mov(R3, ZR);
1889 #else
1890 __ mov(R3, R1_tos_hi);
1891 #endif // AARCH64
1892 __ mov(R1, LR); // save return address
1893
1894 // call tracer
1895 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::trace_bytecode), R1, R2, R3);
1896
1897 __ mov(LR, R0); // restore return address
1898 __ pop(state); // restore tosca
1899
1900 // return
1901 __ ret();
1902
1903 return entry;
1904 }
1905
1906
1907 void TemplateInterpreterGenerator::count_bytecode() {
1908 __ inc_global_counter((address) &BytecodeCounter::_counter_value, 0, Rtemp, R2_tmp, true);
1909 }
1910
1911
1912 void TemplateInterpreterGenerator::histogram_bytecode(Template* t) {
1913 __ inc_global_counter((address)&BytecodeHistogram::_counters[0], sizeof(BytecodeHistogram::_counters[0]) * t->bytecode(), Rtemp, R2_tmp, true);
1914 }
1915
1916
1917 void TemplateInterpreterGenerator::histogram_bytecode_pair(Template* t) {
1918 const Register Rindex_addr = R2_tmp;
1919 Label Lcontinue;
1920 InlinedAddress Lcounters((address)BytecodePairHistogram::_counters);
1921 InlinedAddress Lindex((address)&BytecodePairHistogram::_index);
1922 const Register Rcounters_addr = R2_tmp;
1923 const Register Rindex = R4_tmp;
1924
1925 // calculate new index for counter:
1926 // index = (_index >> log2_number_of_codes) | (bytecode << log2_number_of_codes).
1927 // (_index >> log2_number_of_codes) is previous bytecode
1928
1929 __ ldr_literal(Rindex_addr, Lindex);
1930 __ ldr_s32(Rindex, Address(Rindex_addr));
1931 __ mov_slow(Rtemp, ((int)t->bytecode()) << BytecodePairHistogram::log2_number_of_codes);
1932 __ orr(Rindex, Rtemp, AsmOperand(Rindex, lsr, BytecodePairHistogram::log2_number_of_codes));
1933 __ str_32(Rindex, Address(Rindex_addr));
1934
1935 // Rindex (R4) contains index of counter
1936
1937 __ ldr_literal(Rcounters_addr, Lcounters);
1938 __ ldr_s32(Rtemp, Address::indexed_32(Rcounters_addr, Rindex));
1939 __ adds_32(Rtemp, Rtemp, 1);
1940 __ b(Lcontinue, mi); // avoid overflow
1941 __ str_32(Rtemp, Address::indexed_32(Rcounters_addr, Rindex));
1942
1943 __ b(Lcontinue);
1944
1945 __ bind_literal(Lindex);
1946 __ bind_literal(Lcounters);
1947
1948 __ bind(Lcontinue);
1949 }
1950
1951
1952 void TemplateInterpreterGenerator::trace_bytecode(Template* t) {
1953 // Call a little run-time stub to avoid blow-up for each bytecode.
1954 // The run-time runtime saves the right registers, depending on
1955 // the tosca in-state for the given template.
1956 assert(Interpreter::trace_code(t->tos_in()) != NULL,
1957 "entry must have been generated");
1958 address trace_entry = Interpreter::trace_code(t->tos_in());
1959 __ call(trace_entry, relocInfo::none);
1960 }
1961
1962
1963 void TemplateInterpreterGenerator::stop_interpreter_at() {
1964 Label Lcontinue;
1965 const Register stop_at = R2_tmp;
1966
1967 __ ldr_global_s32(Rtemp, (address) &BytecodeCounter::_counter_value);
1968 __ mov_slow(stop_at, StopInterpreterAt);
1969
1970 // test bytecode counter
1971 __ cmp(Rtemp, stop_at);
1972 __ b(Lcontinue, ne);
1973
1974 __ trace_state("stop_interpreter_at");
1975 __ breakpoint();
1976
1977 __ bind(Lcontinue);
1978 }
1979 #endif // !PRODUCT