1 /*
2 * Copyright (c) 2008, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/barrierSet.inline.hpp"
27 #include "gc/shared/cardTableModRefBS.inline.hpp"
28 #include "gc/shared/collectedHeap.hpp"
29 #include "interp_masm_arm.hpp"
30 #include "interpreter/interpreter.hpp"
31 #include "interpreter/interpreterRuntime.hpp"
32 #include "logging/log.hpp"
33 #include "oops/arrayOop.hpp"
34 #include "oops/markOop.hpp"
35 #include "oops/method.hpp"
36 #include "oops/methodData.hpp"
37 #include "prims/jvmtiExport.hpp"
38 #include "prims/jvmtiThreadState.hpp"
39 #include "runtime/basicLock.hpp"
40 #include "runtime/biasedLocking.hpp"
41 #include "runtime/sharedRuntime.hpp"
42
43 #if INCLUDE_ALL_GCS
44 #include "gc/g1/g1CollectedHeap.inline.hpp"
45 #include "gc/g1/g1SATBCardTableModRefBS.hpp"
46 #include "gc/g1/heapRegion.hpp"
47 #endif // INCLUDE_ALL_GCS
48
49 //--------------------------------------------------------------------
50 // Implementation of InterpreterMacroAssembler
51
52
53
54
55 InterpreterMacroAssembler::InterpreterMacroAssembler(CodeBuffer* code) : MacroAssembler(code) {
56 }
57
58 void InterpreterMacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
59 #if defined(ASSERT) && !defined(AARCH64)
60 // Ensure that last_sp is not filled.
61 { Label L;
62 ldr(Rtemp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
63 cbz(Rtemp, L);
64 stop("InterpreterMacroAssembler::call_VM_helper: last_sp != NULL");
65 bind(L);
66 }
67 #endif // ASSERT && !AARCH64
68
69 // Rbcp must be saved/restored since it may change due to GC.
70 save_bcp();
71
72 #ifdef AARCH64
73 check_no_cached_stack_top(Rtemp);
74 save_stack_top();
75 check_extended_sp(Rtemp);
76 cut_sp_before_call();
77 #endif // AARCH64
78
79 // super call
80 MacroAssembler::call_VM_helper(oop_result, entry_point, number_of_arguments, check_exceptions);
81
82 #ifdef AARCH64
83 // Restore SP to extended SP
84 restore_sp_after_call(Rtemp);
85 check_stack_top();
86 clear_cached_stack_top();
87 #endif // AARCH64
88
89 // Restore interpreter specific registers.
90 restore_bcp();
91 restore_method();
92 }
93
94 void InterpreterMacroAssembler::jump_to_entry(address entry) {
95 assert(entry, "Entry must have been generated by now");
96 b(entry);
97 }
98
99 void InterpreterMacroAssembler::check_and_handle_popframe() {
100 if (can_pop_frame()) {
101 Label L;
102 const Register popframe_cond = R2_tmp;
103
104 // Initiate popframe handling only if it is not already being processed. If the flag
105 // has the popframe_processing bit set, it means that this code is called *during* popframe
106 // handling - we don't want to reenter.
107
108 ldr_s32(popframe_cond, Address(Rthread, JavaThread::popframe_condition_offset()));
109 tbz(popframe_cond, exact_log2(JavaThread::popframe_pending_bit), L);
110 tbnz(popframe_cond, exact_log2(JavaThread::popframe_processing_bit), L);
111
112 // Call Interpreter::remove_activation_preserving_args_entry() to get the
113 // address of the same-named entrypoint in the generated interpreter code.
114 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_preserving_args_entry));
115
116 // Call indirectly to avoid generation ordering problem.
117 jump(R0);
118
119 bind(L);
120 }
121 }
122
123
124 // Blows R2, Rtemp. Sets TOS cached value.
125 void InterpreterMacroAssembler::load_earlyret_value(TosState state) {
126 const Register thread_state = R2_tmp;
127
128 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
129
130 const Address tos_addr(thread_state, JvmtiThreadState::earlyret_tos_offset());
131 const Address oop_addr(thread_state, JvmtiThreadState::earlyret_oop_offset());
132 const Address val_addr(thread_state, JvmtiThreadState::earlyret_value_offset());
133 #ifndef AARCH64
134 const Address val_addr_hi(thread_state, JvmtiThreadState::earlyret_value_offset()
135 + in_ByteSize(wordSize));
136 #endif // !AARCH64
137
138 Register zero = zero_register(Rtemp);
139
140 switch (state) {
141 case atos: ldr(R0_tos, oop_addr);
142 str(zero, oop_addr);
143 interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
144 break;
145
146 #ifdef AARCH64
147 case ltos: ldr(R0_tos, val_addr); break;
148 #else
149 case ltos: ldr(R1_tos_hi, val_addr_hi); // fall through
150 #endif // AARCH64
151 case btos: // fall through
152 case ztos: // fall through
153 case ctos: // fall through
154 case stos: // fall through
155 case itos: ldr_s32(R0_tos, val_addr); break;
156 #ifdef __SOFTFP__
157 case dtos: ldr(R1_tos_hi, val_addr_hi); // fall through
158 case ftos: ldr(R0_tos, val_addr); break;
159 #else
160 case ftos: ldr_float (S0_tos, val_addr); break;
161 case dtos: ldr_double(D0_tos, val_addr); break;
162 #endif // __SOFTFP__
163 case vtos: /* nothing to do */ break;
164 default : ShouldNotReachHere();
165 }
166 // Clean up tos value in the thread object
167 str(zero, val_addr);
168 #ifndef AARCH64
169 str(zero, val_addr_hi);
170 #endif // !AARCH64
171
172 mov(Rtemp, (int) ilgl);
173 str_32(Rtemp, tos_addr);
174 }
175
176
177 // Blows R2, Rtemp.
178 void InterpreterMacroAssembler::check_and_handle_earlyret() {
179 if (can_force_early_return()) {
180 Label L;
181 const Register thread_state = R2_tmp;
182
183 ldr(thread_state, Address(Rthread, JavaThread::jvmti_thread_state_offset()));
184 cbz(thread_state, L); // if (thread->jvmti_thread_state() == NULL) exit;
185
186 // Initiate earlyret handling only if it is not already being processed.
187 // If the flag has the earlyret_processing bit set, it means that this code
188 // is called *during* earlyret handling - we don't want to reenter.
189
190 ldr_s32(Rtemp, Address(thread_state, JvmtiThreadState::earlyret_state_offset()));
191 cmp(Rtemp, JvmtiThreadState::earlyret_pending);
192 b(L, ne);
193
194 // Call Interpreter::remove_activation_early_entry() to get the address of the
195 // same-named entrypoint in the generated interpreter code.
196
197 ldr_s32(R0, Address(thread_state, JvmtiThreadState::earlyret_tos_offset()));
198 call_VM_leaf(CAST_FROM_FN_PTR(address, Interpreter::remove_activation_early_entry), R0);
199
200 jump(R0);
201
202 bind(L);
203 }
204 }
205
206
207 // Sets reg. Blows Rtemp.
208 void InterpreterMacroAssembler::get_unsigned_2_byte_index_at_bcp(Register reg, int bcp_offset) {
209 assert(bcp_offset >= 0, "bcp is still pointing to start of bytecode");
210 assert(reg != Rtemp, "should be different registers");
211
212 ldrb(Rtemp, Address(Rbcp, bcp_offset));
213 ldrb(reg, Address(Rbcp, bcp_offset+1));
214 orr(reg, reg, AsmOperand(Rtemp, lsl, BitsPerByte));
215 }
216
217 void InterpreterMacroAssembler::get_index_at_bcp(Register index, int bcp_offset, Register tmp_reg, size_t index_size) {
218 assert_different_registers(index, tmp_reg);
219 if (index_size == sizeof(u2)) {
220 // load bytes of index separately to avoid unaligned access
221 ldrb(index, Address(Rbcp, bcp_offset+1));
222 ldrb(tmp_reg, Address(Rbcp, bcp_offset));
223 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
224 } else if (index_size == sizeof(u4)) {
225 // TODO-AARCH64: consider using unaligned access here
226 ldrb(index, Address(Rbcp, bcp_offset+3));
227 ldrb(tmp_reg, Address(Rbcp, bcp_offset+2));
228 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
229 ldrb(tmp_reg, Address(Rbcp, bcp_offset+1));
230 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
231 ldrb(tmp_reg, Address(Rbcp, bcp_offset));
232 orr(index, tmp_reg, AsmOperand(index, lsl, BitsPerByte));
233 // Check if the secondary index definition is still ~x, otherwise
234 // we have to change the following assembler code to calculate the
235 // plain index.
236 assert(ConstantPool::decode_invokedynamic_index(~123) == 123, "else change next line");
237 mvn_32(index, index); // convert to plain index
238 } else if (index_size == sizeof(u1)) {
239 ldrb(index, Address(Rbcp, bcp_offset));
240 } else {
241 ShouldNotReachHere();
242 }
243 }
244
245 // Sets cache, index.
246 void InterpreterMacroAssembler::get_cache_and_index_at_bcp(Register cache, Register index, int bcp_offset, size_t index_size) {
247 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
248 assert_different_registers(cache, index);
249
250 get_index_at_bcp(index, bcp_offset, cache, index_size);
251
252 // load constant pool cache pointer
253 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));
254
255 // convert from field index to ConstantPoolCacheEntry index
256 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
257 // TODO-AARCH64 merge this shift with shift "add(..., Rcache, AsmOperand(Rindex, lsl, LogBytesPerWord))" after this method is called
258 logical_shift_left(index, index, 2);
259 }
260
261 // Sets cache, index, bytecode.
262 void InterpreterMacroAssembler::get_cache_and_index_and_bytecode_at_bcp(Register cache, Register index, Register bytecode, int byte_no, int bcp_offset, size_t index_size) {
263 get_cache_and_index_at_bcp(cache, index, bcp_offset, index_size);
264 // caution index and bytecode can be the same
265 add(bytecode, cache, AsmOperand(index, lsl, LogBytesPerWord));
266 #ifdef AARCH64
267 add(bytecode, bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset()));
268 ldarb(bytecode, bytecode);
269 #else
270 ldrb(bytecode, Address(bytecode, (1 + byte_no) + in_bytes(ConstantPoolCache::base_offset() + ConstantPoolCacheEntry::indices_offset())));
271 TemplateTable::volatile_barrier(MacroAssembler::LoadLoad, noreg, true);
272 #endif // AARCH64
273 }
274
275 // Sets cache. Blows reg_tmp.
276 void InterpreterMacroAssembler::get_cache_entry_pointer_at_bcp(Register cache, Register reg_tmp, int bcp_offset, size_t index_size) {
277 assert(bcp_offset > 0, "bcp is still pointing to start of bytecode");
278 assert_different_registers(cache, reg_tmp);
279
280 get_index_at_bcp(reg_tmp, bcp_offset, cache, index_size);
281
282 // load constant pool cache pointer
283 ldr(cache, Address(FP, frame::interpreter_frame_cache_offset * wordSize));
284
285 // skip past the header
286 add(cache, cache, in_bytes(ConstantPoolCache::base_offset()));
287 // convert from field index to ConstantPoolCacheEntry index
288 // and from word offset to byte offset
289 assert(sizeof(ConstantPoolCacheEntry) == 4*wordSize, "adjust code below");
290 add(cache, cache, AsmOperand(reg_tmp, lsl, 2 + LogBytesPerWord));
291 }
292
293 // Load object from cpool->resolved_references(index)
294 void InterpreterMacroAssembler::load_resolved_reference_at_index(
295 Register result, Register index) {
296 assert_different_registers(result, index);
297 get_constant_pool(result);
298
299 Register cache = result;
300 // load pointer for resolved_references[] objArray
301 ldr(cache, Address(result, ConstantPool::cache_offset_in_bytes()));
302 ldr(cache, Address(result, ConstantPoolCache::resolved_references_offset_in_bytes()));
303 // JNIHandles::resolve(result)
304 ldr(cache, Address(cache, 0));
305 // Add in the index
306 // convert from field index to resolved_references() index and from
307 // word index to byte offset. Since this is a java object, it can be compressed
308 add(cache, cache, AsmOperand(index, lsl, LogBytesPerHeapOop));
309 load_heap_oop(result, Address(cache, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
310 }
311
312 void InterpreterMacroAssembler::load_resolved_klass_at_offset(
313 Register Rcpool, Register Rindex, Register Rklass) {
314 add(Rtemp, Rcpool, AsmOperand(Rindex, lsl, LogBytesPerWord));
315 ldrh(Rtemp, Address(Rtemp, sizeof(ConstantPool))); // Rtemp = resolved_klass_index
316 ldr(Rklass, Address(Rcpool, ConstantPool::resolved_klasses_offset_in_bytes())); // Rklass = cpool->_resolved_klasses
317 add(Rklass, Rklass, AsmOperand(Rtemp, lsl, LogBytesPerWord));
318 ldr(Rklass, Address(Rklass, Array<Klass*>::base_offset_in_bytes()));
319 }
320
321 // Generate a subtype check: branch to not_subtype if sub_klass is
322 // not a subtype of super_klass.
323 // Profiling code for the subtype check failure (profile_typecheck_failed)
324 // should be explicitly generated by the caller in the not_subtype case.
325 // Blows Rtemp, tmp1, tmp2.
326 void InterpreterMacroAssembler::gen_subtype_check(Register Rsub_klass,
327 Register Rsuper_klass,
328 Label ¬_subtype,
329 Register tmp1,
330 Register tmp2) {
331
332 assert_different_registers(Rsub_klass, Rsuper_klass, tmp1, tmp2, Rtemp);
333 Label ok_is_subtype, loop, update_cache;
334
335 const Register super_check_offset = tmp1;
336 const Register cached_super = tmp2;
337
338 // Profile the not-null value's klass.
339 profile_typecheck(tmp1, Rsub_klass);
340
341 // Load the super-klass's check offset into
342 ldr_u32(super_check_offset, Address(Rsuper_klass, Klass::super_check_offset_offset()));
343
344 // Check for self
345 cmp(Rsub_klass, Rsuper_klass);
346
347 // Load from the sub-klass's super-class display list, or a 1-word cache of
348 // the secondary superclass list, or a failing value with a sentinel offset
349 // if the super-klass is an interface or exceptionally deep in the Java
350 // hierarchy and we have to scan the secondary superclass list the hard way.
351 // See if we get an immediate positive hit
352 ldr(cached_super, Address(Rsub_klass, super_check_offset));
353
354 cond_cmp(Rsuper_klass, cached_super, ne);
355 b(ok_is_subtype, eq);
356
357 // Check for immediate negative hit
358 cmp(super_check_offset, in_bytes(Klass::secondary_super_cache_offset()));
359 b(not_subtype, ne);
360
361 // Now do a linear scan of the secondary super-klass chain.
362 const Register supers_arr = tmp1;
363 const Register supers_cnt = tmp2;
364 const Register cur_super = Rtemp;
365
366 // Load objArrayOop of secondary supers.
367 ldr(supers_arr, Address(Rsub_klass, Klass::secondary_supers_offset()));
368
369 ldr_u32(supers_cnt, Address(supers_arr, Array<Klass*>::length_offset_in_bytes())); // Load the array length
370 #ifdef AARCH64
371 cbz(supers_cnt, not_subtype);
372 add(supers_arr, supers_arr, Array<Klass*>::base_offset_in_bytes());
373 #else
374 cmp(supers_cnt, 0);
375
376 // Skip to the start of array elements and prefetch the first super-klass.
377 ldr(cur_super, Address(supers_arr, Array<Klass*>::base_offset_in_bytes(), pre_indexed), ne);
378 b(not_subtype, eq);
379 #endif // AARCH64
380
381 bind(loop);
382
383 #ifdef AARCH64
384 ldr(cur_super, Address(supers_arr, wordSize, post_indexed));
385 #endif // AARCH64
386
387 cmp(cur_super, Rsuper_klass);
388 b(update_cache, eq);
389
390 subs(supers_cnt, supers_cnt, 1);
391
392 #ifndef AARCH64
393 ldr(cur_super, Address(supers_arr, wordSize, pre_indexed), ne);
394 #endif // !AARCH64
395
396 b(loop, ne);
397
398 b(not_subtype);
399
400 bind(update_cache);
401 // Must be equal but missed in cache. Update cache.
402 str(Rsuper_klass, Address(Rsub_klass, Klass::secondary_super_cache_offset()));
403
404 bind(ok_is_subtype);
405 }
406
407
408 // The 1st part of the store check.
409 // Sets card_table_base register.
410 void InterpreterMacroAssembler::store_check_part1(Register card_table_base) {
411 // Check barrier set type (should be card table) and element size
412 BarrierSet* bs = Universe::heap()->barrier_set();
413 assert(bs->kind() == BarrierSet::CardTableForRS ||
414 bs->kind() == BarrierSet::CardTableExtension,
415 "Wrong barrier set kind");
416
417 CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(bs);
418 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "Adjust store check code");
419
420 // Load card table base address.
421
422 /* Performance note.
423
424 There is an alternative way of loading card table base address
425 from thread descriptor, which may look more efficient:
426
427 ldr(card_table_base, Address(Rthread, JavaThread::card_table_base_offset()));
428
429 However, performance measurements of micro benchmarks and specJVM98
430 showed that loading of card table base from thread descriptor is
431 7-18% slower compared to loading of literal embedded into the code.
432 Possible cause is a cache miss (card table base address resides in a
433 rarely accessed area of thread descriptor).
434 */
435 // TODO-AARCH64 Investigate if mov_slow is faster than ldr from Rthread on AArch64
436 mov_address(card_table_base, (address)ct->byte_map_base, symbolic_Relocation::card_table_reference);
437 }
438
439 // The 2nd part of the store check.
440 void InterpreterMacroAssembler::store_check_part2(Register obj, Register card_table_base, Register tmp) {
441 assert_different_registers(obj, card_table_base, tmp);
442
443 assert(CardTableModRefBS::dirty_card_val() == 0, "Dirty card value must be 0 due to optimizations.");
444 #ifdef AARCH64
445 add(card_table_base, card_table_base, AsmOperand(obj, lsr, CardTableModRefBS::card_shift));
446 Address card_table_addr(card_table_base);
447 #else
448 Address card_table_addr(card_table_base, obj, lsr, CardTableModRefBS::card_shift);
449 #endif
450
451 if (UseCondCardMark) {
452 if (UseConcMarkSweepGC) {
453 membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad), noreg);
454 }
455 Label already_dirty;
456
457 ldrb(tmp, card_table_addr);
458 cbz(tmp, already_dirty);
459
460 set_card(card_table_base, card_table_addr, tmp);
461 bind(already_dirty);
462
463 } else {
464 if (UseConcMarkSweepGC && CMSPrecleaningEnabled) {
465 membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore), noreg);
466 }
467 set_card(card_table_base, card_table_addr, tmp);
468 }
469 }
470
471 void InterpreterMacroAssembler::set_card(Register card_table_base, Address card_table_addr, Register tmp) {
472 #ifdef AARCH64
473 strb(ZR, card_table_addr);
474 #else
475 CardTableModRefBS* ct = barrier_set_cast<CardTableModRefBS>(Universe::heap()->barrier_set());
476 if ((((uintptr_t)ct->byte_map_base & 0xff) == 0)) {
477 // Card table is aligned so the lowest byte of the table address base is zero.
478 // This works only if the code is not saved for later use, possibly
479 // in a context where the base would no longer be aligned.
480 strb(card_table_base, card_table_addr);
481 } else {
482 mov(tmp, 0);
483 strb(tmp, card_table_addr);
484 }
485 #endif // AARCH64
486 }
487
488 //////////////////////////////////////////////////////////////////////////////////
489
490
491 // Java Expression Stack
492
493 void InterpreterMacroAssembler::pop_ptr(Register r) {
494 assert(r != Rstack_top, "unpredictable instruction");
495 ldr(r, Address(Rstack_top, wordSize, post_indexed));
496 }
497
498 void InterpreterMacroAssembler::pop_i(Register r) {
499 assert(r != Rstack_top, "unpredictable instruction");
500 ldr_s32(r, Address(Rstack_top, wordSize, post_indexed));
501 zap_high_non_significant_bits(r);
502 }
503
504 #ifdef AARCH64
505 void InterpreterMacroAssembler::pop_l(Register r) {
506 assert(r != Rstack_top, "unpredictable instruction");
507 ldr(r, Address(Rstack_top, 2*wordSize, post_indexed));
508 }
509 #else
510 void InterpreterMacroAssembler::pop_l(Register lo, Register hi) {
511 assert_different_registers(lo, hi);
512 assert(lo < hi, "lo must be < hi");
513 pop(RegisterSet(lo) | RegisterSet(hi));
514 }
515 #endif // AARCH64
516
517 void InterpreterMacroAssembler::pop_f(FloatRegister fd) {
518 #ifdef AARCH64
519 ldr_s(fd, Address(Rstack_top, wordSize, post_indexed));
520 #else
521 fpops(fd);
522 #endif // AARCH64
523 }
524
525 void InterpreterMacroAssembler::pop_d(FloatRegister fd) {
526 #ifdef AARCH64
527 ldr_d(fd, Address(Rstack_top, 2*wordSize, post_indexed));
528 #else
529 fpopd(fd);
530 #endif // AARCH64
531 }
532
533
534 // Transition vtos -> state. Blows R0, R1. Sets TOS cached value.
535 void InterpreterMacroAssembler::pop(TosState state) {
536 switch (state) {
537 case atos: pop_ptr(R0_tos); break;
538 case btos: // fall through
539 case ztos: // fall through
540 case ctos: // fall through
541 case stos: // fall through
542 case itos: pop_i(R0_tos); break;
543 #ifdef AARCH64
544 case ltos: pop_l(R0_tos); break;
545 #else
546 case ltos: pop_l(R0_tos_lo, R1_tos_hi); break;
547 #endif // AARCH64
548 #ifdef __SOFTFP__
549 case ftos: pop_i(R0_tos); break;
550 case dtos: pop_l(R0_tos_lo, R1_tos_hi); break;
551 #else
552 case ftos: pop_f(S0_tos); break;
553 case dtos: pop_d(D0_tos); break;
554 #endif // __SOFTFP__
555 case vtos: /* nothing to do */ break;
556 default : ShouldNotReachHere();
557 }
558 interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
559 }
560
561 void InterpreterMacroAssembler::push_ptr(Register r) {
562 assert(r != Rstack_top, "unpredictable instruction");
563 str(r, Address(Rstack_top, -wordSize, pre_indexed));
564 check_stack_top_on_expansion();
565 }
566
567 void InterpreterMacroAssembler::push_i(Register r) {
568 assert(r != Rstack_top, "unpredictable instruction");
569 str_32(r, Address(Rstack_top, -wordSize, pre_indexed));
570 check_stack_top_on_expansion();
571 }
572
573 #ifdef AARCH64
574 void InterpreterMacroAssembler::push_l(Register r) {
575 assert(r != Rstack_top, "unpredictable instruction");
576 stp(r, ZR, Address(Rstack_top, -2*wordSize, pre_indexed));
577 check_stack_top_on_expansion();
578 }
579 #else
580 void InterpreterMacroAssembler::push_l(Register lo, Register hi) {
581 assert_different_registers(lo, hi);
582 assert(lo < hi, "lo must be < hi");
583 push(RegisterSet(lo) | RegisterSet(hi));
584 }
585 #endif // AARCH64
586
587 void InterpreterMacroAssembler::push_f() {
588 #ifdef AARCH64
589 str_s(S0_tos, Address(Rstack_top, -wordSize, pre_indexed));
590 check_stack_top_on_expansion();
591 #else
592 fpushs(S0_tos);
593 #endif // AARCH64
594 }
595
596 void InterpreterMacroAssembler::push_d() {
597 #ifdef AARCH64
598 str_d(D0_tos, Address(Rstack_top, -2*wordSize, pre_indexed));
599 check_stack_top_on_expansion();
600 #else
601 fpushd(D0_tos);
602 #endif // AARCH64
603 }
604
605 // Transition state -> vtos. Blows Rtemp.
606 void InterpreterMacroAssembler::push(TosState state) {
607 interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
608 switch (state) {
609 case atos: push_ptr(R0_tos); break;
610 case btos: // fall through
611 case ztos: // fall through
612 case ctos: // fall through
613 case stos: // fall through
614 case itos: push_i(R0_tos); break;
615 #ifdef AARCH64
616 case ltos: push_l(R0_tos); break;
617 #else
618 case ltos: push_l(R0_tos_lo, R1_tos_hi); break;
619 #endif // AARCH64
620 #ifdef __SOFTFP__
621 case ftos: push_i(R0_tos); break;
622 case dtos: push_l(R0_tos_lo, R1_tos_hi); break;
623 #else
624 case ftos: push_f(); break;
625 case dtos: push_d(); break;
626 #endif // __SOFTFP__
627 case vtos: /* nothing to do */ break;
628 default : ShouldNotReachHere();
629 }
630 }
631
632
633 #ifndef AARCH64
634
635 // Converts return value in R0/R1 (interpreter calling conventions) to TOS cached value.
636 void InterpreterMacroAssembler::convert_retval_to_tos(TosState state) {
637 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
638 // According to interpreter calling conventions, result is returned in R0/R1,
639 // but templates expect ftos in S0, and dtos in D0.
640 if (state == ftos) {
641 fmsr(S0_tos, R0);
642 } else if (state == dtos) {
643 fmdrr(D0_tos, R0, R1);
644 }
645 #endif // !__SOFTFP__ && !__ABI_HARD__
646 }
647
648 // Converts TOS cached value to return value in R0/R1 (according to interpreter calling conventions).
649 void InterpreterMacroAssembler::convert_tos_to_retval(TosState state) {
650 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
651 // According to interpreter calling conventions, result is returned in R0/R1,
652 // so ftos (S0) and dtos (D0) are moved to R0/R1.
653 if (state == ftos) {
654 fmrs(R0, S0_tos);
655 } else if (state == dtos) {
656 fmrrd(R0, R1, D0_tos);
657 }
658 #endif // !__SOFTFP__ && !__ABI_HARD__
659 }
660
661 #endif // !AARCH64
662
663
664 // Helpers for swap and dup
665 void InterpreterMacroAssembler::load_ptr(int n, Register val) {
666 ldr(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n)));
667 }
668
669 void InterpreterMacroAssembler::store_ptr(int n, Register val) {
670 str(val, Address(Rstack_top, Interpreter::expr_offset_in_bytes(n)));
671 }
672
673
674 void InterpreterMacroAssembler::prepare_to_jump_from_interpreted() {
675 #ifdef AARCH64
676 check_no_cached_stack_top(Rtemp);
677 save_stack_top();
678 cut_sp_before_call();
679 mov(Rparams, Rstack_top);
680 #endif // AARCH64
681
682 // set sender sp
683 mov(Rsender_sp, SP);
684
685 #ifndef AARCH64
686 // record last_sp
687 str(Rsender_sp, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
688 #endif // !AARCH64
689 }
690
691 // Jump to from_interpreted entry of a call unless single stepping is possible
692 // in this thread in which case we must call the i2i entry
693 void InterpreterMacroAssembler::jump_from_interpreted(Register method) {
694 assert_different_registers(method, Rtemp);
695
696 prepare_to_jump_from_interpreted();
697
698 if (can_post_interpreter_events()) {
699 // JVMTI events, such as single-stepping, are implemented partly by avoiding running
700 // compiled code in threads for which the event is enabled. Check here for
701 // interp_only_mode if these events CAN be enabled.
702
703 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
704 #ifdef AARCH64
705 {
706 Label not_interp_only_mode;
707
708 cbz(Rtemp, not_interp_only_mode);
709 indirect_jump(Address(method, Method::interpreter_entry_offset()), Rtemp);
710
711 bind(not_interp_only_mode);
712 }
713 #else
714 cmp(Rtemp, 0);
715 ldr(PC, Address(method, Method::interpreter_entry_offset()), ne);
716 #endif // AARCH64
717 }
718
719 indirect_jump(Address(method, Method::from_interpreted_offset()), Rtemp);
720 }
721
722
723 void InterpreterMacroAssembler::restore_dispatch() {
724 mov_slow(RdispatchTable, (address)Interpreter::dispatch_table(vtos));
725 }
726
727
728 // The following two routines provide a hook so that an implementation
729 // can schedule the dispatch in two parts.
730 void InterpreterMacroAssembler::dispatch_prolog(TosState state, int step) {
731 // Nothing ARM-specific to be done here.
732 }
733
734 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int step) {
735 dispatch_next(state, step);
736 }
737
738 void InterpreterMacroAssembler::dispatch_base(TosState state,
739 DispatchTableMode table_mode,
740 bool verifyoop) {
741 if (VerifyActivationFrameSize) {
742 Label L;
743 #ifdef AARCH64
744 mov(Rtemp, SP);
745 sub(Rtemp, FP, Rtemp);
746 #else
747 sub(Rtemp, FP, SP);
748 #endif // AARCH64
749 int min_frame_size = (frame::link_offset - frame::interpreter_frame_initial_sp_offset) * wordSize;
750 cmp(Rtemp, min_frame_size);
751 b(L, ge);
752 stop("broken stack frame");
753 bind(L);
754 }
755
756 if (verifyoop) {
757 interp_verify_oop(R0_tos, state, __FILE__, __LINE__);
758 }
759
760 if((state == itos) || (state == btos) || (state == ztos) || (state == ctos) || (state == stos)) {
761 zap_high_non_significant_bits(R0_tos);
762 }
763
764 #ifdef ASSERT
765 Label L;
766 mov_slow(Rtemp, (address)Interpreter::dispatch_table(vtos));
767 cmp(Rtemp, RdispatchTable);
768 b(L, eq);
769 stop("invalid RdispatchTable");
770 bind(L);
771 #endif
772
773 if (table_mode == DispatchDefault) {
774 if (state == vtos) {
775 indirect_jump(Address::indexed_ptr(RdispatchTable, R3_bytecode), Rtemp);
776 } else {
777 #ifdef AARCH64
778 sub(Rtemp, R3_bytecode, (Interpreter::distance_from_dispatch_table(vtos) -
779 Interpreter::distance_from_dispatch_table(state)));
780 indirect_jump(Address::indexed_ptr(RdispatchTable, Rtemp), Rtemp);
781 #else
782 // on 32-bit ARM this method is faster than the one above.
783 sub(Rtemp, RdispatchTable, (Interpreter::distance_from_dispatch_table(vtos) -
784 Interpreter::distance_from_dispatch_table(state)) * wordSize);
785 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
786 #endif
787 }
788 } else {
789 assert(table_mode == DispatchNormal, "invalid dispatch table mode");
790 address table = (address) Interpreter::normal_table(state);
791 mov_slow(Rtemp, table);
792 indirect_jump(Address::indexed_ptr(Rtemp, R3_bytecode), Rtemp);
793 }
794
795 nop(); // to avoid filling CPU pipeline with invalid instructions
796 nop();
797 }
798
799 void InterpreterMacroAssembler::dispatch_only(TosState state) {
800 dispatch_base(state, DispatchDefault);
801 }
802
803
804 void InterpreterMacroAssembler::dispatch_only_normal(TosState state) {
805 dispatch_base(state, DispatchNormal);
806 }
807
808 void InterpreterMacroAssembler::dispatch_only_noverify(TosState state) {
809 dispatch_base(state, DispatchNormal, false);
810 }
811
812 void InterpreterMacroAssembler::dispatch_next(TosState state, int step) {
813 // load next bytecode and advance Rbcp
814 ldrb(R3_bytecode, Address(Rbcp, step, pre_indexed));
815 dispatch_base(state, DispatchDefault);
816 }
817
818 void InterpreterMacroAssembler::narrow(Register result) {
819 // mask integer result to narrower return type.
820 const Register Rtmp = R2;
821
822 // get method type
823 ldr(Rtmp, Address(Rmethod, Method::const_offset()));
824 ldrb(Rtmp, Address(Rtmp, ConstMethod::result_type_offset()));
825
826 Label notBool, notByte, notChar, done;
827 cmp(Rtmp, T_INT);
828 b(done, eq);
829
830 cmp(Rtmp, T_BOOLEAN);
831 b(notBool, ne);
832 and_32(result, result, 1);
833 b(done);
834
835 bind(notBool);
836 cmp(Rtmp, T_BYTE);
837 b(notByte, ne);
838 sign_extend(result, result, 8);
839 b(done);
840
841 bind(notByte);
842 cmp(Rtmp, T_CHAR);
843 b(notChar, ne);
844 zero_extend(result, result, 16);
845 b(done);
846
847 bind(notChar);
848 // cmp(Rtmp, T_SHORT);
849 // b(done, ne);
850 sign_extend(result, result, 16);
851
852 // Nothing to do
853 bind(done);
854 }
855
856 // remove activation
857 //
858 // Unlock the receiver if this is a synchronized method.
859 // Unlock any Java monitors from syncronized blocks.
860 // Remove the activation from the stack.
861 //
862 // If there are locked Java monitors
863 // If throw_monitor_exception
864 // throws IllegalMonitorStateException
865 // Else if install_monitor_exception
866 // installs IllegalMonitorStateException
867 // Else
868 // no error processing
869 void InterpreterMacroAssembler::remove_activation(TosState state, Register ret_addr,
870 bool throw_monitor_exception,
871 bool install_monitor_exception,
872 bool notify_jvmdi) {
873 Label unlock, unlocked, no_unlock;
874
875 // Note: Registers R0, R1, S0 and D0 (TOS cached value) may be in use for the result.
876
877 const Address do_not_unlock_if_synchronized(Rthread,
878 JavaThread::do_not_unlock_if_synchronized_offset());
879
880 const Register Rflag = R2;
881 const Register Raccess_flags = R3;
882
883 restore_method();
884
885 ldrb(Rflag, do_not_unlock_if_synchronized);
886
887 // get method access flags
888 ldr_u32(Raccess_flags, Address(Rmethod, Method::access_flags_offset()));
889
890 strb(zero_register(Rtemp), do_not_unlock_if_synchronized); // reset the flag
891
892 // check if method is synchronized
893
894 tbz(Raccess_flags, JVM_ACC_SYNCHRONIZED_BIT, unlocked);
895
896 // Don't unlock anything if the _do_not_unlock_if_synchronized flag is set.
897 cbnz(Rflag, no_unlock);
898
899 // unlock monitor
900 push(state); // save result
901
902 // BasicObjectLock will be first in list, since this is a synchronized method. However, need
903 // to check that the object has not been unlocked by an explicit monitorexit bytecode.
904
905 const Register Rmonitor = R1; // fixed in unlock_object()
906 const Register Robj = R2;
907
908 // address of first monitor
909 sub(Rmonitor, FP, - frame::interpreter_frame_monitor_block_bottom_offset * wordSize + (int)sizeof(BasicObjectLock));
910
911 ldr(Robj, Address(Rmonitor, BasicObjectLock::obj_offset_in_bytes()));
912 cbnz(Robj, unlock);
913
914 pop(state);
915
916 if (throw_monitor_exception) {
917 // Entry already unlocked, need to throw exception
918 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
919 should_not_reach_here();
920 } else {
921 // Monitor already unlocked during a stack unroll.
922 // If requested, install an illegal_monitor_state_exception.
923 // Continue with stack unrolling.
924 if (install_monitor_exception) {
925 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
926 }
927 b(unlocked);
928 }
929
930
931 // Exception case for the check that all monitors are unlocked.
932 const Register Rcur = R2;
933 Label restart_check_monitors_unlocked, exception_monitor_is_still_locked;
934
935 bind(exception_monitor_is_still_locked);
936 // Monitor entry is still locked, need to throw exception.
937 // Rcur: monitor entry.
938
939 if (throw_monitor_exception) {
940 // Throw exception
941 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_illegal_monitor_state_exception));
942 should_not_reach_here();
943 } else {
944 // Stack unrolling. Unlock object and install illegal_monitor_exception
945 // Unlock does not block, so don't have to worry about the frame
946
947 push(state);
948 mov(R1, Rcur);
949 unlock_object(R1);
950
951 if (install_monitor_exception) {
952 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::new_illegal_monitor_state_exception));
953 }
954
955 pop(state);
956 b(restart_check_monitors_unlocked);
957 }
958
959 bind(unlock);
960 unlock_object(Rmonitor);
961 pop(state);
962
963 // Check that for block-structured locking (i.e., that all locked objects has been unlocked)
964 bind(unlocked);
965
966 // Check that all monitors are unlocked
967 {
968 Label loop;
969
970 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
971 const Register Rbottom = R3;
972 const Register Rcur_obj = Rtemp;
973
974 bind(restart_check_monitors_unlocked);
975
976 ldr(Rcur, Address(FP, frame::interpreter_frame_monitor_block_top_offset * wordSize));
977 // points to current entry, starting with top-most entry
978 sub(Rbottom, FP, -frame::interpreter_frame_monitor_block_bottom_offset * wordSize);
979 // points to word before bottom of monitor block
980
981 cmp(Rcur, Rbottom); // check if there are no monitors
982 #ifndef AARCH64
983 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
984 // prefetch monitor's object
985 #endif // !AARCH64
986 b(no_unlock, eq);
987
988 bind(loop);
989 #ifdef AARCH64
990 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()));
991 #endif // AARCH64
992 // check if current entry is used
993 cbnz(Rcur_obj, exception_monitor_is_still_locked);
994
995 add(Rcur, Rcur, entry_size); // otherwise advance to next entry
996 cmp(Rcur, Rbottom); // check if bottom reached
997 #ifndef AARCH64
998 ldr(Rcur_obj, Address(Rcur, BasicObjectLock::obj_offset_in_bytes()), ne);
999 // prefetch monitor's object
1000 #endif // !AARCH64
1001 b(loop, ne); // if not at bottom then check this entry
1002 }
1003
1004 bind(no_unlock);
1005
1006 // jvmti support
1007 if (notify_jvmdi) {
1008 notify_method_exit(state, NotifyJVMTI); // preserve TOSCA
1009 } else {
1010 notify_method_exit(state, SkipNotifyJVMTI); // preserve TOSCA
1011 }
1012
1013 // remove activation
1014 #ifdef AARCH64
1015 ldr(Rtemp, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize));
1016 ldp(FP, LR, Address(FP));
1017 mov(SP, Rtemp);
1018 #else
1019 mov(Rtemp, FP);
1020 ldmia(FP, RegisterSet(FP) | RegisterSet(LR));
1021 ldr(SP, Address(Rtemp, frame::interpreter_frame_sender_sp_offset * wordSize));
1022 #endif
1023
1024 if (ret_addr != LR) {
1025 mov(ret_addr, LR);
1026 }
1027 }
1028
1029
1030 // At certain points in the method invocation the monitor of
1031 // synchronized methods hasn't been entered yet.
1032 // To correctly handle exceptions at these points, we set the thread local
1033 // variable _do_not_unlock_if_synchronized to true. The remove_activation will
1034 // check this flag.
1035 void InterpreterMacroAssembler::set_do_not_unlock_if_synchronized(bool flag, Register tmp) {
1036 const Address do_not_unlock_if_synchronized(Rthread,
1037 JavaThread::do_not_unlock_if_synchronized_offset());
1038 if (flag) {
1039 mov(tmp, 1);
1040 strb(tmp, do_not_unlock_if_synchronized);
1041 } else {
1042 strb(zero_register(tmp), do_not_unlock_if_synchronized);
1043 }
1044 }
1045
1046 // Lock object
1047 //
1048 // Argument: R1 : Points to BasicObjectLock to be used for locking.
1049 // Must be initialized with object to lock.
1050 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM.
1051 void InterpreterMacroAssembler::lock_object(Register Rlock) {
1052 assert(Rlock == R1, "the second argument");
1053
1054 if (UseHeavyMonitors) {
1055 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
1056 } else {
1057 Label done;
1058
1059 const Register Robj = R2;
1060 const Register Rmark = R3;
1061 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp);
1062
1063 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
1064 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1065 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes();
1066
1067 Label already_locked, slow_case;
1068
1069 // Load object pointer
1070 ldr(Robj, Address(Rlock, obj_offset));
1071
1072 if (UseBiasedLocking) {
1073 biased_locking_enter(Robj, Rmark/*scratched*/, R0, false, Rtemp, done, slow_case);
1074 }
1075
1076 #ifdef AARCH64
1077 assert(oopDesc::mark_offset_in_bytes() == 0, "must be");
1078 ldr(Rmark, Robj);
1079
1080 // Test if object is already locked
1081 assert(markOopDesc::unlocked_value == 1, "adjust this code");
1082 tbz(Rmark, exact_log2(markOopDesc::unlocked_value), already_locked);
1083
1084 #else // AARCH64
1085
1086 // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
1087 // That would be acceptable as ether CAS or slow case path is taken in that case.
1088 // Exception to that is if the object is locked by the calling thread, then the recursive test will pass (guaranteed as
1089 // loads are satisfied from a store queue if performed on the same processor).
1090
1091 assert(oopDesc::mark_offset_in_bytes() == 0, "must be");
1092 ldr(Rmark, Address(Robj, oopDesc::mark_offset_in_bytes()));
1093
1094 // Test if object is already locked
1095 tst(Rmark, markOopDesc::unlocked_value);
1096 b(already_locked, eq);
1097
1098 #endif // !AARCH64
1099 // Save old object->mark() into BasicLock's displaced header
1100 str(Rmark, Address(Rlock, mark_offset));
1101
1102 cas_for_lock_acquire(Rmark, Rlock, Robj, Rtemp, slow_case);
1103
1104 #ifndef PRODUCT
1105 if (PrintBiasedLockingStatistics) {
1106 cond_atomic_inc32(al, BiasedLocking::fast_path_entry_count_addr());
1107 }
1108 #endif //!PRODUCT
1109
1110 b(done);
1111
1112 // If we got here that means the object is locked by ether calling thread or another thread.
1113 bind(already_locked);
1114 // Handling of locked objects: recursive locks and slow case.
1115
1116 // Fast check for recursive lock.
1117 //
1118 // Can apply the optimization only if this is a stack lock
1119 // allocated in this thread. For efficiency, we can focus on
1120 // recently allocated stack locks (instead of reading the stack
1121 // base and checking whether 'mark' points inside the current
1122 // thread stack):
1123 // 1) (mark & 3) == 0
1124 // 2) SP <= mark < SP + os::pagesize()
1125 //
1126 // Warning: SP + os::pagesize can overflow the stack base. We must
1127 // neither apply the optimization for an inflated lock allocated
1128 // just above the thread stack (this is why condition 1 matters)
1129 // nor apply the optimization if the stack lock is inside the stack
1130 // of another thread. The latter is avoided even in case of overflow
1131 // because we have guard pages at the end of all stacks. Hence, if
1132 // we go over the stack base and hit the stack of another thread,
1133 // this should not be in a writeable area that could contain a
1134 // stack lock allocated by that thread. As a consequence, a stack
1135 // lock less than page size away from SP is guaranteed to be
1136 // owned by the current thread.
1137 //
1138 // Note: assuming SP is aligned, we can check the low bits of
1139 // (mark-SP) instead of the low bits of mark. In that case,
1140 // assuming page size is a power of 2, we can merge the two
1141 // conditions into a single test:
1142 // => ((mark - SP) & (3 - os::pagesize())) == 0
1143
1144 #ifdef AARCH64
1145 // Use the single check since the immediate is OK for AARCH64
1146 sub(R0, Rmark, Rstack_top);
1147 intptr_t mask = ((intptr_t)3) - ((intptr_t)os::vm_page_size());
1148 Assembler::LogicalImmediate imm(mask, false);
1149 ands(R0, R0, imm);
1150
1151 // For recursive case store 0 into lock record.
1152 // It is harmless to store it unconditionally as lock record contains some garbage
1153 // value in its _displaced_header field by this moment.
1154 str(ZR, Address(Rlock, mark_offset));
1155
1156 #else // AARCH64
1157 // (3 - os::pagesize()) cannot be encoded as an ARM immediate operand.
1158 // Check independently the low bits and the distance to SP.
1159 // -1- test low 2 bits
1160 movs(R0, AsmOperand(Rmark, lsl, 30));
1161 // -2- test (mark - SP) if the low two bits are 0
1162 sub(R0, Rmark, SP, eq);
1163 movs(R0, AsmOperand(R0, lsr, exact_log2(os::vm_page_size())), eq);
1164 // If still 'eq' then recursive locking OK: store 0 into lock record
1165 str(R0, Address(Rlock, mark_offset), eq);
1166
1167 #endif // AARCH64
1168
1169 #ifndef PRODUCT
1170 if (PrintBiasedLockingStatistics) {
1171 cond_atomic_inc32(eq, BiasedLocking::fast_path_entry_count_addr());
1172 }
1173 #endif // !PRODUCT
1174
1175 b(done, eq);
1176
1177 bind(slow_case);
1178
1179 // Call the runtime routine for slow case
1180 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter), Rlock);
1181
1182 bind(done);
1183 }
1184 }
1185
1186
1187 // Unlocks an object. Used in monitorexit bytecode and remove_activation.
1188 //
1189 // Argument: R1: Points to BasicObjectLock structure for lock
1190 // Throw an IllegalMonitorException if object is not locked by current thread
1191 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR. Calls VM.
1192 void InterpreterMacroAssembler::unlock_object(Register Rlock) {
1193 assert(Rlock == R1, "the second argument");
1194
1195 if (UseHeavyMonitors) {
1196 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock);
1197 } else {
1198 Label done, slow_case;
1199
1200 const Register Robj = R2;
1201 const Register Rmark = R3;
1202 const Register Rresult = R0;
1203 assert_different_registers(Robj, Rmark, Rlock, R0, Rtemp);
1204
1205 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
1206 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1207 const int mark_offset = lock_offset + BasicLock::displaced_header_offset_in_bytes();
1208
1209 const Register Rzero = zero_register(Rtemp);
1210
1211 // Load oop into Robj
1212 ldr(Robj, Address(Rlock, obj_offset));
1213
1214 // Free entry
1215 str(Rzero, Address(Rlock, obj_offset));
1216
1217 if (UseBiasedLocking) {
1218 biased_locking_exit(Robj, Rmark, done);
1219 }
1220
1221 // Load the old header from BasicLock structure
1222 ldr(Rmark, Address(Rlock, mark_offset));
1223
1224 // Test for recursion (zero mark in BasicLock)
1225 cbz(Rmark, done);
1226
1227 bool allow_fallthrough_on_failure = true;
1228
1229 cas_for_lock_release(Rlock, Rmark, Robj, Rtemp, slow_case, allow_fallthrough_on_failure);
1230
1231 b(done, eq);
1232
1233 bind(slow_case);
1234
1235 // Call the runtime routine for slow case.
1236 str(Robj, Address(Rlock, obj_offset)); // restore obj
1237 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), Rlock);
1238
1239 bind(done);
1240 }
1241 }
1242
1243
1244 // Test ImethodDataPtr. If it is null, continue at the specified label
1245 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp, Label& zero_continue) {
1246 assert(ProfileInterpreter, "must be profiling interpreter");
1247 ldr(mdp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1248 cbz(mdp, zero_continue);
1249 }
1250
1251
1252 // Set the method data pointer for the current bcp.
1253 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64), Rtemp, LR.
1254 void InterpreterMacroAssembler::set_method_data_pointer_for_bcp() {
1255 assert(ProfileInterpreter, "must be profiling interpreter");
1256 Label set_mdp;
1257
1258 // Test MDO to avoid the call if it is NULL.
1259 ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
1260 cbz(Rtemp, set_mdp);
1261
1262 mov(R0, Rmethod);
1263 mov(R1, Rbcp);
1264 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::bcp_to_di), R0, R1);
1265 // R0/W0: mdi
1266
1267 // mdo is guaranteed to be non-zero here, we checked for it before the call.
1268 ldr(Rtemp, Address(Rmethod, Method::method_data_offset()));
1269 add(Rtemp, Rtemp, in_bytes(MethodData::data_offset()));
1270 add_ptr_scaled_int32(Rtemp, Rtemp, R0, 0);
1271
1272 bind(set_mdp);
1273 str(Rtemp, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1274 }
1275
1276
1277 void InterpreterMacroAssembler::verify_method_data_pointer() {
1278 assert(ProfileInterpreter, "must be profiling interpreter");
1279 #ifdef ASSERT
1280 Label verify_continue;
1281 save_caller_save_registers();
1282
1283 const Register Rmdp = R2;
1284 test_method_data_pointer(Rmdp, verify_continue); // If mdp is zero, continue
1285
1286 // If the mdp is valid, it will point to a DataLayout header which is
1287 // consistent with the bcp. The converse is highly probable also.
1288
1289 ldrh(R3, Address(Rmdp, DataLayout::bci_offset()));
1290 ldr(Rtemp, Address(Rmethod, Method::const_offset()));
1291 add(R3, R3, Rtemp);
1292 add(R3, R3, in_bytes(ConstMethod::codes_offset()));
1293 cmp(R3, Rbcp);
1294 b(verify_continue, eq);
1295
1296 mov(R0, Rmethod);
1297 mov(R1, Rbcp);
1298 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::verify_mdp), R0, R1, Rmdp);
1299
1300 bind(verify_continue);
1301 restore_caller_save_registers();
1302 #endif // ASSERT
1303 }
1304
1305
1306 void InterpreterMacroAssembler::set_mdp_data_at(Register mdp_in, int offset, Register value) {
1307 assert(ProfileInterpreter, "must be profiling interpreter");
1308 assert_different_registers(mdp_in, value);
1309 str(value, Address(mdp_in, offset));
1310 }
1311
1312
1313 // Increments mdp data. Sets bumped_count register to adjusted counter.
1314 void InterpreterMacroAssembler::increment_mdp_data_at(Register mdp_in,
1315 int offset,
1316 Register bumped_count,
1317 bool decrement) {
1318 assert(ProfileInterpreter, "must be profiling interpreter");
1319
1320 // Counter address
1321 Address data(mdp_in, offset);
1322 assert_different_registers(mdp_in, bumped_count);
1323
1324 increment_mdp_data_at(data, bumped_count, decrement);
1325 }
1326
1327 void InterpreterMacroAssembler::set_mdp_flag_at(Register mdp_in, int flag_byte_constant) {
1328 assert_different_registers(mdp_in, Rtemp);
1329 assert(ProfileInterpreter, "must be profiling interpreter");
1330 assert((0 < flag_byte_constant) && (flag_byte_constant < (1 << BitsPerByte)), "flag mask is out of range");
1331
1332 // Set the flag
1333 ldrb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset())));
1334 orr(Rtemp, Rtemp, (unsigned)flag_byte_constant);
1335 strb(Rtemp, Address(mdp_in, in_bytes(DataLayout::flags_offset())));
1336 }
1337
1338
1339 // Increments mdp data. Sets bumped_count register to adjusted counter.
1340 void InterpreterMacroAssembler::increment_mdp_data_at(Address data,
1341 Register bumped_count,
1342 bool decrement) {
1343 assert(ProfileInterpreter, "must be profiling interpreter");
1344
1345 ldr(bumped_count, data);
1346 if (decrement) {
1347 // Decrement the register. Set condition codes.
1348 subs(bumped_count, bumped_count, DataLayout::counter_increment);
1349 // Avoid overflow.
1350 #ifdef AARCH64
1351 assert(DataLayout::counter_increment == 1, "required for cinc");
1352 cinc(bumped_count, bumped_count, pl);
1353 #else
1354 add(bumped_count, bumped_count, DataLayout::counter_increment, pl);
1355 #endif // AARCH64
1356 } else {
1357 // Increment the register. Set condition codes.
1358 adds(bumped_count, bumped_count, DataLayout::counter_increment);
1359 // Avoid overflow.
1360 #ifdef AARCH64
1361 assert(DataLayout::counter_increment == 1, "required for cinv");
1362 cinv(bumped_count, bumped_count, mi); // inverts 0x80..00 back to 0x7f..ff
1363 #else
1364 sub(bumped_count, bumped_count, DataLayout::counter_increment, mi);
1365 #endif // AARCH64
1366 }
1367 str(bumped_count, data);
1368 }
1369
1370
1371 void InterpreterMacroAssembler::test_mdp_data_at(Register mdp_in,
1372 int offset,
1373 Register value,
1374 Register test_value_out,
1375 Label& not_equal_continue) {
1376 assert(ProfileInterpreter, "must be profiling interpreter");
1377 assert_different_registers(mdp_in, test_value_out, value);
1378
1379 ldr(test_value_out, Address(mdp_in, offset));
1380 cmp(test_value_out, value);
1381
1382 b(not_equal_continue, ne);
1383 }
1384
1385
1386 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, int offset_of_disp, Register reg_temp) {
1387 assert(ProfileInterpreter, "must be profiling interpreter");
1388 assert_different_registers(mdp_in, reg_temp);
1389
1390 ldr(reg_temp, Address(mdp_in, offset_of_disp));
1391 add(mdp_in, mdp_in, reg_temp);
1392 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1393 }
1394
1395
1396 void InterpreterMacroAssembler::update_mdp_by_offset(Register mdp_in, Register reg_offset, Register reg_tmp) {
1397 assert(ProfileInterpreter, "must be profiling interpreter");
1398 assert_different_registers(mdp_in, reg_offset, reg_tmp);
1399
1400 ldr(reg_tmp, Address(mdp_in, reg_offset));
1401 add(mdp_in, mdp_in, reg_tmp);
1402 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1403 }
1404
1405
1406 void InterpreterMacroAssembler::update_mdp_by_constant(Register mdp_in, int constant) {
1407 assert(ProfileInterpreter, "must be profiling interpreter");
1408 add(mdp_in, mdp_in, constant);
1409 str(mdp_in, Address(FP, frame::interpreter_frame_mdp_offset * wordSize));
1410 }
1411
1412
1413 // Blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR).
1414 void InterpreterMacroAssembler::update_mdp_for_ret(Register return_bci) {
1415 assert(ProfileInterpreter, "must be profiling interpreter");
1416 assert_different_registers(return_bci, R0, R1, R2, R3, Rtemp);
1417
1418 mov(R1, return_bci);
1419 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::update_mdp_for_ret), R1);
1420 }
1421
1422
1423 // Sets mdp, bumped_count registers, blows Rtemp.
1424 void InterpreterMacroAssembler::profile_taken_branch(Register mdp, Register bumped_count) {
1425 assert_different_registers(mdp, bumped_count);
1426
1427 if (ProfileInterpreter) {
1428 Label profile_continue;
1429
1430 // If no method data exists, go to profile_continue.
1431 // Otherwise, assign to mdp
1432 test_method_data_pointer(mdp, profile_continue);
1433
1434 // We are taking a branch. Increment the taken count.
1435 increment_mdp_data_at(mdp, in_bytes(JumpData::taken_offset()), bumped_count);
1436
1437 // The method data pointer needs to be updated to reflect the new target.
1438 update_mdp_by_offset(mdp, in_bytes(JumpData::displacement_offset()), Rtemp);
1439
1440 bind (profile_continue);
1441 }
1442 }
1443
1444
1445 // Sets mdp, blows Rtemp.
1446 void InterpreterMacroAssembler::profile_not_taken_branch(Register mdp) {
1447 assert_different_registers(mdp, Rtemp);
1448
1449 if (ProfileInterpreter) {
1450 Label profile_continue;
1451
1452 // If no method data exists, go to profile_continue.
1453 test_method_data_pointer(mdp, profile_continue);
1454
1455 // We are taking a branch. Increment the not taken count.
1456 increment_mdp_data_at(mdp, in_bytes(BranchData::not_taken_offset()), Rtemp);
1457
1458 // The method data pointer needs to be updated to correspond to the next bytecode
1459 update_mdp_by_constant(mdp, in_bytes(BranchData::branch_data_size()));
1460
1461 bind (profile_continue);
1462 }
1463 }
1464
1465
1466 // Sets mdp, blows Rtemp.
1467 void InterpreterMacroAssembler::profile_call(Register mdp) {
1468 assert_different_registers(mdp, Rtemp);
1469
1470 if (ProfileInterpreter) {
1471 Label profile_continue;
1472
1473 // If no method data exists, go to profile_continue.
1474 test_method_data_pointer(mdp, profile_continue);
1475
1476 // We are making a call. Increment the count.
1477 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1478
1479 // The method data pointer needs to be updated to reflect the new target.
1480 update_mdp_by_constant(mdp, in_bytes(CounterData::counter_data_size()));
1481
1482 bind (profile_continue);
1483 }
1484 }
1485
1486
1487 // Sets mdp, blows Rtemp.
1488 void InterpreterMacroAssembler::profile_final_call(Register mdp) {
1489 if (ProfileInterpreter) {
1490 Label profile_continue;
1491
1492 // If no method data exists, go to profile_continue.
1493 test_method_data_pointer(mdp, profile_continue);
1494
1495 // We are making a call. Increment the count.
1496 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1497
1498 // The method data pointer needs to be updated to reflect the new target.
1499 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1500
1501 bind (profile_continue);
1502 }
1503 }
1504
1505
1506 // Sets mdp, blows Rtemp.
1507 void InterpreterMacroAssembler::profile_virtual_call(Register mdp, Register receiver, bool receiver_can_be_null) {
1508 assert_different_registers(mdp, receiver, Rtemp);
1509
1510 if (ProfileInterpreter) {
1511 Label profile_continue;
1512
1513 // If no method data exists, go to profile_continue.
1514 test_method_data_pointer(mdp, profile_continue);
1515
1516 Label skip_receiver_profile;
1517 if (receiver_can_be_null) {
1518 Label not_null;
1519 cbnz(receiver, not_null);
1520 // We are making a call. Increment the count for null receiver.
1521 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1522 b(skip_receiver_profile);
1523 bind(not_null);
1524 }
1525
1526 // Record the receiver type.
1527 record_klass_in_profile(receiver, mdp, Rtemp, true);
1528 bind(skip_receiver_profile);
1529
1530 // The method data pointer needs to be updated to reflect the new target.
1531 update_mdp_by_constant(mdp, in_bytes(VirtualCallData::virtual_call_data_size()));
1532 bind(profile_continue);
1533 }
1534 }
1535
1536
1537 void InterpreterMacroAssembler::record_klass_in_profile_helper(
1538 Register receiver, Register mdp,
1539 Register reg_tmp,
1540 int start_row, Label& done, bool is_virtual_call) {
1541 if (TypeProfileWidth == 0)
1542 return;
1543
1544 assert_different_registers(receiver, mdp, reg_tmp);
1545
1546 int last_row = VirtualCallData::row_limit() - 1;
1547 assert(start_row <= last_row, "must be work left to do");
1548 // Test this row for both the receiver and for null.
1549 // Take any of three different outcomes:
1550 // 1. found receiver => increment count and goto done
1551 // 2. found null => keep looking for case 1, maybe allocate this cell
1552 // 3. found something else => keep looking for cases 1 and 2
1553 // Case 3 is handled by a recursive call.
1554 for (int row = start_row; row <= last_row; row++) {
1555 Label next_test;
1556
1557 // See if the receiver is receiver[n].
1558 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(row));
1559
1560 test_mdp_data_at(mdp, recvr_offset, receiver, reg_tmp, next_test);
1561
1562 // The receiver is receiver[n]. Increment count[n].
1563 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(row));
1564 increment_mdp_data_at(mdp, count_offset, reg_tmp);
1565 b(done);
1566
1567 bind(next_test);
1568 // reg_tmp now contains the receiver from the CallData.
1569
1570 if (row == start_row) {
1571 Label found_null;
1572 // Failed the equality check on receiver[n]... Test for null.
1573 if (start_row == last_row) {
1574 // The only thing left to do is handle the null case.
1575 if (is_virtual_call) {
1576 cbz(reg_tmp, found_null);
1577 // Receiver did not match any saved receiver and there is no empty row for it.
1578 // Increment total counter to indicate polymorphic case.
1579 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), reg_tmp);
1580 b(done);
1581 bind(found_null);
1582 } else {
1583 cbnz(reg_tmp, done);
1584 }
1585 break;
1586 }
1587 // Since null is rare, make it be the branch-taken case.
1588 cbz(reg_tmp, found_null);
1589
1590 // Put all the "Case 3" tests here.
1591 record_klass_in_profile_helper(receiver, mdp, reg_tmp, start_row + 1, done, is_virtual_call);
1592
1593 // Found a null. Keep searching for a matching receiver,
1594 // but remember that this is an empty (unused) slot.
1595 bind(found_null);
1596 }
1597 }
1598
1599 // In the fall-through case, we found no matching receiver, but we
1600 // observed the receiver[start_row] is NULL.
1601
1602 // Fill in the receiver field and increment the count.
1603 int recvr_offset = in_bytes(VirtualCallData::receiver_offset(start_row));
1604 set_mdp_data_at(mdp, recvr_offset, receiver);
1605 int count_offset = in_bytes(VirtualCallData::receiver_count_offset(start_row));
1606 mov(reg_tmp, DataLayout::counter_increment);
1607 set_mdp_data_at(mdp, count_offset, reg_tmp);
1608 if (start_row > 0) {
1609 b(done);
1610 }
1611 }
1612
1613 void InterpreterMacroAssembler::record_klass_in_profile(Register receiver,
1614 Register mdp,
1615 Register reg_tmp,
1616 bool is_virtual_call) {
1617 assert(ProfileInterpreter, "must be profiling");
1618 assert_different_registers(receiver, mdp, reg_tmp);
1619
1620 Label done;
1621
1622 record_klass_in_profile_helper(receiver, mdp, reg_tmp, 0, done, is_virtual_call);
1623
1624 bind (done);
1625 }
1626
1627 // Sets mdp, blows volatile registers (R0-R3 on 32-bit ARM, R0-R18 on AArch64, Rtemp, LR).
1628 void InterpreterMacroAssembler::profile_ret(Register mdp, Register return_bci) {
1629 assert_different_registers(mdp, return_bci, Rtemp, R0, R1, R2, R3);
1630
1631 if (ProfileInterpreter) {
1632 Label profile_continue;
1633 uint row;
1634
1635 // If no method data exists, go to profile_continue.
1636 test_method_data_pointer(mdp, profile_continue);
1637
1638 // Update the total ret count.
1639 increment_mdp_data_at(mdp, in_bytes(CounterData::count_offset()), Rtemp);
1640
1641 for (row = 0; row < RetData::row_limit(); row++) {
1642 Label next_test;
1643
1644 // See if return_bci is equal to bci[n]:
1645 test_mdp_data_at(mdp, in_bytes(RetData::bci_offset(row)), return_bci,
1646 Rtemp, next_test);
1647
1648 // return_bci is equal to bci[n]. Increment the count.
1649 increment_mdp_data_at(mdp, in_bytes(RetData::bci_count_offset(row)), Rtemp);
1650
1651 // The method data pointer needs to be updated to reflect the new target.
1652 update_mdp_by_offset(mdp, in_bytes(RetData::bci_displacement_offset(row)), Rtemp);
1653 b(profile_continue);
1654 bind(next_test);
1655 }
1656
1657 update_mdp_for_ret(return_bci);
1658
1659 bind(profile_continue);
1660 }
1661 }
1662
1663
1664 // Sets mdp.
1665 void InterpreterMacroAssembler::profile_null_seen(Register mdp) {
1666 if (ProfileInterpreter) {
1667 Label profile_continue;
1668
1669 // If no method data exists, go to profile_continue.
1670 test_method_data_pointer(mdp, profile_continue);
1671
1672 set_mdp_flag_at(mdp, BitData::null_seen_byte_constant());
1673
1674 // The method data pointer needs to be updated.
1675 int mdp_delta = in_bytes(BitData::bit_data_size());
1676 if (TypeProfileCasts) {
1677 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1678 }
1679 update_mdp_by_constant(mdp, mdp_delta);
1680
1681 bind (profile_continue);
1682 }
1683 }
1684
1685
1686 // Sets mdp, blows Rtemp.
1687 void InterpreterMacroAssembler::profile_typecheck_failed(Register mdp) {
1688 assert_different_registers(mdp, Rtemp);
1689
1690 if (ProfileInterpreter && TypeProfileCasts) {
1691 Label profile_continue;
1692
1693 // If no method data exists, go to profile_continue.
1694 test_method_data_pointer(mdp, profile_continue);
1695
1696 int count_offset = in_bytes(CounterData::count_offset());
1697 // Back up the address, since we have already bumped the mdp.
1698 count_offset -= in_bytes(VirtualCallData::virtual_call_data_size());
1699
1700 // *Decrement* the counter. We expect to see zero or small negatives.
1701 increment_mdp_data_at(mdp, count_offset, Rtemp, true);
1702
1703 bind (profile_continue);
1704 }
1705 }
1706
1707
1708 // Sets mdp, blows Rtemp.
1709 void InterpreterMacroAssembler::profile_typecheck(Register mdp, Register klass)
1710 {
1711 assert_different_registers(mdp, klass, Rtemp);
1712
1713 if (ProfileInterpreter) {
1714 Label profile_continue;
1715
1716 // If no method data exists, go to profile_continue.
1717 test_method_data_pointer(mdp, profile_continue);
1718
1719 // The method data pointer needs to be updated.
1720 int mdp_delta = in_bytes(BitData::bit_data_size());
1721 if (TypeProfileCasts) {
1722 mdp_delta = in_bytes(VirtualCallData::virtual_call_data_size());
1723
1724 // Record the object type.
1725 record_klass_in_profile(klass, mdp, Rtemp, false);
1726 }
1727 update_mdp_by_constant(mdp, mdp_delta);
1728
1729 bind(profile_continue);
1730 }
1731 }
1732
1733
1734 // Sets mdp, blows Rtemp.
1735 void InterpreterMacroAssembler::profile_switch_default(Register mdp) {
1736 assert_different_registers(mdp, Rtemp);
1737
1738 if (ProfileInterpreter) {
1739 Label profile_continue;
1740
1741 // If no method data exists, go to profile_continue.
1742 test_method_data_pointer(mdp, profile_continue);
1743
1744 // Update the default case count
1745 increment_mdp_data_at(mdp, in_bytes(MultiBranchData::default_count_offset()), Rtemp);
1746
1747 // The method data pointer needs to be updated.
1748 update_mdp_by_offset(mdp, in_bytes(MultiBranchData::default_displacement_offset()), Rtemp);
1749
1750 bind(profile_continue);
1751 }
1752 }
1753
1754
1755 // Sets mdp. Blows reg_tmp1, reg_tmp2. Index could be the same as reg_tmp2.
1756 void InterpreterMacroAssembler::profile_switch_case(Register mdp, Register index, Register reg_tmp1, Register reg_tmp2) {
1757 assert_different_registers(mdp, reg_tmp1, reg_tmp2);
1758 assert_different_registers(mdp, reg_tmp1, index);
1759
1760 if (ProfileInterpreter) {
1761 Label profile_continue;
1762
1763 const int count_offset = in_bytes(MultiBranchData::case_array_offset()) +
1764 in_bytes(MultiBranchData::relative_count_offset());
1765
1766 const int displacement_offset = in_bytes(MultiBranchData::case_array_offset()) +
1767 in_bytes(MultiBranchData::relative_displacement_offset());
1768
1769 // If no method data exists, go to profile_continue.
1770 test_method_data_pointer(mdp, profile_continue);
1771
1772 // Build the base (index * per_case_size_in_bytes())
1773 logical_shift_left(reg_tmp1, index, exact_log2(in_bytes(MultiBranchData::per_case_size())));
1774
1775 // Update the case count
1776 add(reg_tmp1, reg_tmp1, count_offset);
1777 increment_mdp_data_at(Address(mdp, reg_tmp1), reg_tmp2);
1778
1779 // The method data pointer needs to be updated.
1780 add(reg_tmp1, reg_tmp1, displacement_offset - count_offset);
1781 update_mdp_by_offset(mdp, reg_tmp1, reg_tmp2);
1782
1783 bind (profile_continue);
1784 }
1785 }
1786
1787
1788 void InterpreterMacroAssembler::byteswap_u32(Register r, Register rtmp1, Register rtmp2) {
1789 #ifdef AARCH64
1790 rev_w(r, r);
1791 #else
1792 if (VM_Version::supports_rev()) {
1793 rev(r, r);
1794 } else {
1795 eor(rtmp1, r, AsmOperand(r, ror, 16));
1796 mvn(rtmp2, 0x0000ff00);
1797 andr(rtmp1, rtmp2, AsmOperand(rtmp1, lsr, 8));
1798 eor(r, rtmp1, AsmOperand(r, ror, 8));
1799 }
1800 #endif // AARCH64
1801 }
1802
1803
1804 void InterpreterMacroAssembler::inc_global_counter(address address_of_counter, int offset, Register tmp1, Register tmp2, bool avoid_overflow) {
1805 const intx addr = (intx) (address_of_counter + offset);
1806
1807 assert ((addr & 0x3) == 0, "address of counter should be aligned");
1808 const intx offset_mask = right_n_bits(AARCH64_ONLY(12 + 2) NOT_AARCH64(12));
1809
1810 const address base = (address) (addr & ~offset_mask);
1811 const int offs = (int) (addr & offset_mask);
1812
1813 const Register addr_base = tmp1;
1814 const Register val = tmp2;
1815
1816 mov_slow(addr_base, base);
1817 ldr_s32(val, Address(addr_base, offs));
1818
1819 if (avoid_overflow) {
1820 adds_32(val, val, 1);
1821 #ifdef AARCH64
1822 Label L;
1823 b(L, mi);
1824 str_32(val, Address(addr_base, offs));
1825 bind(L);
1826 #else
1827 str(val, Address(addr_base, offs), pl);
1828 #endif // AARCH64
1829 } else {
1830 add_32(val, val, 1);
1831 str_32(val, Address(addr_base, offs));
1832 }
1833 }
1834
1835 void InterpreterMacroAssembler::interp_verify_oop(Register reg, TosState state, const char *file, int line) {
1836 if (state == atos) { MacroAssembler::_verify_oop(reg, "broken oop", file, line); }
1837 }
1838
1839 // Inline assembly for:
1840 //
1841 // if (thread is in interp_only_mode) {
1842 // InterpreterRuntime::post_method_entry();
1843 // }
1844 // if (DTraceMethodProbes) {
1845 // SharedRuntime::dtrace_method_entry(method, receiver);
1846 // }
1847 // if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) {
1848 // SharedRuntime::rc_trace_method_entry(method, receiver);
1849 // }
1850
1851 void InterpreterMacroAssembler::notify_method_entry() {
1852 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1853 // track stack depth. If it is possible to enter interp_only_mode we add
1854 // the code to check if the event should be sent.
1855 if (can_post_interpreter_events()) {
1856 Label L;
1857
1858 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
1859 cbz(Rtemp, L);
1860
1861 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_entry));
1862
1863 bind(L);
1864 }
1865
1866 // Note: Disable DTrace runtime check for now to eliminate overhead on each method entry
1867 if (DTraceMethodProbes) {
1868 Label Lcontinue;
1869
1870 ldrb_global(Rtemp, (address)&DTraceMethodProbes);
1871 cbz(Rtemp, Lcontinue);
1872
1873 mov(R0, Rthread);
1874 mov(R1, Rmethod);
1875 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry), R0, R1);
1876
1877 bind(Lcontinue);
1878 }
1879 // RedefineClasses() tracing support for obsolete method entry
1880 if (log_is_enabled(Trace, redefine, class, obsolete)) {
1881 mov(R0, Rthread);
1882 mov(R1, Rmethod);
1883 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
1884 R0, R1);
1885 }
1886 }
1887
1888
1889 void InterpreterMacroAssembler::notify_method_exit(
1890 TosState state, NotifyMethodExitMode mode,
1891 bool native, Register result_lo, Register result_hi, FloatRegister result_fp) {
1892 // Whenever JVMTI is interp_only_mode, method entry/exit events are sent to
1893 // track stack depth. If it is possible to enter interp_only_mode we add
1894 // the code to check if the event should be sent.
1895 if (mode == NotifyJVMTI && can_post_interpreter_events()) {
1896 Label L;
1897 // Note: frame::interpreter_frame_result has a dependency on how the
1898 // method result is saved across the call to post_method_exit. If this
1899 // is changed then the interpreter_frame_result implementation will
1900 // need to be updated too.
1901
1902 ldr_s32(Rtemp, Address(Rthread, JavaThread::interp_only_mode_offset()));
1903 cbz(Rtemp, L);
1904
1905 if (native) {
1906 // For c++ and template interpreter push both result registers on the
1907 // stack in native, we don't know the state.
1908 // On AArch64 result registers are stored into the frame at known locations.
1909 // See frame::interpreter_frame_result for code that gets the result values from here.
1910 assert(result_lo != noreg, "result registers should be defined");
1911
1912 #ifdef AARCH64
1913 assert(result_hi == noreg, "result_hi is not used on AArch64");
1914 assert(result_fp != fnoreg, "FP result register must be defined");
1915
1916 str_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize));
1917 str(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize));
1918 #else
1919 assert(result_hi != noreg, "result registers should be defined");
1920
1921 #ifdef __ABI_HARD__
1922 assert(result_fp != fnoreg, "FP result register must be defined");
1923 sub(SP, SP, 2 * wordSize);
1924 fstd(result_fp, Address(SP));
1925 #endif // __ABI_HARD__
1926
1927 push(RegisterSet(result_lo) | RegisterSet(result_hi));
1928 #endif // AARCH64
1929
1930 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
1931
1932 #ifdef AARCH64
1933 ldr_d(result_fp, Address(FP, frame::interpreter_frame_fp_saved_result_offset * wordSize));
1934 ldr(result_lo, Address(FP, frame::interpreter_frame_gp_saved_result_offset * wordSize));
1935 #else
1936 pop(RegisterSet(result_lo) | RegisterSet(result_hi));
1937 #ifdef __ABI_HARD__
1938 fldd(result_fp, Address(SP));
1939 add(SP, SP, 2 * wordSize);
1940 #endif // __ABI_HARD__
1941 #endif // AARCH64
1942
1943 } else {
1944 // For the template interpreter, the value on tos is the size of the
1945 // state. (c++ interpreter calls jvmti somewhere else).
1946 push(state);
1947 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::post_method_exit));
1948 pop(state);
1949 }
1950
1951 bind(L);
1952 }
1953
1954 // Note: Disable DTrace runtime check for now to eliminate overhead on each method exit
1955 if (DTraceMethodProbes) {
1956 Label Lcontinue;
1957
1958 ldrb_global(Rtemp, (address)&DTraceMethodProbes);
1959 cbz(Rtemp, Lcontinue);
1960
1961 push(state);
1962
1963 mov(R0, Rthread);
1964 mov(R1, Rmethod);
1965
1966 call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), R0, R1);
1967
1968 pop(state);
1969
1970 bind(Lcontinue);
1971 }
1972 }
1973
1974
1975 #ifndef PRODUCT
1976
1977 void InterpreterMacroAssembler::trace_state(const char* msg) {
1978 int push_size = save_caller_save_registers();
1979
1980 Label Lcontinue;
1981 InlinedString Lmsg0("%s: FP=" INTPTR_FORMAT ", SP=" INTPTR_FORMAT "\n");
1982 InlinedString Lmsg(msg);
1983 InlinedAddress Lprintf((address)printf);
1984
1985 ldr_literal(R0, Lmsg0);
1986 ldr_literal(R1, Lmsg);
1987 mov(R2, FP);
1988 add(R3, SP, push_size); // original SP (without saved registers)
1989 ldr_literal(Rtemp, Lprintf);
1990 call(Rtemp);
1991
1992 b(Lcontinue);
1993
1994 bind_literal(Lmsg0);
1995 bind_literal(Lmsg);
1996 bind_literal(Lprintf);
1997
1998
1999 bind(Lcontinue);
2000
2001 restore_caller_save_registers();
2002 }
2003
2004 #endif
2005
2006 // Jump if ((*counter_addr += increment) & mask) satisfies the condition.
2007 void InterpreterMacroAssembler::increment_mask_and_jump(Address counter_addr,
2008 int increment, Address mask_addr,
2009 Register scratch, Register scratch2,
2010 AsmCondition cond, Label* where) {
2011 // caution: scratch2 and base address of counter_addr can be the same
2012 assert_different_registers(scratch, scratch2);
2013 ldr_u32(scratch, counter_addr);
2014 add(scratch, scratch, increment);
2015 str_32(scratch, counter_addr);
2016
2017 #ifdef AARCH64
2018 ldr_u32(scratch2, mask_addr);
2019 ands_w(ZR, scratch, scratch2);
2020 #else
2021 ldr(scratch2, mask_addr);
2022 andrs(scratch, scratch, scratch2);
2023 #endif // AARCH64
2024 b(*where, cond);
2025 }
2026
2027 void InterpreterMacroAssembler::get_method_counters(Register method,
2028 Register Rcounters,
2029 Label& skip,
2030 bool saveRegs,
2031 Register reg1,
2032 Register reg2,
2033 Register reg3) {
2034 const Address method_counters(method, Method::method_counters_offset());
2035 Label has_counters;
2036
2037 ldr(Rcounters, method_counters);
2038 cbnz(Rcounters, has_counters);
2039
2040 if (saveRegs) {
2041 // Save and restore in use caller-saved registers since they will be trashed by call_VM
2042 assert(reg1 != noreg, "must specify reg1");
2043 assert(reg2 != noreg, "must specify reg2");
2044 #ifdef AARCH64
2045 assert(reg3 != noreg, "must specify reg3");
2046 stp(reg1, reg2, Address(Rstack_top, -2*wordSize, pre_indexed));
2047 stp(reg3, ZR, Address(Rstack_top, -2*wordSize, pre_indexed));
2048 #else
2049 assert(reg3 == noreg, "must not specify reg3");
2050 push(RegisterSet(reg1) | RegisterSet(reg2));
2051 #endif
2052 }
2053
2054 mov(R1, method);
2055 call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::build_method_counters), R1);
2056
2057 if (saveRegs) {
2058 #ifdef AARCH64
2059 ldp(reg3, ZR, Address(Rstack_top, 2*wordSize, post_indexed));
2060 ldp(reg1, reg2, Address(Rstack_top, 2*wordSize, post_indexed));
2061 #else
2062 pop(RegisterSet(reg1) | RegisterSet(reg2));
2063 #endif
2064 }
2065
2066 ldr(Rcounters, method_counters);
2067 cbz(Rcounters, skip); // No MethodCounters created, OutOfMemory
2068
2069 bind(has_counters);
2070 }