1 /*
2 * Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.hpp"
28 #include "interpreter/interpreter.hpp"
29 #include "interpreter/interpreterRuntime.hpp"
30 #include "interpreter/interp_masm.hpp"
31 #include "interpreter/templateTable.hpp"
32 #include "memory/universe.inline.hpp"
33 #include "oops/methodData.hpp"
34 #include "oops/method.hpp"
35 #include "oops/objArrayKlass.hpp"
36 #include "oops/oop.inline.hpp"
37 #include "prims/methodHandles.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "runtime/stubRoutines.hpp"
40 #include "runtime/synchronizer.hpp"
41
42 #define __ _masm->
43
44 // Platform-dependent initialization
45
46 void TemplateTable::pd_initialize() {
47 // No aarch64 specific initialization
48 }
49
50 // Address computation: local variables
51
52 static inline Address iaddress(int n) {
53 return Address(rlocals, Interpreter::local_offset_in_bytes(n));
54 }
55
56 static inline Address laddress(int n) {
57 return iaddress(n + 1);
58 }
59
60 static inline Address faddress(int n) {
61 return iaddress(n);
62 }
63
64 static inline Address daddress(int n) {
65 return laddress(n);
66 }
67
68 static inline Address aaddress(int n) {
69 return iaddress(n);
70 }
71
72 static inline Address iaddress(Register r) {
73 return Address(rlocals, r, Address::lsl(3));
74 }
75
76 static inline Address laddress(Register r, Register scratch,
77 InterpreterMacroAssembler* _masm) {
78 __ lea(scratch, Address(rlocals, r, Address::lsl(3)));
79 return Address(scratch, Interpreter::local_offset_in_bytes(1));
80 }
81
82 static inline Address faddress(Register r) {
83 return iaddress(r);
84 }
85
86 static inline Address daddress(Register r, Register scratch,
87 InterpreterMacroAssembler* _masm) {
88 return laddress(r, scratch, _masm);
89 }
90
91 static inline Address aaddress(Register r) {
92 return iaddress(r);
93 }
94
95 static inline Address at_rsp() {
96 return Address(esp, 0);
97 }
98
99 // At top of Java expression stack which may be different than esp(). It
100 // isn't for category 1 objects.
101 static inline Address at_tos () {
102 return Address(esp, Interpreter::expr_offset_in_bytes(0));
103 }
104
105 static inline Address at_tos_p1() {
106 return Address(esp, Interpreter::expr_offset_in_bytes(1));
107 }
108
109 static inline Address at_tos_p2() {
110 return Address(esp, Interpreter::expr_offset_in_bytes(2));
111 }
112
113 static inline Address at_tos_p3() {
114 return Address(esp, Interpreter::expr_offset_in_bytes(3));
115 }
116
117 static inline Address at_tos_p4() {
118 return Address(esp, Interpreter::expr_offset_in_bytes(4));
119 }
120
121 static inline Address at_tos_p5() {
122 return Address(esp, Interpreter::expr_offset_in_bytes(5));
123 }
124
125 // Condition conversion
126 static Assembler::Condition j_not(TemplateTable::Condition cc) {
127 switch (cc) {
128 case TemplateTable::equal : return Assembler::NE;
129 case TemplateTable::not_equal : return Assembler::EQ;
130 case TemplateTable::less : return Assembler::GE;
131 case TemplateTable::less_equal : return Assembler::GT;
132 case TemplateTable::greater : return Assembler::LE;
133 case TemplateTable::greater_equal: return Assembler::LT;
134 }
135 ShouldNotReachHere();
136 return Assembler::EQ;
137 }
138
139
140 // Miscelaneous helper routines
141 // Store an oop (or NULL) at the Address described by obj.
142 // If val == noreg this means store a NULL
143 static void do_oop_store(InterpreterMacroAssembler* _masm,
144 Address obj,
145 Register val,
146 BarrierSet::Name barrier,
147 bool precise) {
148 assert(val == noreg || val == r0, "parameter is just for looks");
149 switch (barrier) {
150 #if INCLUDE_ALL_GCS
151 case BarrierSet::G1BarrierSet:
152 {
153 // flatten object address if needed
154 if (obj.index() == noreg && obj.offset() == 0) {
155 if (obj.base() != r3) {
156 __ mov(r3, obj.base());
157 }
158 } else {
159 __ lea(r3, obj);
160 }
161 __ g1_write_barrier_pre(r3 /* obj */,
162 r1 /* pre_val */,
163 rthread /* thread */,
164 r10 /* tmp */,
165 val != noreg /* tosca_live */,
166 false /* expand_call */);
167 if (val == noreg) {
168 __ store_heap_oop_null(Address(r3, 0));
169 } else {
170 // G1 barrier needs uncompressed oop for region cross check.
171 Register new_val = val;
172 if (UseCompressedOops) {
173 new_val = rscratch2;
174 __ mov(new_val, val);
175 }
176 __ store_heap_oop(Address(r3, 0), val);
177 __ g1_write_barrier_post(r3 /* store_adr */,
178 new_val /* new_val */,
179 rthread /* thread */,
180 r10 /* tmp */,
181 r1 /* tmp2 */);
182 }
183
184 }
185 break;
186 #endif // INCLUDE_ALL_GCS
187 case BarrierSet::CardTableModRef:
188 {
189 if (val == noreg) {
190 __ store_heap_oop_null(obj);
191 } else {
192 __ store_heap_oop(obj, val);
193 // flatten object address if needed
194 if (!precise || (obj.index() == noreg && obj.offset() == 0)) {
195 __ store_check(obj.base());
196 } else {
197 __ lea(r3, obj);
198 __ store_check(r3);
199 }
200 }
201 }
202 break;
203 case BarrierSet::ModRef:
204 if (val == noreg) {
205 __ store_heap_oop_null(obj);
206 } else {
207 __ store_heap_oop(obj, val);
208 }
209 break;
210 default :
211 ShouldNotReachHere();
212
213 }
214 }
215
216 Address TemplateTable::at_bcp(int offset) {
217 assert(_desc->uses_bcp(), "inconsistent uses_bcp information");
218 return Address(rbcp, offset);
219 }
220
221 void TemplateTable::patch_bytecode(Bytecodes::Code bc, Register bc_reg,
222 Register temp_reg, bool load_bc_into_bc_reg/*=true*/,
223 int byte_no)
224 {
225 if (!RewriteBytecodes) return;
226 Label L_patch_done;
227
228 switch (bc) {
229 case Bytecodes::_fast_aputfield:
230 case Bytecodes::_fast_bputfield:
231 case Bytecodes::_fast_zputfield:
232 case Bytecodes::_fast_cputfield:
233 case Bytecodes::_fast_dputfield:
234 case Bytecodes::_fast_fputfield:
235 case Bytecodes::_fast_iputfield:
236 case Bytecodes::_fast_lputfield:
237 case Bytecodes::_fast_sputfield:
238 {
239 // We skip bytecode quickening for putfield instructions when
240 // the put_code written to the constant pool cache is zero.
241 // This is required so that every execution of this instruction
242 // calls out to InterpreterRuntime::resolve_get_put to do
243 // additional, required work.
244 assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
245 assert(load_bc_into_bc_reg, "we use bc_reg as temp");
246 __ get_cache_and_index_and_bytecode_at_bcp(temp_reg, bc_reg, temp_reg, byte_no, 1);
247 __ movw(bc_reg, bc);
248 __ cbzw(temp_reg, L_patch_done); // don't patch
249 }
250 break;
251 default:
252 assert(byte_no == -1, "sanity");
253 // the pair bytecodes have already done the load.
254 if (load_bc_into_bc_reg) {
255 __ movw(bc_reg, bc);
256 }
257 }
258
259 if (JvmtiExport::can_post_breakpoint()) {
260 Label L_fast_patch;
261 // if a breakpoint is present we can't rewrite the stream directly
262 __ load_unsigned_byte(temp_reg, at_bcp(0));
263 __ cmpw(temp_reg, Bytecodes::_breakpoint);
264 __ br(Assembler::NE, L_fast_patch);
265 // Let breakpoint table handling rewrite to quicker bytecode
266 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::set_original_bytecode_at), rmethod, rbcp, bc_reg);
267 __ b(L_patch_done);
268 __ bind(L_fast_patch);
269 }
270
271 #ifdef ASSERT
272 Label L_okay;
273 __ load_unsigned_byte(temp_reg, at_bcp(0));
274 __ cmpw(temp_reg, (int) Bytecodes::java_code(bc));
275 __ br(Assembler::EQ, L_okay);
276 __ cmpw(temp_reg, bc_reg);
277 __ br(Assembler::EQ, L_okay);
278 __ stop("patching the wrong bytecode");
279 __ bind(L_okay);
280 #endif
281
282 // patch bytecode
283 __ strb(bc_reg, at_bcp(0));
284 __ bind(L_patch_done);
285 }
286
287
288 // Individual instructions
289
290 void TemplateTable::nop() {
291 transition(vtos, vtos);
292 // nothing to do
293 }
294
295 void TemplateTable::shouldnotreachhere() {
296 transition(vtos, vtos);
297 __ stop("shouldnotreachhere bytecode");
298 }
299
300 void TemplateTable::aconst_null()
301 {
302 transition(vtos, atos);
303 __ mov(r0, 0);
304 }
305
306 void TemplateTable::iconst(int value)
307 {
308 transition(vtos, itos);
309 __ mov(r0, value);
310 }
311
312 void TemplateTable::lconst(int value)
313 {
314 __ mov(r0, value);
315 }
316
317 void TemplateTable::fconst(int value)
318 {
319 transition(vtos, ftos);
320 switch (value) {
321 case 0:
322 __ fmovs(v0, zr);
323 break;
324 case 1:
325 __ fmovs(v0, 1.0);
326 break;
327 case 2:
328 __ fmovs(v0, 2.0);
329 break;
330 default:
331 ShouldNotReachHere();
332 break;
333 }
334 }
335
336 void TemplateTable::dconst(int value)
337 {
338 transition(vtos, dtos);
339 switch (value) {
340 case 0:
341 __ fmovd(v0, zr);
342 break;
343 case 1:
344 __ fmovd(v0, 1.0);
345 break;
346 case 2:
347 __ fmovd(v0, 2.0);
348 break;
349 default:
350 ShouldNotReachHere();
351 break;
352 }
353 }
354
355 void TemplateTable::bipush()
356 {
357 transition(vtos, itos);
358 __ load_signed_byte32(r0, at_bcp(1));
359 }
360
361 void TemplateTable::sipush()
362 {
363 transition(vtos, itos);
364 __ load_unsigned_short(r0, at_bcp(1));
365 __ revw(r0, r0);
366 __ asrw(r0, r0, 16);
367 }
368
369 void TemplateTable::ldc(bool wide)
370 {
371 transition(vtos, vtos);
372 Label call_ldc, notFloat, notClass, Done;
373
374 if (wide) {
375 __ get_unsigned_2_byte_index_at_bcp(r1, 1);
376 } else {
377 __ load_unsigned_byte(r1, at_bcp(1));
378 }
379 __ get_cpool_and_tags(r2, r0);
380
381 const int base_offset = ConstantPool::header_size() * wordSize;
382 const int tags_offset = Array<u1>::base_offset_in_bytes();
383
384 // get type
385 __ add(r3, r1, tags_offset);
386 __ lea(r3, Address(r0, r3));
387 __ ldarb(r3, r3);
388
389 // unresolved class - get the resolved class
390 __ cmp(r3, JVM_CONSTANT_UnresolvedClass);
391 __ br(Assembler::EQ, call_ldc);
392
393 // unresolved class in error state - call into runtime to throw the error
394 // from the first resolution attempt
395 __ cmp(r3, JVM_CONSTANT_UnresolvedClassInError);
396 __ br(Assembler::EQ, call_ldc);
397
398 // resolved class - need to call vm to get java mirror of the class
399 __ cmp(r3, JVM_CONSTANT_Class);
400 __ br(Assembler::NE, notClass);
401
402 __ bind(call_ldc);
403 __ mov(c_rarg1, wide);
404 call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::ldc), c_rarg1);
405 __ push_ptr(r0);
406 __ verify_oop(r0);
407 __ b(Done);
408
409 __ bind(notClass);
410 __ cmp(r3, JVM_CONSTANT_Float);
411 __ br(Assembler::NE, notFloat);
412 // ftos
413 __ adds(r1, r2, r1, Assembler::LSL, 3);
414 __ ldrs(v0, Address(r1, base_offset));
415 __ push_f();
416 __ b(Done);
417
418 __ bind(notFloat);
419 #ifdef ASSERT
420 {
421 Label L;
422 __ cmp(r3, JVM_CONSTANT_Integer);
423 __ br(Assembler::EQ, L);
424 // String and Object are rewritten to fast_aldc
425 __ stop("unexpected tag type in ldc");
426 __ bind(L);
427 }
428 #endif
429 // itos JVM_CONSTANT_Integer only
430 __ adds(r1, r2, r1, Assembler::LSL, 3);
431 __ ldrw(r0, Address(r1, base_offset));
432 __ push_i(r0);
433 __ bind(Done);
434 }
435
436 // Fast path for caching oop constants.
437 void TemplateTable::fast_aldc(bool wide)
438 {
439 transition(vtos, atos);
440
441 Register result = r0;
442 Register tmp = r1;
443 int index_size = wide ? sizeof(u2) : sizeof(u1);
444
445 Label resolved;
446
447 // We are resolved if the resolved reference cache entry contains a
448 // non-null object (String, MethodType, etc.)
449 assert_different_registers(result, tmp);
450 __ get_cache_index_at_bcp(tmp, 1, index_size);
451 __ load_resolved_reference_at_index(result, tmp);
452 __ cbnz(result, resolved);
453
454 address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_ldc);
455
456 // first time invocation - must resolve first
457 __ mov(tmp, (int)bytecode());
458 __ call_VM(result, entry, tmp);
459
460 __ bind(resolved);
461
462 if (VerifyOops) {
463 __ verify_oop(result);
464 }
465 }
466
467 void TemplateTable::ldc2_w()
468 {
469 transition(vtos, vtos);
470 Label Long, Done;
471 __ get_unsigned_2_byte_index_at_bcp(r0, 1);
472
473 __ get_cpool_and_tags(r1, r2);
474 const int base_offset = ConstantPool::header_size() * wordSize;
475 const int tags_offset = Array<u1>::base_offset_in_bytes();
476
477 // get type
478 __ lea(r2, Address(r2, r0, Address::lsl(0)));
479 __ load_unsigned_byte(r2, Address(r2, tags_offset));
480 __ cmpw(r2, (int)JVM_CONSTANT_Double);
481 __ br(Assembler::NE, Long);
482 // dtos
483 __ lea (r2, Address(r1, r0, Address::lsl(3)));
484 __ ldrd(v0, Address(r2, base_offset));
485 __ push_d();
486 __ b(Done);
487
488 __ bind(Long);
489 // ltos
490 __ lea(r0, Address(r1, r0, Address::lsl(3)));
491 __ ldr(r0, Address(r0, base_offset));
492 __ push_l();
493
494 __ bind(Done);
495 }
496
497 void TemplateTable::locals_index(Register reg, int offset)
498 {
499 __ ldrb(reg, at_bcp(offset));
500 __ neg(reg, reg);
501 }
502
503 void TemplateTable::iload() {
504 iload_internal();
505 }
506
507 void TemplateTable::nofast_iload() {
508 iload_internal(may_not_rewrite);
509 }
510
511 void TemplateTable::iload_internal(RewriteControl rc) {
512 transition(vtos, itos);
513 if (RewriteFrequentPairs && rc == may_rewrite) {
514 Label rewrite, done;
515 Register bc = r4;
516
517 // get next bytecode
518 __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_iload)));
519
520 // if _iload, wait to rewrite to iload2. We only want to rewrite the
521 // last two iloads in a pair. Comparing against fast_iload means that
522 // the next bytecode is neither an iload or a caload, and therefore
523 // an iload pair.
524 __ cmpw(r1, Bytecodes::_iload);
525 __ br(Assembler::EQ, done);
526
527 // if _fast_iload rewrite to _fast_iload2
528 __ cmpw(r1, Bytecodes::_fast_iload);
529 __ movw(bc, Bytecodes::_fast_iload2);
530 __ br(Assembler::EQ, rewrite);
531
532 // if _caload rewrite to _fast_icaload
533 __ cmpw(r1, Bytecodes::_caload);
534 __ movw(bc, Bytecodes::_fast_icaload);
535 __ br(Assembler::EQ, rewrite);
536
537 // else rewrite to _fast_iload
538 __ movw(bc, Bytecodes::_fast_iload);
539
540 // rewrite
541 // bc: new bytecode
542 __ bind(rewrite);
543 patch_bytecode(Bytecodes::_iload, bc, r1, false);
544 __ bind(done);
545
546 }
547
548 // do iload, get the local value into tos
549 locals_index(r1);
550 __ ldr(r0, iaddress(r1));
551
552 }
553
554 void TemplateTable::fast_iload2()
555 {
556 transition(vtos, itos);
557 locals_index(r1);
558 __ ldr(r0, iaddress(r1));
559 __ push(itos);
560 locals_index(r1, 3);
561 __ ldr(r0, iaddress(r1));
562 }
563
564 void TemplateTable::fast_iload()
565 {
566 transition(vtos, itos);
567 locals_index(r1);
568 __ ldr(r0, iaddress(r1));
569 }
570
571 void TemplateTable::lload()
572 {
573 transition(vtos, ltos);
574 __ ldrb(r1, at_bcp(1));
575 __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
576 __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
577 }
578
579 void TemplateTable::fload()
580 {
581 transition(vtos, ftos);
582 locals_index(r1);
583 // n.b. we use ldrd here because this is a 64 bit slot
584 // this is comparable to the iload case
585 __ ldrd(v0, faddress(r1));
586 }
587
588 void TemplateTable::dload()
589 {
590 transition(vtos, dtos);
591 __ ldrb(r1, at_bcp(1));
592 __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
593 __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
594 }
595
596 void TemplateTable::aload()
597 {
598 transition(vtos, atos);
599 locals_index(r1);
600 __ ldr(r0, iaddress(r1));
601 }
602
603 void TemplateTable::locals_index_wide(Register reg) {
604 __ ldrh(reg, at_bcp(2));
605 __ rev16w(reg, reg);
606 __ neg(reg, reg);
607 }
608
609 void TemplateTable::wide_iload() {
610 transition(vtos, itos);
611 locals_index_wide(r1);
612 __ ldr(r0, iaddress(r1));
613 }
614
615 void TemplateTable::wide_lload()
616 {
617 transition(vtos, ltos);
618 __ ldrh(r1, at_bcp(2));
619 __ rev16w(r1, r1);
620 __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
621 __ ldr(r0, Address(r1, Interpreter::local_offset_in_bytes(1)));
622 }
623
624 void TemplateTable::wide_fload()
625 {
626 transition(vtos, ftos);
627 locals_index_wide(r1);
628 // n.b. we use ldrd here because this is a 64 bit slot
629 // this is comparable to the iload case
630 __ ldrd(v0, faddress(r1));
631 }
632
633 void TemplateTable::wide_dload()
634 {
635 transition(vtos, dtos);
636 __ ldrh(r1, at_bcp(2));
637 __ rev16w(r1, r1);
638 __ sub(r1, rlocals, r1, ext::uxtw, LogBytesPerWord);
639 __ ldrd(v0, Address(r1, Interpreter::local_offset_in_bytes(1)));
640 }
641
642 void TemplateTable::wide_aload()
643 {
644 transition(vtos, atos);
645 locals_index_wide(r1);
646 __ ldr(r0, aaddress(r1));
647 }
648
649 void TemplateTable::index_check(Register array, Register index)
650 {
651 // destroys r1, rscratch1
652 // check array
653 __ null_check(array, arrayOopDesc::length_offset_in_bytes());
654 // sign extend index for use by indexed load
655 // __ movl2ptr(index, index);
656 // check index
657 Register length = rscratch1;
658 __ ldrw(length, Address(array, arrayOopDesc::length_offset_in_bytes()));
659 __ cmpw(index, length);
660 if (index != r1) {
661 // ??? convention: move aberrant index into r1 for exception message
662 assert(r1 != array, "different registers");
663 __ mov(r1, index);
664 }
665 Label ok;
666 __ br(Assembler::LO, ok);
667 __ mov(rscratch1, Interpreter::_throw_ArrayIndexOutOfBoundsException_entry);
668 __ br(rscratch1);
669 __ bind(ok);
670 }
671
672 void TemplateTable::iaload()
673 {
674 transition(itos, itos);
675 __ mov(r1, r0);
676 __ pop_ptr(r0);
677 // r0: array
678 // r1: index
679 index_check(r0, r1); // leaves index in r1, kills rscratch1
680 __ lea(r1, Address(r0, r1, Address::uxtw(2)));
681 __ ldrw(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_INT)));
682 }
683
684 void TemplateTable::laload()
685 {
686 transition(itos, ltos);
687 __ mov(r1, r0);
688 __ pop_ptr(r0);
689 // r0: array
690 // r1: index
691 index_check(r0, r1); // leaves index in r1, kills rscratch1
692 __ lea(r1, Address(r0, r1, Address::uxtw(3)));
693 __ ldr(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_LONG)));
694 }
695
696 void TemplateTable::faload()
697 {
698 transition(itos, ftos);
699 __ mov(r1, r0);
700 __ pop_ptr(r0);
701 // r0: array
702 // r1: index
703 index_check(r0, r1); // leaves index in r1, kills rscratch1
704 __ lea(r1, Address(r0, r1, Address::uxtw(2)));
705 __ ldrs(v0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
706 }
707
708 void TemplateTable::daload()
709 {
710 transition(itos, dtos);
711 __ mov(r1, r0);
712 __ pop_ptr(r0);
713 // r0: array
714 // r1: index
715 index_check(r0, r1); // leaves index in r1, kills rscratch1
716 __ lea(r1, Address(r0, r1, Address::uxtw(3)));
717 __ ldrd(v0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
718 }
719
720 void TemplateTable::aaload()
721 {
722 transition(itos, atos);
723 __ mov(r1, r0);
724 __ pop_ptr(r0);
725 // r0: array
726 // r1: index
727 index_check(r0, r1); // leaves index in r1, kills rscratch1
728 int s = (UseCompressedOops ? 2 : 3);
729 __ lea(r1, Address(r0, r1, Address::uxtw(s)));
730 __ load_heap_oop(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_OBJECT)));
731 }
732
733 void TemplateTable::baload()
734 {
735 transition(itos, itos);
736 __ mov(r1, r0);
737 __ pop_ptr(r0);
738 // r0: array
739 // r1: index
740 index_check(r0, r1); // leaves index in r1, kills rscratch1
741 __ lea(r1, Address(r0, r1, Address::uxtw(0)));
742 __ load_signed_byte(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_BYTE)));
743 }
744
745 void TemplateTable::caload()
746 {
747 transition(itos, itos);
748 __ mov(r1, r0);
749 __ pop_ptr(r0);
750 // r0: array
751 // r1: index
752 index_check(r0, r1); // leaves index in r1, kills rscratch1
753 __ lea(r1, Address(r0, r1, Address::uxtw(1)));
754 __ load_unsigned_short(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
755 }
756
757 // iload followed by caload frequent pair
758 void TemplateTable::fast_icaload()
759 {
760 transition(vtos, itos);
761 // load index out of locals
762 locals_index(r2);
763 __ ldr(r1, iaddress(r2));
764
765 __ pop_ptr(r0);
766
767 // r0: array
768 // r1: index
769 index_check(r0, r1); // leaves index in r1, kills rscratch1
770 __ lea(r1, Address(r0, r1, Address::uxtw(1)));
771 __ load_unsigned_short(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_CHAR)));
772 }
773
774 void TemplateTable::saload()
775 {
776 transition(itos, itos);
777 __ mov(r1, r0);
778 __ pop_ptr(r0);
779 // r0: array
780 // r1: index
781 index_check(r0, r1); // leaves index in r1, kills rscratch1
782 __ lea(r1, Address(r0, r1, Address::uxtw(1)));
783 __ load_signed_short(r0, Address(r1, arrayOopDesc::base_offset_in_bytes(T_SHORT)));
784 }
785
786 void TemplateTable::iload(int n)
787 {
788 transition(vtos, itos);
789 __ ldr(r0, iaddress(n));
790 }
791
792 void TemplateTable::lload(int n)
793 {
794 transition(vtos, ltos);
795 __ ldr(r0, laddress(n));
796 }
797
798 void TemplateTable::fload(int n)
799 {
800 transition(vtos, ftos);
801 __ ldrs(v0, faddress(n));
802 }
803
804 void TemplateTable::dload(int n)
805 {
806 transition(vtos, dtos);
807 __ ldrd(v0, daddress(n));
808 }
809
810 void TemplateTable::aload(int n)
811 {
812 transition(vtos, atos);
813 __ ldr(r0, iaddress(n));
814 }
815
816 void TemplateTable::aload_0() {
817 aload_0_internal();
818 }
819
820 void TemplateTable::nofast_aload_0() {
821 aload_0_internal(may_not_rewrite);
822 }
823
824 void TemplateTable::aload_0_internal(RewriteControl rc) {
825 // According to bytecode histograms, the pairs:
826 //
827 // _aload_0, _fast_igetfield
828 // _aload_0, _fast_agetfield
829 // _aload_0, _fast_fgetfield
830 //
831 // occur frequently. If RewriteFrequentPairs is set, the (slow)
832 // _aload_0 bytecode checks if the next bytecode is either
833 // _fast_igetfield, _fast_agetfield or _fast_fgetfield and then
834 // rewrites the current bytecode into a pair bytecode; otherwise it
835 // rewrites the current bytecode into _fast_aload_0 that doesn't do
836 // the pair check anymore.
837 //
838 // Note: If the next bytecode is _getfield, the rewrite must be
839 // delayed, otherwise we may miss an opportunity for a pair.
840 //
841 // Also rewrite frequent pairs
842 // aload_0, aload_1
843 // aload_0, iload_1
844 // These bytecodes with a small amount of code are most profitable
845 // to rewrite
846 if (RewriteFrequentPairs && rc == may_rewrite) {
847 Label rewrite, done;
848 const Register bc = r4;
849
850 // get next bytecode
851 __ load_unsigned_byte(r1, at_bcp(Bytecodes::length_for(Bytecodes::_aload_0)));
852
853 // if _getfield then wait with rewrite
854 __ cmpw(r1, Bytecodes::Bytecodes::_getfield);
855 __ br(Assembler::EQ, done);
856
857 // if _igetfield then rewrite to _fast_iaccess_0
858 assert(Bytecodes::java_code(Bytecodes::_fast_iaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
859 __ cmpw(r1, Bytecodes::_fast_igetfield);
860 __ movw(bc, Bytecodes::_fast_iaccess_0);
861 __ br(Assembler::EQ, rewrite);
862
863 // if _agetfield then rewrite to _fast_aaccess_0
864 assert(Bytecodes::java_code(Bytecodes::_fast_aaccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
865 __ cmpw(r1, Bytecodes::_fast_agetfield);
866 __ movw(bc, Bytecodes::_fast_aaccess_0);
867 __ br(Assembler::EQ, rewrite);
868
869 // if _fgetfield then rewrite to _fast_faccess_0
870 assert(Bytecodes::java_code(Bytecodes::_fast_faccess_0) == Bytecodes::_aload_0, "fix bytecode definition");
871 __ cmpw(r1, Bytecodes::_fast_fgetfield);
872 __ movw(bc, Bytecodes::_fast_faccess_0);
873 __ br(Assembler::EQ, rewrite);
874
875 // else rewrite to _fast_aload0
876 assert(Bytecodes::java_code(Bytecodes::_fast_aload_0) == Bytecodes::_aload_0, "fix bytecode definition");
877 __ movw(bc, Bytecodes::Bytecodes::_fast_aload_0);
878
879 // rewrite
880 // bc: new bytecode
881 __ bind(rewrite);
882 patch_bytecode(Bytecodes::_aload_0, bc, r1, false);
883
884 __ bind(done);
885 }
886
887 // Do actual aload_0 (must do this after patch_bytecode which might call VM and GC might change oop).
888 aload(0);
889 }
890
891 void TemplateTable::istore()
892 {
893 transition(itos, vtos);
894 locals_index(r1);
895 // FIXME: We're being very pernickerty here storing a jint in a
896 // local with strw, which costs an extra instruction over what we'd
897 // be able to do with a simple str. We should just store the whole
898 // word.
899 __ lea(rscratch1, iaddress(r1));
900 __ strw(r0, Address(rscratch1));
901 }
902
903 void TemplateTable::lstore()
904 {
905 transition(ltos, vtos);
906 locals_index(r1);
907 __ str(r0, laddress(r1, rscratch1, _masm));
908 }
909
910 void TemplateTable::fstore() {
911 transition(ftos, vtos);
912 locals_index(r1);
913 __ lea(rscratch1, iaddress(r1));
914 __ strs(v0, Address(rscratch1));
915 }
916
917 void TemplateTable::dstore() {
918 transition(dtos, vtos);
919 locals_index(r1);
920 __ strd(v0, daddress(r1, rscratch1, _masm));
921 }
922
923 void TemplateTable::astore()
924 {
925 transition(vtos, vtos);
926 __ pop_ptr(r0);
927 locals_index(r1);
928 __ str(r0, aaddress(r1));
929 }
930
931 void TemplateTable::wide_istore() {
932 transition(vtos, vtos);
933 __ pop_i();
934 locals_index_wide(r1);
935 __ lea(rscratch1, iaddress(r1));
936 __ strw(r0, Address(rscratch1));
937 }
938
939 void TemplateTable::wide_lstore() {
940 transition(vtos, vtos);
941 __ pop_l();
942 locals_index_wide(r1);
943 __ str(r0, laddress(r1, rscratch1, _masm));
944 }
945
946 void TemplateTable::wide_fstore() {
947 transition(vtos, vtos);
948 __ pop_f();
949 locals_index_wide(r1);
950 __ lea(rscratch1, faddress(r1));
951 __ strs(v0, rscratch1);
952 }
953
954 void TemplateTable::wide_dstore() {
955 transition(vtos, vtos);
956 __ pop_d();
957 locals_index_wide(r1);
958 __ strd(v0, daddress(r1, rscratch1, _masm));
959 }
960
961 void TemplateTable::wide_astore() {
962 transition(vtos, vtos);
963 __ pop_ptr(r0);
964 locals_index_wide(r1);
965 __ str(r0, aaddress(r1));
966 }
967
968 void TemplateTable::iastore() {
969 transition(itos, vtos);
970 __ pop_i(r1);
971 __ pop_ptr(r3);
972 // r0: value
973 // r1: index
974 // r3: array
975 index_check(r3, r1); // prefer index in r1
976 __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
977 __ strw(r0, Address(rscratch1,
978 arrayOopDesc::base_offset_in_bytes(T_INT)));
979 }
980
981 void TemplateTable::lastore() {
982 transition(ltos, vtos);
983 __ pop_i(r1);
984 __ pop_ptr(r3);
985 // r0: value
986 // r1: index
987 // r3: array
988 index_check(r3, r1); // prefer index in r1
989 __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
990 __ str(r0, Address(rscratch1,
991 arrayOopDesc::base_offset_in_bytes(T_LONG)));
992 }
993
994 void TemplateTable::fastore() {
995 transition(ftos, vtos);
996 __ pop_i(r1);
997 __ pop_ptr(r3);
998 // v0: value
999 // r1: index
1000 // r3: array
1001 index_check(r3, r1); // prefer index in r1
1002 __ lea(rscratch1, Address(r3, r1, Address::uxtw(2)));
1003 __ strs(v0, Address(rscratch1,
1004 arrayOopDesc::base_offset_in_bytes(T_FLOAT)));
1005 }
1006
1007 void TemplateTable::dastore() {
1008 transition(dtos, vtos);
1009 __ pop_i(r1);
1010 __ pop_ptr(r3);
1011 // v0: value
1012 // r1: index
1013 // r3: array
1014 index_check(r3, r1); // prefer index in r1
1015 __ lea(rscratch1, Address(r3, r1, Address::uxtw(3)));
1016 __ strd(v0, Address(rscratch1,
1017 arrayOopDesc::base_offset_in_bytes(T_DOUBLE)));
1018 }
1019
1020 void TemplateTable::aastore() {
1021 Label is_null, ok_is_subtype, done;
1022 transition(vtos, vtos);
1023 // stack: ..., array, index, value
1024 __ ldr(r0, at_tos()); // value
1025 __ ldr(r2, at_tos_p1()); // index
1026 __ ldr(r3, at_tos_p2()); // array
1027
1028 Address element_address(r4, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
1029
1030 index_check(r3, r2); // kills r1
1031 __ lea(r4, Address(r3, r2, Address::uxtw(UseCompressedOops? 2 : 3)));
1032
1033 // do array store check - check for NULL value first
1034 __ cbz(r0, is_null);
1035
1036 // Move subklass into r1
1037 __ load_klass(r1, r0);
1038 // Move superklass into r0
1039 __ load_klass(r0, r3);
1040 __ ldr(r0, Address(r0,
1041 ObjArrayKlass::element_klass_offset()));
1042 // Compress array + index*oopSize + 12 into a single register. Frees r2.
1043
1044 // Generate subtype check. Blows r2, r5
1045 // Superklass in r0. Subklass in r1.
1046 __ gen_subtype_check(r1, ok_is_subtype);
1047
1048 // Come here on failure
1049 // object is at TOS
1050 __ b(Interpreter::_throw_ArrayStoreException_entry);
1051
1052 // Come here on success
1053 __ bind(ok_is_subtype);
1054
1055 // Get the value we will store
1056 __ ldr(r0, at_tos());
1057 // Now store using the appropriate barrier
1058 do_oop_store(_masm, element_address, r0, _bs->kind(), true);
1059 __ b(done);
1060
1061 // Have a NULL in r0, r3=array, r2=index. Store NULL at ary[idx]
1062 __ bind(is_null);
1063 __ profile_null_seen(r2);
1064
1065 // Store a NULL
1066 do_oop_store(_masm, element_address, noreg, _bs->kind(), true);
1067
1068 // Pop stack arguments
1069 __ bind(done);
1070 __ add(esp, esp, 3 * Interpreter::stackElementSize);
1071 }
1072
1073 void TemplateTable::bastore()
1074 {
1075 transition(itos, vtos);
1076 __ pop_i(r1);
1077 __ pop_ptr(r3);
1078 // r0: value
1079 // r1: index
1080 // r3: array
1081 index_check(r3, r1); // prefer index in r1
1082
1083 // Need to check whether array is boolean or byte
1084 // since both types share the bastore bytecode.
1085 __ load_klass(r2, r3);
1086 __ ldrw(r2, Address(r2, Klass::layout_helper_offset()));
1087 int diffbit_index = exact_log2(Klass::layout_helper_boolean_diffbit());
1088 Label L_skip;
1089 __ tbz(r2, diffbit_index, L_skip);
1090 __ andw(r0, r0, 1); // if it is a T_BOOLEAN array, mask the stored value to 0/1
1091 __ bind(L_skip);
1092
1093 __ lea(rscratch1, Address(r3, r1, Address::uxtw(0)));
1094 __ strb(r0, Address(rscratch1,
1095 arrayOopDesc::base_offset_in_bytes(T_BYTE)));
1096 }
1097
1098 void TemplateTable::castore()
1099 {
1100 transition(itos, vtos);
1101 __ pop_i(r1);
1102 __ pop_ptr(r3);
1103 // r0: value
1104 // r1: index
1105 // r3: array
1106 index_check(r3, r1); // prefer index in r1
1107 __ lea(rscratch1, Address(r3, r1, Address::uxtw(1)));
1108 __ strh(r0, Address(rscratch1,
1109 arrayOopDesc::base_offset_in_bytes(T_CHAR)));
1110 }
1111
1112 void TemplateTable::sastore()
1113 {
1114 castore();
1115 }
1116
1117 void TemplateTable::istore(int n)
1118 {
1119 transition(itos, vtos);
1120 __ str(r0, iaddress(n));
1121 }
1122
1123 void TemplateTable::lstore(int n)
1124 {
1125 transition(ltos, vtos);
1126 __ str(r0, laddress(n));
1127 }
1128
1129 void TemplateTable::fstore(int n)
1130 {
1131 transition(ftos, vtos);
1132 __ strs(v0, faddress(n));
1133 }
1134
1135 void TemplateTable::dstore(int n)
1136 {
1137 transition(dtos, vtos);
1138 __ strd(v0, daddress(n));
1139 }
1140
1141 void TemplateTable::astore(int n)
1142 {
1143 transition(vtos, vtos);
1144 __ pop_ptr(r0);
1145 __ str(r0, iaddress(n));
1146 }
1147
1148 void TemplateTable::pop()
1149 {
1150 transition(vtos, vtos);
1151 __ add(esp, esp, Interpreter::stackElementSize);
1152 }
1153
1154 void TemplateTable::pop2()
1155 {
1156 transition(vtos, vtos);
1157 __ add(esp, esp, 2 * Interpreter::stackElementSize);
1158 }
1159
1160 void TemplateTable::dup()
1161 {
1162 transition(vtos, vtos);
1163 __ ldr(r0, Address(esp, 0));
1164 __ push(r0);
1165 // stack: ..., a, a
1166 }
1167
1168 void TemplateTable::dup_x1()
1169 {
1170 transition(vtos, vtos);
1171 // stack: ..., a, b
1172 __ ldr(r0, at_tos()); // load b
1173 __ ldr(r2, at_tos_p1()); // load a
1174 __ str(r0, at_tos_p1()); // store b
1175 __ str(r2, at_tos()); // store a
1176 __ push(r0); // push b
1177 // stack: ..., b, a, b
1178 }
1179
1180 void TemplateTable::dup_x2()
1181 {
1182 transition(vtos, vtos);
1183 // stack: ..., a, b, c
1184 __ ldr(r0, at_tos()); // load c
1185 __ ldr(r2, at_tos_p2()); // load a
1186 __ str(r0, at_tos_p2()); // store c in a
1187 __ push(r0); // push c
1188 // stack: ..., c, b, c, c
1189 __ ldr(r0, at_tos_p2()); // load b
1190 __ str(r2, at_tos_p2()); // store a in b
1191 // stack: ..., c, a, c, c
1192 __ str(r0, at_tos_p1()); // store b in c
1193 // stack: ..., c, a, b, c
1194 }
1195
1196 void TemplateTable::dup2()
1197 {
1198 transition(vtos, vtos);
1199 // stack: ..., a, b
1200 __ ldr(r0, at_tos_p1()); // load a
1201 __ push(r0); // push a
1202 __ ldr(r0, at_tos_p1()); // load b
1203 __ push(r0); // push b
1204 // stack: ..., a, b, a, b
1205 }
1206
1207 void TemplateTable::dup2_x1()
1208 {
1209 transition(vtos, vtos);
1210 // stack: ..., a, b, c
1211 __ ldr(r2, at_tos()); // load c
1212 __ ldr(r0, at_tos_p1()); // load b
1213 __ push(r0); // push b
1214 __ push(r2); // push c
1215 // stack: ..., a, b, c, b, c
1216 __ str(r2, at_tos_p3()); // store c in b
1217 // stack: ..., a, c, c, b, c
1218 __ ldr(r2, at_tos_p4()); // load a
1219 __ str(r2, at_tos_p2()); // store a in 2nd c
1220 // stack: ..., a, c, a, b, c
1221 __ str(r0, at_tos_p4()); // store b in a
1222 // stack: ..., b, c, a, b, c
1223 }
1224
1225 void TemplateTable::dup2_x2()
1226 {
1227 transition(vtos, vtos);
1228 // stack: ..., a, b, c, d
1229 __ ldr(r2, at_tos()); // load d
1230 __ ldr(r0, at_tos_p1()); // load c
1231 __ push(r0) ; // push c
1232 __ push(r2); // push d
1233 // stack: ..., a, b, c, d, c, d
1234 __ ldr(r0, at_tos_p4()); // load b
1235 __ str(r0, at_tos_p2()); // store b in d
1236 __ str(r2, at_tos_p4()); // store d in b
1237 // stack: ..., a, d, c, b, c, d
1238 __ ldr(r2, at_tos_p5()); // load a
1239 __ ldr(r0, at_tos_p3()); // load c
1240 __ str(r2, at_tos_p3()); // store a in c
1241 __ str(r0, at_tos_p5()); // store c in a
1242 // stack: ..., c, d, a, b, c, d
1243 }
1244
1245 void TemplateTable::swap()
1246 {
1247 transition(vtos, vtos);
1248 // stack: ..., a, b
1249 __ ldr(r2, at_tos_p1()); // load a
1250 __ ldr(r0, at_tos()); // load b
1251 __ str(r2, at_tos()); // store a in b
1252 __ str(r0, at_tos_p1()); // store b in a
1253 // stack: ..., b, a
1254 }
1255
1256 void TemplateTable::iop2(Operation op)
1257 {
1258 transition(itos, itos);
1259 // r0 <== r1 op r0
1260 __ pop_i(r1);
1261 switch (op) {
1262 case add : __ addw(r0, r1, r0); break;
1263 case sub : __ subw(r0, r1, r0); break;
1264 case mul : __ mulw(r0, r1, r0); break;
1265 case _and : __ andw(r0, r1, r0); break;
1266 case _or : __ orrw(r0, r1, r0); break;
1267 case _xor : __ eorw(r0, r1, r0); break;
1268 case shl : __ lslvw(r0, r1, r0); break;
1269 case shr : __ asrvw(r0, r1, r0); break;
1270 case ushr : __ lsrvw(r0, r1, r0);break;
1271 default : ShouldNotReachHere();
1272 }
1273 }
1274
1275 void TemplateTable::lop2(Operation op)
1276 {
1277 transition(ltos, ltos);
1278 // r0 <== r1 op r0
1279 __ pop_l(r1);
1280 switch (op) {
1281 case add : __ add(r0, r1, r0); break;
1282 case sub : __ sub(r0, r1, r0); break;
1283 case mul : __ mul(r0, r1, r0); break;
1284 case _and : __ andr(r0, r1, r0); break;
1285 case _or : __ orr(r0, r1, r0); break;
1286 case _xor : __ eor(r0, r1, r0); break;
1287 default : ShouldNotReachHere();
1288 }
1289 }
1290
1291 void TemplateTable::idiv()
1292 {
1293 transition(itos, itos);
1294 // explicitly check for div0
1295 Label no_div0;
1296 __ cbnzw(r0, no_div0);
1297 __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1298 __ br(rscratch1);
1299 __ bind(no_div0);
1300 __ pop_i(r1);
1301 // r0 <== r1 idiv r0
1302 __ corrected_idivl(r0, r1, r0, /* want_remainder */ false);
1303 }
1304
1305 void TemplateTable::irem()
1306 {
1307 transition(itos, itos);
1308 // explicitly check for div0
1309 Label no_div0;
1310 __ cbnzw(r0, no_div0);
1311 __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1312 __ br(rscratch1);
1313 __ bind(no_div0);
1314 __ pop_i(r1);
1315 // r0 <== r1 irem r0
1316 __ corrected_idivl(r0, r1, r0, /* want_remainder */ true);
1317 }
1318
1319 void TemplateTable::lmul()
1320 {
1321 transition(ltos, ltos);
1322 __ pop_l(r1);
1323 __ mul(r0, r0, r1);
1324 }
1325
1326 void TemplateTable::ldiv()
1327 {
1328 transition(ltos, ltos);
1329 // explicitly check for div0
1330 Label no_div0;
1331 __ cbnz(r0, no_div0);
1332 __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1333 __ br(rscratch1);
1334 __ bind(no_div0);
1335 __ pop_l(r1);
1336 // r0 <== r1 ldiv r0
1337 __ corrected_idivq(r0, r1, r0, /* want_remainder */ false);
1338 }
1339
1340 void TemplateTable::lrem()
1341 {
1342 transition(ltos, ltos);
1343 // explicitly check for div0
1344 Label no_div0;
1345 __ cbnz(r0, no_div0);
1346 __ mov(rscratch1, Interpreter::_throw_ArithmeticException_entry);
1347 __ br(rscratch1);
1348 __ bind(no_div0);
1349 __ pop_l(r1);
1350 // r0 <== r1 lrem r0
1351 __ corrected_idivq(r0, r1, r0, /* want_remainder */ true);
1352 }
1353
1354 void TemplateTable::lshl()
1355 {
1356 transition(itos, ltos);
1357 // shift count is in r0
1358 __ pop_l(r1);
1359 __ lslv(r0, r1, r0);
1360 }
1361
1362 void TemplateTable::lshr()
1363 {
1364 transition(itos, ltos);
1365 // shift count is in r0
1366 __ pop_l(r1);
1367 __ asrv(r0, r1, r0);
1368 }
1369
1370 void TemplateTable::lushr()
1371 {
1372 transition(itos, ltos);
1373 // shift count is in r0
1374 __ pop_l(r1);
1375 __ lsrv(r0, r1, r0);
1376 }
1377
1378 void TemplateTable::fop2(Operation op)
1379 {
1380 transition(ftos, ftos);
1381 switch (op) {
1382 case add:
1383 // n.b. use ldrd because this is a 64 bit slot
1384 __ pop_f(v1);
1385 __ fadds(v0, v1, v0);
1386 break;
1387 case sub:
1388 __ pop_f(v1);
1389 __ fsubs(v0, v1, v0);
1390 break;
1391 case mul:
1392 __ pop_f(v1);
1393 __ fmuls(v0, v1, v0);
1394 break;
1395 case div:
1396 __ pop_f(v1);
1397 __ fdivs(v0, v1, v0);
1398 break;
1399 case rem:
1400 __ fmovs(v1, v0);
1401 __ pop_f(v0);
1402 __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::frem),
1403 0, 2, MacroAssembler::ret_type_float);
1404 break;
1405 default:
1406 ShouldNotReachHere();
1407 break;
1408 }
1409 }
1410
1411 void TemplateTable::dop2(Operation op)
1412 {
1413 transition(dtos, dtos);
1414 switch (op) {
1415 case add:
1416 // n.b. use ldrd because this is a 64 bit slot
1417 __ pop_d(v1);
1418 __ faddd(v0, v1, v0);
1419 break;
1420 case sub:
1421 __ pop_d(v1);
1422 __ fsubd(v0, v1, v0);
1423 break;
1424 case mul:
1425 __ pop_d(v1);
1426 __ fmuld(v0, v1, v0);
1427 break;
1428 case div:
1429 __ pop_d(v1);
1430 __ fdivd(v0, v1, v0);
1431 break;
1432 case rem:
1433 __ fmovd(v1, v0);
1434 __ pop_d(v0);
1435 __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::drem),
1436 0, 2, MacroAssembler::ret_type_double);
1437 break;
1438 default:
1439 ShouldNotReachHere();
1440 break;
1441 }
1442 }
1443
1444 void TemplateTable::ineg()
1445 {
1446 transition(itos, itos);
1447 __ negw(r0, r0);
1448
1449 }
1450
1451 void TemplateTable::lneg()
1452 {
1453 transition(ltos, ltos);
1454 __ neg(r0, r0);
1455 }
1456
1457 void TemplateTable::fneg()
1458 {
1459 transition(ftos, ftos);
1460 __ fnegs(v0, v0);
1461 }
1462
1463 void TemplateTable::dneg()
1464 {
1465 transition(dtos, dtos);
1466 __ fnegd(v0, v0);
1467 }
1468
1469 void TemplateTable::iinc()
1470 {
1471 transition(vtos, vtos);
1472 __ load_signed_byte(r1, at_bcp(2)); // get constant
1473 locals_index(r2);
1474 __ ldr(r0, iaddress(r2));
1475 __ addw(r0, r0, r1);
1476 __ str(r0, iaddress(r2));
1477 }
1478
1479 void TemplateTable::wide_iinc()
1480 {
1481 transition(vtos, vtos);
1482 // __ mov(r1, zr);
1483 __ ldrw(r1, at_bcp(2)); // get constant and index
1484 __ rev16(r1, r1);
1485 __ ubfx(r2, r1, 0, 16);
1486 __ neg(r2, r2);
1487 __ sbfx(r1, r1, 16, 16);
1488 __ ldr(r0, iaddress(r2));
1489 __ addw(r0, r0, r1);
1490 __ str(r0, iaddress(r2));
1491 }
1492
1493 void TemplateTable::convert()
1494 {
1495 // Checking
1496 #ifdef ASSERT
1497 {
1498 TosState tos_in = ilgl;
1499 TosState tos_out = ilgl;
1500 switch (bytecode()) {
1501 case Bytecodes::_i2l: // fall through
1502 case Bytecodes::_i2f: // fall through
1503 case Bytecodes::_i2d: // fall through
1504 case Bytecodes::_i2b: // fall through
1505 case Bytecodes::_i2c: // fall through
1506 case Bytecodes::_i2s: tos_in = itos; break;
1507 case Bytecodes::_l2i: // fall through
1508 case Bytecodes::_l2f: // fall through
1509 case Bytecodes::_l2d: tos_in = ltos; break;
1510 case Bytecodes::_f2i: // fall through
1511 case Bytecodes::_f2l: // fall through
1512 case Bytecodes::_f2d: tos_in = ftos; break;
1513 case Bytecodes::_d2i: // fall through
1514 case Bytecodes::_d2l: // fall through
1515 case Bytecodes::_d2f: tos_in = dtos; break;
1516 default : ShouldNotReachHere();
1517 }
1518 switch (bytecode()) {
1519 case Bytecodes::_l2i: // fall through
1520 case Bytecodes::_f2i: // fall through
1521 case Bytecodes::_d2i: // fall through
1522 case Bytecodes::_i2b: // fall through
1523 case Bytecodes::_i2c: // fall through
1524 case Bytecodes::_i2s: tos_out = itos; break;
1525 case Bytecodes::_i2l: // fall through
1526 case Bytecodes::_f2l: // fall through
1527 case Bytecodes::_d2l: tos_out = ltos; break;
1528 case Bytecodes::_i2f: // fall through
1529 case Bytecodes::_l2f: // fall through
1530 case Bytecodes::_d2f: tos_out = ftos; break;
1531 case Bytecodes::_i2d: // fall through
1532 case Bytecodes::_l2d: // fall through
1533 case Bytecodes::_f2d: tos_out = dtos; break;
1534 default : ShouldNotReachHere();
1535 }
1536 transition(tos_in, tos_out);
1537 }
1538 #endif // ASSERT
1539 // static const int64_t is_nan = 0x8000000000000000L;
1540
1541 // Conversion
1542 switch (bytecode()) {
1543 case Bytecodes::_i2l:
1544 __ sxtw(r0, r0);
1545 break;
1546 case Bytecodes::_i2f:
1547 __ scvtfws(v0, r0);
1548 break;
1549 case Bytecodes::_i2d:
1550 __ scvtfwd(v0, r0);
1551 break;
1552 case Bytecodes::_i2b:
1553 __ sxtbw(r0, r0);
1554 break;
1555 case Bytecodes::_i2c:
1556 __ uxthw(r0, r0);
1557 break;
1558 case Bytecodes::_i2s:
1559 __ sxthw(r0, r0);
1560 break;
1561 case Bytecodes::_l2i:
1562 __ uxtw(r0, r0);
1563 break;
1564 case Bytecodes::_l2f:
1565 __ scvtfs(v0, r0);
1566 break;
1567 case Bytecodes::_l2d:
1568 __ scvtfd(v0, r0);
1569 break;
1570 case Bytecodes::_f2i:
1571 {
1572 Label L_Okay;
1573 __ clear_fpsr();
1574 __ fcvtzsw(r0, v0);
1575 __ get_fpsr(r1);
1576 __ cbzw(r1, L_Okay);
1577 __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2i),
1578 0, 1, MacroAssembler::ret_type_integral);
1579 __ bind(L_Okay);
1580 }
1581 break;
1582 case Bytecodes::_f2l:
1583 {
1584 Label L_Okay;
1585 __ clear_fpsr();
1586 __ fcvtzs(r0, v0);
1587 __ get_fpsr(r1);
1588 __ cbzw(r1, L_Okay);
1589 __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::f2l),
1590 0, 1, MacroAssembler::ret_type_integral);
1591 __ bind(L_Okay);
1592 }
1593 break;
1594 case Bytecodes::_f2d:
1595 __ fcvts(v0, v0);
1596 break;
1597 case Bytecodes::_d2i:
1598 {
1599 Label L_Okay;
1600 __ clear_fpsr();
1601 __ fcvtzdw(r0, v0);
1602 __ get_fpsr(r1);
1603 __ cbzw(r1, L_Okay);
1604 __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2i),
1605 0, 1, MacroAssembler::ret_type_integral);
1606 __ bind(L_Okay);
1607 }
1608 break;
1609 case Bytecodes::_d2l:
1610 {
1611 Label L_Okay;
1612 __ clear_fpsr();
1613 __ fcvtzd(r0, v0);
1614 __ get_fpsr(r1);
1615 __ cbzw(r1, L_Okay);
1616 __ call_VM_leaf_base1(CAST_FROM_FN_PTR(address, SharedRuntime::d2l),
1617 0, 1, MacroAssembler::ret_type_integral);
1618 __ bind(L_Okay);
1619 }
1620 break;
1621 case Bytecodes::_d2f:
1622 __ fcvtd(v0, v0);
1623 break;
1624 default:
1625 ShouldNotReachHere();
1626 }
1627 }
1628
1629 void TemplateTable::lcmp()
1630 {
1631 transition(ltos, itos);
1632 Label done;
1633 __ pop_l(r1);
1634 __ cmp(r1, r0);
1635 __ mov(r0, (u_int64_t)-1L);
1636 __ br(Assembler::LT, done);
1637 // __ mov(r0, 1UL);
1638 // __ csel(r0, r0, zr, Assembler::NE);
1639 // and here is a faster way
1640 __ csinc(r0, zr, zr, Assembler::EQ);
1641 __ bind(done);
1642 }
1643
1644 void TemplateTable::float_cmp(bool is_float, int unordered_result)
1645 {
1646 Label done;
1647 if (is_float) {
1648 // XXX get rid of pop here, use ... reg, mem32
1649 __ pop_f(v1);
1650 __ fcmps(v1, v0);
1651 } else {
1652 // XXX get rid of pop here, use ... reg, mem64
1653 __ pop_d(v1);
1654 __ fcmpd(v1, v0);
1655 }
1656 if (unordered_result < 0) {
1657 // we want -1 for unordered or less than, 0 for equal and 1 for
1658 // greater than.
1659 __ mov(r0, (u_int64_t)-1L);
1660 // for FP LT tests less than or unordered
1661 __ br(Assembler::LT, done);
1662 // install 0 for EQ otherwise 1
1663 __ csinc(r0, zr, zr, Assembler::EQ);
1664 } else {
1665 // we want -1 for less than, 0 for equal and 1 for unordered or
1666 // greater than.
1667 __ mov(r0, 1L);
1668 // for FP HI tests greater than or unordered
1669 __ br(Assembler::HI, done);
1670 // install 0 for EQ otherwise ~0
1671 __ csinv(r0, zr, zr, Assembler::EQ);
1672
1673 }
1674 __ bind(done);
1675 }
1676
1677 void TemplateTable::branch(bool is_jsr, bool is_wide)
1678 {
1679 // We might be moving to a safepoint. The thread which calls
1680 // Interpreter::notice_safepoints() will effectively flush its cache
1681 // when it makes a system call, but we need to do something to
1682 // ensure that we see the changed dispatch table.
1683 __ membar(MacroAssembler::LoadLoad);
1684
1685 __ profile_taken_branch(r0, r1);
1686 const ByteSize be_offset = MethodCounters::backedge_counter_offset() +
1687 InvocationCounter::counter_offset();
1688 const ByteSize inv_offset = MethodCounters::invocation_counter_offset() +
1689 InvocationCounter::counter_offset();
1690
1691 // load branch displacement
1692 if (!is_wide) {
1693 __ ldrh(r2, at_bcp(1));
1694 __ rev16(r2, r2);
1695 // sign extend the 16 bit value in r2
1696 __ sbfm(r2, r2, 0, 15);
1697 } else {
1698 __ ldrw(r2, at_bcp(1));
1699 __ revw(r2, r2);
1700 // sign extend the 32 bit value in r2
1701 __ sbfm(r2, r2, 0, 31);
1702 }
1703
1704 // Handle all the JSR stuff here, then exit.
1705 // It's much shorter and cleaner than intermingling with the non-JSR
1706 // normal-branch stuff occurring below.
1707
1708 if (is_jsr) {
1709 // Pre-load the next target bytecode into rscratch1
1710 __ load_unsigned_byte(rscratch1, Address(rbcp, r2));
1711 // compute return address as bci
1712 __ ldr(rscratch2, Address(rmethod, Method::const_offset()));
1713 __ add(rscratch2, rscratch2,
1714 in_bytes(ConstMethod::codes_offset()) - (is_wide ? 5 : 3));
1715 __ sub(r1, rbcp, rscratch2);
1716 __ push_i(r1);
1717 // Adjust the bcp by the 16-bit displacement in r2
1718 __ add(rbcp, rbcp, r2);
1719 __ dispatch_only(vtos);
1720 return;
1721 }
1722
1723 // Normal (non-jsr) branch handling
1724
1725 // Adjust the bcp by the displacement in r2
1726 __ add(rbcp, rbcp, r2);
1727
1728 assert(UseLoopCounter || !UseOnStackReplacement,
1729 "on-stack-replacement requires loop counters");
1730 Label backedge_counter_overflow;
1731 Label profile_method;
1732 Label dispatch;
1733 if (UseLoopCounter) {
1734 // increment backedge counter for backward branches
1735 // r0: MDO
1736 // w1: MDO bumped taken-count
1737 // r2: target offset
1738 __ cmp(r2, zr);
1739 __ br(Assembler::GT, dispatch); // count only if backward branch
1740
1741 // ECN: FIXME: This code smells
1742 // check if MethodCounters exists
1743 Label has_counters;
1744 __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1745 __ cbnz(rscratch1, has_counters);
1746 __ push(r0);
1747 __ push(r1);
1748 __ push(r2);
1749 __ call_VM(noreg, CAST_FROM_FN_PTR(address,
1750 InterpreterRuntime::build_method_counters), rmethod);
1751 __ pop(r2);
1752 __ pop(r1);
1753 __ pop(r0);
1754 __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1755 __ cbz(rscratch1, dispatch); // No MethodCounters allocated, OutOfMemory
1756 __ bind(has_counters);
1757
1758 if (TieredCompilation) {
1759 Label no_mdo;
1760 int increment = InvocationCounter::count_increment;
1761 if (ProfileInterpreter) {
1762 // Are we profiling?
1763 __ ldr(r1, Address(rmethod, in_bytes(Method::method_data_offset())));
1764 __ cbz(r1, no_mdo);
1765 // Increment the MDO backedge counter
1766 const Address mdo_backedge_counter(r1, in_bytes(MethodData::backedge_counter_offset()) +
1767 in_bytes(InvocationCounter::counter_offset()));
1768 const Address mask(r1, in_bytes(MethodData::backedge_mask_offset()));
1769 __ increment_mask_and_jump(mdo_backedge_counter, increment, mask,
1770 r0, rscratch1, false, Assembler::EQ, &backedge_counter_overflow);
1771 __ b(dispatch);
1772 }
1773 __ bind(no_mdo);
1774 // Increment backedge counter in MethodCounters*
1775 __ ldr(rscratch1, Address(rmethod, Method::method_counters_offset()));
1776 const Address mask(rscratch1, in_bytes(MethodCounters::backedge_mask_offset()));
1777 __ increment_mask_and_jump(Address(rscratch1, be_offset), increment, mask,
1778 r0, rscratch2, false, Assembler::EQ, &backedge_counter_overflow);
1779 } else { // not TieredCompilation
1780 // increment counter
1781 __ ldr(rscratch2, Address(rmethod, Method::method_counters_offset()));
1782 __ ldrw(r0, Address(rscratch2, be_offset)); // load backedge counter
1783 __ addw(rscratch1, r0, InvocationCounter::count_increment); // increment counter
1784 __ strw(rscratch1, Address(rscratch2, be_offset)); // store counter
1785
1786 __ ldrw(r0, Address(rscratch2, inv_offset)); // load invocation counter
1787 __ andw(r0, r0, (unsigned)InvocationCounter::count_mask_value); // and the status bits
1788 __ addw(r0, r0, rscratch1); // add both counters
1789
1790 if (ProfileInterpreter) {
1791 // Test to see if we should create a method data oop
1792 __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_profile_limit_offset())));
1793 __ cmpw(r0, rscratch1);
1794 __ br(Assembler::LT, dispatch);
1795
1796 // if no method data exists, go to profile method
1797 __ test_method_data_pointer(r0, profile_method);
1798
1799 if (UseOnStackReplacement) {
1800 // check for overflow against w1 which is the MDO taken count
1801 __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1802 __ cmpw(r1, rscratch1);
1803 __ br(Assembler::LO, dispatch); // Intel == Assembler::below
1804
1805 // When ProfileInterpreter is on, the backedge_count comes
1806 // from the MethodData*, which value does not get reset on
1807 // the call to frequency_counter_overflow(). To avoid
1808 // excessive calls to the overflow routine while the method is
1809 // being compiled, add a second test to make sure the overflow
1810 // function is called only once every overflow_frequency.
1811 const int overflow_frequency = 1024;
1812 __ andsw(r1, r1, overflow_frequency - 1);
1813 __ br(Assembler::EQ, backedge_counter_overflow);
1814
1815 }
1816 } else {
1817 if (UseOnStackReplacement) {
1818 // check for overflow against w0, which is the sum of the
1819 // counters
1820 __ ldrw(rscratch1, Address(rscratch2, in_bytes(MethodCounters::interpreter_backward_branch_limit_offset())));
1821 __ cmpw(r0, rscratch1);
1822 __ br(Assembler::HS, backedge_counter_overflow); // Intel == Assembler::aboveEqual
1823 }
1824 }
1825 }
1826 }
1827 __ bind(dispatch);
1828
1829 // Pre-load the next target bytecode into rscratch1
1830 __ load_unsigned_byte(rscratch1, Address(rbcp, 0));
1831
1832 // continue with the bytecode @ target
1833 // rscratch1: target bytecode
1834 // rbcp: target bcp
1835 __ dispatch_only(vtos);
1836
1837 if (UseLoopCounter) {
1838 if (ProfileInterpreter) {
1839 // Out-of-line code to allocate method data oop.
1840 __ bind(profile_method);
1841 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::profile_method));
1842 __ load_unsigned_byte(r1, Address(rbcp, 0)); // restore target bytecode
1843 __ set_method_data_pointer_for_bcp();
1844 __ b(dispatch);
1845 }
1846
1847 if (TieredCompilation || UseOnStackReplacement) {
1848 // invocation counter overflow
1849 __ bind(backedge_counter_overflow);
1850 __ neg(r2, r2);
1851 __ add(r2, r2, rbcp); // branch bcp
1852 // IcoResult frequency_counter_overflow([JavaThread*], address branch_bcp)
1853 __ call_VM(noreg,
1854 CAST_FROM_FN_PTR(address,
1855 InterpreterRuntime::frequency_counter_overflow),
1856 r2);
1857 if (!UseOnStackReplacement)
1858 __ b(dispatch);
1859 }
1860
1861 if (UseOnStackReplacement) {
1862 __ load_unsigned_byte(r1, Address(rbcp, 0)); // restore target bytecode
1863
1864 // r0: osr nmethod (osr ok) or NULL (osr not possible)
1865 // w1: target bytecode
1866 // r2: scratch
1867 __ cbz(r0, dispatch); // test result -- no osr if null
1868 // nmethod may have been invalidated (VM may block upon call_VM return)
1869 __ ldrb(r2, Address(r0, nmethod::state_offset()));
1870 if (nmethod::in_use != 0)
1871 __ sub(r2, r2, nmethod::in_use);
1872 __ cbnz(r2, dispatch);
1873
1874 // We have the address of an on stack replacement routine in r0
1875 // We need to prepare to execute the OSR method. First we must
1876 // migrate the locals and monitors off of the stack.
1877
1878 __ mov(r19, r0); // save the nmethod
1879
1880 call_VM(noreg, CAST_FROM_FN_PTR(address, SharedRuntime::OSR_migration_begin));
1881
1882 // r0 is OSR buffer, move it to expected parameter location
1883 __ mov(j_rarg0, r0);
1884
1885 // remove activation
1886 // get sender esp
1887 __ ldr(esp,
1888 Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize));
1889 // remove frame anchor
1890 __ leave();
1891 // Ensure compiled code always sees stack at proper alignment
1892 __ andr(sp, esp, -16);
1893
1894 // and begin the OSR nmethod
1895 __ ldr(rscratch1, Address(r19, nmethod::osr_entry_point_offset()));
1896 __ br(rscratch1);
1897 }
1898 }
1899 }
1900
1901
1902 void TemplateTable::if_0cmp(Condition cc)
1903 {
1904 transition(itos, vtos);
1905 // assume branch is more often taken than not (loops use backward branches)
1906 Label not_taken;
1907 if (cc == equal)
1908 __ cbnzw(r0, not_taken);
1909 else if (cc == not_equal)
1910 __ cbzw(r0, not_taken);
1911 else {
1912 __ andsw(zr, r0, r0);
1913 __ br(j_not(cc), not_taken);
1914 }
1915
1916 branch(false, false);
1917 __ bind(not_taken);
1918 __ profile_not_taken_branch(r0);
1919 }
1920
1921 void TemplateTable::if_icmp(Condition cc)
1922 {
1923 transition(itos, vtos);
1924 // assume branch is more often taken than not (loops use backward branches)
1925 Label not_taken;
1926 __ pop_i(r1);
1927 __ cmpw(r1, r0, Assembler::LSL);
1928 __ br(j_not(cc), not_taken);
1929 branch(false, false);
1930 __ bind(not_taken);
1931 __ profile_not_taken_branch(r0);
1932 }
1933
1934 void TemplateTable::if_nullcmp(Condition cc)
1935 {
1936 transition(atos, vtos);
1937 // assume branch is more often taken than not (loops use backward branches)
1938 Label not_taken;
1939 if (cc == equal)
1940 __ cbnz(r0, not_taken);
1941 else
1942 __ cbz(r0, not_taken);
1943 branch(false, false);
1944 __ bind(not_taken);
1945 __ profile_not_taken_branch(r0);
1946 }
1947
1948 void TemplateTable::if_acmp(Condition cc)
1949 {
1950 transition(atos, vtos);
1951 // assume branch is more often taken than not (loops use backward branches)
1952 Label not_taken;
1953 __ pop_ptr(r1);
1954 __ cmp(r1, r0);
1955 __ br(j_not(cc), not_taken);
1956 branch(false, false);
1957 __ bind(not_taken);
1958 __ profile_not_taken_branch(r0);
1959 }
1960
1961 void TemplateTable::ret() {
1962 transition(vtos, vtos);
1963 // We might be moving to a safepoint. The thread which calls
1964 // Interpreter::notice_safepoints() will effectively flush its cache
1965 // when it makes a system call, but we need to do something to
1966 // ensure that we see the changed dispatch table.
1967 __ membar(MacroAssembler::LoadLoad);
1968
1969 locals_index(r1);
1970 __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
1971 __ profile_ret(r1, r2);
1972 __ ldr(rbcp, Address(rmethod, Method::const_offset()));
1973 __ lea(rbcp, Address(rbcp, r1));
1974 __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
1975 __ dispatch_next(vtos);
1976 }
1977
1978 void TemplateTable::wide_ret() {
1979 transition(vtos, vtos);
1980 locals_index_wide(r1);
1981 __ ldr(r1, aaddress(r1)); // get return bci, compute return bcp
1982 __ profile_ret(r1, r2);
1983 __ ldr(rbcp, Address(rmethod, Method::const_offset()));
1984 __ lea(rbcp, Address(rbcp, r1));
1985 __ add(rbcp, rbcp, in_bytes(ConstMethod::codes_offset()));
1986 __ dispatch_next(vtos);
1987 }
1988
1989
1990 void TemplateTable::tableswitch() {
1991 Label default_case, continue_execution;
1992 transition(itos, vtos);
1993 // align rbcp
1994 __ lea(r1, at_bcp(BytesPerInt));
1995 __ andr(r1, r1, -BytesPerInt);
1996 // load lo & hi
1997 __ ldrw(r2, Address(r1, BytesPerInt));
1998 __ ldrw(r3, Address(r1, 2 * BytesPerInt));
1999 __ rev32(r2, r2);
2000 __ rev32(r3, r3);
2001 // check against lo & hi
2002 __ cmpw(r0, r2);
2003 __ br(Assembler::LT, default_case);
2004 __ cmpw(r0, r3);
2005 __ br(Assembler::GT, default_case);
2006 // lookup dispatch offset
2007 __ subw(r0, r0, r2);
2008 __ lea(r3, Address(r1, r0, Address::uxtw(2)));
2009 __ ldrw(r3, Address(r3, 3 * BytesPerInt));
2010 __ profile_switch_case(r0, r1, r2);
2011 // continue execution
2012 __ bind(continue_execution);
2013 __ rev32(r3, r3);
2014 __ load_unsigned_byte(rscratch1, Address(rbcp, r3, Address::sxtw(0)));
2015 __ add(rbcp, rbcp, r3, ext::sxtw);
2016 __ dispatch_only(vtos);
2017 // handle default
2018 __ bind(default_case);
2019 __ profile_switch_default(r0);
2020 __ ldrw(r3, Address(r1, 0));
2021 __ b(continue_execution);
2022 }
2023
2024 void TemplateTable::lookupswitch() {
2025 transition(itos, itos);
2026 __ stop("lookupswitch bytecode should have been rewritten");
2027 }
2028
2029 void TemplateTable::fast_linearswitch() {
2030 transition(itos, vtos);
2031 Label loop_entry, loop, found, continue_execution;
2032 // bswap r0 so we can avoid bswapping the table entries
2033 __ rev32(r0, r0);
2034 // align rbcp
2035 __ lea(r19, at_bcp(BytesPerInt)); // btw: should be able to get rid of
2036 // this instruction (change offsets
2037 // below)
2038 __ andr(r19, r19, -BytesPerInt);
2039 // set counter
2040 __ ldrw(r1, Address(r19, BytesPerInt));
2041 __ rev32(r1, r1);
2042 __ b(loop_entry);
2043 // table search
2044 __ bind(loop);
2045 __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2046 __ ldrw(rscratch1, Address(rscratch1, 2 * BytesPerInt));
2047 __ cmpw(r0, rscratch1);
2048 __ br(Assembler::EQ, found);
2049 __ bind(loop_entry);
2050 __ subs(r1, r1, 1);
2051 __ br(Assembler::PL, loop);
2052 // default case
2053 __ profile_switch_default(r0);
2054 __ ldrw(r3, Address(r19, 0));
2055 __ b(continue_execution);
2056 // entry found -> get offset
2057 __ bind(found);
2058 __ lea(rscratch1, Address(r19, r1, Address::lsl(3)));
2059 __ ldrw(r3, Address(rscratch1, 3 * BytesPerInt));
2060 __ profile_switch_case(r1, r0, r19);
2061 // continue execution
2062 __ bind(continue_execution);
2063 __ rev32(r3, r3);
2064 __ add(rbcp, rbcp, r3, ext::sxtw);
2065 __ ldrb(rscratch1, Address(rbcp, 0));
2066 __ dispatch_only(vtos);
2067 }
2068
2069 void TemplateTable::fast_binaryswitch() {
2070 transition(itos, vtos);
2071 // Implementation using the following core algorithm:
2072 //
2073 // int binary_search(int key, LookupswitchPair* array, int n) {
2074 // // Binary search according to "Methodik des Programmierens" by
2075 // // Edsger W. Dijkstra and W.H.J. Feijen, Addison Wesley Germany 1985.
2076 // int i = 0;
2077 // int j = n;
2078 // while (i+1 < j) {
2079 // // invariant P: 0 <= i < j <= n and (a[i] <= key < a[j] or Q)
2080 // // with Q: for all i: 0 <= i < n: key < a[i]
2081 // // where a stands for the array and assuming that the (inexisting)
2082 // // element a[n] is infinitely big.
2083 // int h = (i + j) >> 1;
2084 // // i < h < j
2085 // if (key < array[h].fast_match()) {
2086 // j = h;
2087 // } else {
2088 // i = h;
2089 // }
2090 // }
2091 // // R: a[i] <= key < a[i+1] or Q
2092 // // (i.e., if key is within array, i is the correct index)
2093 // return i;
2094 // }
2095
2096 // Register allocation
2097 const Register key = r0; // already set (tosca)
2098 const Register array = r1;
2099 const Register i = r2;
2100 const Register j = r3;
2101 const Register h = rscratch1;
2102 const Register temp = rscratch2;
2103
2104 // Find array start
2105 __ lea(array, at_bcp(3 * BytesPerInt)); // btw: should be able to
2106 // get rid of this
2107 // instruction (change
2108 // offsets below)
2109 __ andr(array, array, -BytesPerInt);
2110
2111 // Initialize i & j
2112 __ mov(i, 0); // i = 0;
2113 __ ldrw(j, Address(array, -BytesPerInt)); // j = length(array);
2114
2115 // Convert j into native byteordering
2116 __ rev32(j, j);
2117
2118 // And start
2119 Label entry;
2120 __ b(entry);
2121
2122 // binary search loop
2123 {
2124 Label loop;
2125 __ bind(loop);
2126 // int h = (i + j) >> 1;
2127 __ addw(h, i, j); // h = i + j;
2128 __ lsrw(h, h, 1); // h = (i + j) >> 1;
2129 // if (key < array[h].fast_match()) {
2130 // j = h;
2131 // } else {
2132 // i = h;
2133 // }
2134 // Convert array[h].match to native byte-ordering before compare
2135 __ ldr(temp, Address(array, h, Address::lsl(3)));
2136 __ rev32(temp, temp);
2137 __ cmpw(key, temp);
2138 // j = h if (key < array[h].fast_match())
2139 __ csel(j, h, j, Assembler::LT);
2140 // i = h if (key >= array[h].fast_match())
2141 __ csel(i, h, i, Assembler::GE);
2142 // while (i+1 < j)
2143 __ bind(entry);
2144 __ addw(h, i, 1); // i+1
2145 __ cmpw(h, j); // i+1 < j
2146 __ br(Assembler::LT, loop);
2147 }
2148
2149 // end of binary search, result index is i (must check again!)
2150 Label default_case;
2151 // Convert array[i].match to native byte-ordering before compare
2152 __ ldr(temp, Address(array, i, Address::lsl(3)));
2153 __ rev32(temp, temp);
2154 __ cmpw(key, temp);
2155 __ br(Assembler::NE, default_case);
2156
2157 // entry found -> j = offset
2158 __ add(j, array, i, ext::uxtx, 3);
2159 __ ldrw(j, Address(j, BytesPerInt));
2160 __ profile_switch_case(i, key, array);
2161 __ rev32(j, j);
2162 __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2163 __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2164 __ dispatch_only(vtos);
2165
2166 // default case -> j = default offset
2167 __ bind(default_case);
2168 __ profile_switch_default(i);
2169 __ ldrw(j, Address(array, -2 * BytesPerInt));
2170 __ rev32(j, j);
2171 __ load_unsigned_byte(rscratch1, Address(rbcp, j, Address::sxtw(0)));
2172 __ lea(rbcp, Address(rbcp, j, Address::sxtw(0)));
2173 __ dispatch_only(vtos);
2174 }
2175
2176
2177 void TemplateTable::_return(TosState state)
2178 {
2179 transition(state, state);
2180 assert(_desc->calls_vm(),
2181 "inconsistent calls_vm information"); // call in remove_activation
2182
2183 if (_desc->bytecode() == Bytecodes::_return_register_finalizer) {
2184 assert(state == vtos, "only valid state");
2185
2186 __ ldr(c_rarg1, aaddress(0));
2187 __ load_klass(r3, c_rarg1);
2188 __ ldrw(r3, Address(r3, Klass::access_flags_offset()));
2189 Label skip_register_finalizer;
2190 __ tbz(r3, exact_log2(JVM_ACC_HAS_FINALIZER), skip_register_finalizer);
2191
2192 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::register_finalizer), c_rarg1);
2193
2194 __ bind(skip_register_finalizer);
2195 }
2196
2197 // Issue a StoreStore barrier after all stores but before return
2198 // from any constructor for any class with a final field. We don't
2199 // know if this is a finalizer, so we always do so.
2200 if (_desc->bytecode() == Bytecodes::_return)
2201 __ membar(MacroAssembler::StoreStore);
2202
2203 // Narrow result if state is itos but result type is smaller.
2204 // Need to narrow in the return bytecode rather than in generate_return_entry
2205 // since compiled code callers expect the result to already be narrowed.
2206 if (state == itos) {
2207 __ narrow(r0);
2208 }
2209
2210 __ remove_activation(state);
2211 __ ret(lr);
2212 }
2213
2214 // ----------------------------------------------------------------------------
2215 // Volatile variables demand their effects be made known to all CPU's
2216 // in order. Store buffers on most chips allow reads & writes to
2217 // reorder; the JMM's ReadAfterWrite.java test fails in -Xint mode
2218 // without some kind of memory barrier (i.e., it's not sufficient that
2219 // the interpreter does not reorder volatile references, the hardware
2220 // also must not reorder them).
2221 //
2222 // According to the new Java Memory Model (JMM):
2223 // (1) All volatiles are serialized wrt to each other. ALSO reads &
2224 // writes act as aquire & release, so:
2225 // (2) A read cannot let unrelated NON-volatile memory refs that
2226 // happen after the read float up to before the read. It's OK for
2227 // non-volatile memory refs that happen before the volatile read to
2228 // float down below it.
2229 // (3) Similar a volatile write cannot let unrelated NON-volatile
2230 // memory refs that happen BEFORE the write float down to after the
2231 // write. It's OK for non-volatile memory refs that happen after the
2232 // volatile write to float up before it.
2233 //
2234 // We only put in barriers around volatile refs (they are expensive),
2235 // not _between_ memory refs (that would require us to track the
2236 // flavor of the previous memory refs). Requirements (2) and (3)
2237 // require some barriers before volatile stores and after volatile
2238 // loads. These nearly cover requirement (1) but miss the
2239 // volatile-store-volatile-load case. This final case is placed after
2240 // volatile-stores although it could just as well go before
2241 // volatile-loads.
2242
2243 void TemplateTable::resolve_cache_and_index(int byte_no,
2244 Register Rcache,
2245 Register index,
2246 size_t index_size) {
2247 const Register temp = r19;
2248 assert_different_registers(Rcache, index, temp);
2249
2250 Label resolved;
2251
2252 Bytecodes::Code code = bytecode();
2253 switch (code) {
2254 case Bytecodes::_nofast_getfield: code = Bytecodes::_getfield; break;
2255 case Bytecodes::_nofast_putfield: code = Bytecodes::_putfield; break;
2256 }
2257
2258 assert(byte_no == f1_byte || byte_no == f2_byte, "byte_no out of range");
2259 __ get_cache_and_index_and_bytecode_at_bcp(Rcache, index, temp, byte_no, 1, index_size);
2260 __ cmp(temp, (int) code); // have we resolved this bytecode?
2261 __ br(Assembler::EQ, resolved);
2262
2263 // resolve first time through
2264 address entry = CAST_FROM_FN_PTR(address, InterpreterRuntime::resolve_from_cache);
2265 __ mov(temp, (int) code);
2266 __ call_VM(noreg, entry, temp);
2267
2268 // Update registers with resolved info
2269 __ get_cache_and_index_at_bcp(Rcache, index, 1, index_size);
2270 // n.b. unlike x86 Rcache is now rcpool plus the indexed offset
2271 // so all clients ofthis method must be modified accordingly
2272 __ bind(resolved);
2273 }
2274
2275 // The Rcache and index registers must be set before call
2276 // n.b unlike x86 cache already includes the index offset
2277 void TemplateTable::load_field_cp_cache_entry(Register obj,
2278 Register cache,
2279 Register index,
2280 Register off,
2281 Register flags,
2282 bool is_static = false) {
2283 assert_different_registers(cache, index, flags, off);
2284
2285 ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2286 // Field offset
2287 __ ldr(off, Address(cache, in_bytes(cp_base_offset +
2288 ConstantPoolCacheEntry::f2_offset())));
2289 // Flags
2290 __ ldrw(flags, Address(cache, in_bytes(cp_base_offset +
2291 ConstantPoolCacheEntry::flags_offset())));
2292
2293 // klass overwrite register
2294 if (is_static) {
2295 __ ldr(obj, Address(cache, in_bytes(cp_base_offset +
2296 ConstantPoolCacheEntry::f1_offset())));
2297 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2298 __ ldr(obj, Address(obj, mirror_offset));
2299 }
2300 }
2301
2302 void TemplateTable::load_invoke_cp_cache_entry(int byte_no,
2303 Register method,
2304 Register itable_index,
2305 Register flags,
2306 bool is_invokevirtual,
2307 bool is_invokevfinal, /*unused*/
2308 bool is_invokedynamic) {
2309 // setup registers
2310 const Register cache = rscratch2;
2311 const Register index = r4;
2312 assert_different_registers(method, flags);
2313 assert_different_registers(method, cache, index);
2314 assert_different_registers(itable_index, flags);
2315 assert_different_registers(itable_index, cache, index);
2316 // determine constant pool cache field offsets
2317 assert(is_invokevirtual == (byte_no == f2_byte), "is_invokevirtual flag redundant");
2318 const int method_offset = in_bytes(
2319 ConstantPoolCache::base_offset() +
2320 (is_invokevirtual
2321 ? ConstantPoolCacheEntry::f2_offset()
2322 : ConstantPoolCacheEntry::f1_offset()));
2323 const int flags_offset = in_bytes(ConstantPoolCache::base_offset() +
2324 ConstantPoolCacheEntry::flags_offset());
2325 // access constant pool cache fields
2326 const int index_offset = in_bytes(ConstantPoolCache::base_offset() +
2327 ConstantPoolCacheEntry::f2_offset());
2328
2329 size_t index_size = (is_invokedynamic ? sizeof(u4) : sizeof(u2));
2330 resolve_cache_and_index(byte_no, cache, index, index_size);
2331 __ ldr(method, Address(cache, method_offset));
2332
2333 if (itable_index != noreg) {
2334 __ ldr(itable_index, Address(cache, index_offset));
2335 }
2336 __ ldrw(flags, Address(cache, flags_offset));
2337 }
2338
2339
2340 // The registers cache and index expected to be set before call.
2341 // Correct values of the cache and index registers are preserved.
2342 void TemplateTable::jvmti_post_field_access(Register cache, Register index,
2343 bool is_static, bool has_tos) {
2344 // do the JVMTI work here to avoid disturbing the register state below
2345 // We use c_rarg registers here because we want to use the register used in
2346 // the call to the VM
2347 if (JvmtiExport::can_post_field_access()) {
2348 // Check to see if a field access watch has been set before we
2349 // take the time to call into the VM.
2350 Label L1;
2351 assert_different_registers(cache, index, r0);
2352 __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2353 __ ldrw(r0, Address(rscratch1));
2354 __ cbzw(r0, L1);
2355
2356 __ get_cache_and_index_at_bcp(c_rarg2, c_rarg3, 1);
2357 __ lea(c_rarg2, Address(c_rarg2, in_bytes(ConstantPoolCache::base_offset())));
2358
2359 if (is_static) {
2360 __ mov(c_rarg1, zr); // NULL object reference
2361 } else {
2362 __ ldr(c_rarg1, at_tos()); // get object pointer without popping it
2363 __ verify_oop(c_rarg1);
2364 }
2365 // c_rarg1: object pointer or NULL
2366 // c_rarg2: cache entry pointer
2367 // c_rarg3: jvalue object on the stack
2368 __ call_VM(noreg, CAST_FROM_FN_PTR(address,
2369 InterpreterRuntime::post_field_access),
2370 c_rarg1, c_rarg2, c_rarg3);
2371 __ get_cache_and_index_at_bcp(cache, index, 1);
2372 __ bind(L1);
2373 }
2374 }
2375
2376 void TemplateTable::pop_and_check_object(Register r)
2377 {
2378 __ pop_ptr(r);
2379 __ null_check(r); // for field access must check obj.
2380 __ verify_oop(r);
2381 }
2382
2383 void TemplateTable::getfield_or_static(int byte_no, bool is_static, RewriteControl rc)
2384 {
2385 const Register cache = r2;
2386 const Register index = r3;
2387 const Register obj = r4;
2388 const Register off = r19;
2389 const Register flags = r0;
2390 const Register raw_flags = r6;
2391 const Register bc = r4; // uses same reg as obj, so don't mix them
2392
2393 resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2394 jvmti_post_field_access(cache, index, is_static, false);
2395 load_field_cp_cache_entry(obj, cache, index, off, raw_flags, is_static);
2396
2397 if (!is_static) {
2398 // obj is on the stack
2399 pop_and_check_object(obj);
2400 }
2401
2402 // 8179954: We need to make sure that the code generated for
2403 // volatile accesses forms a sequentially-consistent set of
2404 // operations when combined with STLR and LDAR. Without a leading
2405 // membar it's possible for a simple Dekker test to fail if loads
2406 // use LDR;DMB but stores use STLR. This can happen if C2 compiles
2407 // the stores in one method and we interpret the loads in another.
2408 if (! UseBarriersForVolatile) {
2409 Label notVolatile;
2410 __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2411 __ membar(MacroAssembler::AnyAny);
2412 __ bind(notVolatile);
2413 }
2414
2415 const Address field(obj, off);
2416
2417 Label Done, notByte, notBool, notInt, notShort, notChar,
2418 notLong, notFloat, notObj, notDouble;
2419
2420 // x86 uses a shift and mask or wings it with a shift plus assert
2421 // the mask is not needed. aarch64 just uses bitfield extract
2422 __ ubfxw(flags, raw_flags, ConstantPoolCacheEntry::tos_state_shift,
2423 ConstantPoolCacheEntry::tos_state_bits);
2424
2425 assert(btos == 0, "change code, btos != 0");
2426 __ cbnz(flags, notByte);
2427
2428 // Don't rewrite getstatic, only getfield
2429 if (is_static) rc = may_not_rewrite;
2430
2431 // btos
2432 __ load_signed_byte(r0, field);
2433 __ push(btos);
2434 // Rewrite bytecode to be faster
2435 if (rc == may_rewrite) {
2436 patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2437 }
2438 __ b(Done);
2439
2440 __ bind(notByte);
2441 __ cmp(flags, ztos);
2442 __ br(Assembler::NE, notBool);
2443
2444 // ztos (same code as btos)
2445 __ ldrsb(r0, field);
2446 __ push(ztos);
2447 // Rewrite bytecode to be faster
2448 if (rc == may_rewrite) {
2449 // use btos rewriting, no truncating to t/f bit is needed for getfield.
2450 patch_bytecode(Bytecodes::_fast_bgetfield, bc, r1);
2451 }
2452 __ b(Done);
2453
2454 __ bind(notBool);
2455 __ cmp(flags, atos);
2456 __ br(Assembler::NE, notObj);
2457 // atos
2458 __ load_heap_oop(r0, field);
2459 __ push(atos);
2460 if (rc == may_rewrite) {
2461 patch_bytecode(Bytecodes::_fast_agetfield, bc, r1);
2462 }
2463 __ b(Done);
2464
2465 __ bind(notObj);
2466 __ cmp(flags, itos);
2467 __ br(Assembler::NE, notInt);
2468 // itos
2469 __ ldrw(r0, field);
2470 __ push(itos);
2471 // Rewrite bytecode to be faster
2472 if (rc == may_rewrite) {
2473 patch_bytecode(Bytecodes::_fast_igetfield, bc, r1);
2474 }
2475 __ b(Done);
2476
2477 __ bind(notInt);
2478 __ cmp(flags, ctos);
2479 __ br(Assembler::NE, notChar);
2480 // ctos
2481 __ load_unsigned_short(r0, field);
2482 __ push(ctos);
2483 // Rewrite bytecode to be faster
2484 if (rc == may_rewrite) {
2485 patch_bytecode(Bytecodes::_fast_cgetfield, bc, r1);
2486 }
2487 __ b(Done);
2488
2489 __ bind(notChar);
2490 __ cmp(flags, stos);
2491 __ br(Assembler::NE, notShort);
2492 // stos
2493 __ load_signed_short(r0, field);
2494 __ push(stos);
2495 // Rewrite bytecode to be faster
2496 if (rc == may_rewrite) {
2497 patch_bytecode(Bytecodes::_fast_sgetfield, bc, r1);
2498 }
2499 __ b(Done);
2500
2501 __ bind(notShort);
2502 __ cmp(flags, ltos);
2503 __ br(Assembler::NE, notLong);
2504 // ltos
2505 __ ldr(r0, field);
2506 __ push(ltos);
2507 // Rewrite bytecode to be faster
2508 if (rc == may_rewrite) {
2509 patch_bytecode(Bytecodes::_fast_lgetfield, bc, r1);
2510 }
2511 __ b(Done);
2512
2513 __ bind(notLong);
2514 __ cmp(flags, ftos);
2515 __ br(Assembler::NE, notFloat);
2516 // ftos
2517 __ ldrs(v0, field);
2518 __ push(ftos);
2519 // Rewrite bytecode to be faster
2520 if (rc == may_rewrite) {
2521 patch_bytecode(Bytecodes::_fast_fgetfield, bc, r1);
2522 }
2523 __ b(Done);
2524
2525 __ bind(notFloat);
2526 #ifdef ASSERT
2527 __ cmp(flags, dtos);
2528 __ br(Assembler::NE, notDouble);
2529 #endif
2530 // dtos
2531 __ ldrd(v0, field);
2532 __ push(dtos);
2533 // Rewrite bytecode to be faster
2534 if (rc == may_rewrite) {
2535 patch_bytecode(Bytecodes::_fast_dgetfield, bc, r1);
2536 }
2537 #ifdef ASSERT
2538 __ b(Done);
2539
2540 __ bind(notDouble);
2541 __ stop("Bad state");
2542 #endif
2543
2544 __ bind(Done);
2545
2546 Label notVolatile;
2547 __ tbz(raw_flags, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2548 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2549 __ bind(notVolatile);
2550 }
2551
2552
2553 void TemplateTable::getfield(int byte_no)
2554 {
2555 getfield_or_static(byte_no, false);
2556 }
2557
2558 void TemplateTable::nofast_getfield(int byte_no) {
2559 getfield_or_static(byte_no, false, may_not_rewrite);
2560 }
2561
2562 void TemplateTable::getstatic(int byte_no)
2563 {
2564 getfield_or_static(byte_no, true);
2565 }
2566
2567 // The registers cache and index expected to be set before call.
2568 // The function may destroy various registers, just not the cache and index registers.
2569 void TemplateTable::jvmti_post_field_mod(Register cache, Register index, bool is_static) {
2570 transition(vtos, vtos);
2571
2572 ByteSize cp_base_offset = ConstantPoolCache::base_offset();
2573
2574 if (JvmtiExport::can_post_field_modification()) {
2575 // Check to see if a field modification watch has been set before
2576 // we take the time to call into the VM.
2577 Label L1;
2578 assert_different_registers(cache, index, r0);
2579 __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2580 __ ldrw(r0, Address(rscratch1));
2581 __ cbz(r0, L1);
2582
2583 __ get_cache_and_index_at_bcp(c_rarg2, rscratch1, 1);
2584
2585 if (is_static) {
2586 // Life is simple. Null out the object pointer.
2587 __ mov(c_rarg1, zr);
2588 } else {
2589 // Life is harder. The stack holds the value on top, followed by
2590 // the object. We don't know the size of the value, though; it
2591 // could be one or two words depending on its type. As a result,
2592 // we must find the type to determine where the object is.
2593 __ ldrw(c_rarg3, Address(c_rarg2,
2594 in_bytes(cp_base_offset +
2595 ConstantPoolCacheEntry::flags_offset())));
2596 __ lsr(c_rarg3, c_rarg3,
2597 ConstantPoolCacheEntry::tos_state_shift);
2598 ConstantPoolCacheEntry::verify_tos_state_shift();
2599 Label nope2, done, ok;
2600 __ ldr(c_rarg1, at_tos_p1()); // initially assume a one word jvalue
2601 __ cmpw(c_rarg3, ltos);
2602 __ br(Assembler::EQ, ok);
2603 __ cmpw(c_rarg3, dtos);
2604 __ br(Assembler::NE, nope2);
2605 __ bind(ok);
2606 __ ldr(c_rarg1, at_tos_p2()); // ltos (two word jvalue)
2607 __ bind(nope2);
2608 }
2609 // cache entry pointer
2610 __ add(c_rarg2, c_rarg2, in_bytes(cp_base_offset));
2611 // object (tos)
2612 __ mov(c_rarg3, esp);
2613 // c_rarg1: object pointer set up above (NULL if static)
2614 // c_rarg2: cache entry pointer
2615 // c_rarg3: jvalue object on the stack
2616 __ call_VM(noreg,
2617 CAST_FROM_FN_PTR(address,
2618 InterpreterRuntime::post_field_modification),
2619 c_rarg1, c_rarg2, c_rarg3);
2620 __ get_cache_and_index_at_bcp(cache, index, 1);
2621 __ bind(L1);
2622 }
2623 }
2624
2625 void TemplateTable::putfield_or_static(int byte_no, bool is_static, RewriteControl rc) {
2626 transition(vtos, vtos);
2627
2628 const Register cache = r2;
2629 const Register index = r3;
2630 const Register obj = r2;
2631 const Register off = r19;
2632 const Register flags = r0;
2633 const Register bc = r4;
2634
2635 resolve_cache_and_index(byte_no, cache, index, sizeof(u2));
2636 jvmti_post_field_mod(cache, index, is_static);
2637 load_field_cp_cache_entry(obj, cache, index, off, flags, is_static);
2638
2639 Label Done;
2640 __ mov(r5, flags);
2641
2642 {
2643 Label notVolatile;
2644 __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2645 __ membar(MacroAssembler::StoreStore);
2646 __ bind(notVolatile);
2647 }
2648
2649 // field address
2650 const Address field(obj, off);
2651
2652 Label notByte, notBool, notInt, notShort, notChar,
2653 notLong, notFloat, notObj, notDouble;
2654
2655 // x86 uses a shift and mask or wings it with a shift plus assert
2656 // the mask is not needed. aarch64 just uses bitfield extract
2657 __ ubfxw(flags, flags, ConstantPoolCacheEntry::tos_state_shift, ConstantPoolCacheEntry::tos_state_bits);
2658
2659 assert(btos == 0, "change code, btos != 0");
2660 __ cbnz(flags, notByte);
2661
2662 // Don't rewrite putstatic, only putfield
2663 if (is_static) rc = may_not_rewrite;
2664
2665 // btos
2666 {
2667 __ pop(btos);
2668 if (!is_static) pop_and_check_object(obj);
2669 __ strb(r0, field);
2670 if (rc == may_rewrite) {
2671 patch_bytecode(Bytecodes::_fast_bputfield, bc, r1, true, byte_no);
2672 }
2673 __ b(Done);
2674 }
2675
2676 __ bind(notByte);
2677 __ cmp(flags, ztos);
2678 __ br(Assembler::NE, notBool);
2679
2680 // ztos
2681 {
2682 __ pop(ztos);
2683 if (!is_static) pop_and_check_object(obj);
2684 __ andw(r0, r0, 0x1);
2685 __ strb(r0, field);
2686 if (rc == may_rewrite) {
2687 patch_bytecode(Bytecodes::_fast_zputfield, bc, r1, true, byte_no);
2688 }
2689 __ b(Done);
2690 }
2691
2692 __ bind(notBool);
2693 __ cmp(flags, atos);
2694 __ br(Assembler::NE, notObj);
2695
2696 // atos
2697 {
2698 __ pop(atos);
2699 if (!is_static) pop_and_check_object(obj);
2700 // Store into the field
2701 do_oop_store(_masm, field, r0, _bs->kind(), false);
2702 if (rc == may_rewrite) {
2703 patch_bytecode(Bytecodes::_fast_aputfield, bc, r1, true, byte_no);
2704 }
2705 __ b(Done);
2706 }
2707
2708 __ bind(notObj);
2709 __ cmp(flags, itos);
2710 __ br(Assembler::NE, notInt);
2711
2712 // itos
2713 {
2714 __ pop(itos);
2715 if (!is_static) pop_and_check_object(obj);
2716 __ strw(r0, field);
2717 if (rc == may_rewrite) {
2718 patch_bytecode(Bytecodes::_fast_iputfield, bc, r1, true, byte_no);
2719 }
2720 __ b(Done);
2721 }
2722
2723 __ bind(notInt);
2724 __ cmp(flags, ctos);
2725 __ br(Assembler::NE, notChar);
2726
2727 // ctos
2728 {
2729 __ pop(ctos);
2730 if (!is_static) pop_and_check_object(obj);
2731 __ strh(r0, field);
2732 if (rc == may_rewrite) {
2733 patch_bytecode(Bytecodes::_fast_cputfield, bc, r1, true, byte_no);
2734 }
2735 __ b(Done);
2736 }
2737
2738 __ bind(notChar);
2739 __ cmp(flags, stos);
2740 __ br(Assembler::NE, notShort);
2741
2742 // stos
2743 {
2744 __ pop(stos);
2745 if (!is_static) pop_and_check_object(obj);
2746 __ strh(r0, field);
2747 if (rc == may_rewrite) {
2748 patch_bytecode(Bytecodes::_fast_sputfield, bc, r1, true, byte_no);
2749 }
2750 __ b(Done);
2751 }
2752
2753 __ bind(notShort);
2754 __ cmp(flags, ltos);
2755 __ br(Assembler::NE, notLong);
2756
2757 // ltos
2758 {
2759 __ pop(ltos);
2760 if (!is_static) pop_and_check_object(obj);
2761 __ str(r0, field);
2762 if (rc == may_rewrite) {
2763 patch_bytecode(Bytecodes::_fast_lputfield, bc, r1, true, byte_no);
2764 }
2765 __ b(Done);
2766 }
2767
2768 __ bind(notLong);
2769 __ cmp(flags, ftos);
2770 __ br(Assembler::NE, notFloat);
2771
2772 // ftos
2773 {
2774 __ pop(ftos);
2775 if (!is_static) pop_and_check_object(obj);
2776 __ strs(v0, field);
2777 if (rc == may_rewrite) {
2778 patch_bytecode(Bytecodes::_fast_fputfield, bc, r1, true, byte_no);
2779 }
2780 __ b(Done);
2781 }
2782
2783 __ bind(notFloat);
2784 #ifdef ASSERT
2785 __ cmp(flags, dtos);
2786 __ br(Assembler::NE, notDouble);
2787 #endif
2788
2789 // dtos
2790 {
2791 __ pop(dtos);
2792 if (!is_static) pop_and_check_object(obj);
2793 __ strd(v0, field);
2794 if (rc == may_rewrite) {
2795 patch_bytecode(Bytecodes::_fast_dputfield, bc, r1, true, byte_no);
2796 }
2797 }
2798
2799 #ifdef ASSERT
2800 __ b(Done);
2801
2802 __ bind(notDouble);
2803 __ stop("Bad state");
2804 #endif
2805
2806 __ bind(Done);
2807
2808 {
2809 Label notVolatile;
2810 __ tbz(r5, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2811 __ membar(MacroAssembler::StoreLoad);
2812 __ bind(notVolatile);
2813 }
2814 }
2815
2816 void TemplateTable::putfield(int byte_no)
2817 {
2818 putfield_or_static(byte_no, false);
2819 }
2820
2821 void TemplateTable::nofast_putfield(int byte_no) {
2822 putfield_or_static(byte_no, false, may_not_rewrite);
2823 }
2824
2825 void TemplateTable::putstatic(int byte_no) {
2826 putfield_or_static(byte_no, true);
2827 }
2828
2829 void TemplateTable::jvmti_post_fast_field_mod()
2830 {
2831 if (JvmtiExport::can_post_field_modification()) {
2832 // Check to see if a field modification watch has been set before
2833 // we take the time to call into the VM.
2834 Label L2;
2835 __ lea(rscratch1, ExternalAddress((address)JvmtiExport::get_field_modification_count_addr()));
2836 __ ldrw(c_rarg3, Address(rscratch1));
2837 __ cbzw(c_rarg3, L2);
2838 __ pop_ptr(r19); // copy the object pointer from tos
2839 __ verify_oop(r19);
2840 __ push_ptr(r19); // put the object pointer back on tos
2841 // Save tos values before call_VM() clobbers them. Since we have
2842 // to do it for every data type, we use the saved values as the
2843 // jvalue object.
2844 switch (bytecode()) { // load values into the jvalue object
2845 case Bytecodes::_fast_aputfield: __ push_ptr(r0); break;
2846 case Bytecodes::_fast_bputfield: // fall through
2847 case Bytecodes::_fast_zputfield: // fall through
2848 case Bytecodes::_fast_sputfield: // fall through
2849 case Bytecodes::_fast_cputfield: // fall through
2850 case Bytecodes::_fast_iputfield: __ push_i(r0); break;
2851 case Bytecodes::_fast_dputfield: __ push_d(); break;
2852 case Bytecodes::_fast_fputfield: __ push_f(); break;
2853 case Bytecodes::_fast_lputfield: __ push_l(r0); break;
2854
2855 default:
2856 ShouldNotReachHere();
2857 }
2858 __ mov(c_rarg3, esp); // points to jvalue on the stack
2859 // access constant pool cache entry
2860 __ get_cache_entry_pointer_at_bcp(c_rarg2, r0, 1);
2861 __ verify_oop(r19);
2862 // r19: object pointer copied above
2863 // c_rarg2: cache entry pointer
2864 // c_rarg3: jvalue object on the stack
2865 __ call_VM(noreg,
2866 CAST_FROM_FN_PTR(address,
2867 InterpreterRuntime::post_field_modification),
2868 r19, c_rarg2, c_rarg3);
2869
2870 switch (bytecode()) { // restore tos values
2871 case Bytecodes::_fast_aputfield: __ pop_ptr(r0); break;
2872 case Bytecodes::_fast_bputfield: // fall through
2873 case Bytecodes::_fast_zputfield: // fall through
2874 case Bytecodes::_fast_sputfield: // fall through
2875 case Bytecodes::_fast_cputfield: // fall through
2876 case Bytecodes::_fast_iputfield: __ pop_i(r0); break;
2877 case Bytecodes::_fast_dputfield: __ pop_d(); break;
2878 case Bytecodes::_fast_fputfield: __ pop_f(); break;
2879 case Bytecodes::_fast_lputfield: __ pop_l(r0); break;
2880 }
2881 __ bind(L2);
2882 }
2883 }
2884
2885 void TemplateTable::fast_storefield(TosState state)
2886 {
2887 transition(state, vtos);
2888
2889 ByteSize base = ConstantPoolCache::base_offset();
2890
2891 jvmti_post_fast_field_mod();
2892
2893 // access constant pool cache
2894 __ get_cache_and_index_at_bcp(r2, r1, 1);
2895
2896 // test for volatile with r3
2897 __ ldrw(r3, Address(r2, in_bytes(base +
2898 ConstantPoolCacheEntry::flags_offset())));
2899
2900 // replace index with field offset from cache entry
2901 __ ldr(r1, Address(r2, in_bytes(base + ConstantPoolCacheEntry::f2_offset())));
2902
2903 {
2904 Label notVolatile;
2905 __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2906 __ membar(MacroAssembler::StoreStore);
2907 __ bind(notVolatile);
2908 }
2909
2910 Label notVolatile;
2911
2912 // Get object from stack
2913 pop_and_check_object(r2);
2914
2915 // field address
2916 const Address field(r2, r1);
2917
2918 // access field
2919 switch (bytecode()) {
2920 case Bytecodes::_fast_aputfield:
2921 do_oop_store(_masm, field, r0, _bs->kind(), false);
2922 break;
2923 case Bytecodes::_fast_lputfield:
2924 __ str(r0, field);
2925 break;
2926 case Bytecodes::_fast_iputfield:
2927 __ strw(r0, field);
2928 break;
2929 case Bytecodes::_fast_zputfield:
2930 __ andw(r0, r0, 0x1); // boolean is true if LSB is 1
2931 // fall through to bputfield
2932 case Bytecodes::_fast_bputfield:
2933 __ strb(r0, field);
2934 break;
2935 case Bytecodes::_fast_sputfield:
2936 // fall through
2937 case Bytecodes::_fast_cputfield:
2938 __ strh(r0, field);
2939 break;
2940 case Bytecodes::_fast_fputfield:
2941 __ strs(v0, field);
2942 break;
2943 case Bytecodes::_fast_dputfield:
2944 __ strd(v0, field);
2945 break;
2946 default:
2947 ShouldNotReachHere();
2948 }
2949
2950 {
2951 Label notVolatile;
2952 __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
2953 __ membar(MacroAssembler::StoreLoad);
2954 __ bind(notVolatile);
2955 }
2956 }
2957
2958
2959 void TemplateTable::fast_accessfield(TosState state)
2960 {
2961 transition(atos, state);
2962 // Do the JVMTI work here to avoid disturbing the register state below
2963 if (JvmtiExport::can_post_field_access()) {
2964 // Check to see if a field access watch has been set before we
2965 // take the time to call into the VM.
2966 Label L1;
2967 __ lea(rscratch1, ExternalAddress((address) JvmtiExport::get_field_access_count_addr()));
2968 __ ldrw(r2, Address(rscratch1));
2969 __ cbzw(r2, L1);
2970 // access constant pool cache entry
2971 __ get_cache_entry_pointer_at_bcp(c_rarg2, rscratch2, 1);
2972 __ verify_oop(r0);
2973 __ push_ptr(r0); // save object pointer before call_VM() clobbers it
2974 __ mov(c_rarg1, r0);
2975 // c_rarg1: object pointer copied above
2976 // c_rarg2: cache entry pointer
2977 __ call_VM(noreg,
2978 CAST_FROM_FN_PTR(address,
2979 InterpreterRuntime::post_field_access),
2980 c_rarg1, c_rarg2);
2981 __ pop_ptr(r0); // restore object pointer
2982 __ bind(L1);
2983 }
2984
2985 // access constant pool cache
2986 __ get_cache_and_index_at_bcp(r2, r1, 1);
2987 __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
2988 ConstantPoolCacheEntry::f2_offset())));
2989 __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
2990 ConstantPoolCacheEntry::flags_offset())));
2991
2992 // r0: object
2993 __ verify_oop(r0);
2994 __ null_check(r0);
2995 const Address field(r0, r1);
2996
2997 // 8179954: We need to make sure that the code generated for
2998 // volatile accesses forms a sequentially-consistent set of
2999 // operations when combined with STLR and LDAR. Without a leading
3000 // membar it's possible for a simple Dekker test to fail if loads
3001 // use LDR;DMB but stores use STLR. This can happen if C2 compiles
3002 // the stores in one method and we interpret the loads in another.
3003 if (! UseBarriersForVolatile) {
3004 Label notVolatile;
3005 __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3006 __ membar(MacroAssembler::AnyAny);
3007 __ bind(notVolatile);
3008 }
3009
3010 // access field
3011 switch (bytecode()) {
3012 case Bytecodes::_fast_agetfield:
3013 __ load_heap_oop(r0, field);
3014 __ verify_oop(r0);
3015 break;
3016 case Bytecodes::_fast_lgetfield:
3017 __ ldr(r0, field);
3018 break;
3019 case Bytecodes::_fast_igetfield:
3020 __ ldrw(r0, field);
3021 break;
3022 case Bytecodes::_fast_bgetfield:
3023 __ load_signed_byte(r0, field);
3024 break;
3025 case Bytecodes::_fast_sgetfield:
3026 __ load_signed_short(r0, field);
3027 break;
3028 case Bytecodes::_fast_cgetfield:
3029 __ load_unsigned_short(r0, field);
3030 break;
3031 case Bytecodes::_fast_fgetfield:
3032 __ ldrs(v0, field);
3033 break;
3034 case Bytecodes::_fast_dgetfield:
3035 __ ldrd(v0, field);
3036 break;
3037 default:
3038 ShouldNotReachHere();
3039 }
3040 {
3041 Label notVolatile;
3042 __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3043 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3044 __ bind(notVolatile);
3045 }
3046 }
3047
3048 void TemplateTable::fast_xaccess(TosState state)
3049 {
3050 transition(vtos, state);
3051
3052 // get receiver
3053 __ ldr(r0, aaddress(0));
3054 // access constant pool cache
3055 __ get_cache_and_index_at_bcp(r2, r3, 2);
3056 __ ldr(r1, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3057 ConstantPoolCacheEntry::f2_offset())));
3058
3059 // 8179954: We need to make sure that the code generated for
3060 // volatile accesses forms a sequentially-consistent set of
3061 // operations when combined with STLR and LDAR. Without a leading
3062 // membar it's possible for a simple Dekker test to fail if loads
3063 // use LDR;DMB but stores use STLR. This can happen if C2 compiles
3064 // the stores in one method and we interpret the loads in another.
3065 if (! UseBarriersForVolatile) {
3066 Label notVolatile;
3067 __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3068 ConstantPoolCacheEntry::flags_offset())));
3069 __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3070 __ membar(MacroAssembler::AnyAny);
3071 __ bind(notVolatile);
3072 }
3073
3074 // make sure exception is reported in correct bcp range (getfield is
3075 // next instruction)
3076 __ increment(rbcp);
3077 __ null_check(r0);
3078 switch (state) {
3079 case itos:
3080 __ ldrw(r0, Address(r0, r1, Address::lsl(0)));
3081 break;
3082 case atos:
3083 __ load_heap_oop(r0, Address(r0, r1, Address::lsl(0)));
3084 __ verify_oop(r0);
3085 break;
3086 case ftos:
3087 __ ldrs(v0, Address(r0, r1, Address::lsl(0)));
3088 break;
3089 default:
3090 ShouldNotReachHere();
3091 }
3092
3093 {
3094 Label notVolatile;
3095 __ ldrw(r3, Address(r2, in_bytes(ConstantPoolCache::base_offset() +
3096 ConstantPoolCacheEntry::flags_offset())));
3097 __ tbz(r3, ConstantPoolCacheEntry::is_volatile_shift, notVolatile);
3098 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
3099 __ bind(notVolatile);
3100 }
3101
3102 __ decrement(rbcp);
3103 }
3104
3105
3106
3107 //-----------------------------------------------------------------------------
3108 // Calls
3109
3110 void TemplateTable::count_calls(Register method, Register temp)
3111 {
3112 __ call_Unimplemented();
3113 }
3114
3115 void TemplateTable::prepare_invoke(int byte_no,
3116 Register method, // linked method (or i-klass)
3117 Register index, // itable index, MethodType, etc.
3118 Register recv, // if caller wants to see it
3119 Register flags // if caller wants to test it
3120 ) {
3121 // determine flags
3122 Bytecodes::Code code = bytecode();
3123 const bool is_invokeinterface = code == Bytecodes::_invokeinterface;
3124 const bool is_invokedynamic = code == Bytecodes::_invokedynamic;
3125 const bool is_invokehandle = code == Bytecodes::_invokehandle;
3126 const bool is_invokevirtual = code == Bytecodes::_invokevirtual;
3127 const bool is_invokespecial = code == Bytecodes::_invokespecial;
3128 const bool load_receiver = (recv != noreg);
3129 const bool save_flags = (flags != noreg);
3130 assert(load_receiver == (code != Bytecodes::_invokestatic && code != Bytecodes::_invokedynamic), "");
3131 assert(save_flags == (is_invokeinterface || is_invokevirtual), "need flags for vfinal");
3132 assert(flags == noreg || flags == r3, "");
3133 assert(recv == noreg || recv == r2, "");
3134
3135 // setup registers & access constant pool cache
3136 if (recv == noreg) recv = r2;
3137 if (flags == noreg) flags = r3;
3138 assert_different_registers(method, index, recv, flags);
3139
3140 // save 'interpreter return address'
3141 __ save_bcp();
3142
3143 load_invoke_cp_cache_entry(byte_no, method, index, flags, is_invokevirtual, false, is_invokedynamic);
3144
3145 // maybe push appendix to arguments (just before return address)
3146 if (is_invokedynamic || is_invokehandle) {
3147 Label L_no_push;
3148 __ tbz(flags, ConstantPoolCacheEntry::has_appendix_shift, L_no_push);
3149 // Push the appendix as a trailing parameter.
3150 // This must be done before we get the receiver,
3151 // since the parameter_size includes it.
3152 __ push(r19);
3153 __ mov(r19, index);
3154 assert(ConstantPoolCacheEntry::_indy_resolved_references_appendix_offset == 0, "appendix expected at index+0");
3155 __ load_resolved_reference_at_index(index, r19);
3156 __ pop(r19);
3157 __ push(index); // push appendix (MethodType, CallSite, etc.)
3158 __ bind(L_no_push);
3159 }
3160
3161 // load receiver if needed (note: no return address pushed yet)
3162 if (load_receiver) {
3163 __ andw(recv, flags, ConstantPoolCacheEntry::parameter_size_mask);
3164 // FIXME -- is this actually correct? looks like it should be 2
3165 // const int no_return_pc_pushed_yet = -1; // argument slot correction before we push return address
3166 // const int receiver_is_at_end = -1; // back off one slot to get receiver
3167 // Address recv_addr = __ argument_address(recv, no_return_pc_pushed_yet + receiver_is_at_end);
3168 // __ movptr(recv, recv_addr);
3169 __ add(rscratch1, esp, recv, ext::uxtx, 3); // FIXME: uxtb here?
3170 __ ldr(recv, Address(rscratch1, -Interpreter::expr_offset_in_bytes(1)));
3171 __ verify_oop(recv);
3172 }
3173
3174 // compute return type
3175 // x86 uses a shift and mask or wings it with a shift plus assert
3176 // the mask is not needed. aarch64 just uses bitfield extract
3177 __ ubfxw(rscratch2, flags, ConstantPoolCacheEntry::tos_state_shift, ConstantPoolCacheEntry::tos_state_bits);
3178 // load return address
3179 {
3180 const address table_addr = (address) Interpreter::invoke_return_entry_table_for(code);
3181 __ mov(rscratch1, table_addr);
3182 __ ldr(lr, Address(rscratch1, rscratch2, Address::lsl(3)));
3183 }
3184 }
3185
3186
3187 void TemplateTable::invokevirtual_helper(Register index,
3188 Register recv,
3189 Register flags)
3190 {
3191 // Uses temporary registers r0, r3
3192 assert_different_registers(index, recv, r0, r3);
3193 // Test for an invoke of a final method
3194 Label notFinal;
3195 __ tbz(flags, ConstantPoolCacheEntry::is_vfinal_shift, notFinal);
3196
3197 const Register method = index; // method must be rmethod
3198 assert(method == rmethod,
3199 "methodOop must be rmethod for interpreter calling convention");
3200
3201 // do the call - the index is actually the method to call
3202 // that is, f2 is a vtable index if !is_vfinal, else f2 is a Method*
3203
3204 // It's final, need a null check here!
3205 __ null_check(recv);
3206
3207 // profile this call
3208 __ profile_final_call(r0);
3209 __ profile_arguments_type(r0, method, r4, true);
3210
3211 __ jump_from_interpreted(method, r0);
3212
3213 __ bind(notFinal);
3214
3215 // get receiver klass
3216 __ null_check(recv, oopDesc::klass_offset_in_bytes());
3217 __ load_klass(r0, recv);
3218
3219 // profile this call
3220 __ profile_virtual_call(r0, rlocals, r3);
3221
3222 // get target methodOop & entry point
3223 __ lookup_virtual_method(r0, index, method);
3224 __ profile_arguments_type(r3, method, r4, true);
3225 // FIXME -- this looks completely redundant. is it?
3226 // __ ldr(r3, Address(method, Method::interpreter_entry_offset()));
3227 __ jump_from_interpreted(method, r3);
3228 }
3229
3230 void TemplateTable::invokevirtual(int byte_no)
3231 {
3232 transition(vtos, vtos);
3233 assert(byte_no == f2_byte, "use this argument");
3234
3235 prepare_invoke(byte_no, rmethod, noreg, r2, r3);
3236
3237 // rmethod: index (actually a Method*)
3238 // r2: receiver
3239 // r3: flags
3240
3241 invokevirtual_helper(rmethod, r2, r3);
3242 }
3243
3244 void TemplateTable::invokespecial(int byte_no)
3245 {
3246 transition(vtos, vtos);
3247 assert(byte_no == f1_byte, "use this argument");
3248
3249 prepare_invoke(byte_no, rmethod, noreg, // get f1 Method*
3250 r2); // get receiver also for null check
3251 __ verify_oop(r2);
3252 __ null_check(r2);
3253 // do the call
3254 __ profile_call(r0);
3255 __ profile_arguments_type(r0, rmethod, rbcp, false);
3256 __ jump_from_interpreted(rmethod, r0);
3257 }
3258
3259 void TemplateTable::invokestatic(int byte_no)
3260 {
3261 transition(vtos, vtos);
3262 assert(byte_no == f1_byte, "use this argument");
3263
3264 prepare_invoke(byte_no, rmethod); // get f1 Method*
3265 // do the call
3266 __ profile_call(r0);
3267 __ profile_arguments_type(r0, rmethod, r4, false);
3268 __ jump_from_interpreted(rmethod, r0);
3269 }
3270
3271 void TemplateTable::fast_invokevfinal(int byte_no)
3272 {
3273 __ call_Unimplemented();
3274 }
3275
3276 void TemplateTable::invokeinterface(int byte_no) {
3277 transition(vtos, vtos);
3278 assert(byte_no == f1_byte, "use this argument");
3279
3280 prepare_invoke(byte_no, r0, rmethod, // get f1 Klass*, f2 itable index
3281 r2, r3); // recv, flags
3282
3283 // r0: interface klass (from f1)
3284 // rmethod: itable index (from f2)
3285 // r2: receiver
3286 // r3: flags
3287
3288 // Special case of invokeinterface called for virtual method of
3289 // java.lang.Object. See cpCacheOop.cpp for details.
3290 // This code isn't produced by javac, but could be produced by
3291 // another compliant java compiler.
3292 Label notMethod;
3293 __ tbz(r3, ConstantPoolCacheEntry::is_forced_virtual_shift, notMethod);
3294
3295 invokevirtual_helper(rmethod, r2, r3);
3296 __ bind(notMethod);
3297
3298 // Get receiver klass into r3 - also a null check
3299 __ restore_locals();
3300 __ null_check(r2, oopDesc::klass_offset_in_bytes());
3301 __ load_klass(r3, r2);
3302
3303 // profile this call
3304 __ profile_virtual_call(r3, r13, r19);
3305
3306 Label no_such_interface, no_such_method;
3307
3308 __ lookup_interface_method(// inputs: rec. class, interface, itable index
3309 r3, r0, rmethod,
3310 // outputs: method, scan temp. reg
3311 rmethod, r13,
3312 no_such_interface);
3313
3314 // rmethod,: methodOop to call
3315 // r2: receiver
3316 // Check for abstract method error
3317 // Note: This should be done more efficiently via a throw_abstract_method_error
3318 // interpreter entry point and a conditional jump to it in case of a null
3319 // method.
3320 __ cbz(rmethod, no_such_method);
3321
3322 __ profile_arguments_type(r3, rmethod, r13, true);
3323
3324 // do the call
3325 // r2: receiver
3326 // rmethod,: methodOop
3327 __ jump_from_interpreted(rmethod, r3);
3328 __ should_not_reach_here();
3329
3330 // exception handling code follows...
3331 // note: must restore interpreter registers to canonical
3332 // state for exception handling to work correctly!
3333
3334 __ bind(no_such_method);
3335 // throw exception
3336 __ restore_bcp(); // bcp must be correct for exception handler (was destroyed)
3337 __ restore_locals(); // make sure locals pointer is correct as well (was destroyed)
3338 __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::throw_AbstractMethodError));
3339 // the call_VM checks for exception, so we should never return here.
3340 __ should_not_reach_here();
3341
3342 __ bind(no_such_interface);
3343 // throw exception
3344 __ restore_bcp(); // bcp must be correct for exception handler (was destroyed)
3345 __ restore_locals(); // make sure locals pointer is correct as well (was destroyed)
3346 __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3347 InterpreterRuntime::throw_IncompatibleClassChangeError));
3348 // the call_VM checks for exception, so we should never return here.
3349 __ should_not_reach_here();
3350 return;
3351 }
3352
3353 void TemplateTable::invokehandle(int byte_no) {
3354 transition(vtos, vtos);
3355 assert(byte_no == f1_byte, "use this argument");
3356
3357 prepare_invoke(byte_no, rmethod, r0, r2);
3358 __ verify_method_ptr(r2);
3359 __ verify_oop(r2);
3360 __ null_check(r2);
3361
3362 // FIXME: profile the LambdaForm also
3363
3364 // r13 is safe to use here as a scratch reg because it is about to
3365 // be clobbered by jump_from_interpreted().
3366 __ profile_final_call(r13);
3367 __ profile_arguments_type(r13, rmethod, r4, true);
3368
3369 __ jump_from_interpreted(rmethod, r0);
3370 }
3371
3372 void TemplateTable::invokedynamic(int byte_no) {
3373 transition(vtos, vtos);
3374 assert(byte_no == f1_byte, "use this argument");
3375
3376 prepare_invoke(byte_no, rmethod, r0);
3377
3378 // r0: CallSite object (from cpool->resolved_references[])
3379 // rmethod: MH.linkToCallSite method (from f2)
3380
3381 // Note: r0_callsite is already pushed by prepare_invoke
3382
3383 // %%% should make a type profile for any invokedynamic that takes a ref argument
3384 // profile this call
3385 __ profile_call(rbcp);
3386 __ profile_arguments_type(r3, rmethod, r13, false);
3387
3388 __ verify_oop(r0);
3389
3390 __ jump_from_interpreted(rmethod, r0);
3391 }
3392
3393
3394 //-----------------------------------------------------------------------------
3395 // Allocation
3396
3397 void TemplateTable::_new() {
3398 transition(vtos, atos);
3399
3400 __ get_unsigned_2_byte_index_at_bcp(r3, 1);
3401 Label slow_case;
3402 Label done;
3403 Label initialize_header;
3404 Label initialize_object; // including clearing the fields
3405 Label allocate_shared;
3406
3407 __ get_cpool_and_tags(r4, r0);
3408 // Make sure the class we're about to instantiate has been resolved.
3409 // This is done before loading InstanceKlass to be consistent with the order
3410 // how Constant Pool is updated (see ConstantPool::klass_at_put)
3411 const int tags_offset = Array<u1>::base_offset_in_bytes();
3412 __ lea(rscratch1, Address(r0, r3, Address::lsl(0)));
3413 __ lea(rscratch1, Address(rscratch1, tags_offset));
3414 __ ldarb(rscratch1, rscratch1);
3415 __ cmp(rscratch1, JVM_CONSTANT_Class);
3416 __ br(Assembler::NE, slow_case);
3417
3418 // get InstanceKlass
3419 __ load_resolved_klass_at_offset(r4, r3, r4, rscratch1);
3420
3421 // make sure klass is initialized & doesn't have finalizer
3422 // make sure klass is fully initialized
3423 __ ldrb(rscratch1, Address(r4, InstanceKlass::init_state_offset()));
3424 __ cmp(rscratch1, InstanceKlass::fully_initialized);
3425 __ br(Assembler::NE, slow_case);
3426
3427 // get instance_size in InstanceKlass (scaled to a count of bytes)
3428 __ ldrw(r3,
3429 Address(r4,
3430 Klass::layout_helper_offset()));
3431 // test to see if it has a finalizer or is malformed in some way
3432 __ tbnz(r3, exact_log2(Klass::_lh_instance_slow_path_bit), slow_case);
3433
3434 // Allocate the instance
3435 // 1) Try to allocate in the TLAB
3436 // 2) if fail and the object is large allocate in the shared Eden
3437 // 3) if the above fails (or is not applicable), go to a slow case
3438 // (creates a new TLAB, etc.)
3439
3440 const bool allow_shared_alloc =
3441 Universe::heap()->supports_inline_contig_alloc();
3442
3443 if (UseTLAB) {
3444 __ tlab_allocate(r0, r3, 0, noreg, r1,
3445 allow_shared_alloc ? allocate_shared : slow_case);
3446
3447 if (ZeroTLAB) {
3448 // the fields have been already cleared
3449 __ b(initialize_header);
3450 } else {
3451 // initialize both the header and fields
3452 __ b(initialize_object);
3453 }
3454 }
3455
3456 // Allocation in the shared Eden, if allowed.
3457 //
3458 // r3: instance size in bytes
3459 if (allow_shared_alloc) {
3460 __ bind(allocate_shared);
3461
3462 __ eden_allocate(r0, r3, 0, r10, slow_case);
3463 __ incr_allocated_bytes(rthread, r3, 0, rscratch1);
3464 }
3465
3466 if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) {
3467 // The object is initialized before the header. If the object size is
3468 // zero, go directly to the header initialization.
3469 __ bind(initialize_object);
3470 __ sub(r3, r3, sizeof(oopDesc));
3471 __ cbz(r3, initialize_header);
3472
3473 // Initialize object fields
3474 {
3475 __ add(r2, r0, sizeof(oopDesc));
3476 Label loop;
3477 __ bind(loop);
3478 __ str(zr, Address(__ post(r2, BytesPerLong)));
3479 __ sub(r3, r3, BytesPerLong);
3480 __ cbnz(r3, loop);
3481 }
3482
3483 // initialize object header only.
3484 __ bind(initialize_header);
3485 if (UseBiasedLocking) {
3486 __ ldr(rscratch1, Address(r4, Klass::prototype_header_offset()));
3487 } else {
3488 __ mov(rscratch1, (intptr_t)markOopDesc::prototype());
3489 }
3490 __ str(rscratch1, Address(r0, oopDesc::mark_offset_in_bytes()));
3491 __ store_klass_gap(r0, zr); // zero klass gap for compressed oops
3492 __ store_klass(r0, r4); // store klass last
3493
3494 {
3495 SkipIfEqual skip(_masm, &DTraceAllocProbes, false);
3496 // Trigger dtrace event for fastpath
3497 __ push(atos); // save the return value
3498 __ call_VM_leaf(
3499 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), r0);
3500 __ pop(atos); // restore the return value
3501
3502 }
3503 __ b(done);
3504 }
3505
3506 // slow case
3507 __ bind(slow_case);
3508 __ get_constant_pool(c_rarg1);
3509 __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3510 call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::_new), c_rarg1, c_rarg2);
3511 __ verify_oop(r0);
3512
3513 // continue
3514 __ bind(done);
3515 // Must prevent reordering of stores for object initialization with stores that publish the new object.
3516 __ membar(Assembler::StoreStore);
3517 }
3518
3519 void TemplateTable::newarray() {
3520 transition(itos, atos);
3521 __ load_unsigned_byte(c_rarg1, at_bcp(1));
3522 __ mov(c_rarg2, r0);
3523 call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::newarray),
3524 c_rarg1, c_rarg2);
3525 // Must prevent reordering of stores for object initialization with stores that publish the new object.
3526 __ membar(Assembler::StoreStore);
3527 }
3528
3529 void TemplateTable::anewarray() {
3530 transition(itos, atos);
3531 __ get_unsigned_2_byte_index_at_bcp(c_rarg2, 1);
3532 __ get_constant_pool(c_rarg1);
3533 __ mov(c_rarg3, r0);
3534 call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::anewarray),
3535 c_rarg1, c_rarg2, c_rarg3);
3536 // Must prevent reordering of stores for object initialization with stores that publish the new object.
3537 __ membar(Assembler::StoreStore);
3538 }
3539
3540 void TemplateTable::arraylength() {
3541 transition(atos, itos);
3542 __ null_check(r0, arrayOopDesc::length_offset_in_bytes());
3543 __ ldrw(r0, Address(r0, arrayOopDesc::length_offset_in_bytes()));
3544 }
3545
3546 void TemplateTable::checkcast()
3547 {
3548 transition(atos, atos);
3549 Label done, is_null, ok_is_subtype, quicked, resolved;
3550 __ cbz(r0, is_null);
3551
3552 // Get cpool & tags index
3553 __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3554 __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3555 // See if bytecode has already been quicked
3556 __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3557 __ lea(r1, Address(rscratch1, r19));
3558 __ ldarb(r1, r1);
3559 __ cmp(r1, JVM_CONSTANT_Class);
3560 __ br(Assembler::EQ, quicked);
3561
3562 __ push(atos); // save receiver for result, and for GC
3563 call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3564 // vm_result_2 has metadata result
3565 __ get_vm_result_2(r0, rthread);
3566 __ pop(r3); // restore receiver
3567 __ b(resolved);
3568
3569 // Get superklass in r0 and subklass in r3
3570 __ bind(quicked);
3571 __ mov(r3, r0); // Save object in r3; r0 needed for subtype check
3572 __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1); // r0 = klass
3573
3574 __ bind(resolved);
3575 __ load_klass(r19, r3);
3576
3577 // Generate subtype check. Blows r2, r5. Object in r3.
3578 // Superklass in r0. Subklass in r19.
3579 __ gen_subtype_check(r19, ok_is_subtype);
3580
3581 // Come here on failure
3582 __ push(r3);
3583 // object is at TOS
3584 __ b(Interpreter::_throw_ClassCastException_entry);
3585
3586 // Come here on success
3587 __ bind(ok_is_subtype);
3588 __ mov(r0, r3); // Restore object in r3
3589
3590 // Collect counts on whether this test sees NULLs a lot or not.
3591 if (ProfileInterpreter) {
3592 __ b(done);
3593 __ bind(is_null);
3594 __ profile_null_seen(r2);
3595 } else {
3596 __ bind(is_null); // same as 'done'
3597 }
3598 __ bind(done);
3599 }
3600
3601 void TemplateTable::instanceof() {
3602 transition(atos, itos);
3603 Label done, is_null, ok_is_subtype, quicked, resolved;
3604 __ cbz(r0, is_null);
3605
3606 // Get cpool & tags index
3607 __ get_cpool_and_tags(r2, r3); // r2=cpool, r3=tags array
3608 __ get_unsigned_2_byte_index_at_bcp(r19, 1); // r19=index
3609 // See if bytecode has already been quicked
3610 __ add(rscratch1, r3, Array<u1>::base_offset_in_bytes());
3611 __ lea(r1, Address(rscratch1, r19));
3612 __ ldarb(r1, r1);
3613 __ cmp(r1, JVM_CONSTANT_Class);
3614 __ br(Assembler::EQ, quicked);
3615
3616 __ push(atos); // save receiver for result, and for GC
3617 call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::quicken_io_cc));
3618 // vm_result_2 has metadata result
3619 __ get_vm_result_2(r0, rthread);
3620 __ pop(r3); // restore receiver
3621 __ verify_oop(r3);
3622 __ load_klass(r3, r3);
3623 __ b(resolved);
3624
3625 // Get superklass in r0 and subklass in r3
3626 __ bind(quicked);
3627 __ load_klass(r3, r0);
3628 __ load_resolved_klass_at_offset(r2, r19, r0, rscratch1);
3629
3630 __ bind(resolved);
3631
3632 // Generate subtype check. Blows r2, r5
3633 // Superklass in r0. Subklass in r3.
3634 __ gen_subtype_check(r3, ok_is_subtype);
3635
3636 // Come here on failure
3637 __ mov(r0, 0);
3638 __ b(done);
3639 // Come here on success
3640 __ bind(ok_is_subtype);
3641 __ mov(r0, 1);
3642
3643 // Collect counts on whether this test sees NULLs a lot or not.
3644 if (ProfileInterpreter) {
3645 __ b(done);
3646 __ bind(is_null);
3647 __ profile_null_seen(r2);
3648 } else {
3649 __ bind(is_null); // same as 'done'
3650 }
3651 __ bind(done);
3652 // r0 = 0: obj == NULL or obj is not an instanceof the specified klass
3653 // r0 = 1: obj != NULL and obj is an instanceof the specified klass
3654 }
3655
3656 //-----------------------------------------------------------------------------
3657 // Breakpoints
3658 void TemplateTable::_breakpoint() {
3659 // Note: We get here even if we are single stepping..
3660 // jbug inists on setting breakpoints at every bytecode
3661 // even if we are in single step mode.
3662
3663 transition(vtos, vtos);
3664
3665 // get the unpatched byte code
3666 __ get_method(c_rarg1);
3667 __ call_VM(noreg,
3668 CAST_FROM_FN_PTR(address,
3669 InterpreterRuntime::get_original_bytecode_at),
3670 c_rarg1, rbcp);
3671 __ mov(r19, r0);
3672
3673 // post the breakpoint event
3674 __ call_VM(noreg,
3675 CAST_FROM_FN_PTR(address, InterpreterRuntime::_breakpoint),
3676 rmethod, rbcp);
3677
3678 // complete the execution of original bytecode
3679 __ mov(rscratch1, r19);
3680 __ dispatch_only_normal(vtos);
3681 }
3682
3683 //-----------------------------------------------------------------------------
3684 // Exceptions
3685
3686 void TemplateTable::athrow() {
3687 transition(atos, vtos);
3688 __ null_check(r0);
3689 __ b(Interpreter::throw_exception_entry());
3690 }
3691
3692 //-----------------------------------------------------------------------------
3693 // Synchronization
3694 //
3695 // Note: monitorenter & exit are symmetric routines; which is reflected
3696 // in the assembly code structure as well
3697 //
3698 // Stack layout:
3699 //
3700 // [expressions ] <--- esp = expression stack top
3701 // ..
3702 // [expressions ]
3703 // [monitor entry] <--- monitor block top = expression stack bot
3704 // ..
3705 // [monitor entry]
3706 // [frame data ] <--- monitor block bot
3707 // ...
3708 // [saved rbp ] <--- rbp
3709 void TemplateTable::monitorenter()
3710 {
3711 transition(atos, vtos);
3712
3713 // check for NULL object
3714 __ null_check(r0);
3715
3716 const Address monitor_block_top(
3717 rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3718 const Address monitor_block_bot(
3719 rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3720 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3721
3722 Label allocated;
3723
3724 // initialize entry pointer
3725 __ mov(c_rarg1, zr); // points to free slot or NULL
3726
3727 // find a free slot in the monitor block (result in c_rarg1)
3728 {
3729 Label entry, loop, exit;
3730 __ ldr(c_rarg3, monitor_block_top); // points to current entry,
3731 // starting with top-most entry
3732 __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3733
3734 __ b(entry);
3735
3736 __ bind(loop);
3737 // check if current entry is used
3738 // if not used then remember entry in c_rarg1
3739 __ ldr(rscratch1, Address(c_rarg3, BasicObjectLock::obj_offset_in_bytes()));
3740 __ cmp(zr, rscratch1);
3741 __ csel(c_rarg1, c_rarg3, c_rarg1, Assembler::EQ);
3742 // check if current entry is for same object
3743 __ cmp(r0, rscratch1);
3744 // if same object then stop searching
3745 __ br(Assembler::EQ, exit);
3746 // otherwise advance to next entry
3747 __ add(c_rarg3, c_rarg3, entry_size);
3748 __ bind(entry);
3749 // check if bottom reached
3750 __ cmp(c_rarg3, c_rarg2);
3751 // if not at bottom then check this entry
3752 __ br(Assembler::NE, loop);
3753 __ bind(exit);
3754 }
3755
3756 __ cbnz(c_rarg1, allocated); // check if a slot has been found and
3757 // if found, continue with that on
3758
3759 // allocate one if there's no free slot
3760 {
3761 Label entry, loop;
3762 // 1. compute new pointers // rsp: old expression stack top
3763 __ ldr(c_rarg1, monitor_block_bot); // c_rarg1: old expression stack bottom
3764 __ sub(esp, esp, entry_size); // move expression stack top
3765 __ sub(c_rarg1, c_rarg1, entry_size); // move expression stack bottom
3766 __ mov(c_rarg3, esp); // set start value for copy loop
3767 __ str(c_rarg1, monitor_block_bot); // set new monitor block bottom
3768
3769 __ sub(sp, sp, entry_size); // make room for the monitor
3770
3771 __ b(entry);
3772 // 2. move expression stack contents
3773 __ bind(loop);
3774 __ ldr(c_rarg2, Address(c_rarg3, entry_size)); // load expression stack
3775 // word from old location
3776 __ str(c_rarg2, Address(c_rarg3, 0)); // and store it at new location
3777 __ add(c_rarg3, c_rarg3, wordSize); // advance to next word
3778 __ bind(entry);
3779 __ cmp(c_rarg3, c_rarg1); // check if bottom reached
3780 __ br(Assembler::NE, loop); // if not at bottom then
3781 // copy next word
3782 }
3783
3784 // call run-time routine
3785 // c_rarg1: points to monitor entry
3786 __ bind(allocated);
3787
3788 // Increment bcp to point to the next bytecode, so exception
3789 // handling for async. exceptions work correctly.
3790 // The object has already been poped from the stack, so the
3791 // expression stack looks correct.
3792 __ increment(rbcp);
3793
3794 // store object
3795 __ str(r0, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3796 __ lock_object(c_rarg1);
3797
3798 // check to make sure this monitor doesn't cause stack overflow after locking
3799 __ save_bcp(); // in case of exception
3800 __ generate_stack_overflow_check(0);
3801
3802 // The bcp has already been incremented. Just need to dispatch to
3803 // next instruction.
3804 __ dispatch_next(vtos);
3805 }
3806
3807
3808 void TemplateTable::monitorexit()
3809 {
3810 transition(atos, vtos);
3811
3812 // check for NULL object
3813 __ null_check(r0);
3814
3815 const Address monitor_block_top(
3816 rfp, frame::interpreter_frame_monitor_block_top_offset * wordSize);
3817 const Address monitor_block_bot(
3818 rfp, frame::interpreter_frame_initial_sp_offset * wordSize);
3819 const int entry_size = frame::interpreter_frame_monitor_size() * wordSize;
3820
3821 Label found;
3822
3823 // find matching slot
3824 {
3825 Label entry, loop;
3826 __ ldr(c_rarg1, monitor_block_top); // points to current entry,
3827 // starting with top-most entry
3828 __ lea(c_rarg2, monitor_block_bot); // points to word before bottom
3829 // of monitor block
3830 __ b(entry);
3831
3832 __ bind(loop);
3833 // check if current entry is for same object
3834 __ ldr(rscratch1, Address(c_rarg1, BasicObjectLock::obj_offset_in_bytes()));
3835 __ cmp(r0, rscratch1);
3836 // if same object then stop searching
3837 __ br(Assembler::EQ, found);
3838 // otherwise advance to next entry
3839 __ add(c_rarg1, c_rarg1, entry_size);
3840 __ bind(entry);
3841 // check if bottom reached
3842 __ cmp(c_rarg1, c_rarg2);
3843 // if not at bottom then check this entry
3844 __ br(Assembler::NE, loop);
3845 }
3846
3847 // error handling. Unlocking was not block-structured
3848 __ call_VM(noreg, CAST_FROM_FN_PTR(address,
3849 InterpreterRuntime::throw_illegal_monitor_state_exception));
3850 __ should_not_reach_here();
3851
3852 // call run-time routine
3853 __ bind(found);
3854 __ push_ptr(r0); // make sure object is on stack (contract with oopMaps)
3855 __ unlock_object(c_rarg1);
3856 __ pop_ptr(r0); // discard object
3857 }
3858
3859
3860 // Wide instructions
3861 void TemplateTable::wide()
3862 {
3863 __ load_unsigned_byte(r19, at_bcp(1));
3864 __ mov(rscratch1, (address)Interpreter::_wentry_point);
3865 __ ldr(rscratch1, Address(rscratch1, r19, Address::uxtw(3)));
3866 __ br(rscratch1);
3867 }
3868
3869
3870 // Multi arrays
3871 void TemplateTable::multianewarray() {
3872 transition(vtos, atos);
3873 __ load_unsigned_byte(r0, at_bcp(3)); // get number of dimensions
3874 // last dim is on top of stack; we want address of first one:
3875 // first_addr = last_addr + (ndims - 1) * wordSize
3876 __ lea(c_rarg1, Address(esp, r0, Address::uxtw(3)));
3877 __ sub(c_rarg1, c_rarg1, wordSize);
3878 call_VM(r0,
3879 CAST_FROM_FN_PTR(address, InterpreterRuntime::multianewarray),
3880 c_rarg1);
3881 __ load_unsigned_byte(r1, at_bcp(3));
3882 __ lea(esp, Address(esp, r1, Address::uxtw(3)));
3883 }