1 /*
2 * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "c1/c1_Defs.hpp"
28 #include "c1/c1_LIRAssembler.hpp"
29 #include "c1/c1_MacroAssembler.hpp"
30 #include "c1/c1_Runtime1.hpp"
31 #include "ci/ciUtilities.hpp"
32 #include "gc/shared/cardTable.hpp"
33 #include "gc/shared/cardTableBarrierSet.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "nativeInst_arm.hpp"
36 #include "oops/compiledICHolder.hpp"
37 #include "oops/oop.inline.hpp"
38 #include "prims/jvmtiExport.hpp"
39 #include "register_arm.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #include "runtime/signature.hpp"
42 #include "runtime/vframeArray.hpp"
43 #include "utilities/align.hpp"
44 #include "vmreg_arm.inline.hpp"
45
46 // Note: Rtemp usage is this file should not impact C2 and should be
47 // correct as long as it is not implicitly used in lower layers (the
48 // arm [macro]assembler) and used with care in the other C1 specific
49 // files.
50
51 // Implementation of StubAssembler
52
53 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
54 mov(R0, Rthread);
55
56 int call_offset = set_last_Java_frame(SP, FP, false, Rtemp);
57
58 call(entry);
59 if (call_offset == -1) { // PC not saved
60 call_offset = offset();
61 }
62 reset_last_Java_frame(Rtemp);
63
64 assert(frame_size() != no_frame_size, "frame must be fixed");
65 if (_stub_id != Runtime1::forward_exception_id) {
66 ldr(R3, Address(Rthread, Thread::pending_exception_offset()));
67 }
68
69 if (oop_result1->is_valid()) {
70 assert_different_registers(oop_result1, R3, Rtemp);
71 get_vm_result(oop_result1, Rtemp);
72 }
73 if (metadata_result->is_valid()) {
74 assert_different_registers(metadata_result, R3, Rtemp);
75 get_vm_result_2(metadata_result, Rtemp);
76 }
77
78 // Check for pending exception
79 // unpack_with_exception_in_tls path is taken through
80 // Runtime1::exception_handler_for_pc
81 if (_stub_id != Runtime1::forward_exception_id) {
82 assert(frame_size() != no_frame_size, "cannot directly call forward_exception_id");
83 #ifdef AARCH64
84 Label skip;
85 cbz(R3, skip);
86 jump(Runtime1::entry_for(Runtime1::forward_exception_id), relocInfo::runtime_call_type, Rtemp);
87 bind(skip);
88 #else
89 cmp(R3, 0);
90 jump(Runtime1::entry_for(Runtime1::forward_exception_id), relocInfo::runtime_call_type, Rtemp, ne);
91 #endif // AARCH64
92 } else {
93 #ifdef ASSERT
94 // Should not have pending exception in forward_exception stub
95 ldr(R3, Address(Rthread, Thread::pending_exception_offset()));
96 cmp(R3, 0);
97 breakpoint(ne);
98 #endif // ASSERT
99 }
100 return call_offset;
101 }
102
103
104 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {
105 if (arg1 != R1) {
106 mov(R1, arg1);
107 }
108 return call_RT(oop_result1, metadata_result, entry, 1);
109 }
110
111
112 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {
113 assert(arg1 == R1 && arg2 == R2, "cannot handle otherwise");
114 return call_RT(oop_result1, metadata_result, entry, 2);
115 }
116
117
118 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
119 assert(arg1 == R1 && arg2 == R2 && arg3 == R3, "cannot handle otherwise");
120 return call_RT(oop_result1, metadata_result, entry, 3);
121 }
122
123
124 #define __ sasm->
125
126 // TODO: ARM - does this duplicate RegisterSaver in SharedRuntime?
127 #ifdef AARCH64
128
129 //
130 // On AArch64 registers save area has the following layout:
131 //
132 // |---------------------|
133 // | return address (LR) |
134 // | FP |
135 // |---------------------|
136 // | D31 |
137 // | ... |
138 // | D0 |
139 // |---------------------|
140 // | padding |
141 // |---------------------|
142 // | R28 |
143 // | ... |
144 // | R0 |
145 // |---------------------| <-- SP
146 //
147
148 enum RegisterLayout {
149 number_of_saved_gprs = 29,
150 number_of_saved_fprs = FloatRegisterImpl::number_of_registers,
151
152 R0_offset = 0,
153 D0_offset = R0_offset + number_of_saved_gprs + 1,
154 FP_offset = D0_offset + number_of_saved_fprs,
155 LR_offset = FP_offset + 1,
156
157 reg_save_size = LR_offset + 1,
158
159 arg1_offset = reg_save_size * wordSize,
160 arg2_offset = (reg_save_size + 1) * wordSize
161 };
162
163 #else
164
165 enum RegisterLayout {
166 fpu_save_size = pd_nof_fpu_regs_reg_alloc,
167 #ifndef __SOFTFP__
168 D0_offset = 0,
169 #endif
170 R0_offset = fpu_save_size,
171 R1_offset,
172 R2_offset,
173 R3_offset,
174 R4_offset,
175 R5_offset,
176 R6_offset,
177 #if (FP_REG_NUM != 7)
178 R7_offset,
179 #endif
180 R8_offset,
181 R9_offset,
182 R10_offset,
183 #if (FP_REG_NUM != 11)
184 R11_offset,
185 #endif
186 R12_offset,
187 FP_offset,
188 LR_offset,
189 reg_save_size,
190 arg1_offset = reg_save_size * wordSize,
191 arg2_offset = (reg_save_size + 1) * wordSize
192 };
193
194 #endif // AARCH64
195
196 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers = HaveVFP) {
197 sasm->set_frame_size(reg_save_size /* in words */);
198
199 // Record saved value locations in an OopMap.
200 // Locations are offsets from sp after runtime call.
201 OopMap* map = new OopMap(VMRegImpl::slots_per_word * reg_save_size, 0);
202
203 #ifdef AARCH64
204 for (int i = 0; i < number_of_saved_gprs; i++) {
205 map->set_callee_saved(VMRegImpl::stack2reg((R0_offset + i) * VMRegImpl::slots_per_word), as_Register(i)->as_VMReg());
206 }
207 map->set_callee_saved(VMRegImpl::stack2reg(FP_offset * VMRegImpl::slots_per_word), FP->as_VMReg());
208 map->set_callee_saved(VMRegImpl::stack2reg(LR_offset * VMRegImpl::slots_per_word), LR->as_VMReg());
209
210 if (save_fpu_registers) {
211 for (int i = 0; i < number_of_saved_fprs; i++) {
212 map->set_callee_saved(VMRegImpl::stack2reg((D0_offset + i) * VMRegImpl::slots_per_word), as_FloatRegister(i)->as_VMReg());
213 }
214 }
215 #else
216 int j=0;
217 for (int i = R0_offset; i < R10_offset; i++) {
218 if (j == FP_REG_NUM) {
219 // skip the FP register, saved below
220 j++;
221 }
222 map->set_callee_saved(VMRegImpl::stack2reg(i), as_Register(j)->as_VMReg());
223 j++;
224 }
225 assert(j == R10->encoding(), "must be");
226 #if (FP_REG_NUM != 11)
227 // add R11, if not saved as FP
228 map->set_callee_saved(VMRegImpl::stack2reg(R11_offset), R11->as_VMReg());
229 #endif
230 map->set_callee_saved(VMRegImpl::stack2reg(FP_offset), FP->as_VMReg());
231 map->set_callee_saved(VMRegImpl::stack2reg(LR_offset), LR->as_VMReg());
232
233 if (save_fpu_registers) {
234 for (int i = 0; i < fpu_save_size; i++) {
235 map->set_callee_saved(VMRegImpl::stack2reg(i), as_FloatRegister(i)->as_VMReg());
236 }
237 }
238 #endif // AARCH64
239
240 return map;
241 }
242
243 static OopMap* save_live_registers(StubAssembler* sasm, bool save_fpu_registers = HaveVFP) {
244 __ block_comment("save_live_registers");
245 sasm->set_frame_size(reg_save_size /* in words */);
246
247 #ifdef AARCH64
248 assert((reg_save_size * wordSize) % StackAlignmentInBytes == 0, "SP should be aligned");
249
250 __ raw_push(FP, LR);
251
252 __ sub(SP, SP, (reg_save_size - 2) * wordSize);
253
254 for (int i = 0; i < align_down((int)number_of_saved_gprs, 2); i += 2) {
255 __ stp(as_Register(i), as_Register(i+1), Address(SP, (R0_offset + i) * wordSize));
256 }
257
258 if (is_odd(number_of_saved_gprs)) {
259 int i = number_of_saved_gprs - 1;
260 __ str(as_Register(i), Address(SP, (R0_offset + i) * wordSize));
261 }
262
263 if (save_fpu_registers) {
264 assert (is_even(number_of_saved_fprs), "adjust this code");
265 for (int i = 0; i < number_of_saved_fprs; i += 2) {
266 __ stp_d(as_FloatRegister(i), as_FloatRegister(i+1), Address(SP, (D0_offset + i) * wordSize));
267 }
268 }
269 #else
270 __ push(RegisterSet(FP) | RegisterSet(LR));
271 __ push(RegisterSet(R0, R6) | RegisterSet(R8, R10) | R12 | altFP_7_11);
272 if (save_fpu_registers) {
273 __ fstmdbd(SP, FloatRegisterSet(D0, fpu_save_size / 2), writeback);
274 } else {
275 __ sub(SP, SP, fpu_save_size * wordSize);
276 }
277 #endif // AARCH64
278
279 return generate_oop_map(sasm, save_fpu_registers);
280 }
281
282
283 static void restore_live_registers(StubAssembler* sasm,
284 bool restore_R0,
285 bool restore_FP_LR,
286 bool do_return,
287 bool restore_fpu_registers = HaveVFP) {
288 __ block_comment("restore_live_registers");
289
290 #ifdef AARCH64
291 if (restore_R0) {
292 __ ldr(R0, Address(SP, R0_offset * wordSize));
293 }
294
295 assert(is_odd(number_of_saved_gprs), "adjust this code");
296 for (int i = 1; i < number_of_saved_gprs; i += 2) {
297 __ ldp(as_Register(i), as_Register(i+1), Address(SP, (R0_offset + i) * wordSize));
298 }
299
300 if (restore_fpu_registers) {
301 assert (is_even(number_of_saved_fprs), "adjust this code");
302 for (int i = 0; i < number_of_saved_fprs; i += 2) {
303 __ ldp_d(as_FloatRegister(i), as_FloatRegister(i+1), Address(SP, (D0_offset + i) * wordSize));
304 }
305 }
306
307 __ add(SP, SP, (reg_save_size - 2) * wordSize);
308
309 if (restore_FP_LR) {
310 __ raw_pop(FP, LR);
311 if (do_return) {
312 __ ret();
313 }
314 } else {
315 assert (!do_return, "return without restoring FP/LR");
316 }
317 #else
318 if (restore_fpu_registers) {
319 __ fldmiad(SP, FloatRegisterSet(D0, fpu_save_size / 2), writeback);
320 if (!restore_R0) {
321 __ add(SP, SP, (R1_offset - fpu_save_size) * wordSize);
322 }
323 } else {
324 __ add(SP, SP, (restore_R0 ? fpu_save_size : R1_offset) * wordSize);
325 }
326 __ pop(RegisterSet((restore_R0 ? R0 : R1), R6) | RegisterSet(R8, R10) | R12 | altFP_7_11);
327 if (restore_FP_LR) {
328 __ pop(RegisterSet(FP) | RegisterSet(do_return ? PC : LR));
329 } else {
330 assert (!do_return, "return without restoring FP/LR");
331 }
332 #endif // AARCH64
333 }
334
335
336 static void restore_live_registers_except_R0(StubAssembler* sasm, bool restore_fpu_registers = HaveVFP) {
337 restore_live_registers(sasm, false, true, true, restore_fpu_registers);
338 }
339
340 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = HaveVFP) {
341 restore_live_registers(sasm, true, true, true, restore_fpu_registers);
342 }
343
344 #ifndef AARCH64
345 static void restore_live_registers_except_FP_LR(StubAssembler* sasm, bool restore_fpu_registers = HaveVFP) {
346 restore_live_registers(sasm, true, false, false, restore_fpu_registers);
347 }
348 #endif // !AARCH64
349
350 static void restore_live_registers_without_return(StubAssembler* sasm, bool restore_fpu_registers = HaveVFP) {
351 restore_live_registers(sasm, true, true, false, restore_fpu_registers);
352 }
353
354 void StubAssembler::save_live_registers() {
355 ::save_live_registers(this);
356 }
357
358 void StubAssembler::restore_live_registers_without_return() {
359 ::restore_live_registers_without_return(this);
360 }
361
362 void Runtime1::initialize_pd() {
363 }
364
365
366 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
367 OopMap* oop_map = save_live_registers(sasm);
368
369 int call_offset;
370 if (has_argument) {
371 __ ldr(R1, Address(SP, arg1_offset));
372 __ ldr(R2, Address(SP, arg2_offset));
373 call_offset = __ call_RT(noreg, noreg, target, R1, R2);
374 } else {
375 call_offset = __ call_RT(noreg, noreg, target);
376 }
377
378 OopMapSet* oop_maps = new OopMapSet();
379 oop_maps->add_gc_map(call_offset, oop_map);
380
381 DEBUG_ONLY(STOP("generate_exception_throw");) // Should not reach here
382 return oop_maps;
383 }
384
385
386 static void restore_sp_for_method_handle(StubAssembler* sasm) {
387 // Restore SP from its saved reg (FP) if the exception PC is a MethodHandle call site.
388 __ ldr_s32(Rtemp, Address(Rthread, JavaThread::is_method_handle_return_offset()));
389 #ifdef AARCH64
390 Label skip;
391 __ cbz(Rtemp, skip);
392 __ mov(SP, Rmh_SP_save);
393 __ bind(skip);
394 #else
395 __ cmp(Rtemp, 0);
396 __ mov(SP, Rmh_SP_save, ne);
397 #endif // AARCH64
398 }
399
400
401 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler* sasm) {
402 __ block_comment("generate_handle_exception");
403
404 bool save_fpu_registers = false;
405
406 // Save registers, if required.
407 OopMapSet* oop_maps = new OopMapSet();
408 OopMap* oop_map = NULL;
409
410 switch (id) {
411 case forward_exception_id: {
412 save_fpu_registers = HaveVFP;
413 oop_map = generate_oop_map(sasm);
414 __ ldr(Rexception_obj, Address(Rthread, Thread::pending_exception_offset()));
415 __ ldr(Rexception_pc, Address(SP, LR_offset * wordSize));
416 Register zero = __ zero_register(Rtemp);
417 __ str(zero, Address(Rthread, Thread::pending_exception_offset()));
418 break;
419 }
420 case handle_exception_id:
421 save_fpu_registers = HaveVFP;
422 // fall-through
423 case handle_exception_nofpu_id:
424 // At this point all registers MAY be live.
425 oop_map = save_live_registers(sasm, save_fpu_registers);
426 break;
427 case handle_exception_from_callee_id:
428 // At this point all registers except exception oop (R4/R19) and
429 // exception pc (R5/R20) are dead.
430 oop_map = save_live_registers(sasm); // TODO it's not required to save all registers
431 break;
432 default: ShouldNotReachHere();
433 }
434
435 __ str(Rexception_obj, Address(Rthread, JavaThread::exception_oop_offset()));
436 __ str(Rexception_pc, Address(Rthread, JavaThread::exception_pc_offset()));
437
438 __ str(Rexception_pc, Address(SP, LR_offset * wordSize)); // patch throwing pc into return address
439
440 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
441 oop_maps->add_gc_map(call_offset, oop_map);
442
443 // Exception handler found
444 __ str(R0, Address(SP, LR_offset * wordSize)); // patch the return address
445
446 // Restore the registers that were saved at the beginning, remove
447 // frame and jump to the exception handler.
448 switch (id) {
449 case forward_exception_id:
450 case handle_exception_nofpu_id:
451 case handle_exception_id:
452 restore_live_registers(sasm, save_fpu_registers);
453 // Note: the restore live registers includes the jump to LR (patched to R0)
454 break;
455 case handle_exception_from_callee_id:
456 restore_live_registers_without_return(sasm); // must not jump immediatly to handler
457 restore_sp_for_method_handle(sasm);
458 __ ret();
459 break;
460 default: ShouldNotReachHere();
461 }
462
463 DEBUG_ONLY(STOP("generate_handle_exception");) // Should not reach here
464
465 return oop_maps;
466 }
467
468
469 void Runtime1::generate_unwind_exception(StubAssembler* sasm) {
470 // FP no longer used to find the frame start
471 // on entry, remove_frame() has already been called (restoring FP and LR)
472
473 // search the exception handler address of the caller (using the return address)
474 __ mov(c_rarg0, Rthread);
475 __ mov(Rexception_pc, LR);
476 __ mov(c_rarg1, LR);
477 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), c_rarg0, c_rarg1);
478
479 // Exception oop should be still in Rexception_obj and pc in Rexception_pc
480 // Jump to handler
481 __ verify_not_null_oop(Rexception_obj);
482
483 // JSR292 extension
484 restore_sp_for_method_handle(sasm);
485
486 __ jump(R0);
487 }
488
489
490 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
491 OopMap* oop_map = save_live_registers(sasm);
492
493 // call the runtime patching routine, returns non-zero if nmethod got deopted.
494 int call_offset = __ call_RT(noreg, noreg, target);
495 OopMapSet* oop_maps = new OopMapSet();
496 oop_maps->add_gc_map(call_offset, oop_map);
497
498 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
499 assert(deopt_blob != NULL, "deoptimization blob must have been created");
500
501 __ cmp_32(R0, 0);
502
503 #ifdef AARCH64
504 Label call_deopt;
505
506 restore_live_registers_without_return(sasm);
507 __ b(call_deopt, ne);
508 __ ret();
509
510 __ bind(call_deopt);
511 #else
512 restore_live_registers_except_FP_LR(sasm);
513 __ pop(RegisterSet(FP) | RegisterSet(PC), eq);
514
515 // Deoptimization needed
516 // TODO: ARM - no need to restore FP & LR because unpack_with_reexecution() stores them back
517 __ pop(RegisterSet(FP) | RegisterSet(LR));
518 #endif // AARCH64
519
520 __ jump(deopt_blob->unpack_with_reexecution(), relocInfo::runtime_call_type, Rtemp);
521
522 DEBUG_ONLY(STOP("generate_patching");) // Should not reach here
523 return oop_maps;
524 }
525
526
527 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
528 const bool must_gc_arguments = true;
529 const bool dont_gc_arguments = false;
530
531 OopMapSet* oop_maps = NULL;
532 bool save_fpu_registers = HaveVFP;
533
534 switch (id) {
535 case forward_exception_id:
536 {
537 oop_maps = generate_handle_exception(id, sasm);
538 // does not return on ARM
539 }
540 break;
541
542 case new_instance_id:
543 case fast_new_instance_id:
544 case fast_new_instance_init_check_id:
545 {
546 const Register result = R0;
547 const Register klass = R1;
548
549 if (UseTLAB && Universe::heap()->supports_inline_contig_alloc() && id != new_instance_id) {
550 // We come here when TLAB allocation failed.
551 // In this case we try to allocate directly from eden.
552 Label slow_case, slow_case_no_pop;
553
554 // Make sure the class is fully initialized
555 if (id == fast_new_instance_init_check_id) {
556 __ ldrb(result, Address(klass, InstanceKlass::init_state_offset()));
557 __ cmp(result, InstanceKlass::fully_initialized);
558 __ b(slow_case_no_pop, ne);
559 }
560
561 // Free some temporary registers
562 const Register obj_size = R4;
563 const Register tmp1 = R5;
564 const Register tmp2 = LR;
565 const Register obj_end = Rtemp;
566
567 __ raw_push(R4, R5, LR);
568
569 __ ldr_u32(obj_size, Address(klass, Klass::layout_helper_offset()));
570 __ eden_allocate(result, obj_end, tmp1, tmp2, obj_size, slow_case); // initializes result and obj_end
571 __ incr_allocated_bytes(obj_size, tmp2);
572 __ initialize_object(result, obj_end, klass, noreg /* len */, tmp1, tmp2,
573 instanceOopDesc::header_size() * HeapWordSize, -1,
574 /* is_tlab_allocated */ false);
575 __ raw_pop_and_ret(R4, R5);
576
577 __ bind(slow_case);
578 __ raw_pop(R4, R5, LR);
579
580 __ bind(slow_case_no_pop);
581 }
582
583 OopMap* map = save_live_registers(sasm);
584 int call_offset = __ call_RT(result, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
585 oop_maps = new OopMapSet();
586 oop_maps->add_gc_map(call_offset, map);
587
588 // MacroAssembler::StoreStore useless (included in the runtime exit path)
589
590 restore_live_registers_except_R0(sasm);
591 }
592 break;
593
594 case counter_overflow_id:
595 {
596 OopMap* oop_map = save_live_registers(sasm);
597 __ ldr(R1, Address(SP, arg1_offset));
598 __ ldr(R2, Address(SP, arg2_offset));
599 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), R1, R2);
600 oop_maps = new OopMapSet();
601 oop_maps->add_gc_map(call_offset, oop_map);
602 restore_live_registers(sasm);
603 }
604 break;
605
606 case new_type_array_id:
607 case new_object_array_id:
608 {
609 if (id == new_type_array_id) {
610 __ set_info("new_type_array", dont_gc_arguments);
611 } else {
612 __ set_info("new_object_array", dont_gc_arguments);
613 }
614
615 const Register result = R0;
616 const Register klass = R1;
617 const Register length = R2;
618
619 if (UseTLAB && Universe::heap()->supports_inline_contig_alloc()) {
620 // We come here when TLAB allocation failed.
621 // In this case we try to allocate directly from eden.
622 Label slow_case, slow_case_no_pop;
623
624 #ifdef AARCH64
625 __ mov_slow(Rtemp, C1_MacroAssembler::max_array_allocation_length);
626 __ cmp_32(length, Rtemp);
627 #else
628 __ cmp_32(length, C1_MacroAssembler::max_array_allocation_length);
629 #endif // AARCH64
630 __ b(slow_case_no_pop, hs);
631
632 // Free some temporary registers
633 const Register arr_size = R4;
634 const Register tmp1 = R5;
635 const Register tmp2 = LR;
636 const Register tmp3 = Rtemp;
637 const Register obj_end = tmp3;
638
639 __ raw_push(R4, R5, LR);
640
641 // Get the allocation size: round_up((length << (layout_helper & 0xff)) + header_size)
642 __ ldr_u32(tmp1, Address(klass, Klass::layout_helper_offset()));
643 __ mov(arr_size, MinObjAlignmentInBytesMask);
644 __ and_32(tmp2, tmp1, (unsigned int)(Klass::_lh_header_size_mask << Klass::_lh_header_size_shift));
645
646 #ifdef AARCH64
647 __ lslv_w(tmp3, length, tmp1);
648 __ add(arr_size, arr_size, tmp3);
649 #else
650 __ add(arr_size, arr_size, AsmOperand(length, lsl, tmp1));
651 #endif // AARCH64
652
653 __ add(arr_size, arr_size, AsmOperand(tmp2, lsr, Klass::_lh_header_size_shift));
654 __ align_reg(arr_size, arr_size, MinObjAlignmentInBytes);
655
656 // eden_allocate destroys tmp2, so reload header_size after allocation
657 // eden_allocate initializes result and obj_end
658 __ eden_allocate(result, obj_end, tmp1, tmp2, arr_size, slow_case);
659 __ incr_allocated_bytes(arr_size, tmp2);
660 __ ldrb(tmp2, Address(klass, in_bytes(Klass::layout_helper_offset()) +
661 Klass::_lh_header_size_shift / BitsPerByte));
662 __ initialize_object(result, obj_end, klass, length, tmp1, tmp2, tmp2, -1, /* is_tlab_allocated */ false);
663 __ raw_pop_and_ret(R4, R5);
664
665 __ bind(slow_case);
666 __ raw_pop(R4, R5, LR);
667 __ bind(slow_case_no_pop);
668 }
669
670 OopMap* map = save_live_registers(sasm);
671 int call_offset;
672 if (id == new_type_array_id) {
673 call_offset = __ call_RT(result, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
674 } else {
675 call_offset = __ call_RT(result, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
676 }
677 oop_maps = new OopMapSet();
678 oop_maps->add_gc_map(call_offset, map);
679
680 // MacroAssembler::StoreStore useless (included in the runtime exit path)
681
682 restore_live_registers_except_R0(sasm);
683 }
684 break;
685
686 case new_multi_array_id:
687 {
688 __ set_info("new_multi_array", dont_gc_arguments);
689
690 // R0: klass
691 // R2: rank
692 // SP: address of 1st dimension
693 const Register result = R0;
694 OopMap* map = save_live_registers(sasm);
695
696 __ mov(R1, R0);
697 __ add(R3, SP, arg1_offset);
698 int call_offset = __ call_RT(result, noreg, CAST_FROM_FN_PTR(address, new_multi_array), R1, R2, R3);
699
700 oop_maps = new OopMapSet();
701 oop_maps->add_gc_map(call_offset, map);
702
703 // MacroAssembler::StoreStore useless (included in the runtime exit path)
704
705 restore_live_registers_except_R0(sasm);
706 }
707 break;
708
709 case register_finalizer_id:
710 {
711 __ set_info("register_finalizer", dont_gc_arguments);
712
713 // Do not call runtime if JVM_ACC_HAS_FINALIZER flag is not set
714 __ load_klass(Rtemp, R0);
715 __ ldr_u32(Rtemp, Address(Rtemp, Klass::access_flags_offset()));
716
717 #ifdef AARCH64
718 Label L;
719 __ tbnz(Rtemp, exact_log2(JVM_ACC_HAS_FINALIZER), L);
720 __ ret();
721 __ bind(L);
722 #else
723 __ tst(Rtemp, JVM_ACC_HAS_FINALIZER);
724 __ bx(LR, eq);
725 #endif // AARCH64
726
727 // Call VM
728 OopMap* map = save_live_registers(sasm);
729 oop_maps = new OopMapSet();
730 int call_offset = __ call_RT(noreg, noreg,
731 CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), R0);
732 oop_maps->add_gc_map(call_offset, map);
733 restore_live_registers(sasm);
734 }
735 break;
736
737 case throw_range_check_failed_id:
738 {
739 __ set_info("range_check_failed", dont_gc_arguments);
740 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
741 }
742 break;
743
744 case throw_index_exception_id:
745 {
746 __ set_info("index_range_check_failed", dont_gc_arguments);
747 #ifdef AARCH64
748 __ NOT_TESTED();
749 #endif
750 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
751 }
752 break;
753
754 case throw_div0_exception_id:
755 {
756 __ set_info("throw_div0_exception", dont_gc_arguments);
757 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
758 }
759 break;
760
761 case throw_null_pointer_exception_id:
762 {
763 __ set_info("throw_null_pointer_exception", dont_gc_arguments);
764 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
765 }
766 break;
767
768 case handle_exception_nofpu_id:
769 case handle_exception_id:
770 {
771 __ set_info("handle_exception", dont_gc_arguments);
772 oop_maps = generate_handle_exception(id, sasm);
773 }
774 break;
775
776 case handle_exception_from_callee_id:
777 {
778 __ set_info("handle_exception_from_callee", dont_gc_arguments);
779 oop_maps = generate_handle_exception(id, sasm);
780 }
781 break;
782
783 case unwind_exception_id:
784 {
785 __ set_info("unwind_exception", dont_gc_arguments);
786 generate_unwind_exception(sasm);
787 }
788 break;
789
790 case throw_array_store_exception_id:
791 {
792 __ set_info("throw_array_store_exception", dont_gc_arguments);
793 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
794 }
795 break;
796
797 case throw_class_cast_exception_id:
798 {
799 __ set_info("throw_class_cast_exception", dont_gc_arguments);
800 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
801 }
802 break;
803
804 case throw_incompatible_class_change_error_id:
805 {
806 __ set_info("throw_incompatible_class_cast_exception", dont_gc_arguments);
807 #ifdef AARCH64
808 __ NOT_TESTED();
809 #endif
810 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
811 }
812 break;
813
814 case slow_subtype_check_id:
815 {
816 // (in) R0 - sub, destroyed,
817 // (in) R1 - super, not changed
818 // (out) R0 - result: 1 if check passed, 0 otherwise
819 __ raw_push(R2, R3, LR);
820
821 // Load an array of secondary_supers
822 __ ldr(R2, Address(R0, Klass::secondary_supers_offset()));
823 // Length goes to R3
824 __ ldr_s32(R3, Address(R2, Array<Klass*>::length_offset_in_bytes()));
825 __ add(R2, R2, Array<Klass*>::base_offset_in_bytes());
826
827 Label loop, miss;
828 __ bind(loop);
829 __ cbz(R3, miss);
830 __ ldr(LR, Address(R2, wordSize, post_indexed));
831 __ sub(R3, R3, 1);
832 __ cmp(LR, R1);
833 __ b(loop, ne);
834
835 // We get here if an equal cache entry is found
836 __ str(R1, Address(R0, Klass::secondary_super_cache_offset()));
837 __ mov(R0, 1);
838 __ raw_pop_and_ret(R2, R3);
839
840 // A cache entry not found - return false
841 __ bind(miss);
842 __ mov(R0, 0);
843 __ raw_pop_and_ret(R2, R3);
844 }
845 break;
846
847 case monitorenter_nofpu_id:
848 save_fpu_registers = false;
849 // fall through
850 case monitorenter_id:
851 {
852 __ set_info("monitorenter", dont_gc_arguments);
853 const Register obj = R1;
854 const Register lock = R2;
855 OopMap* map = save_live_registers(sasm, save_fpu_registers);
856 __ ldr(obj, Address(SP, arg1_offset));
857 __ ldr(lock, Address(SP, arg2_offset));
858 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), obj, lock);
859 oop_maps = new OopMapSet();
860 oop_maps->add_gc_map(call_offset, map);
861 restore_live_registers(sasm, save_fpu_registers);
862 }
863 break;
864
865 case monitorexit_nofpu_id:
866 save_fpu_registers = false;
867 // fall through
868 case monitorexit_id:
869 {
870 __ set_info("monitorexit", dont_gc_arguments);
871 const Register lock = R1;
872 OopMap* map = save_live_registers(sasm, save_fpu_registers);
873 __ ldr(lock, Address(SP, arg1_offset));
874 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), lock);
875 oop_maps = new OopMapSet();
876 oop_maps->add_gc_map(call_offset, map);
877 restore_live_registers(sasm, save_fpu_registers);
878 }
879 break;
880
881 case deoptimize_id:
882 {
883 __ set_info("deoptimize", dont_gc_arguments);
884 OopMap* oop_map = save_live_registers(sasm);
885 const Register trap_request = R1;
886 __ ldr(trap_request, Address(SP, arg1_offset));
887 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), trap_request);
888 oop_maps = new OopMapSet();
889 oop_maps->add_gc_map(call_offset, oop_map);
890 restore_live_registers_without_return(sasm);
891 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
892 assert(deopt_blob != NULL, "deoptimization blob must have been created");
893 __ jump(deopt_blob->unpack_with_reexecution(), relocInfo::runtime_call_type, AARCH64_ONLY(Rtemp) NOT_AARCH64(noreg));
894 }
895 break;
896
897 case access_field_patching_id:
898 {
899 __ set_info("access_field_patching", dont_gc_arguments);
900 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
901 }
902 break;
903
904 case load_klass_patching_id:
905 {
906 __ set_info("load_klass_patching", dont_gc_arguments);
907 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
908 }
909 break;
910
911 case load_appendix_patching_id:
912 {
913 __ set_info("load_appendix_patching", dont_gc_arguments);
914 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
915 }
916 break;
917
918 case load_mirror_patching_id:
919 {
920 __ set_info("load_mirror_patching", dont_gc_arguments);
921 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
922 }
923 break;
924
925 case predicate_failed_trap_id:
926 {
927 __ set_info("predicate_failed_trap", dont_gc_arguments);
928
929 OopMap* oop_map = save_live_registers(sasm);
930 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
931
932 oop_maps = new OopMapSet();
933 oop_maps->add_gc_map(call_offset, oop_map);
934
935 restore_live_registers_without_return(sasm);
936
937 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
938 assert(deopt_blob != NULL, "deoptimization blob must have been created");
939 __ jump(deopt_blob->unpack_with_reexecution(), relocInfo::runtime_call_type, Rtemp);
940 }
941 break;
942
943 default:
944 {
945 __ set_info("unimplemented entry", dont_gc_arguments);
946 STOP("unimplemented entry");
947 }
948 break;
949 }
950 return oop_maps;
951 }
952
953 #undef __
954
955 #ifdef __SOFTFP__
956 const char *Runtime1::pd_name_for_address(address entry) {
957
958 #define FUNCTION_CASE(a, f) \
959 if ((intptr_t)a == CAST_FROM_FN_PTR(intptr_t, f)) return #f
960
961 FUNCTION_CASE(entry, __aeabi_fadd_glibc);
962 FUNCTION_CASE(entry, __aeabi_fmul);
963 FUNCTION_CASE(entry, __aeabi_fsub_glibc);
964 FUNCTION_CASE(entry, __aeabi_fdiv);
965
966 // __aeabi_XXXX_glibc: Imported code from glibc soft-fp bundle for calculation accuracy improvement. See CR 6757269.
967 FUNCTION_CASE(entry, __aeabi_dadd_glibc);
968 FUNCTION_CASE(entry, __aeabi_dmul);
969 FUNCTION_CASE(entry, __aeabi_dsub_glibc);
970 FUNCTION_CASE(entry, __aeabi_ddiv);
971
972 FUNCTION_CASE(entry, __aeabi_f2d);
973 FUNCTION_CASE(entry, __aeabi_d2f);
974 FUNCTION_CASE(entry, __aeabi_i2f);
975 FUNCTION_CASE(entry, __aeabi_i2d);
976 FUNCTION_CASE(entry, __aeabi_f2iz);
977
978 FUNCTION_CASE(entry, SharedRuntime::fcmpl);
979 FUNCTION_CASE(entry, SharedRuntime::fcmpg);
980 FUNCTION_CASE(entry, SharedRuntime::dcmpl);
981 FUNCTION_CASE(entry, SharedRuntime::dcmpg);
982
983 FUNCTION_CASE(entry, SharedRuntime::unordered_fcmplt);
984 FUNCTION_CASE(entry, SharedRuntime::unordered_dcmplt);
985 FUNCTION_CASE(entry, SharedRuntime::unordered_fcmple);
986 FUNCTION_CASE(entry, SharedRuntime::unordered_dcmple);
987 FUNCTION_CASE(entry, SharedRuntime::unordered_fcmpge);
988 FUNCTION_CASE(entry, SharedRuntime::unordered_dcmpge);
989 FUNCTION_CASE(entry, SharedRuntime::unordered_fcmpgt);
990 FUNCTION_CASE(entry, SharedRuntime::unordered_dcmpgt);
991
992 FUNCTION_CASE(entry, SharedRuntime::fneg);
993 FUNCTION_CASE(entry, SharedRuntime::dneg);
994
995 FUNCTION_CASE(entry, __aeabi_fcmpeq);
996 FUNCTION_CASE(entry, __aeabi_fcmplt);
997 FUNCTION_CASE(entry, __aeabi_fcmple);
998 FUNCTION_CASE(entry, __aeabi_fcmpge);
999 FUNCTION_CASE(entry, __aeabi_fcmpgt);
1000
1001 FUNCTION_CASE(entry, __aeabi_dcmpeq);
1002 FUNCTION_CASE(entry, __aeabi_dcmplt);
1003 FUNCTION_CASE(entry, __aeabi_dcmple);
1004 FUNCTION_CASE(entry, __aeabi_dcmpge);
1005 FUNCTION_CASE(entry, __aeabi_dcmpgt);
1006 #undef FUNCTION_CASE
1007 return "";
1008 }
1009 #else // __SOFTFP__
1010 const char *Runtime1::pd_name_for_address(address entry) {
1011 return "<unknown function>";
1012 }
1013 #endif // __SOFTFP__