1 /*
2 * Copyright (c) 2008, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_Compilation.hpp"
28 #include "c1/c1_LIRAssembler.hpp"
29 #include "c1/c1_MacroAssembler.hpp"
30 #include "c1/c1_Runtime1.hpp"
31 #include "c1/c1_ValueStack.hpp"
32 #include "ci/ciArrayKlass.hpp"
33 #include "ci/ciInstance.hpp"
34 #include "gc/shared/barrierSet.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "nativeInst_arm.hpp"
38 #include "oops/objArrayKlass.hpp"
39 #include "runtime/frame.inline.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #include "vmreg_arm.inline.hpp"
42
43 #define __ _masm->
44
45 // Note: Rtemp usage is this file should not impact C2 and should be
46 // correct as long as it is not implicitly used in lower layers (the
47 // arm [macro]assembler) and used with care in the other C1 specific
48 // files.
49
50 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
51 ShouldNotCallThis(); // Not used on ARM
52 return false;
53 }
54
55
56 LIR_Opr LIR_Assembler::receiverOpr() {
57 // The first register in Java calling conventions
58 return FrameMap::R0_oop_opr;
59 }
60
61 LIR_Opr LIR_Assembler::osrBufferPointer() {
62 return FrameMap::as_pointer_opr(R0);
63 }
64
65 #ifndef PRODUCT
66 void LIR_Assembler::verify_reserved_argument_area_size(int args_count) {
67 assert(args_count * wordSize <= frame_map()->reserved_argument_area_size(), "not enough space for arguments");
68 }
69 #endif // !PRODUCT
70
71 void LIR_Assembler::store_parameter(jint c, int offset_from_sp_in_words) {
72 assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
73 int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
74 assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "not enough space");
75 __ mov_slow(Rtemp, c);
76 __ str(Rtemp, Address(SP, offset_from_sp_in_bytes));
77 }
78
79 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_sp_in_words) {
80 assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
81 int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
82 assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "not enough space");
83 __ mov_metadata(Rtemp, m);
84 __ str(Rtemp, Address(SP, offset_from_sp_in_bytes));
85 }
86
87 //--------------fpu register translations-----------------------
88
89
90 void LIR_Assembler::set_24bit_FPU() {
91 ShouldNotReachHere();
92 }
93
94 void LIR_Assembler::reset_FPU() {
95 ShouldNotReachHere();
96 }
97
98 void LIR_Assembler::fpop() {
99 Unimplemented();
100 }
101
102 void LIR_Assembler::fxch(int i) {
103 Unimplemented();
104 }
105
106 void LIR_Assembler::fld(int i) {
107 Unimplemented();
108 }
109
110 void LIR_Assembler::ffree(int i) {
111 Unimplemented();
112 }
113
114 void LIR_Assembler::breakpoint() {
115 __ breakpoint();
116 }
117
118 void LIR_Assembler::push(LIR_Opr opr) {
119 Unimplemented();
120 }
121
122 void LIR_Assembler::pop(LIR_Opr opr) {
123 Unimplemented();
124 }
125
126 //-------------------------------------------
127 Address LIR_Assembler::as_Address(LIR_Address* addr) {
128 Register base = addr->base()->as_pointer_register();
129
130 #ifdef AARCH64
131 int align = exact_log2(type2aelembytes(addr->type(), true));
132 #endif
133
134 if (addr->index()->is_illegal() || addr->index()->is_constant()) {
135 int offset = addr->disp();
136 if (addr->index()->is_constant()) {
137 offset += addr->index()->as_constant_ptr()->as_jint() << addr->scale();
138 }
139
140 #ifdef AARCH64
141 if (!Assembler::is_unsigned_imm_in_range(offset, 12, align) && !Assembler::is_imm_in_range(offset, 9, 0)) {
142 BAILOUT_("offset not in range", Address(base));
143 }
144 assert(UseUnalignedAccesses || (offset & right_n_bits(align)) == 0, "offset should be aligned");
145 #else
146 if ((offset <= -4096) || (offset >= 4096)) {
147 BAILOUT_("offset not in range", Address(base));
148 }
149 #endif // AARCH64
150
151 return Address(base, offset);
152
153 } else {
154 assert(addr->disp() == 0, "can't have both");
155 int scale = addr->scale();
156
157 #ifdef AARCH64
158 assert((scale == 0) || (scale == align), "scale should be zero or equal to embedded shift");
159
160 bool is_index_extended = (addr->index()->type() == T_INT);
161 if (is_index_extended) {
162 assert(addr->index()->is_single_cpu(), "should be");
163 return Address(base, addr->index()->as_register(), ex_sxtw, scale);
164 } else {
165 assert(addr->index()->is_double_cpu(), "should be");
166 return Address(base, addr->index()->as_register_lo(), ex_lsl, scale);
167 }
168 #else
169 assert(addr->index()->is_single_cpu(), "should be");
170 return scale >= 0 ? Address(base, addr->index()->as_register(), lsl, scale) :
171 Address(base, addr->index()->as_register(), lsr, -scale);
172 #endif // AARCH64
173 }
174 }
175
176 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
177 #ifdef AARCH64
178 ShouldNotCallThis(); // Not used on AArch64
179 return Address();
180 #else
181 Address base = as_Address(addr);
182 assert(base.index() == noreg, "must be");
183 if (base.disp() + BytesPerWord >= 4096) { BAILOUT_("offset not in range", Address(base.base(),0)); }
184 return Address(base.base(), base.disp() + BytesPerWord);
185 #endif // AARCH64
186 }
187
188 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
189 #ifdef AARCH64
190 ShouldNotCallThis(); // Not used on AArch64
191 return Address();
192 #else
193 return as_Address(addr);
194 #endif // AARCH64
195 }
196
197
198 void LIR_Assembler::osr_entry() {
199 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
200 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
201 ValueStack* entry_state = osr_entry->end()->state();
202 int number_of_locks = entry_state->locks_size();
203
204 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
205 Register OSR_buf = osrBufferPointer()->as_pointer_register();
206
207 assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
208 int monitor_offset = (method()->max_locals() + 2 * (number_of_locks - 1)) * BytesPerWord;
209 for (int i = 0; i < number_of_locks; i++) {
210 int slot_offset = monitor_offset - (i * 2 * BytesPerWord);
211 __ ldr(R1, Address(OSR_buf, slot_offset + 0*BytesPerWord));
212 __ ldr(R2, Address(OSR_buf, slot_offset + 1*BytesPerWord));
213 __ str(R1, frame_map()->address_for_monitor_lock(i));
214 __ str(R2, frame_map()->address_for_monitor_object(i));
215 }
216 }
217
218
219 int LIR_Assembler::check_icache() {
220 Register receiver = LIR_Assembler::receiverOpr()->as_register();
221 int offset = __ offset();
222 __ inline_cache_check(receiver, Ricklass);
223 return offset;
224 }
225
226
227 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
228 jobject o = (jobject)Universe::non_oop_word();
229 int index = __ oop_recorder()->allocate_oop_index(o);
230
231 PatchingStub* patch = new PatchingStub(_masm, patching_id(info), index);
232
233 __ patchable_mov_oop(reg, o, index);
234 patching_epilog(patch, lir_patch_normal, reg, info);
235 }
236
237
238 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
239 Metadata* o = (Metadata*)Universe::non_oop_word();
240 int index = __ oop_recorder()->allocate_metadata_index(o);
241 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index);
242
243 __ patchable_mov_metadata(reg, o, index);
244 patching_epilog(patch, lir_patch_normal, reg, info);
245 }
246
247
248 int LIR_Assembler::initial_frame_size_in_bytes() const {
249 // Subtracts two words to account for return address and link
250 return frame_map()->framesize()*VMRegImpl::stack_slot_size - 2*wordSize;
251 }
252
253
254 int LIR_Assembler::emit_exception_handler() {
255 // TODO: ARM
256 __ nop(); // See comments in other ports
257
258 address handler_base = __ start_a_stub(exception_handler_size());
259 if (handler_base == NULL) {
260 bailout("exception handler overflow");
261 return -1;
262 }
263
264 int offset = code_offset();
265
266 // check that there is really an exception
267 __ verify_not_null_oop(Rexception_obj);
268
269 __ call(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id), relocInfo::runtime_call_type);
270 __ should_not_reach_here();
271
272 assert(code_offset() - offset <= exception_handler_size(), "overflow");
273 __ end_a_stub();
274
275 return offset;
276 }
277
278 // Emit the code to remove the frame from the stack in the exception
279 // unwind path.
280 int LIR_Assembler::emit_unwind_handler() {
281 #ifndef PRODUCT
282 if (CommentedAssembly) {
283 _masm->block_comment("Unwind handler");
284 }
285 #endif
286
287 int offset = code_offset();
288
289 // Fetch the exception from TLS and clear out exception related thread state
290 Register zero = __ zero_register(Rtemp);
291 __ ldr(Rexception_obj, Address(Rthread, JavaThread::exception_oop_offset()));
292 __ str(zero, Address(Rthread, JavaThread::exception_oop_offset()));
293 __ str(zero, Address(Rthread, JavaThread::exception_pc_offset()));
294
295 __ bind(_unwind_handler_entry);
296 __ verify_not_null_oop(Rexception_obj);
297
298 // Preform needed unlocking
299 MonitorExitStub* stub = NULL;
300 if (method()->is_synchronized()) {
301 monitor_address(0, FrameMap::R0_opr);
302 stub = new MonitorExitStub(FrameMap::R0_opr, true, 0);
303 __ unlock_object(R2, R1, R0, Rtemp, *stub->entry());
304 __ bind(*stub->continuation());
305 }
306
307 // remove the activation and dispatch to the unwind handler
308 __ remove_frame(initial_frame_size_in_bytes()); // restores FP and LR
309 __ jump(Runtime1::entry_for(Runtime1::unwind_exception_id), relocInfo::runtime_call_type, Rtemp);
310
311 // Emit the slow path assembly
312 if (stub != NULL) {
313 stub->emit_code(this);
314 }
315
316 return offset;
317 }
318
319
320 int LIR_Assembler::emit_deopt_handler() {
321 address handler_base = __ start_a_stub(deopt_handler_size());
322 if (handler_base == NULL) {
323 bailout("deopt handler overflow");
324 return -1;
325 }
326
327 int offset = code_offset();
328
329 __ mov_relative_address(LR, __ pc());
330 #ifdef AARCH64
331 __ raw_push(LR, LR);
332 __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, Rtemp);
333 #else
334 __ push(LR); // stub expects LR to be saved
335 __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg);
336 #endif // AARCH64
337
338 assert(code_offset() - offset <= deopt_handler_size(), "overflow");
339 __ end_a_stub();
340
341 return offset;
342 }
343
344
345 void LIR_Assembler::return_op(LIR_Opr result) {
346 // Pop the frame before safepoint polling
347 __ remove_frame(initial_frame_size_in_bytes());
348
349 // mov_slow here is usually one or two instruction
350 // TODO-AARCH64 3 instructions on AArch64, so try to load polling page by ldr_literal
351 __ mov_address(Rtemp, os::get_polling_page(), symbolic_Relocation::polling_page_reference);
352 __ relocate(relocInfo::poll_return_type);
353 __ ldr(Rtemp, Address(Rtemp));
354 __ ret();
355 }
356
357
358 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
359 __ mov_address(Rtemp, os::get_polling_page(), symbolic_Relocation::polling_page_reference);
360 if (info != NULL) {
361 add_debug_info_for_branch(info);
362 }
363 int offset = __ offset();
364 __ relocate(relocInfo::poll_type);
365 __ ldr(Rtemp, Address(Rtemp));
366 return offset;
367 }
368
369
370 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
371 if (from_reg != to_reg) {
372 __ mov(to_reg, from_reg);
373 }
374 }
375
376 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
377 assert(src->is_constant() && dest->is_register(), "must be");
378 LIR_Const* c = src->as_constant_ptr();
379
380 switch (c->type()) {
381 case T_ADDRESS:
382 case T_INT:
383 assert(patch_code == lir_patch_none, "no patching handled here");
384 __ mov_slow(dest->as_register(), c->as_jint());
385 break;
386
387 case T_LONG:
388 assert(patch_code == lir_patch_none, "no patching handled here");
389 #ifdef AARCH64
390 __ mov_slow(dest->as_pointer_register(), (intptr_t)c->as_jlong());
391 #else
392 __ mov_slow(dest->as_register_lo(), c->as_jint_lo());
393 __ mov_slow(dest->as_register_hi(), c->as_jint_hi());
394 #endif // AARCH64
395 break;
396
397 case T_OBJECT:
398 if (patch_code == lir_patch_none) {
399 __ mov_oop(dest->as_register(), c->as_jobject());
400 } else {
401 jobject2reg_with_patching(dest->as_register(), info);
402 }
403 break;
404
405 case T_METADATA:
406 if (patch_code == lir_patch_none) {
407 __ mov_metadata(dest->as_register(), c->as_metadata());
408 } else {
409 klass2reg_with_patching(dest->as_register(), info);
410 }
411 break;
412
413 case T_FLOAT:
414 if (dest->is_single_fpu()) {
415 __ mov_float(dest->as_float_reg(), c->as_jfloat());
416 } else {
417 #ifdef AARCH64
418 ShouldNotReachHere();
419 #else
420 // Simple getters can return float constant directly into r0
421 __ mov_slow(dest->as_register(), c->as_jint_bits());
422 #endif // AARCH64
423 }
424 break;
425
426 case T_DOUBLE:
427 if (dest->is_double_fpu()) {
428 __ mov_double(dest->as_double_reg(), c->as_jdouble());
429 } else {
430 #ifdef AARCH64
431 ShouldNotReachHere();
432 #else
433 // Simple getters can return double constant directly into r1r0
434 __ mov_slow(dest->as_register_lo(), c->as_jint_lo_bits());
435 __ mov_slow(dest->as_register_hi(), c->as_jint_hi_bits());
436 #endif // AARCH64
437 }
438 break;
439
440 default:
441 ShouldNotReachHere();
442 }
443 }
444
445 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
446 assert(src->is_constant(), "must be");
447 assert(dest->is_stack(), "must be");
448 LIR_Const* c = src->as_constant_ptr();
449
450 switch (c->type()) {
451 case T_INT: // fall through
452 case T_FLOAT:
453 __ mov_slow(Rtemp, c->as_jint_bits());
454 __ str_32(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
455 break;
456
457 case T_ADDRESS:
458 __ mov_slow(Rtemp, c->as_jint());
459 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
460 break;
461
462 case T_OBJECT:
463 __ mov_oop(Rtemp, c->as_jobject());
464 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
465 break;
466
467 case T_LONG: // fall through
468 case T_DOUBLE:
469 #ifdef AARCH64
470 __ mov_slow(Rtemp, c->as_jlong_bits());
471 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix()));
472 #else
473 __ mov_slow(Rtemp, c->as_jint_lo_bits());
474 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes));
475 if (c->as_jint_hi_bits() != c->as_jint_lo_bits()) {
476 __ mov_slow(Rtemp, c->as_jint_hi_bits());
477 }
478 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes));
479 #endif // AARCH64
480 break;
481
482 default:
483 ShouldNotReachHere();
484 }
485 }
486
487 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type,
488 CodeEmitInfo* info, bool wide) {
489 #ifdef AARCH64
490 assert((src->as_constant_ptr()->type() == T_OBJECT && src->as_constant_ptr()->as_jobject() == NULL) ||
491 (src->as_constant_ptr()->type() == T_INT && src->as_constant_ptr()->as_jint() == 0) ||
492 (src->as_constant_ptr()->type() == T_LONG && src->as_constant_ptr()->as_jlong() == 0) ||
493 (src->as_constant_ptr()->type() == T_FLOAT && src->as_constant_ptr()->as_jint_bits() == 0) ||
494 (src->as_constant_ptr()->type() == T_DOUBLE && src->as_constant_ptr()->as_jlong_bits() == 0),
495 "cannot handle otherwise");
496 assert(dest->as_address_ptr()->type() == type, "should be");
497
498 Address addr = as_Address(dest->as_address_ptr());
499 int null_check_offset = code_offset();
500 switch (type) {
501 case T_OBJECT: // fall through
502 case T_ARRAY:
503 if (UseCompressedOops && !wide) {
504 __ str_w(ZR, addr);
505 } else {
506 __ str(ZR, addr);
507 }
508 break;
509 case T_ADDRESS: // fall through
510 case T_DOUBLE: // fall through
511 case T_LONG: __ str(ZR, addr); break;
512 case T_FLOAT: // fall through
513 case T_INT: __ str_w(ZR, addr); break;
514 case T_BOOLEAN: // fall through
515 case T_BYTE: __ strb(ZR, addr); break;
516 case T_CHAR: // fall through
517 case T_SHORT: __ strh(ZR, addr); break;
518 default: ShouldNotReachHere();
519 }
520 #else
521 assert((src->as_constant_ptr()->type() == T_OBJECT && src->as_constant_ptr()->as_jobject() == NULL),"cannot handle otherwise");
522 __ mov(Rtemp, 0);
523
524 int null_check_offset = code_offset();
525 __ str(Rtemp, as_Address(dest->as_address_ptr()));
526 #endif // AARCH64
527
528 if (info != NULL) {
529 #ifndef AARCH64
530 assert(false, "arm32 didn't support this before, investigate if bug");
531 #endif
532 add_debug_info_for_null_check(null_check_offset, info);
533 }
534 }
535
536 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
537 assert(src->is_register() && dest->is_register(), "must be");
538
539 if (src->is_single_cpu()) {
540 if (dest->is_single_cpu()) {
541 move_regs(src->as_register(), dest->as_register());
542 #ifdef AARCH64
543 } else if (dest->is_double_cpu()) {
544 assert ((src->type() == T_OBJECT) || (src->type() == T_ARRAY) || (src->type() == T_ADDRESS), "invalid src type");
545 move_regs(src->as_register(), dest->as_register_lo());
546 #else
547 } else if (dest->is_single_fpu()) {
548 __ fmsr(dest->as_float_reg(), src->as_register());
549 #endif // AARCH64
550 } else {
551 ShouldNotReachHere();
552 }
553 } else if (src->is_double_cpu()) {
554 #ifdef AARCH64
555 move_regs(src->as_register_lo(), dest->as_register_lo());
556 #else
557 if (dest->is_double_cpu()) {
558 __ long_move(dest->as_register_lo(), dest->as_register_hi(), src->as_register_lo(), src->as_register_hi());
559 } else {
560 __ fmdrr(dest->as_double_reg(), src->as_register_lo(), src->as_register_hi());
561 }
562 #endif // AARCH64
563 } else if (src->is_single_fpu()) {
564 if (dest->is_single_fpu()) {
565 __ mov_float(dest->as_float_reg(), src->as_float_reg());
566 } else if (dest->is_single_cpu()) {
567 __ mov_fpr2gpr_float(dest->as_register(), src->as_float_reg());
568 } else {
569 ShouldNotReachHere();
570 }
571 } else if (src->is_double_fpu()) {
572 if (dest->is_double_fpu()) {
573 __ mov_double(dest->as_double_reg(), src->as_double_reg());
574 } else if (dest->is_double_cpu()) {
575 #ifdef AARCH64
576 __ fmov_xd(dest->as_register_lo(), src->as_double_reg());
577 #else
578 __ fmrrd(dest->as_register_lo(), dest->as_register_hi(), src->as_double_reg());
579 #endif // AARCH64
580 } else {
581 ShouldNotReachHere();
582 }
583 } else {
584 ShouldNotReachHere();
585 }
586 }
587
588 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
589 assert(src->is_register(), "should not call otherwise");
590 assert(dest->is_stack(), "should not call otherwise");
591
592 Address addr = dest->is_single_word() ?
593 frame_map()->address_for_slot(dest->single_stack_ix()) :
594 frame_map()->address_for_slot(dest->double_stack_ix());
595
596 #ifndef AARCH64
597 assert(lo_word_offset_in_bytes == 0 && hi_word_offset_in_bytes == 4, "little ending");
598 if (src->is_single_fpu() || src->is_double_fpu()) {
599 if (addr.disp() >= 1024) { BAILOUT("Too exotic case to handle here"); }
600 }
601 #endif // !AARCH64
602
603 if (src->is_single_cpu()) {
604 switch (type) {
605 case T_OBJECT:
606 case T_ARRAY: __ verify_oop(src->as_register()); // fall through
607 case T_ADDRESS:
608 case T_METADATA: __ str(src->as_register(), addr); break;
609 case T_FLOAT: // used in intBitsToFloat intrinsic implementation, fall through
610 case T_INT: __ str_32(src->as_register(), addr); break;
611 default:
612 ShouldNotReachHere();
613 }
614 } else if (src->is_double_cpu()) {
615 __ str(src->as_register_lo(), addr);
616 #ifndef AARCH64
617 __ str(src->as_register_hi(), frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes));
618 #endif // !AARCH64
619 } else if (src->is_single_fpu()) {
620 __ str_float(src->as_float_reg(), addr);
621 } else if (src->is_double_fpu()) {
622 __ str_double(src->as_double_reg(), addr);
623 } else {
624 ShouldNotReachHere();
625 }
626 }
627
628
629 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type,
630 LIR_PatchCode patch_code, CodeEmitInfo* info,
631 bool pop_fpu_stack, bool wide,
632 bool unaligned) {
633 LIR_Address* to_addr = dest->as_address_ptr();
634 Register base_reg = to_addr->base()->as_pointer_register();
635 const bool needs_patching = (patch_code != lir_patch_none);
636
637 PatchingStub* patch = NULL;
638 if (needs_patching) {
639 #ifdef AARCH64
640 // Same alignment of reg2mem code and PatchingStub code. Required to make copied bind_literal() code properly aligned.
641 __ align(wordSize);
642 #endif
643 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
644 #ifdef AARCH64
645 // Extra nop for MT safe patching
646 __ nop();
647 #endif // AARCH64
648 }
649
650 int null_check_offset = code_offset();
651
652 switch (type) {
653 case T_ARRAY:
654 case T_OBJECT:
655 if (UseCompressedOops && !wide) {
656 #ifdef AARCH64
657 const Register temp_src = Rtemp;
658 assert_different_registers(temp_src, src->as_register());
659 __ encode_heap_oop(temp_src, src->as_register());
660 null_check_offset = code_offset();
661 __ str_32(temp_src, as_Address(to_addr));
662 #else
663 ShouldNotReachHere();
664 #endif // AARCH64
665 } else {
666 __ str(src->as_register(), as_Address(to_addr));
667 }
668 break;
669
670 case T_ADDRESS:
671 #ifdef AARCH64
672 case T_LONG:
673 #endif // AARCH64
674 __ str(src->as_pointer_register(), as_Address(to_addr));
675 break;
676
677 case T_BYTE:
678 case T_BOOLEAN:
679 __ strb(src->as_register(), as_Address(to_addr));
680 break;
681
682 case T_CHAR:
683 case T_SHORT:
684 __ strh(src->as_register(), as_Address(to_addr));
685 break;
686
687 case T_INT:
688 #ifdef __SOFTFP__
689 case T_FLOAT:
690 #endif // __SOFTFP__
691 __ str_32(src->as_register(), as_Address(to_addr));
692 break;
693
694 #ifdef AARCH64
695
696 case T_FLOAT:
697 __ str_s(src->as_float_reg(), as_Address(to_addr));
698 break;
699
700 case T_DOUBLE:
701 __ str_d(src->as_double_reg(), as_Address(to_addr));
702 break;
703
704 #else // AARCH64
705
706 #ifdef __SOFTFP__
707 case T_DOUBLE:
708 #endif // __SOFTFP__
709 case T_LONG: {
710 Register from_lo = src->as_register_lo();
711 Register from_hi = src->as_register_hi();
712 if (to_addr->index()->is_register()) {
713 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
714 assert(to_addr->disp() == 0, "Not yet supporting both");
715 __ add(Rtemp, base_reg, to_addr->index()->as_register());
716 base_reg = Rtemp;
717 __ str(from_lo, Address(Rtemp));
718 if (patch != NULL) {
719 patching_epilog(patch, lir_patch_low, base_reg, info);
720 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
721 patch_code = lir_patch_high;
722 }
723 __ str(from_hi, Address(Rtemp, BytesPerWord));
724 } else if (base_reg == from_lo) {
725 __ str(from_hi, as_Address_hi(to_addr));
726 if (patch != NULL) {
727 patching_epilog(patch, lir_patch_high, base_reg, info);
728 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
729 patch_code = lir_patch_low;
730 }
731 __ str(from_lo, as_Address_lo(to_addr));
732 } else {
733 __ str(from_lo, as_Address_lo(to_addr));
734 if (patch != NULL) {
735 patching_epilog(patch, lir_patch_low, base_reg, info);
736 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
737 patch_code = lir_patch_high;
738 }
739 __ str(from_hi, as_Address_hi(to_addr));
740 }
741 break;
742 }
743
744 #ifndef __SOFTFP__
745 case T_FLOAT:
746 if (to_addr->index()->is_register()) {
747 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
748 __ add(Rtemp, base_reg, to_addr->index()->as_register());
749 if ((to_addr->disp() <= -4096) || (to_addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
750 __ fsts(src->as_float_reg(), Address(Rtemp, to_addr->disp()));
751 } else {
752 __ fsts(src->as_float_reg(), as_Address(to_addr));
753 }
754 break;
755
756 case T_DOUBLE:
757 if (to_addr->index()->is_register()) {
758 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
759 __ add(Rtemp, base_reg, to_addr->index()->as_register());
760 if ((to_addr->disp() <= -4096) || (to_addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
761 __ fstd(src->as_double_reg(), Address(Rtemp, to_addr->disp()));
762 } else {
763 __ fstd(src->as_double_reg(), as_Address(to_addr));
764 }
765 break;
766 #endif // __SOFTFP__
767
768 #endif // AARCH64
769
770 default:
771 ShouldNotReachHere();
772 }
773
774 if (info != NULL) {
775 add_debug_info_for_null_check(null_check_offset, info);
776 }
777
778 if (patch != NULL) {
779 // Offset embeedded into LDR/STR instruction may appear not enough
780 // to address a field. So, provide a space for one more instruction
781 // that will deal with larger offsets.
782 __ nop();
783 patching_epilog(patch, patch_code, base_reg, info);
784 }
785 }
786
787
788 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
789 assert(src->is_stack(), "should not call otherwise");
790 assert(dest->is_register(), "should not call otherwise");
791
792 Address addr = src->is_single_word() ?
793 frame_map()->address_for_slot(src->single_stack_ix()) :
794 frame_map()->address_for_slot(src->double_stack_ix());
795
796 #ifndef AARCH64
797 assert(lo_word_offset_in_bytes == 0 && hi_word_offset_in_bytes == 4, "little ending");
798 if (dest->is_single_fpu() || dest->is_double_fpu()) {
799 if (addr.disp() >= 1024) { BAILOUT("Too exotic case to handle here"); }
800 }
801 #endif // !AARCH64
802
803 if (dest->is_single_cpu()) {
804 switch (type) {
805 case T_OBJECT:
806 case T_ARRAY:
807 case T_ADDRESS:
808 case T_METADATA: __ ldr(dest->as_register(), addr); break;
809 case T_FLOAT: // used in floatToRawIntBits intrinsic implemenation
810 case T_INT: __ ldr_u32(dest->as_register(), addr); break;
811 default:
812 ShouldNotReachHere();
813 }
814 if ((type == T_OBJECT) || (type == T_ARRAY)) {
815 __ verify_oop(dest->as_register());
816 }
817 } else if (dest->is_double_cpu()) {
818 __ ldr(dest->as_register_lo(), addr);
819 #ifndef AARCH64
820 __ ldr(dest->as_register_hi(), frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes));
821 #endif // !AARCH64
822 } else if (dest->is_single_fpu()) {
823 __ ldr_float(dest->as_float_reg(), addr);
824 } else if (dest->is_double_fpu()) {
825 __ ldr_double(dest->as_double_reg(), addr);
826 } else {
827 ShouldNotReachHere();
828 }
829 }
830
831
832 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
833 if (src->is_single_stack()) {
834 switch (src->type()) {
835 case T_OBJECT:
836 case T_ARRAY:
837 case T_ADDRESS:
838 case T_METADATA:
839 __ ldr(Rtemp, frame_map()->address_for_slot(src->single_stack_ix()));
840 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
841 break;
842
843 case T_INT:
844 case T_FLOAT:
845 __ ldr_u32(Rtemp, frame_map()->address_for_slot(src->single_stack_ix()));
846 __ str_32(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
847 break;
848
849 default:
850 ShouldNotReachHere();
851 }
852 } else {
853 assert(src->is_double_stack(), "must be");
854 __ ldr(Rtemp, frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes));
855 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes));
856 #ifdef AARCH64
857 assert(lo_word_offset_in_bytes == 0, "adjust this code");
858 #else
859 __ ldr(Rtemp, frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes));
860 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes));
861 #endif // AARCH64
862 }
863 }
864
865
866 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type,
867 LIR_PatchCode patch_code, CodeEmitInfo* info,
868 bool wide, bool unaligned) {
869 assert(src->is_address(), "should not call otherwise");
870 assert(dest->is_register(), "should not call otherwise");
871 LIR_Address* addr = src->as_address_ptr();
872
873 Register base_reg = addr->base()->as_pointer_register();
874
875 PatchingStub* patch = NULL;
876 if (patch_code != lir_patch_none) {
877 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
878 #ifdef AARCH64
879 // Extra nop for MT safe patching
880 __ nop();
881 #endif // AARCH64
882 }
883 if (info != NULL) {
884 add_debug_info_for_null_check_here(info);
885 }
886
887 switch (type) {
888 case T_OBJECT: // fall through
889 case T_ARRAY:
890 if (UseCompressedOops && !wide) {
891 __ ldr_u32(dest->as_register(), as_Address(addr));
892 } else {
893 __ ldr(dest->as_register(), as_Address(addr));
894 }
895 break;
896
897 case T_ADDRESS:
898 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
899 __ ldr_u32(dest->as_pointer_register(), as_Address(addr));
900 } else {
901 __ ldr(dest->as_pointer_register(), as_Address(addr));
902 }
903 break;
904
905 #ifdef AARCH64
906 case T_LONG:
907 #else
908 case T_INT:
909 #ifdef __SOFTFP__
910 case T_FLOAT:
911 #endif // __SOFTFP__
912 #endif // AARCH64
913 __ ldr(dest->as_pointer_register(), as_Address(addr));
914 break;
915
916 case T_BOOLEAN:
917 __ ldrb(dest->as_register(), as_Address(addr));
918 break;
919
920 case T_BYTE:
921 __ ldrsb(dest->as_register(), as_Address(addr));
922 break;
923
924 case T_CHAR:
925 __ ldrh(dest->as_register(), as_Address(addr));
926 break;
927
928 case T_SHORT:
929 __ ldrsh(dest->as_register(), as_Address(addr));
930 break;
931
932 #ifdef AARCH64
933
934 case T_INT:
935 __ ldr_w(dest->as_register(), as_Address(addr));
936 break;
937
938 case T_FLOAT:
939 __ ldr_s(dest->as_float_reg(), as_Address(addr));
940 break;
941
942 case T_DOUBLE:
943 __ ldr_d(dest->as_double_reg(), as_Address(addr));
944 break;
945
946 #else // AARCH64
947
948 #ifdef __SOFTFP__
949 case T_DOUBLE:
950 #endif // __SOFTFP__
951 case T_LONG: {
952 Register to_lo = dest->as_register_lo();
953 Register to_hi = dest->as_register_hi();
954 if (addr->index()->is_register()) {
955 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
956 assert(addr->disp() == 0, "Not yet supporting both");
957 __ add(Rtemp, base_reg, addr->index()->as_register());
958 base_reg = Rtemp;
959 __ ldr(to_lo, Address(Rtemp));
960 if (patch != NULL) {
961 patching_epilog(patch, lir_patch_low, base_reg, info);
962 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
963 patch_code = lir_patch_high;
964 }
965 __ ldr(to_hi, Address(Rtemp, BytesPerWord));
966 } else if (base_reg == to_lo) {
967 __ ldr(to_hi, as_Address_hi(addr));
968 if (patch != NULL) {
969 patching_epilog(patch, lir_patch_high, base_reg, info);
970 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
971 patch_code = lir_patch_low;
972 }
973 __ ldr(to_lo, as_Address_lo(addr));
974 } else {
975 __ ldr(to_lo, as_Address_lo(addr));
976 if (patch != NULL) {
977 patching_epilog(patch, lir_patch_low, base_reg, info);
978 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
979 patch_code = lir_patch_high;
980 }
981 __ ldr(to_hi, as_Address_hi(addr));
982 }
983 break;
984 }
985
986 #ifndef __SOFTFP__
987 case T_FLOAT:
988 if (addr->index()->is_register()) {
989 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
990 __ add(Rtemp, base_reg, addr->index()->as_register());
991 if ((addr->disp() <= -4096) || (addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
992 __ flds(dest->as_float_reg(), Address(Rtemp, addr->disp()));
993 } else {
994 __ flds(dest->as_float_reg(), as_Address(addr));
995 }
996 break;
997
998 case T_DOUBLE:
999 if (addr->index()->is_register()) {
1000 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
1001 __ add(Rtemp, base_reg, addr->index()->as_register());
1002 if ((addr->disp() <= -4096) || (addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
1003 __ fldd(dest->as_double_reg(), Address(Rtemp, addr->disp()));
1004 } else {
1005 __ fldd(dest->as_double_reg(), as_Address(addr));
1006 }
1007 break;
1008 #endif // __SOFTFP__
1009
1010 #endif // AARCH64
1011
1012 default:
1013 ShouldNotReachHere();
1014 }
1015
1016 if (patch != NULL) {
1017 // Offset embeedded into LDR/STR instruction may appear not enough
1018 // to address a field. So, provide a space for one more instruction
1019 // that will deal with larger offsets.
1020 __ nop();
1021 patching_epilog(patch, patch_code, base_reg, info);
1022 }
1023
1024 #ifdef AARCH64
1025 switch (type) {
1026 case T_ARRAY:
1027 case T_OBJECT:
1028 if (UseCompressedOops && !wide) {
1029 __ decode_heap_oop(dest->as_register());
1030 }
1031 __ verify_oop(dest->as_register());
1032 break;
1033
1034 case T_ADDRESS:
1035 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1036 __ decode_klass_not_null(dest->as_register());
1037 }
1038 break;
1039 }
1040 #endif // AARCH64
1041 }
1042
1043
1044 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1045 bool is_32 = op->result_opr()->is_single_cpu();
1046
1047 if (op->code() == lir_idiv && op->in_opr2()->is_constant() && is_32) {
1048 int c = op->in_opr2()->as_constant_ptr()->as_jint();
1049 assert(is_power_of_2(c), "non power-of-2 constant should be put in a register");
1050
1051 Register left = op->in_opr1()->as_register();
1052 Register dest = op->result_opr()->as_register();
1053 if (c == 1) {
1054 __ mov(dest, left);
1055 } else if (c == 2) {
1056 __ add_32(dest, left, AsmOperand(left, lsr, 31));
1057 __ asr_32(dest, dest, 1);
1058 } else if (c != (int) 0x80000000) {
1059 int power = log2_intptr(c);
1060 __ asr_32(Rtemp, left, 31);
1061 __ add_32(dest, left, AsmOperand(Rtemp, lsr, 32-power)); // dest = left + (left < 0 ? 2^power - 1 : 0);
1062 __ asr_32(dest, dest, power); // dest = dest >>> power;
1063 } else {
1064 // x/0x80000000 is a special case, since dividend is a power of two, but is negative.
1065 // The only possible result values are 0 and 1, with 1 only for dividend == divisor == 0x80000000.
1066 __ cmp_32(left, c);
1067 #ifdef AARCH64
1068 __ cset(dest, eq);
1069 #else
1070 __ mov(dest, 0, ne);
1071 __ mov(dest, 1, eq);
1072 #endif // AARCH64
1073 }
1074 } else {
1075 #ifdef AARCH64
1076 Register left = op->in_opr1()->as_pointer_register();
1077 Register right = op->in_opr2()->as_pointer_register();
1078 Register dest = op->result_opr()->as_pointer_register();
1079
1080 switch (op->code()) {
1081 case lir_idiv:
1082 if (is_32) {
1083 __ sdiv_w(dest, left, right);
1084 } else {
1085 __ sdiv(dest, left, right);
1086 }
1087 break;
1088 case lir_irem: {
1089 Register tmp = op->in_opr3()->as_pointer_register();
1090 assert_different_registers(left, tmp);
1091 assert_different_registers(right, tmp);
1092 if (is_32) {
1093 __ sdiv_w(tmp, left, right);
1094 __ msub_w(dest, right, tmp, left);
1095 } else {
1096 __ sdiv(tmp, left, right);
1097 __ msub(dest, right, tmp, left);
1098 }
1099 break;
1100 }
1101 default:
1102 ShouldNotReachHere();
1103 }
1104 #else
1105 assert(op->code() == lir_idiv || op->code() == lir_irem, "unexpected op3");
1106 __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type);
1107 add_debug_info_for_div0_here(op->info());
1108 #endif // AARCH64
1109 }
1110 }
1111
1112
1113 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1114 #ifdef ASSERT
1115 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1116 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1117 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1118 assert(op->info() == NULL, "CodeEmitInfo?");
1119 #endif // ASSERT
1120
1121 #ifdef __SOFTFP__
1122 assert (op->code() != lir_cond_float_branch, "this should be impossible");
1123 #else
1124 if (op->code() == lir_cond_float_branch) {
1125 #ifndef AARCH64
1126 __ fmstat();
1127 #endif // !AARCH64
1128 __ b(*(op->ublock()->label()), vs);
1129 }
1130 #endif // __SOFTFP__
1131
1132 AsmCondition acond = al;
1133 switch (op->cond()) {
1134 case lir_cond_equal: acond = eq; break;
1135 case lir_cond_notEqual: acond = ne; break;
1136 case lir_cond_less: acond = lt; break;
1137 case lir_cond_lessEqual: acond = le; break;
1138 case lir_cond_greaterEqual: acond = ge; break;
1139 case lir_cond_greater: acond = gt; break;
1140 case lir_cond_aboveEqual: acond = hs; break;
1141 case lir_cond_belowEqual: acond = ls; break;
1142 default: assert(op->cond() == lir_cond_always, "must be");
1143 }
1144 __ b(*(op->label()), acond);
1145 }
1146
1147
1148 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1149 LIR_Opr src = op->in_opr();
1150 LIR_Opr dest = op->result_opr();
1151
1152 switch (op->bytecode()) {
1153 case Bytecodes::_i2l:
1154 #ifdef AARCH64
1155 __ sign_extend(dest->as_register_lo(), src->as_register(), 32);
1156 #else
1157 move_regs(src->as_register(), dest->as_register_lo());
1158 __ mov(dest->as_register_hi(), AsmOperand(src->as_register(), asr, 31));
1159 #endif // AARCH64
1160 break;
1161 case Bytecodes::_l2i:
1162 move_regs(src->as_register_lo(), dest->as_register());
1163 break;
1164 case Bytecodes::_i2b:
1165 __ sign_extend(dest->as_register(), src->as_register(), 8);
1166 break;
1167 case Bytecodes::_i2s:
1168 __ sign_extend(dest->as_register(), src->as_register(), 16);
1169 break;
1170 case Bytecodes::_i2c:
1171 __ zero_extend(dest->as_register(), src->as_register(), 16);
1172 break;
1173 case Bytecodes::_f2d:
1174 __ convert_f2d(dest->as_double_reg(), src->as_float_reg());
1175 break;
1176 case Bytecodes::_d2f:
1177 __ convert_d2f(dest->as_float_reg(), src->as_double_reg());
1178 break;
1179 case Bytecodes::_i2f:
1180 #ifdef AARCH64
1181 __ scvtf_sw(dest->as_float_reg(), src->as_register());
1182 #else
1183 __ fmsr(Stemp, src->as_register());
1184 __ fsitos(dest->as_float_reg(), Stemp);
1185 #endif // AARCH64
1186 break;
1187 case Bytecodes::_i2d:
1188 #ifdef AARCH64
1189 __ scvtf_dw(dest->as_double_reg(), src->as_register());
1190 #else
1191 __ fmsr(Stemp, src->as_register());
1192 __ fsitod(dest->as_double_reg(), Stemp);
1193 #endif // AARCH64
1194 break;
1195 case Bytecodes::_f2i:
1196 #ifdef AARCH64
1197 __ fcvtzs_ws(dest->as_register(), src->as_float_reg());
1198 #else
1199 __ ftosizs(Stemp, src->as_float_reg());
1200 __ fmrs(dest->as_register(), Stemp);
1201 #endif // AARCH64
1202 break;
1203 case Bytecodes::_d2i:
1204 #ifdef AARCH64
1205 __ fcvtzs_wd(dest->as_register(), src->as_double_reg());
1206 #else
1207 __ ftosizd(Stemp, src->as_double_reg());
1208 __ fmrs(dest->as_register(), Stemp);
1209 #endif // AARCH64
1210 break;
1211 #ifdef AARCH64
1212 case Bytecodes::_l2f:
1213 __ scvtf_sx(dest->as_float_reg(), src->as_register_lo());
1214 break;
1215 case Bytecodes::_l2d:
1216 __ scvtf_dx(dest->as_double_reg(), src->as_register_lo());
1217 break;
1218 case Bytecodes::_f2l:
1219 __ fcvtzs_xs(dest->as_register_lo(), src->as_float_reg());
1220 break;
1221 case Bytecodes::_d2l:
1222 __ fcvtzs_xd(dest->as_register_lo(), src->as_double_reg());
1223 break;
1224 #endif // AARCH64
1225 default:
1226 ShouldNotReachHere();
1227 }
1228 }
1229
1230
1231 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1232 if (op->init_check()) {
1233 Register tmp = op->tmp1()->as_register();
1234 __ ldrb(tmp, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
1235 add_debug_info_for_null_check_here(op->stub()->info());
1236 __ cmp(tmp, InstanceKlass::fully_initialized);
1237 __ b(*op->stub()->entry(), ne);
1238 }
1239 __ allocate_object(op->obj()->as_register(),
1240 op->tmp1()->as_register(),
1241 op->tmp2()->as_register(),
1242 op->tmp3()->as_register(),
1243 op->header_size(),
1244 op->object_size(),
1245 op->klass()->as_register(),
1246 *op->stub()->entry());
1247 __ bind(*op->stub()->continuation());
1248 }
1249
1250 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1251 if (UseSlowPath ||
1252 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1253 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1254 __ b(*op->stub()->entry());
1255 } else {
1256 __ allocate_array(op->obj()->as_register(),
1257 op->len()->as_register(),
1258 op->tmp1()->as_register(),
1259 op->tmp2()->as_register(),
1260 op->tmp3()->as_register(),
1261 arrayOopDesc::header_size(op->type()),
1262 type2aelembytes(op->type()),
1263 op->klass()->as_register(),
1264 *op->stub()->entry());
1265 }
1266 __ bind(*op->stub()->continuation());
1267 }
1268
1269 void LIR_Assembler::type_profile_helper(Register mdo, int mdo_offset_bias,
1270 ciMethodData *md, ciProfileData *data,
1271 Register recv, Register tmp1, Label* update_done) {
1272 assert_different_registers(mdo, recv, tmp1);
1273 uint i;
1274 for (i = 0; i < VirtualCallData::row_limit(); i++) {
1275 Label next_test;
1276 // See if the receiver is receiver[n].
1277 Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
1278 mdo_offset_bias);
1279 __ ldr(tmp1, receiver_addr);
1280 __ verify_klass_ptr(tmp1);
1281 __ cmp(recv, tmp1);
1282 __ b(next_test, ne);
1283 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
1284 mdo_offset_bias);
1285 __ ldr(tmp1, data_addr);
1286 __ add(tmp1, tmp1, DataLayout::counter_increment);
1287 __ str(tmp1, data_addr);
1288 __ b(*update_done);
1289 __ bind(next_test);
1290 }
1291
1292 // Didn't find receiver; find next empty slot and fill it in
1293 for (i = 0; i < VirtualCallData::row_limit(); i++) {
1294 Label next_test;
1295 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
1296 mdo_offset_bias);
1297 __ ldr(tmp1, recv_addr);
1298 __ cbnz(tmp1, next_test);
1299 __ str(recv, recv_addr);
1300 __ mov(tmp1, DataLayout::counter_increment);
1301 __ str(tmp1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
1302 mdo_offset_bias));
1303 __ b(*update_done);
1304 __ bind(next_test);
1305 }
1306 }
1307
1308 void LIR_Assembler::setup_md_access(ciMethod* method, int bci,
1309 ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) {
1310 md = method->method_data_or_null();
1311 assert(md != NULL, "Sanity");
1312 data = md->bci_to_data(bci);
1313 assert(data != NULL, "need data for checkcast");
1314 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1315 if (md->byte_offset_of_slot(data, DataLayout::header_offset()) + data->size_in_bytes() >= 4096) {
1316 // The offset is large so bias the mdo by the base of the slot so
1317 // that the ldr can use an immediate offset to reference the slots of the data
1318 mdo_offset_bias = md->byte_offset_of_slot(data, DataLayout::header_offset());
1319 }
1320 }
1321
1322 // On 32-bit ARM, code before this helper should test obj for null (ZF should be set if obj is null).
1323 void LIR_Assembler::typecheck_profile_helper1(ciMethod* method, int bci,
1324 ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias,
1325 Register obj, Register mdo, Register data_val, Label* obj_is_null) {
1326 assert(method != NULL, "Should have method");
1327 assert_different_registers(obj, mdo, data_val);
1328 setup_md_access(method, bci, md, data, mdo_offset_bias);
1329 Label not_null;
1330 #ifdef AARCH64
1331 __ cbnz(obj, not_null);
1332 #else
1333 __ b(not_null, ne);
1334 #endif // AARCH64
1335 __ mov_metadata(mdo, md->constant_encoding());
1336 if (mdo_offset_bias > 0) {
1337 __ mov_slow(data_val, mdo_offset_bias);
1338 __ add(mdo, mdo, data_val);
1339 }
1340 Address flags_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias);
1341 __ ldrb(data_val, flags_addr);
1342 __ orr(data_val, data_val, (uint)BitData::null_seen_byte_constant());
1343 __ strb(data_val, flags_addr);
1344 __ b(*obj_is_null);
1345 __ bind(not_null);
1346 }
1347
1348 void LIR_Assembler::typecheck_profile_helper2(ciMethodData* md, ciProfileData* data, int mdo_offset_bias,
1349 Register mdo, Register recv, Register value, Register tmp1,
1350 Label* profile_cast_success, Label* profile_cast_failure,
1351 Label* success, Label* failure) {
1352 assert_different_registers(mdo, value, tmp1);
1353 __ bind(*profile_cast_success);
1354 __ mov_metadata(mdo, md->constant_encoding());
1355 if (mdo_offset_bias > 0) {
1356 __ mov_slow(tmp1, mdo_offset_bias);
1357 __ add(mdo, mdo, tmp1);
1358 }
1359 __ load_klass(recv, value);
1360 type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, success);
1361 __ b(*success);
1362 // Cast failure case
1363 __ bind(*profile_cast_failure);
1364 __ mov_metadata(mdo, md->constant_encoding());
1365 if (mdo_offset_bias > 0) {
1366 __ mov_slow(tmp1, mdo_offset_bias);
1367 __ add(mdo, mdo, tmp1);
1368 }
1369 Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
1370 __ ldr(tmp1, data_addr);
1371 __ sub(tmp1, tmp1, DataLayout::counter_increment);
1372 __ str(tmp1, data_addr);
1373 __ b(*failure);
1374 }
1375
1376 // Sets `res` to true, if `cond` holds. On AArch64 also sets `res` to false if `cond` does not hold.
1377 static void set_instanceof_result(MacroAssembler* _masm, Register res, AsmCondition cond) {
1378 #ifdef AARCH64
1379 __ cset(res, cond);
1380 #else
1381 __ mov(res, 1, cond);
1382 #endif // AARCH64
1383 }
1384
1385
1386 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1387 // TODO: ARM - can be more effective with one more register
1388 switch (op->code()) {
1389 case lir_store_check: {
1390 CodeStub* stub = op->stub();
1391 Register value = op->object()->as_register();
1392 Register array = op->array()->as_register();
1393 Register klass_RInfo = op->tmp1()->as_register();
1394 Register k_RInfo = op->tmp2()->as_register();
1395 assert_different_registers(klass_RInfo, k_RInfo, Rtemp);
1396 if (op->should_profile()) {
1397 assert_different_registers(value, klass_RInfo, k_RInfo, Rtemp);
1398 }
1399
1400 // check if it needs to be profiled
1401 ciMethodData* md;
1402 ciProfileData* data;
1403 int mdo_offset_bias = 0;
1404 Label profile_cast_success, profile_cast_failure, done;
1405 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1406 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1407
1408 if (op->should_profile()) {
1409 #ifndef AARCH64
1410 __ cmp(value, 0);
1411 #endif // !AARCH64
1412 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, value, k_RInfo, Rtemp, &done);
1413 } else {
1414 __ cbz(value, done);
1415 }
1416 assert_different_registers(k_RInfo, value);
1417 add_debug_info_for_null_check_here(op->info_for_exception());
1418 __ load_klass(k_RInfo, array);
1419 __ load_klass(klass_RInfo, value);
1420 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1421 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1422 // check for immediate positive hit
1423 __ ldr(Rtemp, Address(klass_RInfo, Rtemp));
1424 __ cmp(klass_RInfo, k_RInfo);
1425 __ cond_cmp(Rtemp, k_RInfo, ne);
1426 __ b(*success_target, eq);
1427 // check for immediate negative hit
1428 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1429 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset()));
1430 __ b(*failure_target, ne);
1431 // slow case
1432 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1433 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1434 __ cbz(R0, *failure_target);
1435 if (op->should_profile()) {
1436 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp;
1437 if (mdo == value) {
1438 mdo = k_RInfo;
1439 recv = klass_RInfo;
1440 }
1441 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, value, tmp1,
1442 &profile_cast_success, &profile_cast_failure,
1443 &done, stub->entry());
1444 }
1445 __ bind(done);
1446 break;
1447 }
1448
1449 case lir_checkcast: {
1450 CodeStub* stub = op->stub();
1451 Register obj = op->object()->as_register();
1452 Register res = op->result_opr()->as_register();
1453 Register klass_RInfo = op->tmp1()->as_register();
1454 Register k_RInfo = op->tmp2()->as_register();
1455 ciKlass* k = op->klass();
1456 assert_different_registers(res, k_RInfo, klass_RInfo, Rtemp);
1457
1458 if (stub->is_simple_exception_stub()) {
1459 // TODO: ARM - Late binding is used to prevent confusion of register allocator
1460 assert(stub->is_exception_throw_stub(), "must be");
1461 ((SimpleExceptionStub*)stub)->set_obj(op->result_opr());
1462 }
1463 ciMethodData* md;
1464 ciProfileData* data;
1465 int mdo_offset_bias = 0;
1466
1467 Label done;
1468
1469 Label profile_cast_failure, profile_cast_success;
1470 Label *failure_target = op->should_profile() ? &profile_cast_failure : op->stub()->entry();
1471 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1472
1473 #ifdef AARCH64
1474 move_regs(obj, res);
1475 if (op->should_profile()) {
1476 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done);
1477 } else {
1478 __ cbz(obj, done);
1479 }
1480 if (k->is_loaded()) {
1481 __ mov_metadata(k_RInfo, k->constant_encoding());
1482 } else {
1483 if (res != obj) {
1484 op->info_for_patch()->add_register_oop(FrameMap::as_oop_opr(res));
1485 }
1486 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1487 }
1488 __ load_klass(klass_RInfo, res);
1489
1490 if (op->fast_check()) {
1491 __ cmp(klass_RInfo, k_RInfo);
1492 __ b(*failure_target, ne);
1493 } else if (k->is_loaded()) {
1494 __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset()));
1495 if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) {
1496 __ cmp(Rtemp, k_RInfo);
1497 __ b(*failure_target, ne);
1498 } else {
1499 __ cmp(klass_RInfo, k_RInfo);
1500 __ cond_cmp(Rtemp, k_RInfo, ne);
1501 __ b(*success_target, eq);
1502 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1503 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1504 __ cbz(R0, *failure_target);
1505 }
1506 } else {
1507 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1508 // check for immediate positive hit
1509 __ ldr(Rtemp, Address(klass_RInfo, Rtemp));
1510 __ cmp(klass_RInfo, k_RInfo);
1511 __ cond_cmp(Rtemp, k_RInfo, ne);
1512 __ b(*success_target, eq);
1513 // check for immediate negative hit
1514 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1515 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset()));
1516 __ b(*failure_target, ne);
1517 // slow case
1518 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1519 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1520 __ cbz(R0, *failure_target);
1521 }
1522
1523 #else // AARCH64
1524
1525 __ movs(res, obj);
1526 if (op->should_profile()) {
1527 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done);
1528 } else {
1529 __ b(done, eq);
1530 }
1531 if (k->is_loaded()) {
1532 __ mov_metadata(k_RInfo, k->constant_encoding());
1533 } else if (k_RInfo != obj) {
1534 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1535 __ movs(res, obj);
1536 } else {
1537 // Patching doesn't update "res" register after GC, so do patching first
1538 klass2reg_with_patching(Rtemp, op->info_for_patch());
1539 __ movs(res, obj);
1540 __ mov(k_RInfo, Rtemp);
1541 }
1542 __ load_klass(klass_RInfo, res, ne);
1543
1544 if (op->fast_check()) {
1545 __ cmp(klass_RInfo, k_RInfo, ne);
1546 __ b(*failure_target, ne);
1547 } else if (k->is_loaded()) {
1548 __ b(*success_target, eq);
1549 __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset()));
1550 if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) {
1551 __ cmp(Rtemp, k_RInfo);
1552 __ b(*failure_target, ne);
1553 } else {
1554 __ cmp(klass_RInfo, k_RInfo);
1555 __ cmp(Rtemp, k_RInfo, ne);
1556 __ b(*success_target, eq);
1557 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1558 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1559 __ cbz(R0, *failure_target);
1560 }
1561 } else {
1562 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1563 __ b(*success_target, eq);
1564 // check for immediate positive hit
1565 __ ldr(Rtemp, Address(klass_RInfo, Rtemp));
1566 __ cmp(klass_RInfo, k_RInfo);
1567 __ cmp(Rtemp, k_RInfo, ne);
1568 __ b(*success_target, eq);
1569 // check for immediate negative hit
1570 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1571 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset()));
1572 __ b(*failure_target, ne);
1573 // slow case
1574 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1575 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1576 __ cbz(R0, *failure_target);
1577 }
1578 #endif // AARCH64
1579
1580 if (op->should_profile()) {
1581 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp;
1582 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, res, tmp1,
1583 &profile_cast_success, &profile_cast_failure,
1584 &done, stub->entry());
1585 }
1586 __ bind(done);
1587 break;
1588 }
1589
1590 case lir_instanceof: {
1591 Register obj = op->object()->as_register();
1592 Register res = op->result_opr()->as_register();
1593 Register klass_RInfo = op->tmp1()->as_register();
1594 Register k_RInfo = op->tmp2()->as_register();
1595 ciKlass* k = op->klass();
1596 assert_different_registers(res, klass_RInfo, k_RInfo, Rtemp);
1597
1598 ciMethodData* md;
1599 ciProfileData* data;
1600 int mdo_offset_bias = 0;
1601
1602 Label done;
1603
1604 Label profile_cast_failure, profile_cast_success;
1605 Label *failure_target = op->should_profile() ? &profile_cast_failure : &done;
1606 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1607
1608 #ifdef AARCH64
1609 move_regs(obj, res);
1610 #else
1611 __ movs(res, obj);
1612 #endif // AARCH64
1613
1614 if (op->should_profile()) {
1615 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done);
1616 } else {
1617 #ifdef AARCH64
1618 __ cbz(obj, done); // If obj == NULL, res is false
1619 #else
1620 __ b(done, eq);
1621 #endif // AARCH64
1622 }
1623
1624 if (k->is_loaded()) {
1625 __ mov_metadata(k_RInfo, k->constant_encoding());
1626 } else {
1627 op->info_for_patch()->add_register_oop(FrameMap::as_oop_opr(res));
1628 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1629 }
1630 __ load_klass(klass_RInfo, res);
1631
1632 #ifndef AARCH64
1633 if (!op->should_profile()) {
1634 __ mov(res, 0);
1635 }
1636 #endif // !AARCH64
1637
1638 if (op->fast_check()) {
1639 __ cmp(klass_RInfo, k_RInfo);
1640 if (!op->should_profile()) {
1641 set_instanceof_result(_masm, res, eq);
1642 } else {
1643 __ b(profile_cast_failure, ne);
1644 }
1645 } else if (k->is_loaded()) {
1646 __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset()));
1647 if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) {
1648 __ cmp(Rtemp, k_RInfo);
1649 if (!op->should_profile()) {
1650 set_instanceof_result(_masm, res, eq);
1651 } else {
1652 __ b(profile_cast_failure, ne);
1653 }
1654 } else {
1655 __ cmp(klass_RInfo, k_RInfo);
1656 __ cond_cmp(Rtemp, k_RInfo, ne);
1657 if (!op->should_profile()) {
1658 set_instanceof_result(_masm, res, eq);
1659 }
1660 __ b(*success_target, eq);
1661 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1662 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1663 if (!op->should_profile()) {
1664 move_regs(R0, res);
1665 } else {
1666 __ cbz(R0, *failure_target);
1667 }
1668 }
1669 } else {
1670 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1671 // check for immediate positive hit
1672 __ cmp(klass_RInfo, k_RInfo);
1673 if (!op->should_profile()) {
1674 #ifdef AARCH64
1675 // TODO-AARCH64 check if separate conditional branch is more efficient than ldr+cond_cmp
1676 __ ldr(res, Address(klass_RInfo, Rtemp));
1677 #else
1678 __ ldr(res, Address(klass_RInfo, Rtemp), ne);
1679 #endif // AARCH64
1680 __ cond_cmp(res, k_RInfo, ne);
1681 set_instanceof_result(_masm, res, eq);
1682 } else {
1683 #ifdef AARCH64
1684 // TODO-AARCH64 check if separate conditional branch is more efficient than ldr+cond_cmp
1685 __ ldr(Rtemp, Address(klass_RInfo, Rtemp));
1686 #else
1687 __ ldr(Rtemp, Address(klass_RInfo, Rtemp), ne);
1688 #endif // AARCH64
1689 __ cond_cmp(Rtemp, k_RInfo, ne);
1690 }
1691 __ b(*success_target, eq);
1692 // check for immediate negative hit
1693 if (op->should_profile()) {
1694 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1695 }
1696 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset()));
1697 if (!op->should_profile()) {
1698 #ifdef AARCH64
1699 __ mov(res, 0);
1700 #else
1701 __ mov(res, 0, ne);
1702 #endif // AARCH64
1703 }
1704 __ b(*failure_target, ne);
1705 // slow case
1706 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1707 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1708 if (!op->should_profile()) {
1709 move_regs(R0, res);
1710 }
1711 if (op->should_profile()) {
1712 __ cbz(R0, *failure_target);
1713 }
1714 }
1715
1716 if (op->should_profile()) {
1717 Label done_ok, done_failure;
1718 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp;
1719 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, res, tmp1,
1720 &profile_cast_success, &profile_cast_failure,
1721 &done_ok, &done_failure);
1722 __ bind(done_failure);
1723 __ mov(res, 0);
1724 __ b(done);
1725 __ bind(done_ok);
1726 __ mov(res, 1);
1727 }
1728 __ bind(done);
1729 break;
1730 }
1731 default:
1732 ShouldNotReachHere();
1733 }
1734 }
1735
1736
1737 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1738 // if (*addr == cmpval) {
1739 // *addr = newval;
1740 // dest = 1;
1741 // } else {
1742 // dest = 0;
1743 // }
1744 #ifdef AARCH64
1745 Label retry, done;
1746 Register addr = op->addr()->as_pointer_register();
1747 Register cmpval = op->cmp_value()->as_pointer_register();
1748 Register newval = op->new_value()->as_pointer_register();
1749 Register dest = op->result_opr()->as_pointer_register();
1750 assert_different_registers(dest, addr, cmpval, newval, Rtemp);
1751
1752 if (UseCompressedOops && op->code() == lir_cas_obj) {
1753 Register tmp1 = op->tmp1()->as_pointer_register();
1754 Register tmp2 = op->tmp2()->as_pointer_register();
1755 assert_different_registers(dest, addr, cmpval, newval, tmp1, tmp2, Rtemp);
1756 __ encode_heap_oop(tmp1, cmpval); cmpval = tmp1;
1757 __ encode_heap_oop(tmp2, newval); newval = tmp2;
1758 }
1759
1760 __ mov(dest, ZR);
1761 __ bind(retry);
1762 if (((op->code() == lir_cas_obj) && !UseCompressedOops) || op->code() == lir_cas_long) {
1763 __ ldaxr(Rtemp, addr);
1764 __ cmp(Rtemp, cmpval);
1765 __ b(done, ne);
1766 __ stlxr(Rtemp, newval, addr);
1767 } else if (((op->code() == lir_cas_obj) && UseCompressedOops) || op->code() == lir_cas_int) {
1768 __ ldaxr_w(Rtemp, addr);
1769 __ cmp_w(Rtemp, cmpval);
1770 __ b(done, ne);
1771 __ stlxr_w(Rtemp, newval, addr);
1772 } else {
1773 ShouldNotReachHere();
1774 }
1775 __ cbnz_w(Rtemp, retry);
1776 __ mov(dest, 1);
1777 __ bind(done);
1778 #else
1779 // FIXME: membar_release
1780 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp);
1781 Register addr = op->addr()->is_register() ?
1782 op->addr()->as_pointer_register() :
1783 op->addr()->as_address_ptr()->base()->as_pointer_register();
1784 assert(op->addr()->is_register() || op->addr()->as_address_ptr()->disp() == 0, "unexpected disp");
1785 assert(op->addr()->is_register() || op->addr()->as_address_ptr()->index() == LIR_OprDesc::illegalOpr(), "unexpected index");
1786 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1787 Register cmpval = op->cmp_value()->as_register();
1788 Register newval = op->new_value()->as_register();
1789 Register dest = op->result_opr()->as_register();
1790 assert_different_registers(dest, addr, cmpval, newval, Rtemp);
1791
1792 __ atomic_cas_bool(cmpval, newval, addr, 0, Rtemp); // Rtemp free by default at C1 LIR layer
1793 __ mov(dest, 1, eq);
1794 __ mov(dest, 0, ne);
1795 } else if (op->code() == lir_cas_long) {
1796 assert(VM_Version::supports_cx8(), "wrong machine");
1797 Register cmp_value_lo = op->cmp_value()->as_register_lo();
1798 Register cmp_value_hi = op->cmp_value()->as_register_hi();
1799 Register new_value_lo = op->new_value()->as_register_lo();
1800 Register new_value_hi = op->new_value()->as_register_hi();
1801 Register dest = op->result_opr()->as_register();
1802 Register tmp_lo = op->tmp1()->as_register_lo();
1803 Register tmp_hi = op->tmp1()->as_register_hi();
1804
1805 assert_different_registers(tmp_lo, tmp_hi, cmp_value_lo, cmp_value_hi, dest, new_value_lo, new_value_hi, addr);
1806 assert(tmp_hi->encoding() == tmp_lo->encoding() + 1, "non aligned register pair");
1807 assert(new_value_hi->encoding() == new_value_lo->encoding() + 1, "non aligned register pair");
1808 assert((tmp_lo->encoding() & 0x1) == 0, "misaligned register pair");
1809 assert((new_value_lo->encoding() & 0x1) == 0, "misaligned register pair");
1810 __ atomic_cas64(tmp_lo, tmp_hi, dest, cmp_value_lo, cmp_value_hi,
1811 new_value_lo, new_value_hi, addr, 0);
1812 } else {
1813 Unimplemented();
1814 }
1815 #endif // AARCH64
1816 // FIXME: is full membar really needed instead of just membar_acquire?
1817 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp);
1818 }
1819
1820
1821 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1822 AsmCondition acond = al;
1823 AsmCondition ncond = nv;
1824 if (opr1 != opr2) {
1825 switch (condition) {
1826 case lir_cond_equal: acond = eq; ncond = ne; break;
1827 case lir_cond_notEqual: acond = ne; ncond = eq; break;
1828 case lir_cond_less: acond = lt; ncond = ge; break;
1829 case lir_cond_lessEqual: acond = le; ncond = gt; break;
1830 case lir_cond_greaterEqual: acond = ge; ncond = lt; break;
1831 case lir_cond_greater: acond = gt; ncond = le; break;
1832 case lir_cond_aboveEqual: acond = hs; ncond = lo; break;
1833 case lir_cond_belowEqual: acond = ls; ncond = hi; break;
1834 default: ShouldNotReachHere();
1835 }
1836 }
1837
1838 #ifdef AARCH64
1839
1840 // TODO-AARCH64 implement it more efficiently
1841
1842 if (opr1->is_register()) {
1843 reg2reg(opr1, result);
1844 } else if (opr1->is_stack()) {
1845 stack2reg(opr1, result, result->type());
1846 } else if (opr1->is_constant()) {
1847 const2reg(opr1, result, lir_patch_none, NULL);
1848 } else {
1849 ShouldNotReachHere();
1850 }
1851
1852 Label skip;
1853 __ b(skip, acond);
1854
1855 if (opr2->is_register()) {
1856 reg2reg(opr2, result);
1857 } else if (opr2->is_stack()) {
1858 stack2reg(opr2, result, result->type());
1859 } else if (opr2->is_constant()) {
1860 const2reg(opr2, result, lir_patch_none, NULL);
1861 } else {
1862 ShouldNotReachHere();
1863 }
1864
1865 __ bind(skip);
1866
1867 #else
1868 for (;;) { // two iterations only
1869 if (opr1 == result) {
1870 // do nothing
1871 } else if (opr1->is_single_cpu()) {
1872 __ mov(result->as_register(), opr1->as_register(), acond);
1873 } else if (opr1->is_double_cpu()) {
1874 __ long_move(result->as_register_lo(), result->as_register_hi(),
1875 opr1->as_register_lo(), opr1->as_register_hi(), acond);
1876 } else if (opr1->is_single_stack()) {
1877 __ ldr(result->as_register(), frame_map()->address_for_slot(opr1->single_stack_ix()), acond);
1878 } else if (opr1->is_double_stack()) {
1879 __ ldr(result->as_register_lo(),
1880 frame_map()->address_for_slot(opr1->double_stack_ix(), lo_word_offset_in_bytes), acond);
1881 __ ldr(result->as_register_hi(),
1882 frame_map()->address_for_slot(opr1->double_stack_ix(), hi_word_offset_in_bytes), acond);
1883 } else if (opr1->is_illegal()) {
1884 // do nothing: this part of the cmove has been optimized away in the peephole optimizer
1885 } else {
1886 assert(opr1->is_constant(), "must be");
1887 LIR_Const* c = opr1->as_constant_ptr();
1888
1889 switch (c->type()) {
1890 case T_INT:
1891 __ mov_slow(result->as_register(), c->as_jint(), acond);
1892 break;
1893 case T_LONG:
1894 __ mov_slow(result->as_register_lo(), c->as_jint_lo(), acond);
1895 __ mov_slow(result->as_register_hi(), c->as_jint_hi(), acond);
1896 break;
1897 case T_OBJECT:
1898 __ mov_oop(result->as_register(), c->as_jobject(), 0, acond);
1899 break;
1900 case T_FLOAT:
1901 #ifdef __SOFTFP__
1902 // not generated now.
1903 __ mov_slow(result->as_register(), c->as_jint(), acond);
1904 #else
1905 __ mov_float(result->as_float_reg(), c->as_jfloat(), acond);
1906 #endif // __SOFTFP__
1907 break;
1908 case T_DOUBLE:
1909 #ifdef __SOFTFP__
1910 // not generated now.
1911 __ mov_slow(result->as_register_lo(), c->as_jint_lo(), acond);
1912 __ mov_slow(result->as_register_hi(), c->as_jint_hi(), acond);
1913 #else
1914 __ mov_double(result->as_double_reg(), c->as_jdouble(), acond);
1915 #endif // __SOFTFP__
1916 break;
1917 default:
1918 ShouldNotReachHere();
1919 }
1920 }
1921
1922 // Negate the condition and repeat the algorithm with the second operand
1923 if (opr1 == opr2) { break; }
1924 opr1 = opr2;
1925 acond = ncond;
1926 }
1927 #endif // AARCH64
1928 }
1929
1930 #if defined(AARCH64) || defined(ASSERT)
1931 static int reg_size(LIR_Opr op) {
1932 switch (op->type()) {
1933 case T_FLOAT:
1934 case T_INT: return BytesPerInt;
1935 case T_LONG:
1936 case T_DOUBLE: return BytesPerLong;
1937 case T_OBJECT:
1938 case T_ARRAY:
1939 case T_METADATA: return BytesPerWord;
1940 case T_ADDRESS:
1941 case T_ILLEGAL: // fall through
1942 default: ShouldNotReachHere(); return -1;
1943 }
1944 }
1945 #endif
1946
1947 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1948 assert(info == NULL, "unused on this code path");
1949 assert(dest->is_register(), "wrong items state");
1950
1951 if (right->is_address()) {
1952 // special case for adding shifted/extended register
1953 const Register res = dest->as_pointer_register();
1954 const Register lreg = left->as_pointer_register();
1955 const LIR_Address* addr = right->as_address_ptr();
1956
1957 assert(addr->base()->as_pointer_register() == lreg && addr->index()->is_register() && addr->disp() == 0, "must be");
1958
1959 int scale = addr->scale();
1960 AsmShift shift = lsl;
1961
1962 #ifdef AARCH64
1963 bool is_index_extended = reg_size(addr->base()) > reg_size(addr->index());
1964 if (scale < 0) {
1965 scale = -scale;
1966 shift = lsr;
1967 }
1968 assert(shift == lsl || !is_index_extended, "could not have extend and right shift in one operand");
1969 assert(0 <= scale && scale <= 63, "scale is too large");
1970
1971 if (is_index_extended) {
1972 assert(scale <= 4, "scale is too large for add with extended register");
1973 assert(addr->index()->is_single_cpu(), "should be");
1974 assert(addr->index()->type() == T_INT, "should be");
1975 assert(dest->is_double_cpu(), "should be");
1976 assert(code == lir_add, "special case of add with extended register");
1977
1978 __ add(res, lreg, addr->index()->as_register(), ex_sxtw, scale);
1979 return;
1980 } else if (reg_size(dest) == BytesPerInt) {
1981 assert(reg_size(addr->base()) == reg_size(addr->index()), "should be");
1982 assert(reg_size(addr->base()) == reg_size(dest), "should be");
1983
1984 AsmOperand operand(addr->index()->as_pointer_register(), shift, scale);
1985 switch (code) {
1986 case lir_add: __ add_32(res, lreg, operand); break;
1987 case lir_sub: __ sub_32(res, lreg, operand); break;
1988 default: ShouldNotReachHere();
1989 }
1990 return;
1991 }
1992 #endif // AARCH64
1993
1994 assert(reg_size(addr->base()) == reg_size(addr->index()), "should be");
1995 assert(reg_size(addr->base()) == reg_size(dest), "should be");
1996 assert(reg_size(dest) == wordSize, "should be");
1997
1998 AsmOperand operand(addr->index()->as_pointer_register(), shift, scale);
1999 switch (code) {
2000 case lir_add: __ add(res, lreg, operand); break;
2001 case lir_sub: __ sub(res, lreg, operand); break;
2002 default: ShouldNotReachHere();
2003 }
2004
2005 #ifndef AARCH64
2006 } else if (left->is_address()) {
2007 assert(code == lir_sub && right->is_single_cpu(), "special case used by strength_reduce_multiply()");
2008 const LIR_Address* addr = left->as_address_ptr();
2009 const Register res = dest->as_register();
2010 const Register rreg = right->as_register();
2011 assert(addr->base()->as_register() == rreg && addr->index()->is_register() && addr->disp() == 0, "must be");
2012 __ rsb(res, rreg, AsmOperand(addr->index()->as_register(), lsl, addr->scale()));
2013 #endif // !AARCH64
2014
2015 } else if (dest->is_single_cpu()) {
2016 assert(left->is_single_cpu(), "unexpected left operand");
2017 #ifdef AARCH64
2018 assert(dest->type() == T_INT, "unexpected dest type");
2019 assert(left->type() == T_INT, "unexpected left type");
2020 assert(right->type() == T_INT, "unexpected right type");
2021 #endif // AARCH64
2022
2023 const Register res = dest->as_register();
2024 const Register lreg = left->as_register();
2025
2026 if (right->is_single_cpu()) {
2027 const Register rreg = right->as_register();
2028 switch (code) {
2029 case lir_add: __ add_32(res, lreg, rreg); break;
2030 case lir_sub: __ sub_32(res, lreg, rreg); break;
2031 case lir_mul: __ mul_32(res, lreg, rreg); break;
2032 default: ShouldNotReachHere();
2033 }
2034 } else {
2035 assert(right->is_constant(), "must be");
2036 const jint c = right->as_constant_ptr()->as_jint();
2037 if (!Assembler::is_arith_imm_in_range(c)) {
2038 BAILOUT("illegal arithmetic operand");
2039 }
2040 switch (code) {
2041 case lir_add: __ add_32(res, lreg, c); break;
2042 case lir_sub: __ sub_32(res, lreg, c); break;
2043 default: ShouldNotReachHere();
2044 }
2045 }
2046
2047 } else if (dest->is_double_cpu()) {
2048 #ifdef AARCH64
2049 assert(left->is_double_cpu() ||
2050 (left->is_single_cpu() && ((left->type() == T_OBJECT) || (left->type() == T_ARRAY) || (left->type() == T_ADDRESS))),
2051 "unexpected left operand");
2052
2053 const Register res = dest->as_register_lo();
2054 const Register lreg = left->as_pointer_register();
2055
2056 if (right->is_constant()) {
2057 assert(right->type() == T_LONG, "unexpected right type");
2058 assert((right->as_constant_ptr()->as_jlong() >> 24) == 0, "out of range");
2059 jint imm = (jint)right->as_constant_ptr()->as_jlong();
2060 switch (code) {
2061 case lir_add: __ add(res, lreg, imm); break;
2062 case lir_sub: __ sub(res, lreg, imm); break;
2063 default: ShouldNotReachHere();
2064 }
2065 } else {
2066 assert(right->is_double_cpu() ||
2067 (right->is_single_cpu() && ((right->type() == T_OBJECT) || (right->type() == T_ARRAY) || (right->type() == T_ADDRESS))),
2068 "unexpected right operand");
2069 const Register rreg = right->as_pointer_register();
2070 switch (code) {
2071 case lir_add: __ add(res, lreg, rreg); break;
2072 case lir_sub: __ sub(res, lreg, rreg); break;
2073 case lir_mul: __ mul(res, lreg, rreg); break;
2074 default: ShouldNotReachHere();
2075 }
2076 }
2077 #else // AARCH64
2078 Register res_lo = dest->as_register_lo();
2079 Register res_hi = dest->as_register_hi();
2080 Register lreg_lo = left->as_register_lo();
2081 Register lreg_hi = left->as_register_hi();
2082 if (right->is_double_cpu()) {
2083 Register rreg_lo = right->as_register_lo();
2084 Register rreg_hi = right->as_register_hi();
2085 if (res_lo == lreg_hi || res_lo == rreg_hi) {
2086 res_lo = Rtemp;
2087 }
2088 switch (code) {
2089 case lir_add:
2090 __ adds(res_lo, lreg_lo, rreg_lo);
2091 __ adc(res_hi, lreg_hi, rreg_hi);
2092 break;
2093 case lir_sub:
2094 __ subs(res_lo, lreg_lo, rreg_lo);
2095 __ sbc(res_hi, lreg_hi, rreg_hi);
2096 break;
2097 default:
2098 ShouldNotReachHere();
2099 }
2100 } else {
2101 assert(right->is_constant(), "must be");
2102 assert((right->as_constant_ptr()->as_jlong() >> 32) == 0, "out of range");
2103 const jint c = (jint) right->as_constant_ptr()->as_jlong();
2104 if (res_lo == lreg_hi) {
2105 res_lo = Rtemp;
2106 }
2107 switch (code) {
2108 case lir_add:
2109 __ adds(res_lo, lreg_lo, c);
2110 __ adc(res_hi, lreg_hi, 0);
2111 break;
2112 case lir_sub:
2113 __ subs(res_lo, lreg_lo, c);
2114 __ sbc(res_hi, lreg_hi, 0);
2115 break;
2116 default:
2117 ShouldNotReachHere();
2118 }
2119 }
2120 move_regs(res_lo, dest->as_register_lo());
2121 #endif // AARCH64
2122
2123 } else if (dest->is_single_fpu()) {
2124 assert(left->is_single_fpu(), "must be");
2125 assert(right->is_single_fpu(), "must be");
2126 const FloatRegister res = dest->as_float_reg();
2127 const FloatRegister lreg = left->as_float_reg();
2128 const FloatRegister rreg = right->as_float_reg();
2129 switch (code) {
2130 case lir_add: __ add_float(res, lreg, rreg); break;
2131 case lir_sub: __ sub_float(res, lreg, rreg); break;
2132 case lir_mul_strictfp: // fall through
2133 case lir_mul: __ mul_float(res, lreg, rreg); break;
2134 case lir_div_strictfp: // fall through
2135 case lir_div: __ div_float(res, lreg, rreg); break;
2136 default: ShouldNotReachHere();
2137 }
2138 } else if (dest->is_double_fpu()) {
2139 assert(left->is_double_fpu(), "must be");
2140 assert(right->is_double_fpu(), "must be");
2141 const FloatRegister res = dest->as_double_reg();
2142 const FloatRegister lreg = left->as_double_reg();
2143 const FloatRegister rreg = right->as_double_reg();
2144 switch (code) {
2145 case lir_add: __ add_double(res, lreg, rreg); break;
2146 case lir_sub: __ sub_double(res, lreg, rreg); break;
2147 case lir_mul_strictfp: // fall through
2148 case lir_mul: __ mul_double(res, lreg, rreg); break;
2149 case lir_div_strictfp: // fall through
2150 case lir_div: __ div_double(res, lreg, rreg); break;
2151 default: ShouldNotReachHere();
2152 }
2153 } else {
2154 ShouldNotReachHere();
2155 }
2156 }
2157
2158
2159 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
2160 switch (code) {
2161 case lir_abs:
2162 __ abs_double(dest->as_double_reg(), value->as_double_reg());
2163 break;
2164 case lir_sqrt:
2165 __ sqrt_double(dest->as_double_reg(), value->as_double_reg());
2166 break;
2167 default:
2168 ShouldNotReachHere();
2169 }
2170 }
2171
2172
2173 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) {
2174 assert(dest->is_register(), "wrong items state");
2175 assert(left->is_register(), "wrong items state");
2176
2177 if (dest->is_single_cpu()) {
2178 #ifdef AARCH64
2179 assert (dest->type() == T_INT, "unexpected result type");
2180 assert (left->type() == T_INT, "unexpected left type");
2181 assert (right->type() == T_INT, "unexpected right type");
2182 #endif // AARCH64
2183
2184 const Register res = dest->as_register();
2185 const Register lreg = left->as_register();
2186
2187 if (right->is_single_cpu()) {
2188 const Register rreg = right->as_register();
2189 switch (code) {
2190 case lir_logic_and: __ and_32(res, lreg, rreg); break;
2191 case lir_logic_or: __ orr_32(res, lreg, rreg); break;
2192 case lir_logic_xor: __ eor_32(res, lreg, rreg); break;
2193 default: ShouldNotReachHere();
2194 }
2195 } else {
2196 assert(right->is_constant(), "must be");
2197 const uint c = (uint)right->as_constant_ptr()->as_jint();
2198 switch (code) {
2199 case lir_logic_and: __ and_32(res, lreg, c); break;
2200 case lir_logic_or: __ orr_32(res, lreg, c); break;
2201 case lir_logic_xor: __ eor_32(res, lreg, c); break;
2202 default: ShouldNotReachHere();
2203 }
2204 }
2205 } else {
2206 assert(dest->is_double_cpu(), "should be");
2207 Register res_lo = dest->as_register_lo();
2208
2209 #ifdef AARCH64
2210 assert ((left->is_single_cpu() && left->is_oop_register()) || left->is_double_cpu(), "should be");
2211 const Register lreg_lo = left->as_pointer_register();
2212 #else
2213 assert (dest->type() == T_LONG, "unexpected result type");
2214 assert (left->type() == T_LONG, "unexpected left type");
2215 assert (right->type() == T_LONG, "unexpected right type");
2216
2217 const Register res_hi = dest->as_register_hi();
2218 const Register lreg_lo = left->as_register_lo();
2219 const Register lreg_hi = left->as_register_hi();
2220 #endif // AARCH64
2221
2222 if (right->is_register()) {
2223 #ifdef AARCH64
2224 assert ((right->is_single_cpu() && right->is_oop_register()) || right->is_double_cpu(), "should be");
2225 const Register rreg_lo = right->as_pointer_register();
2226 switch (code) {
2227 case lir_logic_and: __ andr(res_lo, lreg_lo, rreg_lo); break;
2228 case lir_logic_or: __ orr (res_lo, lreg_lo, rreg_lo); break;
2229 case lir_logic_xor: __ eor (res_lo, lreg_lo, rreg_lo); break;
2230 default: ShouldNotReachHere();
2231 }
2232 #else
2233 const Register rreg_lo = right->as_register_lo();
2234 const Register rreg_hi = right->as_register_hi();
2235 if (res_lo == lreg_hi || res_lo == rreg_hi) {
2236 res_lo = Rtemp; // Temp register helps to avoid overlap between result and input
2237 }
2238 switch (code) {
2239 case lir_logic_and:
2240 __ andr(res_lo, lreg_lo, rreg_lo);
2241 __ andr(res_hi, lreg_hi, rreg_hi);
2242 break;
2243 case lir_logic_or:
2244 __ orr(res_lo, lreg_lo, rreg_lo);
2245 __ orr(res_hi, lreg_hi, rreg_hi);
2246 break;
2247 case lir_logic_xor:
2248 __ eor(res_lo, lreg_lo, rreg_lo);
2249 __ eor(res_hi, lreg_hi, rreg_hi);
2250 break;
2251 default:
2252 ShouldNotReachHere();
2253 }
2254 move_regs(res_lo, dest->as_register_lo());
2255 #endif // AARCH64
2256 } else {
2257 assert(right->is_constant(), "must be");
2258 #ifdef AARCH64
2259 const julong c = (julong)right->as_constant_ptr()->as_jlong();
2260 Assembler::LogicalImmediate imm(c, false);
2261 if (imm.is_encoded()) {
2262 switch (code) {
2263 case lir_logic_and: __ andr(res_lo, lreg_lo, imm); break;
2264 case lir_logic_or: __ orr (res_lo, lreg_lo, imm); break;
2265 case lir_logic_xor: __ eor (res_lo, lreg_lo, imm); break;
2266 default: ShouldNotReachHere();
2267 }
2268 } else {
2269 BAILOUT("64 bit constant cannot be inlined");
2270 }
2271 #else
2272 const jint c_lo = (jint) right->as_constant_ptr()->as_jlong();
2273 const jint c_hi = (jint) (right->as_constant_ptr()->as_jlong() >> 32);
2274 // Case for logic_or from do_ClassIDIntrinsic()
2275 if (c_hi == 0 && AsmOperand::is_rotated_imm(c_lo)) {
2276 switch (code) {
2277 case lir_logic_and:
2278 __ andr(res_lo, lreg_lo, c_lo);
2279 __ mov(res_hi, 0);
2280 break;
2281 case lir_logic_or:
2282 __ orr(res_lo, lreg_lo, c_lo);
2283 break;
2284 case lir_logic_xor:
2285 __ eor(res_lo, lreg_lo, c_lo);
2286 break;
2287 default:
2288 ShouldNotReachHere();
2289 }
2290 } else if (code == lir_logic_and &&
2291 c_hi == -1 &&
2292 (AsmOperand::is_rotated_imm(c_lo) ||
2293 AsmOperand::is_rotated_imm(~c_lo))) {
2294 // Another case which handles logic_and from do_ClassIDIntrinsic()
2295 if (AsmOperand::is_rotated_imm(c_lo)) {
2296 __ andr(res_lo, lreg_lo, c_lo);
2297 } else {
2298 __ bic(res_lo, lreg_lo, ~c_lo);
2299 }
2300 if (res_hi != lreg_hi) {
2301 __ mov(res_hi, lreg_hi);
2302 }
2303 } else {
2304 BAILOUT("64 bit constant cannot be inlined");
2305 }
2306 #endif // AARCH64
2307 }
2308 }
2309 }
2310
2311
2312 #ifdef AARCH64
2313
2314 void LIR_Assembler::long_compare_helper(LIR_Opr opr1, LIR_Opr opr2) {
2315 assert(opr1->is_double_cpu(), "should be");
2316 Register x = opr1->as_register_lo();
2317
2318 if (opr2->is_double_cpu()) {
2319 Register y = opr2->as_register_lo();
2320 __ cmp(x, y);
2321
2322 } else {
2323 assert(opr2->is_constant(), "should be");
2324 assert(opr2->as_constant_ptr()->type() == T_LONG, "long constant expected");
2325 jlong c = opr2->as_jlong();
2326 assert(((c >> 31) == 0) || ((c >> 31) == -1), "immediate is out of range");
2327 if (c >= 0) {
2328 __ cmp(x, (jint)c);
2329 } else {
2330 __ cmn(x, (jint)(-c));
2331 }
2332 }
2333 }
2334
2335 #endif // AARCH64
2336
2337 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2338 if (opr1->is_single_cpu()) {
2339 if (opr2->is_constant()) {
2340 switch (opr2->as_constant_ptr()->type()) {
2341 case T_INT: {
2342 const jint c = opr2->as_constant_ptr()->as_jint();
2343 if (Assembler::is_arith_imm_in_range(c)) {
2344 __ cmp_32(opr1->as_register(), c);
2345 } else if (Assembler::is_arith_imm_in_range(-c)) {
2346 __ cmn_32(opr1->as_register(), -c);
2347 } else {
2348 // This can happen when compiling lookupswitch
2349 __ mov_slow(Rtemp, c);
2350 __ cmp_32(opr1->as_register(), Rtemp);
2351 }
2352 break;
2353 }
2354 case T_OBJECT:
2355 assert(opr2->as_constant_ptr()->as_jobject() == NULL, "cannot handle otherwise");
2356 __ cmp(opr1->as_register(), 0);
2357 break;
2358 default:
2359 ShouldNotReachHere();
2360 }
2361 } else if (opr2->is_single_cpu()) {
2362 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY || opr1->type() == T_METADATA || opr1->type() == T_ADDRESS) {
2363 assert(opr2->type() == T_OBJECT || opr2->type() == T_ARRAY || opr2->type() == T_METADATA || opr2->type() == T_ADDRESS, "incompatibe type");
2364 __ cmp(opr1->as_register(), opr2->as_register());
2365 } else {
2366 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY && opr2->type() != T_METADATA && opr2->type() != T_ADDRESS, "incompatibe type");
2367 __ cmp_32(opr1->as_register(), opr2->as_register());
2368 }
2369 } else {
2370 ShouldNotReachHere();
2371 }
2372 } else if (opr1->is_double_cpu()) {
2373 #ifdef AARCH64
2374 long_compare_helper(opr1, opr2);
2375 #else
2376 Register xlo = opr1->as_register_lo();
2377 Register xhi = opr1->as_register_hi();
2378 if (opr2->is_constant() && opr2->as_jlong() == 0) {
2379 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "cannot handle otherwise");
2380 __ orrs(Rtemp, xlo, xhi);
2381 } else if (opr2->is_register()) {
2382 Register ylo = opr2->as_register_lo();
2383 Register yhi = opr2->as_register_hi();
2384 if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
2385 __ teq(xhi, yhi);
2386 __ teq(xlo, ylo, eq);
2387 } else {
2388 __ subs(xlo, xlo, ylo);
2389 __ sbcs(xhi, xhi, yhi);
2390 }
2391 } else {
2392 ShouldNotReachHere();
2393 }
2394 #endif // AARCH64
2395 } else if (opr1->is_single_fpu()) {
2396 if (opr2->is_constant()) {
2397 assert(opr2->as_jfloat() == 0.0f, "cannot handle otherwise");
2398 __ cmp_zero_float(opr1->as_float_reg());
2399 } else {
2400 __ cmp_float(opr1->as_float_reg(), opr2->as_float_reg());
2401 }
2402 } else if (opr1->is_double_fpu()) {
2403 if (opr2->is_constant()) {
2404 assert(opr2->as_jdouble() == 0.0, "cannot handle otherwise");
2405 __ cmp_zero_double(opr1->as_double_reg());
2406 } else {
2407 __ cmp_double(opr1->as_double_reg(), opr2->as_double_reg());
2408 }
2409 } else {
2410 ShouldNotReachHere();
2411 }
2412 }
2413
2414 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2415 const Register res = dst->as_register();
2416 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2417 comp_op(lir_cond_unknown, left, right, op);
2418 #ifdef AARCH64
2419 if (code == lir_ucmp_fd2i) { // unordered is less
2420 __ cset(res, gt); // 1 if '>', else 0
2421 __ csinv(res, res, ZR, ge); // previous value if '>=', else -1
2422 } else {
2423 __ cset(res, hi); // 1 if '>' or unordered, else 0
2424 __ csinv(res, res, ZR, pl); // previous value if '>=' or unordered, else -1
2425 }
2426 #else
2427 __ fmstat();
2428 if (code == lir_ucmp_fd2i) { // unordered is less
2429 __ mvn(res, 0, lt);
2430 __ mov(res, 1, ge);
2431 } else { // unordered is greater
2432 __ mov(res, 1, cs);
2433 __ mvn(res, 0, cc);
2434 }
2435 __ mov(res, 0, eq);
2436 #endif // AARCH64
2437
2438 } else {
2439 assert(code == lir_cmp_l2i, "must be");
2440
2441 #ifdef AARCH64
2442 long_compare_helper(left, right);
2443
2444 __ cset(res, gt); // 1 if '>', else 0
2445 __ csinv(res, res, ZR, ge); // previous value if '>=', else -1
2446 #else
2447 Label done;
2448 const Register xlo = left->as_register_lo();
2449 const Register xhi = left->as_register_hi();
2450 const Register ylo = right->as_register_lo();
2451 const Register yhi = right->as_register_hi();
2452 __ cmp(xhi, yhi);
2453 __ mov(res, 1, gt);
2454 __ mvn(res, 0, lt);
2455 __ b(done, ne);
2456 __ subs(res, xlo, ylo);
2457 __ mov(res, 1, hi);
2458 __ mvn(res, 0, lo);
2459 __ bind(done);
2460 #endif // AARCH64
2461 }
2462 }
2463
2464
2465 void LIR_Assembler::align_call(LIR_Code code) {
2466 // Not needed
2467 }
2468
2469
2470 void LIR_Assembler::call(LIR_OpJavaCall *op, relocInfo::relocType rtype) {
2471 int ret_addr_offset = __ patchable_call(op->addr(), rtype);
2472 assert(ret_addr_offset == __ offset(), "embedded return address not allowed");
2473 add_call_info_here(op->info());
2474 }
2475
2476
2477 void LIR_Assembler::ic_call(LIR_OpJavaCall *op) {
2478 bool near_range = __ cache_fully_reachable();
2479 address oop_address = pc();
2480
2481 bool use_movw = AARCH64_ONLY(false) NOT_AARCH64(VM_Version::supports_movw());
2482
2483 // Ricklass may contain something that is not a metadata pointer so
2484 // mov_metadata can't be used
2485 InlinedAddress value((address)Universe::non_oop_word());
2486 InlinedAddress addr(op->addr());
2487 if (use_movw) {
2488 #ifdef AARCH64
2489 ShouldNotReachHere();
2490 #else
2491 __ movw(Ricklass, ((unsigned int)Universe::non_oop_word()) & 0xffff);
2492 __ movt(Ricklass, ((unsigned int)Universe::non_oop_word()) >> 16);
2493 #endif // AARCH64
2494 } else {
2495 // No movw/movt, must be load a pc relative value but no
2496 // relocation so no metadata table to load from.
2497 // Use a b instruction rather than a bl, inline constant after the
2498 // branch, use a PC relative ldr to load the constant, arrange for
2499 // the call to return after the constant(s).
2500 __ ldr_literal(Ricklass, value);
2501 }
2502 __ relocate(virtual_call_Relocation::spec(oop_address));
2503 if (near_range && use_movw) {
2504 __ bl(op->addr());
2505 } else {
2506 Label call_return;
2507 __ adr(LR, call_return);
2508 if (near_range) {
2509 __ b(op->addr());
2510 } else {
2511 __ indirect_jump(addr, Rtemp);
2512 __ bind_literal(addr);
2513 }
2514 if (!use_movw) {
2515 __ bind_literal(value);
2516 }
2517 __ bind(call_return);
2518 }
2519 add_call_info(code_offset(), op->info());
2520 }
2521
2522
2523 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2524 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2525 ShouldNotReachHere();
2526 }
2527
2528 void LIR_Assembler::emit_static_call_stub() {
2529 address call_pc = __ pc();
2530 address stub = __ start_a_stub(call_stub_size());
2531 if (stub == NULL) {
2532 BAILOUT("static call stub overflow");
2533 }
2534
2535 DEBUG_ONLY(int offset = code_offset();)
2536
2537 InlinedMetadata metadata_literal(NULL);
2538 __ relocate(static_stub_Relocation::spec(call_pc));
2539 // If not a single instruction, NativeMovConstReg::next_instruction_address()
2540 // must jump over the whole following ldr_literal.
2541 // (See CompiledStaticCall::set_to_interpreted())
2542 #ifdef ASSERT
2543 address ldr_site = __ pc();
2544 #endif
2545 __ ldr_literal(Rmethod, metadata_literal);
2546 assert(nativeMovConstReg_at(ldr_site)->next_instruction_address() == __ pc(), "Fix ldr_literal or its parsing");
2547 bool near_range = __ cache_fully_reachable();
2548 InlinedAddress dest((address)-1);
2549 if (near_range) {
2550 address branch_site = __ pc();
2551 __ b(branch_site); // b to self maps to special NativeJump -1 destination
2552 } else {
2553 __ indirect_jump(dest, Rtemp);
2554 }
2555 __ bind_literal(metadata_literal); // includes spec_for_immediate reloc
2556 if (!near_range) {
2557 __ bind_literal(dest); // special NativeJump -1 destination
2558 }
2559
2560 assert(code_offset() - offset <= call_stub_size(), "overflow");
2561 __ end_a_stub();
2562 }
2563
2564 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2565 assert(exceptionOop->as_register() == Rexception_obj, "must match");
2566 assert(exceptionPC->as_register() == Rexception_pc, "must match");
2567 info->add_register_oop(exceptionOop);
2568
2569 Runtime1::StubID handle_id = compilation()->has_fpu_code() ?
2570 Runtime1::handle_exception_id :
2571 Runtime1::handle_exception_nofpu_id;
2572 Label return_address;
2573 __ adr(Rexception_pc, return_address);
2574 __ call(Runtime1::entry_for(handle_id), relocInfo::runtime_call_type);
2575 __ bind(return_address);
2576 add_call_info_here(info); // for exception handler
2577 }
2578
2579 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2580 assert(exceptionOop->as_register() == Rexception_obj, "must match");
2581 __ b(_unwind_handler_entry);
2582 }
2583
2584 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2585 #ifdef AARCH64
2586 if (dest->is_single_cpu()) {
2587 Register res = dest->as_register();
2588 Register x = left->as_register();
2589 Register y = count->as_register();
2590 assert (dest->type() == T_INT, "unexpected result type");
2591 assert (left->type() == T_INT, "unexpected left type");
2592
2593 switch (code) {
2594 case lir_shl: __ lslv_w(res, x, y); break;
2595 case lir_shr: __ asrv_w(res, x, y); break;
2596 case lir_ushr: __ lsrv_w(res, x, y); break;
2597 default: ShouldNotReachHere();
2598 }
2599 } else if (dest->is_double_cpu()) {
2600 Register res = dest->as_register_lo();
2601 Register x = left->as_register_lo();
2602 Register y = count->as_register();
2603
2604 switch (code) {
2605 case lir_shl: __ lslv(res, x, y); break;
2606 case lir_shr: __ asrv(res, x, y); break;
2607 case lir_ushr: __ lsrv(res, x, y); break;
2608 default: ShouldNotReachHere();
2609 }
2610 } else {
2611 ShouldNotReachHere();
2612 }
2613 #else
2614 AsmShift shift = lsl;
2615 switch (code) {
2616 case lir_shl: shift = lsl; break;
2617 case lir_shr: shift = asr; break;
2618 case lir_ushr: shift = lsr; break;
2619 default: ShouldNotReachHere();
2620 }
2621
2622 if (dest->is_single_cpu()) {
2623 __ andr(Rtemp, count->as_register(), 31);
2624 __ mov(dest->as_register(), AsmOperand(left->as_register(), shift, Rtemp));
2625 } else if (dest->is_double_cpu()) {
2626 Register dest_lo = dest->as_register_lo();
2627 Register dest_hi = dest->as_register_hi();
2628 Register src_lo = left->as_register_lo();
2629 Register src_hi = left->as_register_hi();
2630 Register Rcount = count->as_register();
2631 // Resolve possible register conflicts
2632 if (shift == lsl && dest_hi == src_lo) {
2633 dest_hi = Rtemp;
2634 } else if (shift != lsl && dest_lo == src_hi) {
2635 dest_lo = Rtemp;
2636 } else if (dest_lo == src_lo && dest_hi == src_hi) {
2637 dest_lo = Rtemp;
2638 } else if (dest_lo == Rcount || dest_hi == Rcount) {
2639 Rcount = Rtemp;
2640 }
2641 __ andr(Rcount, count->as_register(), 63);
2642 __ long_shift(dest_lo, dest_hi, src_lo, src_hi, shift, Rcount);
2643 move_regs(dest_lo, dest->as_register_lo());
2644 move_regs(dest_hi, dest->as_register_hi());
2645 } else {
2646 ShouldNotReachHere();
2647 }
2648 #endif // AARCH64
2649 }
2650
2651
2652 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2653 #ifdef AARCH64
2654 if (dest->is_single_cpu()) {
2655 assert (dest->type() == T_INT, "unexpected result type");
2656 assert (left->type() == T_INT, "unexpected left type");
2657 count &= 31;
2658 if (count != 0) {
2659 switch (code) {
2660 case lir_shl: __ _lsl_w(dest->as_register(), left->as_register(), count); break;
2661 case lir_shr: __ _asr_w(dest->as_register(), left->as_register(), count); break;
2662 case lir_ushr: __ _lsr_w(dest->as_register(), left->as_register(), count); break;
2663 default: ShouldNotReachHere();
2664 }
2665 } else {
2666 move_regs(left->as_register(), dest->as_register());
2667 }
2668 } else if (dest->is_double_cpu()) {
2669 count &= 63;
2670 if (count != 0) {
2671 switch (code) {
2672 case lir_shl: __ _lsl(dest->as_register_lo(), left->as_register_lo(), count); break;
2673 case lir_shr: __ _asr(dest->as_register_lo(), left->as_register_lo(), count); break;
2674 case lir_ushr: __ _lsr(dest->as_register_lo(), left->as_register_lo(), count); break;
2675 default: ShouldNotReachHere();
2676 }
2677 } else {
2678 move_regs(left->as_register_lo(), dest->as_register_lo());
2679 }
2680 } else {
2681 ShouldNotReachHere();
2682 }
2683
2684 #else
2685 AsmShift shift = lsl;
2686 switch (code) {
2687 case lir_shl: shift = lsl; break;
2688 case lir_shr: shift = asr; break;
2689 case lir_ushr: shift = lsr; break;
2690 default: ShouldNotReachHere();
2691 }
2692
2693 if (dest->is_single_cpu()) {
2694 count &= 31;
2695 if (count != 0) {
2696 __ mov(dest->as_register(), AsmOperand(left->as_register(), shift, count));
2697 } else {
2698 move_regs(left->as_register(), dest->as_register());
2699 }
2700 } else if (dest->is_double_cpu()) {
2701 count &= 63;
2702 if (count != 0) {
2703 Register dest_lo = dest->as_register_lo();
2704 Register dest_hi = dest->as_register_hi();
2705 Register src_lo = left->as_register_lo();
2706 Register src_hi = left->as_register_hi();
2707 // Resolve possible register conflicts
2708 if (shift == lsl && dest_hi == src_lo) {
2709 dest_hi = Rtemp;
2710 } else if (shift != lsl && dest_lo == src_hi) {
2711 dest_lo = Rtemp;
2712 }
2713 __ long_shift(dest_lo, dest_hi, src_lo, src_hi, shift, count);
2714 move_regs(dest_lo, dest->as_register_lo());
2715 move_regs(dest_hi, dest->as_register_hi());
2716 } else {
2717 __ long_move(dest->as_register_lo(), dest->as_register_hi(),
2718 left->as_register_lo(), left->as_register_hi());
2719 }
2720 } else {
2721 ShouldNotReachHere();
2722 }
2723 #endif // AARCH64
2724 }
2725
2726
2727 // Saves 4 given registers in reserved argument area.
2728 void LIR_Assembler::save_in_reserved_area(Register r1, Register r2, Register r3, Register r4) {
2729 verify_reserved_argument_area_size(4);
2730 #ifdef AARCH64
2731 __ stp(r1, r2, Address(SP, 0));
2732 __ stp(r3, r4, Address(SP, 2*wordSize));
2733 #else
2734 __ stmia(SP, RegisterSet(r1) | RegisterSet(r2) | RegisterSet(r3) | RegisterSet(r4));
2735 #endif // AARCH64
2736 }
2737
2738 // Restores 4 given registers from reserved argument area.
2739 void LIR_Assembler::restore_from_reserved_area(Register r1, Register r2, Register r3, Register r4) {
2740 #ifdef AARCH64
2741 __ ldp(r1, r2, Address(SP, 0));
2742 __ ldp(r3, r4, Address(SP, 2*wordSize));
2743 #else
2744 __ ldmia(SP, RegisterSet(r1) | RegisterSet(r2) | RegisterSet(r3) | RegisterSet(r4), no_writeback);
2745 #endif // AARCH64
2746 }
2747
2748
2749 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2750 ciArrayKlass* default_type = op->expected_type();
2751 Register src = op->src()->as_register();
2752 Register src_pos = op->src_pos()->as_register();
2753 Register dst = op->dst()->as_register();
2754 Register dst_pos = op->dst_pos()->as_register();
2755 Register length = op->length()->as_register();
2756 Register tmp = op->tmp()->as_register();
2757 Register tmp2 = Rtemp;
2758
2759 assert(src == R0 && src_pos == R1 && dst == R2 && dst_pos == R3, "code assumption");
2760 #ifdef AARCH64
2761 assert(length == R4, "code assumption");
2762 #endif // AARCH64
2763
2764 __ resolve(ACCESS_READ, src);
2765 __ resolve(ACCESS_WRITE, dst);
2766
2767 CodeStub* stub = op->stub();
2768
2769 int flags = op->flags();
2770 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2771 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2772
2773 // If we don't know anything or it's an object array, just go through the generic arraycopy
2774 if (default_type == NULL) {
2775
2776 // save arguments, because they will be killed by a runtime call
2777 save_in_reserved_area(R0, R1, R2, R3);
2778
2779 #ifdef AARCH64
2780 // save length argument, will be killed by a runtime call
2781 __ raw_push(length, ZR);
2782 #else
2783 // pass length argument on SP[0]
2784 __ str(length, Address(SP, -2*wordSize, pre_indexed)); // 2 words for a proper stack alignment
2785 #endif // AARCH64
2786
2787 address copyfunc_addr = StubRoutines::generic_arraycopy();
2788 assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2789 #ifndef PRODUCT
2790 if (PrintC1Statistics) {
2791 __ inc_counter((address)&Runtime1::_generic_arraycopystub_cnt, tmp, tmp2);
2792 }
2793 #endif // !PRODUCT
2794 // the stub is in the code cache so close enough
2795 __ call(copyfunc_addr, relocInfo::runtime_call_type);
2796
2797 #ifdef AARCH64
2798 __ raw_pop(length, ZR);
2799 #else
2800 __ add(SP, SP, 2*wordSize);
2801 #endif // AARCH64
2802
2803 __ cbz_32(R0, *stub->continuation());
2804
2805 __ mvn_32(tmp, R0);
2806 restore_from_reserved_area(R0, R1, R2, R3); // load saved arguments in slow case only
2807 __ sub_32(length, length, tmp);
2808 __ add_32(src_pos, src_pos, tmp);
2809 __ add_32(dst_pos, dst_pos, tmp);
2810
2811 __ b(*stub->entry());
2812
2813 __ bind(*stub->continuation());
2814 return;
2815 }
2816
2817 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(),
2818 "must be true at this point");
2819 int elem_size = type2aelembytes(basic_type);
2820 int shift = exact_log2(elem_size);
2821
2822 // Check for NULL
2823 if (flags & LIR_OpArrayCopy::src_null_check) {
2824 if (flags & LIR_OpArrayCopy::dst_null_check) {
2825 __ cmp(src, 0);
2826 __ cond_cmp(dst, 0, ne); // make one instruction shorter if both checks are needed
2827 __ b(*stub->entry(), eq);
2828 } else {
2829 __ cbz(src, *stub->entry());
2830 }
2831 } else if (flags & LIR_OpArrayCopy::dst_null_check) {
2832 __ cbz(dst, *stub->entry());
2833 }
2834
2835 // If the compiler was not able to prove that exact type of the source or the destination
2836 // of the arraycopy is an array type, check at runtime if the source or the destination is
2837 // an instance type.
2838 if (flags & LIR_OpArrayCopy::type_check) {
2839 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2840 __ load_klass(tmp, dst);
2841 __ ldr_u32(tmp2, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2842 __ mov_slow(tmp, Klass::_lh_neutral_value);
2843 __ cmp_32(tmp2, tmp);
2844 __ b(*stub->entry(), ge);
2845 }
2846
2847 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2848 __ load_klass(tmp, src);
2849 __ ldr_u32(tmp2, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2850 __ mov_slow(tmp, Klass::_lh_neutral_value);
2851 __ cmp_32(tmp2, tmp);
2852 __ b(*stub->entry(), ge);
2853 }
2854 }
2855
2856 // Check if negative
2857 const int all_positive_checks = LIR_OpArrayCopy::src_pos_positive_check |
2858 LIR_OpArrayCopy::dst_pos_positive_check |
2859 LIR_OpArrayCopy::length_positive_check;
2860 switch (flags & all_positive_checks) {
2861 case LIR_OpArrayCopy::src_pos_positive_check:
2862 __ branch_if_negative_32(src_pos, *stub->entry());
2863 break;
2864 case LIR_OpArrayCopy::dst_pos_positive_check:
2865 __ branch_if_negative_32(dst_pos, *stub->entry());
2866 break;
2867 case LIR_OpArrayCopy::length_positive_check:
2868 __ branch_if_negative_32(length, *stub->entry());
2869 break;
2870 case LIR_OpArrayCopy::src_pos_positive_check | LIR_OpArrayCopy::dst_pos_positive_check:
2871 __ branch_if_any_negative_32(src_pos, dst_pos, tmp, *stub->entry());
2872 break;
2873 case LIR_OpArrayCopy::src_pos_positive_check | LIR_OpArrayCopy::length_positive_check:
2874 __ branch_if_any_negative_32(src_pos, length, tmp, *stub->entry());
2875 break;
2876 case LIR_OpArrayCopy::dst_pos_positive_check | LIR_OpArrayCopy::length_positive_check:
2877 __ branch_if_any_negative_32(dst_pos, length, tmp, *stub->entry());
2878 break;
2879 case all_positive_checks:
2880 __ branch_if_any_negative_32(src_pos, dst_pos, length, tmp, *stub->entry());
2881 break;
2882 default:
2883 assert((flags & all_positive_checks) == 0, "the last option");
2884 }
2885
2886 // Range checks
2887 if (flags & LIR_OpArrayCopy::src_range_check) {
2888 __ ldr_s32(tmp2, Address(src, arrayOopDesc::length_offset_in_bytes()));
2889 __ add_32(tmp, src_pos, length);
2890 __ cmp_32(tmp, tmp2);
2891 __ b(*stub->entry(), hi);
2892 }
2893 if (flags & LIR_OpArrayCopy::dst_range_check) {
2894 __ ldr_s32(tmp2, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2895 __ add_32(tmp, dst_pos, length);
2896 __ cmp_32(tmp, tmp2);
2897 __ b(*stub->entry(), hi);
2898 }
2899
2900 // Check if src and dst are of the same type
2901 if (flags & LIR_OpArrayCopy::type_check) {
2902 // We don't know the array types are compatible
2903 if (basic_type != T_OBJECT) {
2904 // Simple test for basic type arrays
2905 if (UseCompressedClassPointers) {
2906 // We don't need decode because we just need to compare
2907 __ ldr_u32(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
2908 __ ldr_u32(tmp2, Address(dst, oopDesc::klass_offset_in_bytes()));
2909 __ cmp_32(tmp, tmp2);
2910 } else {
2911 __ load_klass(tmp, src);
2912 __ load_klass(tmp2, dst);
2913 __ cmp(tmp, tmp2);
2914 }
2915 __ b(*stub->entry(), ne);
2916 } else {
2917 // For object arrays, if src is a sub class of dst then we can
2918 // safely do the copy.
2919 Label cont, slow;
2920
2921 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2922
2923 __ load_klass(tmp, src);
2924 __ load_klass(tmp2, dst);
2925
2926 // We are at a call so all live registers are saved before we
2927 // get here
2928 assert_different_registers(tmp, tmp2, R6, altFP_7_11);
2929
2930 __ check_klass_subtype_fast_path(tmp, tmp2, R6, altFP_7_11, &cont, copyfunc_addr == NULL ? stub->entry() : &slow, NULL);
2931
2932 __ mov(R6, R0);
2933 __ mov(altFP_7_11, R1);
2934 __ mov(R0, tmp);
2935 __ mov(R1, tmp2);
2936 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); // does not blow any registers except R0, LR and Rtemp
2937 __ cmp_32(R0, 0);
2938 __ mov(R0, R6);
2939 __ mov(R1, altFP_7_11);
2940
2941 if (copyfunc_addr != NULL) { // use stub if available
2942 // src is not a sub class of dst so we have to do a
2943 // per-element check.
2944
2945 __ b(cont, ne);
2946
2947 __ bind(slow);
2948
2949 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2950 if ((flags & mask) != mask) {
2951 // Check that at least both of them object arrays.
2952 assert(flags & mask, "one of the two should be known to be an object array");
2953
2954 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2955 __ load_klass(tmp, src);
2956 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2957 __ load_klass(tmp, dst);
2958 }
2959 int lh_offset = in_bytes(Klass::layout_helper_offset());
2960
2961 __ ldr_u32(tmp2, Address(tmp, lh_offset));
2962
2963 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2964 __ mov_slow(tmp, objArray_lh);
2965 __ cmp_32(tmp, tmp2);
2966 __ b(*stub->entry(), ne);
2967 }
2968
2969 save_in_reserved_area(R0, R1, R2, R3);
2970
2971 Register src_ptr = R0;
2972 Register dst_ptr = R1;
2973 Register len = R2;
2974 Register chk_off = R3;
2975 Register super_k = AARCH64_ONLY(R4) NOT_AARCH64(tmp);
2976
2977 __ add(src_ptr, src, arrayOopDesc::base_offset_in_bytes(basic_type));
2978 __ add_ptr_scaled_int32(src_ptr, src_ptr, src_pos, shift);
2979
2980 __ add(dst_ptr, dst, arrayOopDesc::base_offset_in_bytes(basic_type));
2981 __ add_ptr_scaled_int32(dst_ptr, dst_ptr, dst_pos, shift);
2982 __ load_klass(tmp, dst);
2983
2984 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2985 int sco_offset = in_bytes(Klass::super_check_offset_offset());
2986
2987 #ifdef AARCH64
2988 __ raw_push(length, ZR); // Preserve length around *copyfunc_addr call
2989
2990 __ mov(len, length);
2991 __ ldr(super_k, Address(tmp, ek_offset)); // super_k == R4 == length, so this load cannot be performed earlier
2992 // TODO-AARCH64: check whether it is faster to load super klass early by using tmp and additional mov.
2993 __ ldr_u32(chk_off, Address(super_k, sco_offset));
2994 #else // AARCH64
2995 __ ldr(super_k, Address(tmp, ek_offset));
2996
2997 __ mov(len, length);
2998 __ ldr_u32(chk_off, Address(super_k, sco_offset));
2999 __ push(super_k);
3000 #endif // AARCH64
3001
3002 __ call(copyfunc_addr, relocInfo::runtime_call_type);
3003
3004 #ifndef PRODUCT
3005 if (PrintC1Statistics) {
3006 Label failed;
3007 __ cbnz_32(R0, failed);
3008 __ inc_counter((address)&Runtime1::_arraycopy_checkcast_cnt, tmp, tmp2);
3009 __ bind(failed);
3010 }
3011 #endif // PRODUCT
3012
3013 #ifdef AARCH64
3014 __ raw_pop(length, ZR);
3015 #else
3016 __ add(SP, SP, wordSize); // Drop super_k argument
3017 #endif // AARCH64
3018
3019 __ cbz_32(R0, *stub->continuation());
3020 __ mvn_32(tmp, R0);
3021
3022 // load saved arguments in slow case only
3023 restore_from_reserved_area(R0, R1, R2, R3);
3024
3025 __ sub_32(length, length, tmp);
3026 __ add_32(src_pos, src_pos, tmp);
3027 __ add_32(dst_pos, dst_pos, tmp);
3028
3029 #ifndef PRODUCT
3030 if (PrintC1Statistics) {
3031 __ inc_counter((address)&Runtime1::_arraycopy_checkcast_attempt_cnt, tmp, tmp2);
3032 }
3033 #endif
3034
3035 __ b(*stub->entry());
3036
3037 __ bind(cont);
3038 } else {
3039 __ b(*stub->entry(), eq);
3040 __ bind(cont);
3041 }
3042 }
3043 }
3044
3045 #ifndef PRODUCT
3046 if (PrintC1Statistics) {
3047 address counter = Runtime1::arraycopy_count_address(basic_type);
3048 __ inc_counter(counter, tmp, tmp2);
3049 }
3050 #endif // !PRODUCT
3051
3052 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
3053 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
3054 const char *name;
3055 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
3056
3057 Register src_ptr = R0;
3058 Register dst_ptr = R1;
3059 Register len = R2;
3060
3061 __ add(src_ptr, src, arrayOopDesc::base_offset_in_bytes(basic_type));
3062 __ add_ptr_scaled_int32(src_ptr, src_ptr, src_pos, shift);
3063
3064 __ add(dst_ptr, dst, arrayOopDesc::base_offset_in_bytes(basic_type));
3065 __ add_ptr_scaled_int32(dst_ptr, dst_ptr, dst_pos, shift);
3066
3067 __ mov(len, length);
3068
3069 __ call(entry, relocInfo::runtime_call_type);
3070
3071 __ bind(*stub->continuation());
3072 }
3073
3074 #ifdef ASSERT
3075 // emit run-time assertion
3076 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3077 assert(op->code() == lir_assert, "must be");
3078
3079 #ifdef AARCH64
3080 __ NOT_IMPLEMENTED();
3081 #else
3082 if (op->in_opr1()->is_valid()) {
3083 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3084 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3085 } else {
3086 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3087 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3088 }
3089
3090 Label ok;
3091 if (op->condition() != lir_cond_always) {
3092 AsmCondition acond = al;
3093 switch (op->condition()) {
3094 case lir_cond_equal: acond = eq; break;
3095 case lir_cond_notEqual: acond = ne; break;
3096 case lir_cond_less: acond = lt; break;
3097 case lir_cond_lessEqual: acond = le; break;
3098 case lir_cond_greaterEqual: acond = ge; break;
3099 case lir_cond_greater: acond = gt; break;
3100 case lir_cond_aboveEqual: acond = hs; break;
3101 case lir_cond_belowEqual: acond = ls; break;
3102 default: ShouldNotReachHere();
3103 }
3104 __ b(ok, acond);
3105 }
3106 if (op->halt()) {
3107 const char* str = __ code_string(op->msg());
3108 __ stop(str);
3109 } else {
3110 breakpoint();
3111 }
3112 __ bind(ok);
3113 #endif // AARCH64
3114 }
3115 #endif // ASSERT
3116
3117 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3118 fatal("CRC32 intrinsic is not implemented on this platform");
3119 }
3120
3121 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
3122 Register obj = op->obj_opr()->as_pointer_register();
3123 Register hdr = op->hdr_opr()->as_pointer_register();
3124 Register lock = op->lock_opr()->as_pointer_register();
3125 Register tmp = op->scratch_opr()->is_illegal() ? noreg :
3126 op->scratch_opr()->as_pointer_register();
3127
3128 if (!UseFastLocking) {
3129 __ b(*op->stub()->entry());
3130 } else if (op->code() == lir_lock) {
3131 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3132 __ resolve(ACCESS_READ | ACCESS_WRITE, obj);
3133 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
3134 if (op->info() != NULL) {
3135 add_debug_info_for_null_check(null_check_offset, op->info());
3136 }
3137 } else if (op->code() == lir_unlock) {
3138 __ unlock_object(hdr, obj, lock, tmp, *op->stub()->entry());
3139 } else {
3140 ShouldNotReachHere();
3141 }
3142 __ bind(*op->stub()->continuation());
3143 }
3144
3145
3146 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3147 ciMethod* method = op->profiled_method();
3148 int bci = op->profiled_bci();
3149 ciMethod* callee = op->profiled_callee();
3150
3151 // Update counter for all call types
3152 ciMethodData* md = method->method_data_or_null();
3153 assert(md != NULL, "Sanity");
3154 ciProfileData* data = md->bci_to_data(bci);
3155 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
3156 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
3157 Register mdo = op->mdo()->as_register();
3158 assert(op->tmp1()->is_register(), "tmp1 must be allocated");
3159 Register tmp1 = op->tmp1()->as_pointer_register();
3160 assert_different_registers(mdo, tmp1);
3161 __ mov_metadata(mdo, md->constant_encoding());
3162 int mdo_offset_bias = 0;
3163 int max_offset = AARCH64_ONLY(4096 << LogBytesPerWord) NOT_AARCH64(4096);
3164 if (md->byte_offset_of_slot(data, CounterData::count_offset()) + data->size_in_bytes() >= max_offset) {
3165 // The offset is large so bias the mdo by the base of the slot so
3166 // that the ldr can use an immediate offset to reference the slots of the data
3167 mdo_offset_bias = md->byte_offset_of_slot(data, CounterData::count_offset());
3168 __ mov_slow(tmp1, mdo_offset_bias);
3169 __ add(mdo, mdo, tmp1);
3170 }
3171
3172 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
3173 // Perform additional virtual call profiling for invokevirtual and
3174 // invokeinterface bytecodes
3175 if (op->should_profile_receiver_type()) {
3176 assert(op->recv()->is_single_cpu(), "recv must be allocated");
3177 Register recv = op->recv()->as_register();
3178 assert_different_registers(mdo, tmp1, recv);
3179 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3180 ciKlass* known_klass = op->known_holder();
3181 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
3182 // We know the type that will be seen at this call site; we can
3183 // statically update the MethodData* rather than needing to do
3184 // dynamic tests on the receiver type
3185
3186 // NOTE: we should probably put a lock around this search to
3187 // avoid collisions by concurrent compilations
3188 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3189 uint i;
3190 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3191 ciKlass* receiver = vc_data->receiver(i);
3192 if (known_klass->equals(receiver)) {
3193 Address data_addr(mdo, md->byte_offset_of_slot(data,
3194 VirtualCallData::receiver_count_offset(i)) -
3195 mdo_offset_bias);
3196 __ ldr(tmp1, data_addr);
3197 __ add(tmp1, tmp1, DataLayout::counter_increment);
3198 __ str(tmp1, data_addr);
3199 return;
3200 }
3201 }
3202
3203 // Receiver type not found in profile data; select an empty slot
3204
3205 // Note that this is less efficient than it should be because it
3206 // always does a write to the receiver part of the
3207 // VirtualCallData rather than just the first time
3208 for (i = 0; i < VirtualCallData::row_limit(); i++) {
3209 ciKlass* receiver = vc_data->receiver(i);
3210 if (receiver == NULL) {
3211 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)) -
3212 mdo_offset_bias);
3213 __ mov_metadata(tmp1, known_klass->constant_encoding());
3214 __ str(tmp1, recv_addr);
3215 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)) -
3216 mdo_offset_bias);
3217 __ ldr(tmp1, data_addr);
3218 __ add(tmp1, tmp1, DataLayout::counter_increment);
3219 __ str(tmp1, data_addr);
3220 return;
3221 }
3222 }
3223 } else {
3224 __ load_klass(recv, recv);
3225 Label update_done;
3226 type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &update_done);
3227 // Receiver did not match any saved receiver and there is no empty row for it.
3228 // Increment total counter to indicate polymorphic case.
3229 __ ldr(tmp1, counter_addr);
3230 __ add(tmp1, tmp1, DataLayout::counter_increment);
3231 __ str(tmp1, counter_addr);
3232
3233 __ bind(update_done);
3234 }
3235 } else {
3236 // Static call
3237 __ ldr(tmp1, counter_addr);
3238 __ add(tmp1, tmp1, DataLayout::counter_increment);
3239 __ str(tmp1, counter_addr);
3240 }
3241 }
3242
3243 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3244 fatal("Type profiling not implemented on this platform");
3245 }
3246
3247 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3248 Unimplemented();
3249 }
3250
3251
3252 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3253 Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no);
3254 __ add_slow(dst->as_pointer_register(), mon_addr.base(), mon_addr.disp());
3255 }
3256
3257
3258 void LIR_Assembler::align_backward_branch_target() {
3259 // TODO-AARCH64 review it
3260 // Some ARM processors do better with 8-byte branch target alignment
3261 __ align(8);
3262 }
3263
3264
3265 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
3266
3267 if (left->is_single_cpu()) {
3268 assert (dest->type() == T_INT, "unexpected result type");
3269 assert (left->type() == T_INT, "unexpected left type");
3270 __ neg_32(dest->as_register(), left->as_register());
3271 } else if (left->is_double_cpu()) {
3272 #ifdef AARCH64
3273 __ neg(dest->as_register_lo(), left->as_register_lo());
3274 #else
3275 Register dest_lo = dest->as_register_lo();
3276 Register dest_hi = dest->as_register_hi();
3277 Register src_lo = left->as_register_lo();
3278 Register src_hi = left->as_register_hi();
3279 if (dest_lo == src_hi) {
3280 dest_lo = Rtemp;
3281 }
3282 __ rsbs(dest_lo, src_lo, 0);
3283 __ rsc(dest_hi, src_hi, 0);
3284 move_regs(dest_lo, dest->as_register_lo());
3285 #endif // AARCH64
3286 } else if (left->is_single_fpu()) {
3287 __ neg_float(dest->as_float_reg(), left->as_float_reg());
3288 } else if (left->is_double_fpu()) {
3289 __ neg_double(dest->as_double_reg(), left->as_double_reg());
3290 } else {
3291 ShouldNotReachHere();
3292 }
3293 }
3294
3295
3296 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3297 assert(patch_code == lir_patch_none, "Patch code not supported");
3298 LIR_Address* addr = addr_opr->as_address_ptr();
3299 if (addr->index()->is_illegal()) {
3300 jint c = addr->disp();
3301 if (!Assembler::is_arith_imm_in_range(c)) {
3302 BAILOUT("illegal arithmetic operand");
3303 }
3304 __ add(dest->as_pointer_register(), addr->base()->as_pointer_register(), c);
3305 } else {
3306 assert(addr->disp() == 0, "cannot handle otherwise");
3307 #ifdef AARCH64
3308 assert(addr->index()->is_double_cpu(), "should be");
3309 #endif // AARCH64
3310 __ add(dest->as_pointer_register(), addr->base()->as_pointer_register(),
3311 AsmOperand(addr->index()->as_pointer_register(), lsl, addr->scale()));
3312 }
3313 }
3314
3315
3316 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3317 assert(!tmp->is_valid(), "don't need temporary");
3318 __ call(dest);
3319 if (info != NULL) {
3320 add_call_info_here(info);
3321 }
3322 }
3323
3324
3325 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3326 #ifdef AARCH64
3327 Unimplemented(); // TODO-AARCH64: Use stlr/ldar instructions for volatile load/store
3328 #else
3329 assert(src->is_double_cpu() && dest->is_address() ||
3330 src->is_address() && dest->is_double_cpu(),
3331 "Simple move_op is called for all other cases");
3332
3333 int null_check_offset;
3334 if (dest->is_address()) {
3335 // Store
3336 const LIR_Address* addr = dest->as_address_ptr();
3337 const Register src_lo = src->as_register_lo();
3338 const Register src_hi = src->as_register_hi();
3339 assert(addr->index()->is_illegal() && addr->disp() == 0, "The address is simple already");
3340
3341 if (src_lo < src_hi) {
3342 null_check_offset = __ offset();
3343 __ stmia(addr->base()->as_register(), RegisterSet(src_lo) | RegisterSet(src_hi));
3344 } else {
3345 assert(src_lo < Rtemp, "Rtemp is higher than any allocatable register");
3346 __ mov(Rtemp, src_hi);
3347 null_check_offset = __ offset();
3348 __ stmia(addr->base()->as_register(), RegisterSet(src_lo) | RegisterSet(Rtemp));
3349 }
3350 } else {
3351 // Load
3352 const LIR_Address* addr = src->as_address_ptr();
3353 const Register dest_lo = dest->as_register_lo();
3354 const Register dest_hi = dest->as_register_hi();
3355 assert(addr->index()->is_illegal() && addr->disp() == 0, "The address is simple already");
3356
3357 null_check_offset = __ offset();
3358 if (dest_lo < dest_hi) {
3359 __ ldmia(addr->base()->as_register(), RegisterSet(dest_lo) | RegisterSet(dest_hi));
3360 } else {
3361 assert(dest_lo < Rtemp, "Rtemp is higher than any allocatable register");
3362 __ ldmia(addr->base()->as_register(), RegisterSet(dest_lo) | RegisterSet(Rtemp));
3363 __ mov(dest_hi, Rtemp);
3364 }
3365 }
3366
3367 if (info != NULL) {
3368 add_debug_info_for_null_check(null_check_offset, info);
3369 }
3370 #endif // AARCH64
3371 }
3372
3373
3374 void LIR_Assembler::membar() {
3375 __ membar(MacroAssembler::StoreLoad, Rtemp);
3376 }
3377
3378 void LIR_Assembler::membar_acquire() {
3379 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp);
3380 }
3381
3382 void LIR_Assembler::membar_release() {
3383 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp);
3384 }
3385
3386 void LIR_Assembler::membar_loadload() {
3387 __ membar(MacroAssembler::LoadLoad, Rtemp);
3388 }
3389
3390 void LIR_Assembler::membar_storestore() {
3391 __ membar(MacroAssembler::StoreStore, Rtemp);
3392 }
3393
3394 void LIR_Assembler::membar_loadstore() {
3395 __ membar(MacroAssembler::LoadStore, Rtemp);
3396 }
3397
3398 void LIR_Assembler::membar_storeload() {
3399 __ membar(MacroAssembler::StoreLoad, Rtemp);
3400 }
3401
3402 void LIR_Assembler::on_spin_wait() {
3403 Unimplemented();
3404 }
3405
3406 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3407 // Not used on ARM
3408 Unimplemented();
3409 }
3410
3411 void LIR_Assembler::peephole(LIR_List* lir) {
3412 #ifdef AARCH64
3413 return; // TODO-AARCH64 implement peephole optimizations
3414 #endif
3415 LIR_OpList* inst = lir->instructions_list();
3416 const int inst_length = inst->length();
3417 for (int i = 0; i < inst_length; i++) {
3418 LIR_Op* op = inst->at(i);
3419 switch (op->code()) {
3420 case lir_cmp: {
3421 // Replace:
3422 // cmp rX, y
3423 // cmove [EQ] y, z, rX
3424 // with
3425 // cmp rX, y
3426 // cmove [EQ] illegalOpr, z, rX
3427 //
3428 // or
3429 // cmp rX, y
3430 // cmove [NE] z, y, rX
3431 // with
3432 // cmp rX, y
3433 // cmove [NE] z, illegalOpr, rX
3434 //
3435 // moves from illegalOpr should be removed when converting LIR to native assembly
3436
3437 LIR_Op2* cmp = op->as_Op2();
3438 assert(cmp != NULL, "cmp LIR instruction is not an op2");
3439
3440 if (i + 1 < inst_length) {
3441 LIR_Op2* cmove = inst->at(i + 1)->as_Op2();
3442 if (cmove != NULL && cmove->code() == lir_cmove) {
3443 LIR_Opr cmove_res = cmove->result_opr();
3444 bool res_is_op1 = cmove_res == cmp->in_opr1();
3445 bool res_is_op2 = cmove_res == cmp->in_opr2();
3446 LIR_Opr cmp_res, cmp_arg;
3447 if (res_is_op1) {
3448 cmp_res = cmp->in_opr1();
3449 cmp_arg = cmp->in_opr2();
3450 } else if (res_is_op2) {
3451 cmp_res = cmp->in_opr2();
3452 cmp_arg = cmp->in_opr1();
3453 } else {
3454 cmp_res = LIR_OprFact::illegalOpr;
3455 cmp_arg = LIR_OprFact::illegalOpr;
3456 }
3457
3458 if (cmp_res != LIR_OprFact::illegalOpr) {
3459 LIR_Condition cond = cmove->condition();
3460 if (cond == lir_cond_equal && cmove->in_opr1() == cmp_arg) {
3461 cmove->set_in_opr1(LIR_OprFact::illegalOpr);
3462 } else if (cond == lir_cond_notEqual && cmove->in_opr2() == cmp_arg) {
3463 cmove->set_in_opr2(LIR_OprFact::illegalOpr);
3464 }
3465 }
3466 }
3467 }
3468 break;
3469 }
3470
3471 default:
3472 break;
3473 }
3474 }
3475 }
3476
3477 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3478 #ifdef AARCH64
3479 Register ptr = src->as_pointer_register();
3480 #else
3481 assert(src->is_address(), "sanity");
3482 Address addr = as_Address(src->as_address_ptr());
3483 #endif
3484
3485 if (code == lir_xchg) {
3486 #ifdef AARCH64
3487 if (UseCompressedOops && data->is_oop()) {
3488 __ encode_heap_oop(tmp->as_pointer_register(), data->as_register());
3489 }
3490 #endif // AARCH64
3491 } else {
3492 assert (!data->is_oop(), "xadd for oops");
3493 }
3494
3495 #ifndef AARCH64
3496 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp);
3497 #endif // !AARCH64
3498
3499 Label retry;
3500 __ bind(retry);
3501
3502 if ((data->type() == T_INT) || (data->is_oop() AARCH64_ONLY(&& UseCompressedOops))) {
3503 Register dst = dest->as_register();
3504 Register new_val = noreg;
3505 #ifdef AARCH64
3506 __ ldaxr_w(dst, ptr);
3507 #else
3508 __ ldrex(dst, addr);
3509 #endif
3510 if (code == lir_xadd) {
3511 Register tmp_reg = tmp->as_register();
3512 if (data->is_constant()) {
3513 assert_different_registers(dst, tmp_reg);
3514 __ add_32(tmp_reg, dst, data->as_constant_ptr()->as_jint());
3515 } else {
3516 assert_different_registers(dst, tmp_reg, data->as_register());
3517 __ add_32(tmp_reg, dst, data->as_register());
3518 }
3519 new_val = tmp_reg;
3520 } else {
3521 if (UseCompressedOops && data->is_oop()) {
3522 new_val = tmp->as_pointer_register();
3523 } else {
3524 new_val = data->as_register();
3525 }
3526 assert_different_registers(dst, new_val);
3527 }
3528 #ifdef AARCH64
3529 __ stlxr_w(Rtemp, new_val, ptr);
3530 #else
3531 __ strex(Rtemp, new_val, addr);
3532 #endif // AARCH64
3533
3534 #ifdef AARCH64
3535 } else if ((data->type() == T_LONG) || (data->is_oop() && !UseCompressedOops)) {
3536 Register dst = dest->as_pointer_register();
3537 Register new_val = noreg;
3538 __ ldaxr(dst, ptr);
3539 if (code == lir_xadd) {
3540 Register tmp_reg = tmp->as_pointer_register();
3541 if (data->is_constant()) {
3542 assert_different_registers(dst, ptr, tmp_reg);
3543 jlong c = data->as_constant_ptr()->as_jlong();
3544 assert((jlong)((jint)c) == c, "overflow");
3545 __ add(tmp_reg, dst, (jint)c);
3546 } else {
3547 assert_different_registers(dst, ptr, tmp_reg, data->as_pointer_register());
3548 __ add(tmp_reg, dst, data->as_pointer_register());
3549 }
3550 new_val = tmp_reg;
3551 } else {
3552 new_val = data->as_pointer_register();
3553 assert_different_registers(dst, ptr, new_val);
3554 }
3555 __ stlxr(Rtemp, new_val, ptr);
3556 #else
3557 } else if (data->type() == T_LONG) {
3558 Register dst_lo = dest->as_register_lo();
3559 Register new_val_lo = noreg;
3560 Register dst_hi = dest->as_register_hi();
3561
3562 assert(dst_hi->encoding() == dst_lo->encoding() + 1, "non aligned register pair");
3563 assert((dst_lo->encoding() & 0x1) == 0, "misaligned register pair");
3564
3565 __ bind(retry);
3566 __ ldrexd(dst_lo, addr);
3567 if (code == lir_xadd) {
3568 Register tmp_lo = tmp->as_register_lo();
3569 Register tmp_hi = tmp->as_register_hi();
3570
3571 assert(tmp_hi->encoding() == tmp_lo->encoding() + 1, "non aligned register pair");
3572 assert((tmp_lo->encoding() & 0x1) == 0, "misaligned register pair");
3573
3574 if (data->is_constant()) {
3575 jlong c = data->as_constant_ptr()->as_jlong();
3576 assert((jlong)((jint)c) == c, "overflow");
3577 assert_different_registers(dst_lo, dst_hi, tmp_lo, tmp_hi);
3578 __ adds(tmp_lo, dst_lo, (jint)c);
3579 __ adc(tmp_hi, dst_hi, 0);
3580 } else {
3581 Register new_val_lo = data->as_register_lo();
3582 Register new_val_hi = data->as_register_hi();
3583 __ adds(tmp_lo, dst_lo, new_val_lo);
3584 __ adc(tmp_hi, dst_hi, new_val_hi);
3585 assert_different_registers(dst_lo, dst_hi, tmp_lo, tmp_hi, new_val_lo, new_val_hi);
3586 }
3587 new_val_lo = tmp_lo;
3588 } else {
3589 new_val_lo = data->as_register_lo();
3590 Register new_val_hi = data->as_register_hi();
3591
3592 assert_different_registers(dst_lo, dst_hi, new_val_lo, new_val_hi);
3593 assert(new_val_hi->encoding() == new_val_lo->encoding() + 1, "non aligned register pair");
3594 assert((new_val_lo->encoding() & 0x1) == 0, "misaligned register pair");
3595 }
3596 __ strexd(Rtemp, new_val_lo, addr);
3597 #endif // AARCH64
3598 } else {
3599 ShouldNotReachHere();
3600 }
3601
3602 __ cbnz_32(Rtemp, retry);
3603 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp);
3604
3605 #ifdef AARCH64
3606 if (UseCompressedOops && data->is_oop()) {
3607 __ decode_heap_oop(dest->as_register());
3608 }
3609 #endif // AARCH64
3610 }
3611
3612 #undef __