1 /*
2 * Copyright (c) 2008, 2019, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_Compilation.hpp"
28 #include "c1/c1_LIRAssembler.hpp"
29 #include "c1/c1_MacroAssembler.hpp"
30 #include "c1/c1_Runtime1.hpp"
31 #include "c1/c1_ValueStack.hpp"
32 #include "ci/ciArrayKlass.hpp"
33 #include "ci/ciInstance.hpp"
34 #include "gc/shared/barrierSet.hpp"
35 #include "gc/shared/cardTableBarrierSet.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "memory/universe.hpp"
38 #include "nativeInst_arm.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "runtime/frame.inline.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "vmreg_arm.inline.hpp"
43
44 #define __ _masm->
45
46 // Note: Rtemp usage is this file should not impact C2 and should be
47 // correct as long as it is not implicitly used in lower layers (the
48 // arm [macro]assembler) and used with care in the other C1 specific
49 // files.
50
51 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
52 ShouldNotCallThis(); // Not used on ARM
53 return false;
54 }
55
56
57 LIR_Opr LIR_Assembler::receiverOpr() {
58 // The first register in Java calling conventions
59 return FrameMap::R0_oop_opr;
60 }
61
62 LIR_Opr LIR_Assembler::osrBufferPointer() {
63 return FrameMap::as_pointer_opr(R0);
64 }
65
66 #ifndef PRODUCT
67 void LIR_Assembler::verify_reserved_argument_area_size(int args_count) {
68 assert(args_count * wordSize <= frame_map()->reserved_argument_area_size(), "not enough space for arguments");
69 }
70 #endif // !PRODUCT
71
72 void LIR_Assembler::store_parameter(jint c, int offset_from_sp_in_words) {
73 assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
74 int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
75 assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "not enough space");
76 __ mov_slow(Rtemp, c);
77 __ str(Rtemp, Address(SP, offset_from_sp_in_bytes));
78 }
79
80 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_sp_in_words) {
81 assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
82 int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
83 assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "not enough space");
84 __ mov_metadata(Rtemp, m);
85 __ str(Rtemp, Address(SP, offset_from_sp_in_bytes));
86 }
87
88 //--------------fpu register translations-----------------------
89
90
91 void LIR_Assembler::set_24bit_FPU() {
92 ShouldNotReachHere();
93 }
94
95 void LIR_Assembler::reset_FPU() {
96 ShouldNotReachHere();
97 }
98
99 void LIR_Assembler::fpop() {
100 Unimplemented();
101 }
102
103 void LIR_Assembler::fxch(int i) {
104 Unimplemented();
105 }
106
107 void LIR_Assembler::fld(int i) {
108 Unimplemented();
109 }
110
111 void LIR_Assembler::ffree(int i) {
112 Unimplemented();
113 }
114
115 void LIR_Assembler::breakpoint() {
116 __ breakpoint();
117 }
118
119 void LIR_Assembler::push(LIR_Opr opr) {
120 Unimplemented();
121 }
122
123 void LIR_Assembler::pop(LIR_Opr opr) {
124 Unimplemented();
125 }
126
127 //-------------------------------------------
128 Address LIR_Assembler::as_Address(LIR_Address* addr) {
129 Register base = addr->base()->as_pointer_register();
130
131
132 if (addr->index()->is_illegal() || addr->index()->is_constant()) {
133 int offset = addr->disp();
134 if (addr->index()->is_constant()) {
135 offset += addr->index()->as_constant_ptr()->as_jint() << addr->scale();
136 }
137
138 if ((offset <= -4096) || (offset >= 4096)) {
139 BAILOUT_("offset not in range", Address(base));
140 }
141
142 return Address(base, offset);
143
144 } else {
145 assert(addr->disp() == 0, "can't have both");
146 int scale = addr->scale();
147
148 assert(addr->index()->is_single_cpu(), "should be");
149 return scale >= 0 ? Address(base, addr->index()->as_register(), lsl, scale) :
150 Address(base, addr->index()->as_register(), lsr, -scale);
151 }
152 }
153
154 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
155 Address base = as_Address(addr);
156 assert(base.index() == noreg, "must be");
157 if (base.disp() + BytesPerWord >= 4096) { BAILOUT_("offset not in range", Address(base.base(),0)); }
158 return Address(base.base(), base.disp() + BytesPerWord);
159 }
160
161 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
162 return as_Address(addr);
163 }
164
165
166 void LIR_Assembler::osr_entry() {
167 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
168 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
169 ValueStack* entry_state = osr_entry->end()->state();
170 int number_of_locks = entry_state->locks_size();
171
172 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
173 Register OSR_buf = osrBufferPointer()->as_pointer_register();
174
175 assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
176 int monitor_offset = (method()->max_locals() + 2 * (number_of_locks - 1)) * BytesPerWord;
177 for (int i = 0; i < number_of_locks; i++) {
178 int slot_offset = monitor_offset - (i * 2 * BytesPerWord);
179 __ ldr(R1, Address(OSR_buf, slot_offset + 0*BytesPerWord));
180 __ ldr(R2, Address(OSR_buf, slot_offset + 1*BytesPerWord));
181 __ str(R1, frame_map()->address_for_monitor_lock(i));
182 __ str(R2, frame_map()->address_for_monitor_object(i));
183 }
184 }
185
186
187 int LIR_Assembler::check_icache() {
188 Register receiver = LIR_Assembler::receiverOpr()->as_register();
189 int offset = __ offset();
190 __ inline_cache_check(receiver, Ricklass);
191 return offset;
192 }
193
194 void LIR_Assembler::clinit_barrier(ciMethod* method) {
195 ShouldNotReachHere(); // not implemented
196 }
197
198 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
199 jobject o = (jobject)Universe::non_oop_word();
200 int index = __ oop_recorder()->allocate_oop_index(o);
201
202 PatchingStub* patch = new PatchingStub(_masm, patching_id(info), index);
203
204 __ patchable_mov_oop(reg, o, index);
205 patching_epilog(patch, lir_patch_normal, reg, info);
206 }
207
208
209 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
210 Metadata* o = (Metadata*)Universe::non_oop_word();
211 int index = __ oop_recorder()->allocate_metadata_index(o);
212 PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index);
213
214 __ patchable_mov_metadata(reg, o, index);
215 patching_epilog(patch, lir_patch_normal, reg, info);
216 }
217
218
219 int LIR_Assembler::initial_frame_size_in_bytes() const {
220 // Subtracts two words to account for return address and link
221 return frame_map()->framesize()*VMRegImpl::stack_slot_size - 2*wordSize;
222 }
223
224
225 int LIR_Assembler::emit_exception_handler() {
226 // TODO: ARM
227 __ nop(); // See comments in other ports
228
229 address handler_base = __ start_a_stub(exception_handler_size());
230 if (handler_base == NULL) {
231 bailout("exception handler overflow");
232 return -1;
233 }
234
235 int offset = code_offset();
236
237 // check that there is really an exception
238 __ verify_not_null_oop(Rexception_obj);
239
240 __ call(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id), relocInfo::runtime_call_type);
241 __ should_not_reach_here();
242
243 assert(code_offset() - offset <= exception_handler_size(), "overflow");
244 __ end_a_stub();
245
246 return offset;
247 }
248
249 // Emit the code to remove the frame from the stack in the exception
250 // unwind path.
251 int LIR_Assembler::emit_unwind_handler() {
252 #ifndef PRODUCT
253 if (CommentedAssembly) {
254 _masm->block_comment("Unwind handler");
255 }
256 #endif
257
258 int offset = code_offset();
259
260 // Fetch the exception from TLS and clear out exception related thread state
261 Register zero = __ zero_register(Rtemp);
262 __ ldr(Rexception_obj, Address(Rthread, JavaThread::exception_oop_offset()));
263 __ str(zero, Address(Rthread, JavaThread::exception_oop_offset()));
264 __ str(zero, Address(Rthread, JavaThread::exception_pc_offset()));
265
266 __ bind(_unwind_handler_entry);
267 __ verify_not_null_oop(Rexception_obj);
268
269 // Preform needed unlocking
270 MonitorExitStub* stub = NULL;
271 if (method()->is_synchronized()) {
272 monitor_address(0, FrameMap::R0_opr);
273 stub = new MonitorExitStub(FrameMap::R0_opr, true, 0);
274 __ unlock_object(R2, R1, R0, Rtemp, *stub->entry());
275 __ bind(*stub->continuation());
276 }
277
278 // remove the activation and dispatch to the unwind handler
279 __ remove_frame(initial_frame_size_in_bytes()); // restores FP and LR
280 __ jump(Runtime1::entry_for(Runtime1::unwind_exception_id), relocInfo::runtime_call_type, Rtemp);
281
282 // Emit the slow path assembly
283 if (stub != NULL) {
284 stub->emit_code(this);
285 }
286
287 return offset;
288 }
289
290
291 int LIR_Assembler::emit_deopt_handler() {
292 address handler_base = __ start_a_stub(deopt_handler_size());
293 if (handler_base == NULL) {
294 bailout("deopt handler overflow");
295 return -1;
296 }
297
298 int offset = code_offset();
299
300 __ mov_relative_address(LR, __ pc());
301 __ push(LR); // stub expects LR to be saved
302 __ jump(SharedRuntime::deopt_blob()->unpack(), relocInfo::runtime_call_type, noreg);
303
304 assert(code_offset() - offset <= deopt_handler_size(), "overflow");
305 __ end_a_stub();
306
307 return offset;
308 }
309
310
311 void LIR_Assembler::return_op(LIR_Opr result) {
312 // Pop the frame before safepoint polling
313 __ remove_frame(initial_frame_size_in_bytes());
314
315 // mov_slow here is usually one or two instruction
316 __ mov_address(Rtemp, os::get_polling_page());
317 __ relocate(relocInfo::poll_return_type);
318 __ ldr(Rtemp, Address(Rtemp));
319 __ ret();
320 }
321
322
323 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
324 __ mov_address(Rtemp, os::get_polling_page());
325 if (info != NULL) {
326 add_debug_info_for_branch(info);
327 }
328 int offset = __ offset();
329 __ relocate(relocInfo::poll_type);
330 __ ldr(Rtemp, Address(Rtemp));
331 return offset;
332 }
333
334
335 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
336 if (from_reg != to_reg) {
337 __ mov(to_reg, from_reg);
338 }
339 }
340
341 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
342 assert(src->is_constant() && dest->is_register(), "must be");
343 LIR_Const* c = src->as_constant_ptr();
344
345 switch (c->type()) {
346 case T_ADDRESS:
347 case T_INT:
348 assert(patch_code == lir_patch_none, "no patching handled here");
349 __ mov_slow(dest->as_register(), c->as_jint());
350 break;
351
352 case T_LONG:
353 assert(patch_code == lir_patch_none, "no patching handled here");
354 __ mov_slow(dest->as_register_lo(), c->as_jint_lo());
355 __ mov_slow(dest->as_register_hi(), c->as_jint_hi());
356 break;
357
358 case T_OBJECT:
359 if (patch_code == lir_patch_none) {
360 __ mov_oop(dest->as_register(), c->as_jobject());
361 } else {
362 jobject2reg_with_patching(dest->as_register(), info);
363 }
364 break;
365
366 case T_METADATA:
367 if (patch_code == lir_patch_none) {
368 __ mov_metadata(dest->as_register(), c->as_metadata());
369 } else {
370 klass2reg_with_patching(dest->as_register(), info);
371 }
372 break;
373
374 case T_FLOAT:
375 if (dest->is_single_fpu()) {
376 __ mov_float(dest->as_float_reg(), c->as_jfloat());
377 } else {
378 // Simple getters can return float constant directly into r0
379 __ mov_slow(dest->as_register(), c->as_jint_bits());
380 }
381 break;
382
383 case T_DOUBLE:
384 if (dest->is_double_fpu()) {
385 __ mov_double(dest->as_double_reg(), c->as_jdouble());
386 } else {
387 // Simple getters can return double constant directly into r1r0
388 __ mov_slow(dest->as_register_lo(), c->as_jint_lo_bits());
389 __ mov_slow(dest->as_register_hi(), c->as_jint_hi_bits());
390 }
391 break;
392
393 default:
394 ShouldNotReachHere();
395 }
396 }
397
398 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
399 assert(src->is_constant(), "must be");
400 assert(dest->is_stack(), "must be");
401 LIR_Const* c = src->as_constant_ptr();
402
403 switch (c->type()) {
404 case T_INT: // fall through
405 case T_FLOAT:
406 __ mov_slow(Rtemp, c->as_jint_bits());
407 __ str_32(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
408 break;
409
410 case T_ADDRESS:
411 __ mov_slow(Rtemp, c->as_jint());
412 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
413 break;
414
415 case T_OBJECT:
416 __ mov_oop(Rtemp, c->as_jobject());
417 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
418 break;
419
420 case T_LONG: // fall through
421 case T_DOUBLE:
422 __ mov_slow(Rtemp, c->as_jint_lo_bits());
423 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes));
424 if (c->as_jint_hi_bits() != c->as_jint_lo_bits()) {
425 __ mov_slow(Rtemp, c->as_jint_hi_bits());
426 }
427 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes));
428 break;
429
430 default:
431 ShouldNotReachHere();
432 }
433 }
434
435 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type,
436 CodeEmitInfo* info, bool wide) {
437 assert((src->as_constant_ptr()->type() == T_OBJECT && src->as_constant_ptr()->as_jobject() == NULL),"cannot handle otherwise");
438 __ mov(Rtemp, 0);
439
440 int null_check_offset = code_offset();
441 __ str(Rtemp, as_Address(dest->as_address_ptr()));
442
443 if (info != NULL) {
444 assert(false, "arm32 didn't support this before, investigate if bug");
445 add_debug_info_for_null_check(null_check_offset, info);
446 }
447 }
448
449 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
450 assert(src->is_register() && dest->is_register(), "must be");
451
452 if (src->is_single_cpu()) {
453 if (dest->is_single_cpu()) {
454 move_regs(src->as_register(), dest->as_register());
455 } else if (dest->is_single_fpu()) {
456 __ fmsr(dest->as_float_reg(), src->as_register());
457 } else {
458 ShouldNotReachHere();
459 }
460 } else if (src->is_double_cpu()) {
461 if (dest->is_double_cpu()) {
462 __ long_move(dest->as_register_lo(), dest->as_register_hi(), src->as_register_lo(), src->as_register_hi());
463 } else {
464 __ fmdrr(dest->as_double_reg(), src->as_register_lo(), src->as_register_hi());
465 }
466 } else if (src->is_single_fpu()) {
467 if (dest->is_single_fpu()) {
468 __ mov_float(dest->as_float_reg(), src->as_float_reg());
469 } else if (dest->is_single_cpu()) {
470 __ mov_fpr2gpr_float(dest->as_register(), src->as_float_reg());
471 } else {
472 ShouldNotReachHere();
473 }
474 } else if (src->is_double_fpu()) {
475 if (dest->is_double_fpu()) {
476 __ mov_double(dest->as_double_reg(), src->as_double_reg());
477 } else if (dest->is_double_cpu()) {
478 __ fmrrd(dest->as_register_lo(), dest->as_register_hi(), src->as_double_reg());
479 } else {
480 ShouldNotReachHere();
481 }
482 } else {
483 ShouldNotReachHere();
484 }
485 }
486
487 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
488 assert(src->is_register(), "should not call otherwise");
489 assert(dest->is_stack(), "should not call otherwise");
490
491 Address addr = dest->is_single_word() ?
492 frame_map()->address_for_slot(dest->single_stack_ix()) :
493 frame_map()->address_for_slot(dest->double_stack_ix());
494
495 assert(lo_word_offset_in_bytes == 0 && hi_word_offset_in_bytes == 4, "little ending");
496 if (src->is_single_fpu() || src->is_double_fpu()) {
497 if (addr.disp() >= 1024) { BAILOUT("Too exotic case to handle here"); }
498 }
499
500 if (src->is_single_cpu()) {
501 switch (type) {
502 case T_OBJECT:
503 case T_ARRAY: __ verify_oop(src->as_register()); // fall through
504 case T_ADDRESS:
505 case T_METADATA: __ str(src->as_register(), addr); break;
506 case T_FLOAT: // used in intBitsToFloat intrinsic implementation, fall through
507 case T_INT: __ str_32(src->as_register(), addr); break;
508 default:
509 ShouldNotReachHere();
510 }
511 } else if (src->is_double_cpu()) {
512 __ str(src->as_register_lo(), addr);
513 __ str(src->as_register_hi(), frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes));
514 } else if (src->is_single_fpu()) {
515 __ str_float(src->as_float_reg(), addr);
516 } else if (src->is_double_fpu()) {
517 __ str_double(src->as_double_reg(), addr);
518 } else {
519 ShouldNotReachHere();
520 }
521 }
522
523
524 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type,
525 LIR_PatchCode patch_code, CodeEmitInfo* info,
526 bool pop_fpu_stack, bool wide,
527 bool unaligned) {
528 LIR_Address* to_addr = dest->as_address_ptr();
529 Register base_reg = to_addr->base()->as_pointer_register();
530 const bool needs_patching = (patch_code != lir_patch_none);
531
532 PatchingStub* patch = NULL;
533 if (needs_patching) {
534 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
535 }
536
537 int null_check_offset = code_offset();
538
539 switch (type) {
540 case T_ARRAY:
541 case T_OBJECT:
542 if (UseCompressedOops && !wide) {
543 ShouldNotReachHere();
544 } else {
545 __ str(src->as_register(), as_Address(to_addr));
546 }
547 break;
548
549 case T_ADDRESS:
550 __ str(src->as_pointer_register(), as_Address(to_addr));
551 break;
552
553 case T_BYTE:
554 case T_BOOLEAN:
555 __ strb(src->as_register(), as_Address(to_addr));
556 break;
557
558 case T_CHAR:
559 case T_SHORT:
560 __ strh(src->as_register(), as_Address(to_addr));
561 break;
562
563 case T_INT:
564 #ifdef __SOFTFP__
565 case T_FLOAT:
566 #endif // __SOFTFP__
567 __ str_32(src->as_register(), as_Address(to_addr));
568 break;
569
570
571 #ifdef __SOFTFP__
572 case T_DOUBLE:
573 #endif // __SOFTFP__
574 case T_LONG: {
575 Register from_lo = src->as_register_lo();
576 Register from_hi = src->as_register_hi();
577 if (to_addr->index()->is_register()) {
578 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
579 assert(to_addr->disp() == 0, "Not yet supporting both");
580 __ add(Rtemp, base_reg, to_addr->index()->as_register());
581 base_reg = Rtemp;
582 __ str(from_lo, Address(Rtemp));
583 if (patch != NULL) {
584 __ nop(); // see comment before patching_epilog for 2nd str
585 patching_epilog(patch, lir_patch_low, base_reg, info);
586 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
587 patch_code = lir_patch_high;
588 }
589 __ str(from_hi, Address(Rtemp, BytesPerWord));
590 } else if (base_reg == from_lo) {
591 __ str(from_hi, as_Address_hi(to_addr));
592 if (patch != NULL) {
593 __ nop(); // see comment before patching_epilog for 2nd str
594 patching_epilog(patch, lir_patch_high, base_reg, info);
595 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
596 patch_code = lir_patch_low;
597 }
598 __ str(from_lo, as_Address_lo(to_addr));
599 } else {
600 __ str(from_lo, as_Address_lo(to_addr));
601 if (patch != NULL) {
602 __ nop(); // see comment before patching_epilog for 2nd str
603 patching_epilog(patch, lir_patch_low, base_reg, info);
604 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
605 patch_code = lir_patch_high;
606 }
607 __ str(from_hi, as_Address_hi(to_addr));
608 }
609 break;
610 }
611
612 #ifndef __SOFTFP__
613 case T_FLOAT:
614 if (to_addr->index()->is_register()) {
615 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
616 __ add(Rtemp, base_reg, to_addr->index()->as_register());
617 if ((to_addr->disp() <= -4096) || (to_addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
618 __ fsts(src->as_float_reg(), Address(Rtemp, to_addr->disp()));
619 } else {
620 __ fsts(src->as_float_reg(), as_Address(to_addr));
621 }
622 break;
623
624 case T_DOUBLE:
625 if (to_addr->index()->is_register()) {
626 assert(to_addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
627 __ add(Rtemp, base_reg, to_addr->index()->as_register());
628 if ((to_addr->disp() <= -4096) || (to_addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
629 __ fstd(src->as_double_reg(), Address(Rtemp, to_addr->disp()));
630 } else {
631 __ fstd(src->as_double_reg(), as_Address(to_addr));
632 }
633 break;
634 #endif // __SOFTFP__
635
636
637 default:
638 ShouldNotReachHere();
639 }
640
641 if (info != NULL) {
642 add_debug_info_for_null_check(null_check_offset, info);
643 }
644
645 if (patch != NULL) {
646 // Offset embedded into LDR/STR instruction may appear not enough
647 // to address a field. So, provide a space for one more instruction
648 // that will deal with larger offsets.
649 __ nop();
650 patching_epilog(patch, patch_code, base_reg, info);
651 }
652 }
653
654
655 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
656 assert(src->is_stack(), "should not call otherwise");
657 assert(dest->is_register(), "should not call otherwise");
658
659 Address addr = src->is_single_word() ?
660 frame_map()->address_for_slot(src->single_stack_ix()) :
661 frame_map()->address_for_slot(src->double_stack_ix());
662
663 assert(lo_word_offset_in_bytes == 0 && hi_word_offset_in_bytes == 4, "little ending");
664 if (dest->is_single_fpu() || dest->is_double_fpu()) {
665 if (addr.disp() >= 1024) { BAILOUT("Too exotic case to handle here"); }
666 }
667
668 if (dest->is_single_cpu()) {
669 switch (type) {
670 case T_OBJECT:
671 case T_ARRAY:
672 case T_ADDRESS:
673 case T_METADATA: __ ldr(dest->as_register(), addr); break;
674 case T_FLOAT: // used in floatToRawIntBits intrinsic implemenation
675 case T_INT: __ ldr_u32(dest->as_register(), addr); break;
676 default:
677 ShouldNotReachHere();
678 }
679 if ((type == T_OBJECT) || (type == T_ARRAY)) {
680 __ verify_oop(dest->as_register());
681 }
682 } else if (dest->is_double_cpu()) {
683 __ ldr(dest->as_register_lo(), addr);
684 __ ldr(dest->as_register_hi(), frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes));
685 } else if (dest->is_single_fpu()) {
686 __ ldr_float(dest->as_float_reg(), addr);
687 } else if (dest->is_double_fpu()) {
688 __ ldr_double(dest->as_double_reg(), addr);
689 } else {
690 ShouldNotReachHere();
691 }
692 }
693
694
695 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
696 if (src->is_single_stack()) {
697 switch (src->type()) {
698 case T_OBJECT:
699 case T_ARRAY:
700 case T_ADDRESS:
701 case T_METADATA:
702 __ ldr(Rtemp, frame_map()->address_for_slot(src->single_stack_ix()));
703 __ str(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
704 break;
705
706 case T_INT:
707 case T_FLOAT:
708 __ ldr_u32(Rtemp, frame_map()->address_for_slot(src->single_stack_ix()));
709 __ str_32(Rtemp, frame_map()->address_for_slot(dest->single_stack_ix()));
710 break;
711
712 default:
713 ShouldNotReachHere();
714 }
715 } else {
716 assert(src->is_double_stack(), "must be");
717 __ ldr(Rtemp, frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes));
718 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes));
719 __ ldr(Rtemp, frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes));
720 __ str(Rtemp, frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes));
721 }
722 }
723
724
725 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type,
726 LIR_PatchCode patch_code, CodeEmitInfo* info,
727 bool wide, bool unaligned) {
728 assert(src->is_address(), "should not call otherwise");
729 assert(dest->is_register(), "should not call otherwise");
730 LIR_Address* addr = src->as_address_ptr();
731
732 Register base_reg = addr->base()->as_pointer_register();
733
734 PatchingStub* patch = NULL;
735 if (patch_code != lir_patch_none) {
736 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
737 }
738 if (info != NULL) {
739 add_debug_info_for_null_check_here(info);
740 }
741
742 switch (type) {
743 case T_OBJECT: // fall through
744 case T_ARRAY:
745 if (UseCompressedOops && !wide) {
746 __ ldr_u32(dest->as_register(), as_Address(addr));
747 } else {
748 __ ldr(dest->as_register(), as_Address(addr));
749 }
750 break;
751
752 case T_ADDRESS:
753 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
754 __ ldr_u32(dest->as_pointer_register(), as_Address(addr));
755 } else {
756 __ ldr(dest->as_pointer_register(), as_Address(addr));
757 }
758 break;
759
760 case T_INT:
761 #ifdef __SOFTFP__
762 case T_FLOAT:
763 #endif // __SOFTFP__
764 __ ldr(dest->as_pointer_register(), as_Address(addr));
765 break;
766
767 case T_BOOLEAN:
768 __ ldrb(dest->as_register(), as_Address(addr));
769 break;
770
771 case T_BYTE:
772 __ ldrsb(dest->as_register(), as_Address(addr));
773 break;
774
775 case T_CHAR:
776 __ ldrh(dest->as_register(), as_Address(addr));
777 break;
778
779 case T_SHORT:
780 __ ldrsh(dest->as_register(), as_Address(addr));
781 break;
782
783
784 #ifdef __SOFTFP__
785 case T_DOUBLE:
786 #endif // __SOFTFP__
787 case T_LONG: {
788 Register to_lo = dest->as_register_lo();
789 Register to_hi = dest->as_register_hi();
790 if (addr->index()->is_register()) {
791 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
792 assert(addr->disp() == 0, "Not yet supporting both");
793 __ add(Rtemp, base_reg, addr->index()->as_register());
794 base_reg = Rtemp;
795 __ ldr(to_lo, Address(Rtemp));
796 if (patch != NULL) {
797 __ nop(); // see comment before patching_epilog for 2nd ldr
798 patching_epilog(patch, lir_patch_low, base_reg, info);
799 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
800 patch_code = lir_patch_high;
801 }
802 __ ldr(to_hi, Address(Rtemp, BytesPerWord));
803 } else if (base_reg == to_lo) {
804 __ ldr(to_hi, as_Address_hi(addr));
805 if (patch != NULL) {
806 __ nop(); // see comment before patching_epilog for 2nd ldr
807 patching_epilog(patch, lir_patch_high, base_reg, info);
808 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
809 patch_code = lir_patch_low;
810 }
811 __ ldr(to_lo, as_Address_lo(addr));
812 } else {
813 __ ldr(to_lo, as_Address_lo(addr));
814 if (patch != NULL) {
815 __ nop(); // see comment before patching_epilog for 2nd ldr
816 patching_epilog(patch, lir_patch_low, base_reg, info);
817 patch = new PatchingStub(_masm, PatchingStub::access_field_id);
818 patch_code = lir_patch_high;
819 }
820 __ ldr(to_hi, as_Address_hi(addr));
821 }
822 break;
823 }
824
825 #ifndef __SOFTFP__
826 case T_FLOAT:
827 if (addr->index()->is_register()) {
828 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
829 __ add(Rtemp, base_reg, addr->index()->as_register());
830 if ((addr->disp() <= -4096) || (addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
831 __ flds(dest->as_float_reg(), Address(Rtemp, addr->disp()));
832 } else {
833 __ flds(dest->as_float_reg(), as_Address(addr));
834 }
835 break;
836
837 case T_DOUBLE:
838 if (addr->index()->is_register()) {
839 assert(addr->scale() == LIR_Address::times_1,"Unexpected scaled register");
840 __ add(Rtemp, base_reg, addr->index()->as_register());
841 if ((addr->disp() <= -4096) || (addr->disp() >= 4096)) { BAILOUT("offset not in range"); }
842 __ fldd(dest->as_double_reg(), Address(Rtemp, addr->disp()));
843 } else {
844 __ fldd(dest->as_double_reg(), as_Address(addr));
845 }
846 break;
847 #endif // __SOFTFP__
848
849
850 default:
851 ShouldNotReachHere();
852 }
853
854 if (patch != NULL) {
855 // Offset embedded into LDR/STR instruction may appear not enough
856 // to address a field. So, provide a space for one more instruction
857 // that will deal with larger offsets.
858 __ nop();
859 patching_epilog(patch, patch_code, base_reg, info);
860 }
861
862 }
863
864
865 void LIR_Assembler::emit_op3(LIR_Op3* op) {
866 bool is_32 = op->result_opr()->is_single_cpu();
867
868 if (op->code() == lir_idiv && op->in_opr2()->is_constant() && is_32) {
869 int c = op->in_opr2()->as_constant_ptr()->as_jint();
870 assert(is_power_of_2(c), "non power-of-2 constant should be put in a register");
871
872 Register left = op->in_opr1()->as_register();
873 Register dest = op->result_opr()->as_register();
874 if (c == 1) {
875 __ mov(dest, left);
876 } else if (c == 2) {
877 __ add_32(dest, left, AsmOperand(left, lsr, 31));
878 __ asr_32(dest, dest, 1);
879 } else if (c != (int) 0x80000000) {
880 int power = log2_intptr(c);
881 __ asr_32(Rtemp, left, 31);
882 __ add_32(dest, left, AsmOperand(Rtemp, lsr, 32-power)); // dest = left + (left < 0 ? 2^power - 1 : 0);
883 __ asr_32(dest, dest, power); // dest = dest >>> power;
884 } else {
885 // x/0x80000000 is a special case, since dividend is a power of two, but is negative.
886 // The only possible result values are 0 and 1, with 1 only for dividend == divisor == 0x80000000.
887 __ cmp_32(left, c);
888 __ mov(dest, 0, ne);
889 __ mov(dest, 1, eq);
890 }
891 } else {
892 assert(op->code() == lir_idiv || op->code() == lir_irem, "unexpected op3");
893 __ call(StubRoutines::Arm::idiv_irem_entry(), relocInfo::runtime_call_type);
894 add_debug_info_for_div0_here(op->info());
895 }
896 }
897
898
899 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
900 #ifdef ASSERT
901 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
902 if (op->block() != NULL) _branch_target_blocks.append(op->block());
903 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
904 assert(op->info() == NULL, "CodeEmitInfo?");
905 #endif // ASSERT
906
907 #ifdef __SOFTFP__
908 assert (op->code() != lir_cond_float_branch, "this should be impossible");
909 #else
910 if (op->code() == lir_cond_float_branch) {
911 __ fmstat();
912 __ b(*(op->ublock()->label()), vs);
913 }
914 #endif // __SOFTFP__
915
916 AsmCondition acond = al;
917 switch (op->cond()) {
918 case lir_cond_equal: acond = eq; break;
919 case lir_cond_notEqual: acond = ne; break;
920 case lir_cond_less: acond = lt; break;
921 case lir_cond_lessEqual: acond = le; break;
922 case lir_cond_greaterEqual: acond = ge; break;
923 case lir_cond_greater: acond = gt; break;
924 case lir_cond_aboveEqual: acond = hs; break;
925 case lir_cond_belowEqual: acond = ls; break;
926 default: assert(op->cond() == lir_cond_always, "must be");
927 }
928 __ b(*(op->label()), acond);
929 }
930
931
932 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
933 LIR_Opr src = op->in_opr();
934 LIR_Opr dest = op->result_opr();
935
936 switch (op->bytecode()) {
937 case Bytecodes::_i2l:
938 move_regs(src->as_register(), dest->as_register_lo());
939 __ mov(dest->as_register_hi(), AsmOperand(src->as_register(), asr, 31));
940 break;
941 case Bytecodes::_l2i:
942 move_regs(src->as_register_lo(), dest->as_register());
943 break;
944 case Bytecodes::_i2b:
945 __ sign_extend(dest->as_register(), src->as_register(), 8);
946 break;
947 case Bytecodes::_i2s:
948 __ sign_extend(dest->as_register(), src->as_register(), 16);
949 break;
950 case Bytecodes::_i2c:
951 __ zero_extend(dest->as_register(), src->as_register(), 16);
952 break;
953 case Bytecodes::_f2d:
954 __ convert_f2d(dest->as_double_reg(), src->as_float_reg());
955 break;
956 case Bytecodes::_d2f:
957 __ convert_d2f(dest->as_float_reg(), src->as_double_reg());
958 break;
959 case Bytecodes::_i2f:
960 __ fmsr(Stemp, src->as_register());
961 __ fsitos(dest->as_float_reg(), Stemp);
962 break;
963 case Bytecodes::_i2d:
964 __ fmsr(Stemp, src->as_register());
965 __ fsitod(dest->as_double_reg(), Stemp);
966 break;
967 case Bytecodes::_f2i:
968 __ ftosizs(Stemp, src->as_float_reg());
969 __ fmrs(dest->as_register(), Stemp);
970 break;
971 case Bytecodes::_d2i:
972 __ ftosizd(Stemp, src->as_double_reg());
973 __ fmrs(dest->as_register(), Stemp);
974 break;
975 default:
976 ShouldNotReachHere();
977 }
978 }
979
980
981 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
982 if (op->init_check()) {
983 Register tmp = op->tmp1()->as_register();
984 __ ldrb(tmp, Address(op->klass()->as_register(), InstanceKlass::init_state_offset()));
985 add_debug_info_for_null_check_here(op->stub()->info());
986 __ cmp(tmp, InstanceKlass::fully_initialized);
987 __ b(*op->stub()->entry(), ne);
988 }
989 __ allocate_object(op->obj()->as_register(),
990 op->tmp1()->as_register(),
991 op->tmp2()->as_register(),
992 op->tmp3()->as_register(),
993 op->header_size(),
994 op->object_size(),
995 op->klass()->as_register(),
996 *op->stub()->entry());
997 __ bind(*op->stub()->continuation());
998 }
999
1000 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1001 if (UseSlowPath ||
1002 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1003 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1004 __ b(*op->stub()->entry());
1005 } else {
1006 __ allocate_array(op->obj()->as_register(),
1007 op->len()->as_register(),
1008 op->tmp1()->as_register(),
1009 op->tmp2()->as_register(),
1010 op->tmp3()->as_register(),
1011 arrayOopDesc::header_size(op->type()),
1012 type2aelembytes(op->type()),
1013 op->klass()->as_register(),
1014 *op->stub()->entry());
1015 }
1016 __ bind(*op->stub()->continuation());
1017 }
1018
1019 void LIR_Assembler::type_profile_helper(Register mdo, int mdo_offset_bias,
1020 ciMethodData *md, ciProfileData *data,
1021 Register recv, Register tmp1, Label* update_done) {
1022 assert_different_registers(mdo, recv, tmp1);
1023 uint i;
1024 for (i = 0; i < VirtualCallData::row_limit(); i++) {
1025 Label next_test;
1026 // See if the receiver is receiver[n].
1027 Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
1028 mdo_offset_bias);
1029 __ ldr(tmp1, receiver_addr);
1030 __ verify_klass_ptr(tmp1);
1031 __ cmp(recv, tmp1);
1032 __ b(next_test, ne);
1033 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
1034 mdo_offset_bias);
1035 __ ldr(tmp1, data_addr);
1036 __ add(tmp1, tmp1, DataLayout::counter_increment);
1037 __ str(tmp1, data_addr);
1038 __ b(*update_done);
1039 __ bind(next_test);
1040 }
1041
1042 // Didn't find receiver; find next empty slot and fill it in
1043 for (i = 0; i < VirtualCallData::row_limit(); i++) {
1044 Label next_test;
1045 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
1046 mdo_offset_bias);
1047 __ ldr(tmp1, recv_addr);
1048 __ cbnz(tmp1, next_test);
1049 __ str(recv, recv_addr);
1050 __ mov(tmp1, DataLayout::counter_increment);
1051 __ str(tmp1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
1052 mdo_offset_bias));
1053 __ b(*update_done);
1054 __ bind(next_test);
1055 }
1056 }
1057
1058 void LIR_Assembler::setup_md_access(ciMethod* method, int bci,
1059 ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) {
1060 md = method->method_data_or_null();
1061 assert(md != NULL, "Sanity");
1062 data = md->bci_to_data(bci);
1063 assert(data != NULL, "need data for checkcast");
1064 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1065 if (md->byte_offset_of_slot(data, DataLayout::header_offset()) + data->size_in_bytes() >= 4096) {
1066 // The offset is large so bias the mdo by the base of the slot so
1067 // that the ldr can use an immediate offset to reference the slots of the data
1068 mdo_offset_bias = md->byte_offset_of_slot(data, DataLayout::header_offset());
1069 }
1070 }
1071
1072 // On 32-bit ARM, code before this helper should test obj for null (ZF should be set if obj is null).
1073 void LIR_Assembler::typecheck_profile_helper1(ciMethod* method, int bci,
1074 ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias,
1075 Register obj, Register mdo, Register data_val, Label* obj_is_null) {
1076 assert(method != NULL, "Should have method");
1077 assert_different_registers(obj, mdo, data_val);
1078 setup_md_access(method, bci, md, data, mdo_offset_bias);
1079 Label not_null;
1080 __ b(not_null, ne);
1081 __ mov_metadata(mdo, md->constant_encoding());
1082 if (mdo_offset_bias > 0) {
1083 __ mov_slow(data_val, mdo_offset_bias);
1084 __ add(mdo, mdo, data_val);
1085 }
1086 Address flags_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias);
1087 __ ldrb(data_val, flags_addr);
1088 __ orr(data_val, data_val, (uint)BitData::null_seen_byte_constant());
1089 __ strb(data_val, flags_addr);
1090 __ b(*obj_is_null);
1091 __ bind(not_null);
1092 }
1093
1094 void LIR_Assembler::typecheck_profile_helper2(ciMethodData* md, ciProfileData* data, int mdo_offset_bias,
1095 Register mdo, Register recv, Register value, Register tmp1,
1096 Label* profile_cast_success, Label* profile_cast_failure,
1097 Label* success, Label* failure) {
1098 assert_different_registers(mdo, value, tmp1);
1099 __ bind(*profile_cast_success);
1100 __ mov_metadata(mdo, md->constant_encoding());
1101 if (mdo_offset_bias > 0) {
1102 __ mov_slow(tmp1, mdo_offset_bias);
1103 __ add(mdo, mdo, tmp1);
1104 }
1105 __ load_klass(recv, value);
1106 type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, success);
1107 __ b(*success);
1108 // Cast failure case
1109 __ bind(*profile_cast_failure);
1110 __ mov_metadata(mdo, md->constant_encoding());
1111 if (mdo_offset_bias > 0) {
1112 __ mov_slow(tmp1, mdo_offset_bias);
1113 __ add(mdo, mdo, tmp1);
1114 }
1115 Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
1116 __ ldr(tmp1, data_addr);
1117 __ sub(tmp1, tmp1, DataLayout::counter_increment);
1118 __ str(tmp1, data_addr);
1119 __ b(*failure);
1120 }
1121
1122 // Sets `res` to true, if `cond` holds.
1123 static void set_instanceof_result(MacroAssembler* _masm, Register res, AsmCondition cond) {
1124 __ mov(res, 1, cond);
1125 }
1126
1127
1128 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1129 // TODO: ARM - can be more effective with one more register
1130 switch (op->code()) {
1131 case lir_store_check: {
1132 CodeStub* stub = op->stub();
1133 Register value = op->object()->as_register();
1134 Register array = op->array()->as_register();
1135 Register klass_RInfo = op->tmp1()->as_register();
1136 Register k_RInfo = op->tmp2()->as_register();
1137 assert_different_registers(klass_RInfo, k_RInfo, Rtemp);
1138 if (op->should_profile()) {
1139 assert_different_registers(value, klass_RInfo, k_RInfo, Rtemp);
1140 }
1141
1142 // check if it needs to be profiled
1143 ciMethodData* md;
1144 ciProfileData* data;
1145 int mdo_offset_bias = 0;
1146 Label profile_cast_success, profile_cast_failure, done;
1147 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1148 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1149
1150 if (op->should_profile()) {
1151 __ cmp(value, 0);
1152 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, value, k_RInfo, Rtemp, &done);
1153 } else {
1154 __ cbz(value, done);
1155 }
1156 assert_different_registers(k_RInfo, value);
1157 add_debug_info_for_null_check_here(op->info_for_exception());
1158 __ load_klass(k_RInfo, array);
1159 __ load_klass(klass_RInfo, value);
1160 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1161 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1162 // check for immediate positive hit
1163 __ ldr(Rtemp, Address(klass_RInfo, Rtemp));
1164 __ cmp(klass_RInfo, k_RInfo);
1165 __ cond_cmp(Rtemp, k_RInfo, ne);
1166 __ b(*success_target, eq);
1167 // check for immediate negative hit
1168 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1169 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset()));
1170 __ b(*failure_target, ne);
1171 // slow case
1172 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1173 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1174 __ cbz(R0, *failure_target);
1175 if (op->should_profile()) {
1176 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp;
1177 if (mdo == value) {
1178 mdo = k_RInfo;
1179 recv = klass_RInfo;
1180 }
1181 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, value, tmp1,
1182 &profile_cast_success, &profile_cast_failure,
1183 &done, stub->entry());
1184 }
1185 __ bind(done);
1186 break;
1187 }
1188
1189 case lir_checkcast: {
1190 CodeStub* stub = op->stub();
1191 Register obj = op->object()->as_register();
1192 Register res = op->result_opr()->as_register();
1193 Register klass_RInfo = op->tmp1()->as_register();
1194 Register k_RInfo = op->tmp2()->as_register();
1195 ciKlass* k = op->klass();
1196 assert_different_registers(res, k_RInfo, klass_RInfo, Rtemp);
1197
1198 if (stub->is_simple_exception_stub()) {
1199 // TODO: ARM - Late binding is used to prevent confusion of register allocator
1200 assert(stub->is_exception_throw_stub(), "must be");
1201 ((SimpleExceptionStub*)stub)->set_obj(op->result_opr());
1202 }
1203 ciMethodData* md;
1204 ciProfileData* data;
1205 int mdo_offset_bias = 0;
1206
1207 Label done;
1208
1209 Label profile_cast_failure, profile_cast_success;
1210 Label *failure_target = op->should_profile() ? &profile_cast_failure : op->stub()->entry();
1211 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1212
1213
1214 __ movs(res, obj);
1215 if (op->should_profile()) {
1216 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done);
1217 } else {
1218 __ b(done, eq);
1219 }
1220 if (k->is_loaded()) {
1221 __ mov_metadata(k_RInfo, k->constant_encoding());
1222 } else if (k_RInfo != obj) {
1223 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1224 __ movs(res, obj);
1225 } else {
1226 // Patching doesn't update "res" register after GC, so do patching first
1227 klass2reg_with_patching(Rtemp, op->info_for_patch());
1228 __ movs(res, obj);
1229 __ mov(k_RInfo, Rtemp);
1230 }
1231 __ load_klass(klass_RInfo, res, ne);
1232
1233 if (op->fast_check()) {
1234 __ cmp(klass_RInfo, k_RInfo, ne);
1235 __ b(*failure_target, ne);
1236 } else if (k->is_loaded()) {
1237 __ b(*success_target, eq);
1238 __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset()));
1239 if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) {
1240 __ cmp(Rtemp, k_RInfo);
1241 __ b(*failure_target, ne);
1242 } else {
1243 __ cmp(klass_RInfo, k_RInfo);
1244 __ cmp(Rtemp, k_RInfo, ne);
1245 __ b(*success_target, eq);
1246 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1247 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1248 __ cbz(R0, *failure_target);
1249 }
1250 } else {
1251 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1252 __ b(*success_target, eq);
1253 // check for immediate positive hit
1254 __ ldr(Rtemp, Address(klass_RInfo, Rtemp));
1255 __ cmp(klass_RInfo, k_RInfo);
1256 __ cmp(Rtemp, k_RInfo, ne);
1257 __ b(*success_target, eq);
1258 // check for immediate negative hit
1259 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1260 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset()));
1261 __ b(*failure_target, ne);
1262 // slow case
1263 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1264 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1265 __ cbz(R0, *failure_target);
1266 }
1267
1268 if (op->should_profile()) {
1269 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp;
1270 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, res, tmp1,
1271 &profile_cast_success, &profile_cast_failure,
1272 &done, stub->entry());
1273 }
1274 __ bind(done);
1275 break;
1276 }
1277
1278 case lir_instanceof: {
1279 Register obj = op->object()->as_register();
1280 Register res = op->result_opr()->as_register();
1281 Register klass_RInfo = op->tmp1()->as_register();
1282 Register k_RInfo = op->tmp2()->as_register();
1283 ciKlass* k = op->klass();
1284 assert_different_registers(res, klass_RInfo, k_RInfo, Rtemp);
1285
1286 ciMethodData* md;
1287 ciProfileData* data;
1288 int mdo_offset_bias = 0;
1289
1290 Label done;
1291
1292 Label profile_cast_failure, profile_cast_success;
1293 Label *failure_target = op->should_profile() ? &profile_cast_failure : &done;
1294 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1295
1296 __ movs(res, obj);
1297
1298 if (op->should_profile()) {
1299 typecheck_profile_helper1(op->profiled_method(), op->profiled_bci(), md, data, mdo_offset_bias, res, klass_RInfo, Rtemp, &done);
1300 } else {
1301 __ b(done, eq);
1302 }
1303
1304 if (k->is_loaded()) {
1305 __ mov_metadata(k_RInfo, k->constant_encoding());
1306 } else {
1307 op->info_for_patch()->add_register_oop(FrameMap::as_oop_opr(res));
1308 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1309 }
1310 __ load_klass(klass_RInfo, res);
1311
1312 if (!op->should_profile()) {
1313 __ mov(res, 0);
1314 }
1315
1316 if (op->fast_check()) {
1317 __ cmp(klass_RInfo, k_RInfo);
1318 if (!op->should_profile()) {
1319 set_instanceof_result(_masm, res, eq);
1320 } else {
1321 __ b(profile_cast_failure, ne);
1322 }
1323 } else if (k->is_loaded()) {
1324 __ ldr(Rtemp, Address(klass_RInfo, k->super_check_offset()));
1325 if (in_bytes(Klass::secondary_super_cache_offset()) != (int) k->super_check_offset()) {
1326 __ cmp(Rtemp, k_RInfo);
1327 if (!op->should_profile()) {
1328 set_instanceof_result(_masm, res, eq);
1329 } else {
1330 __ b(profile_cast_failure, ne);
1331 }
1332 } else {
1333 __ cmp(klass_RInfo, k_RInfo);
1334 __ cond_cmp(Rtemp, k_RInfo, ne);
1335 if (!op->should_profile()) {
1336 set_instanceof_result(_masm, res, eq);
1337 }
1338 __ b(*success_target, eq);
1339 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1340 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1341 if (!op->should_profile()) {
1342 move_regs(R0, res);
1343 } else {
1344 __ cbz(R0, *failure_target);
1345 }
1346 }
1347 } else {
1348 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1349 // check for immediate positive hit
1350 __ cmp(klass_RInfo, k_RInfo);
1351 if (!op->should_profile()) {
1352 __ ldr(res, Address(klass_RInfo, Rtemp), ne);
1353 __ cond_cmp(res, k_RInfo, ne);
1354 set_instanceof_result(_masm, res, eq);
1355 } else {
1356 __ ldr(Rtemp, Address(klass_RInfo, Rtemp), ne);
1357 __ cond_cmp(Rtemp, k_RInfo, ne);
1358 }
1359 __ b(*success_target, eq);
1360 // check for immediate negative hit
1361 if (op->should_profile()) {
1362 __ ldr_u32(Rtemp, Address(k_RInfo, Klass::super_check_offset_offset()));
1363 }
1364 __ cmp(Rtemp, in_bytes(Klass::secondary_super_cache_offset()));
1365 if (!op->should_profile()) {
1366 __ mov(res, 0, ne);
1367 }
1368 __ b(*failure_target, ne);
1369 // slow case
1370 assert(klass_RInfo == R0 && k_RInfo == R1, "runtime call setup");
1371 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
1372 if (!op->should_profile()) {
1373 move_regs(R0, res);
1374 }
1375 if (op->should_profile()) {
1376 __ cbz(R0, *failure_target);
1377 }
1378 }
1379
1380 if (op->should_profile()) {
1381 Label done_ok, done_failure;
1382 Register mdo = klass_RInfo, recv = k_RInfo, tmp1 = Rtemp;
1383 typecheck_profile_helper2(md, data, mdo_offset_bias, mdo, recv, res, tmp1,
1384 &profile_cast_success, &profile_cast_failure,
1385 &done_ok, &done_failure);
1386 __ bind(done_failure);
1387 __ mov(res, 0);
1388 __ b(done);
1389 __ bind(done_ok);
1390 __ mov(res, 1);
1391 }
1392 __ bind(done);
1393 break;
1394 }
1395 default:
1396 ShouldNotReachHere();
1397 }
1398 }
1399
1400
1401 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1402 // if (*addr == cmpval) {
1403 // *addr = newval;
1404 // dest = 1;
1405 // } else {
1406 // dest = 0;
1407 // }
1408 // FIXME: membar_release
1409 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp);
1410 Register addr = op->addr()->is_register() ?
1411 op->addr()->as_pointer_register() :
1412 op->addr()->as_address_ptr()->base()->as_pointer_register();
1413 assert(op->addr()->is_register() || op->addr()->as_address_ptr()->disp() == 0, "unexpected disp");
1414 assert(op->addr()->is_register() || op->addr()->as_address_ptr()->index() == LIR_OprDesc::illegalOpr(), "unexpected index");
1415 if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1416 Register cmpval = op->cmp_value()->as_register();
1417 Register newval = op->new_value()->as_register();
1418 Register dest = op->result_opr()->as_register();
1419 assert_different_registers(dest, addr, cmpval, newval, Rtemp);
1420
1421 __ atomic_cas_bool(cmpval, newval, addr, 0, Rtemp); // Rtemp free by default at C1 LIR layer
1422 __ mov(dest, 1, eq);
1423 __ mov(dest, 0, ne);
1424 } else if (op->code() == lir_cas_long) {
1425 assert(VM_Version::supports_cx8(), "wrong machine");
1426 Register cmp_value_lo = op->cmp_value()->as_register_lo();
1427 Register cmp_value_hi = op->cmp_value()->as_register_hi();
1428 Register new_value_lo = op->new_value()->as_register_lo();
1429 Register new_value_hi = op->new_value()->as_register_hi();
1430 Register dest = op->result_opr()->as_register();
1431 Register tmp_lo = op->tmp1()->as_register_lo();
1432 Register tmp_hi = op->tmp1()->as_register_hi();
1433
1434 assert_different_registers(tmp_lo, tmp_hi, cmp_value_lo, cmp_value_hi, dest, new_value_lo, new_value_hi, addr);
1435 assert(tmp_hi->encoding() == tmp_lo->encoding() + 1, "non aligned register pair");
1436 assert(new_value_hi->encoding() == new_value_lo->encoding() + 1, "non aligned register pair");
1437 assert((tmp_lo->encoding() & 0x1) == 0, "misaligned register pair");
1438 assert((new_value_lo->encoding() & 0x1) == 0, "misaligned register pair");
1439 __ atomic_cas64(tmp_lo, tmp_hi, dest, cmp_value_lo, cmp_value_hi,
1440 new_value_lo, new_value_hi, addr, 0);
1441 } else {
1442 Unimplemented();
1443 }
1444 // FIXME: is full membar really needed instead of just membar_acquire?
1445 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp);
1446 }
1447
1448
1449 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1450 AsmCondition acond = al;
1451 AsmCondition ncond = nv;
1452 if (opr1 != opr2) {
1453 switch (condition) {
1454 case lir_cond_equal: acond = eq; ncond = ne; break;
1455 case lir_cond_notEqual: acond = ne; ncond = eq; break;
1456 case lir_cond_less: acond = lt; ncond = ge; break;
1457 case lir_cond_lessEqual: acond = le; ncond = gt; break;
1458 case lir_cond_greaterEqual: acond = ge; ncond = lt; break;
1459 case lir_cond_greater: acond = gt; ncond = le; break;
1460 case lir_cond_aboveEqual: acond = hs; ncond = lo; break;
1461 case lir_cond_belowEqual: acond = ls; ncond = hi; break;
1462 default: ShouldNotReachHere();
1463 }
1464 }
1465
1466 for (;;) { // two iterations only
1467 if (opr1 == result) {
1468 // do nothing
1469 } else if (opr1->is_single_cpu()) {
1470 __ mov(result->as_register(), opr1->as_register(), acond);
1471 } else if (opr1->is_double_cpu()) {
1472 __ long_move(result->as_register_lo(), result->as_register_hi(),
1473 opr1->as_register_lo(), opr1->as_register_hi(), acond);
1474 } else if (opr1->is_single_stack()) {
1475 __ ldr(result->as_register(), frame_map()->address_for_slot(opr1->single_stack_ix()), acond);
1476 } else if (opr1->is_double_stack()) {
1477 __ ldr(result->as_register_lo(),
1478 frame_map()->address_for_slot(opr1->double_stack_ix(), lo_word_offset_in_bytes), acond);
1479 __ ldr(result->as_register_hi(),
1480 frame_map()->address_for_slot(opr1->double_stack_ix(), hi_word_offset_in_bytes), acond);
1481 } else if (opr1->is_illegal()) {
1482 // do nothing: this part of the cmove has been optimized away in the peephole optimizer
1483 } else {
1484 assert(opr1->is_constant(), "must be");
1485 LIR_Const* c = opr1->as_constant_ptr();
1486
1487 switch (c->type()) {
1488 case T_INT:
1489 __ mov_slow(result->as_register(), c->as_jint(), acond);
1490 break;
1491 case T_LONG:
1492 __ mov_slow(result->as_register_lo(), c->as_jint_lo(), acond);
1493 __ mov_slow(result->as_register_hi(), c->as_jint_hi(), acond);
1494 break;
1495 case T_OBJECT:
1496 __ mov_oop(result->as_register(), c->as_jobject(), 0, acond);
1497 break;
1498 case T_FLOAT:
1499 #ifdef __SOFTFP__
1500 // not generated now.
1501 __ mov_slow(result->as_register(), c->as_jint(), acond);
1502 #else
1503 __ mov_float(result->as_float_reg(), c->as_jfloat(), acond);
1504 #endif // __SOFTFP__
1505 break;
1506 case T_DOUBLE:
1507 #ifdef __SOFTFP__
1508 // not generated now.
1509 __ mov_slow(result->as_register_lo(), c->as_jint_lo(), acond);
1510 __ mov_slow(result->as_register_hi(), c->as_jint_hi(), acond);
1511 #else
1512 __ mov_double(result->as_double_reg(), c->as_jdouble(), acond);
1513 #endif // __SOFTFP__
1514 break;
1515 default:
1516 ShouldNotReachHere();
1517 }
1518 }
1519
1520 // Negate the condition and repeat the algorithm with the second operand
1521 if (opr1 == opr2) { break; }
1522 opr1 = opr2;
1523 acond = ncond;
1524 }
1525 }
1526
1527 #ifdef ASSERT
1528 static int reg_size(LIR_Opr op) {
1529 switch (op->type()) {
1530 case T_FLOAT:
1531 case T_INT: return BytesPerInt;
1532 case T_LONG:
1533 case T_DOUBLE: return BytesPerLong;
1534 case T_OBJECT:
1535 case T_ARRAY:
1536 case T_METADATA: return BytesPerWord;
1537 case T_ADDRESS:
1538 case T_ILLEGAL: // fall through
1539 default: ShouldNotReachHere(); return -1;
1540 }
1541 }
1542 #endif
1543
1544 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1545 assert(info == NULL, "unused on this code path");
1546 assert(dest->is_register(), "wrong items state");
1547
1548 if (right->is_address()) {
1549 // special case for adding shifted/extended register
1550 const Register res = dest->as_pointer_register();
1551 const Register lreg = left->as_pointer_register();
1552 const LIR_Address* addr = right->as_address_ptr();
1553
1554 assert(addr->base()->as_pointer_register() == lreg && addr->index()->is_register() && addr->disp() == 0, "must be");
1555
1556 int scale = addr->scale();
1557 AsmShift shift = lsl;
1558
1559
1560 assert(reg_size(addr->base()) == reg_size(addr->index()), "should be");
1561 assert(reg_size(addr->base()) == reg_size(dest), "should be");
1562 assert(reg_size(dest) == wordSize, "should be");
1563
1564 AsmOperand operand(addr->index()->as_pointer_register(), shift, scale);
1565 switch (code) {
1566 case lir_add: __ add(res, lreg, operand); break;
1567 case lir_sub: __ sub(res, lreg, operand); break;
1568 default: ShouldNotReachHere();
1569 }
1570
1571 } else if (left->is_address()) {
1572 assert(code == lir_sub && right->is_single_cpu(), "special case used by strength_reduce_multiply()");
1573 const LIR_Address* addr = left->as_address_ptr();
1574 const Register res = dest->as_register();
1575 const Register rreg = right->as_register();
1576 assert(addr->base()->as_register() == rreg && addr->index()->is_register() && addr->disp() == 0, "must be");
1577 __ rsb(res, rreg, AsmOperand(addr->index()->as_register(), lsl, addr->scale()));
1578
1579 } else if (dest->is_single_cpu()) {
1580 assert(left->is_single_cpu(), "unexpected left operand");
1581
1582 const Register res = dest->as_register();
1583 const Register lreg = left->as_register();
1584
1585 if (right->is_single_cpu()) {
1586 const Register rreg = right->as_register();
1587 switch (code) {
1588 case lir_add: __ add_32(res, lreg, rreg); break;
1589 case lir_sub: __ sub_32(res, lreg, rreg); break;
1590 case lir_mul: __ mul_32(res, lreg, rreg); break;
1591 default: ShouldNotReachHere();
1592 }
1593 } else {
1594 assert(right->is_constant(), "must be");
1595 const jint c = right->as_constant_ptr()->as_jint();
1596 if (!Assembler::is_arith_imm_in_range(c)) {
1597 BAILOUT("illegal arithmetic operand");
1598 }
1599 switch (code) {
1600 case lir_add: __ add_32(res, lreg, c); break;
1601 case lir_sub: __ sub_32(res, lreg, c); break;
1602 default: ShouldNotReachHere();
1603 }
1604 }
1605
1606 } else if (dest->is_double_cpu()) {
1607 Register res_lo = dest->as_register_lo();
1608 Register res_hi = dest->as_register_hi();
1609 Register lreg_lo = left->as_register_lo();
1610 Register lreg_hi = left->as_register_hi();
1611 if (right->is_double_cpu()) {
1612 Register rreg_lo = right->as_register_lo();
1613 Register rreg_hi = right->as_register_hi();
1614 if (res_lo == lreg_hi || res_lo == rreg_hi) {
1615 res_lo = Rtemp;
1616 }
1617 switch (code) {
1618 case lir_add:
1619 __ adds(res_lo, lreg_lo, rreg_lo);
1620 __ adc(res_hi, lreg_hi, rreg_hi);
1621 break;
1622 case lir_sub:
1623 __ subs(res_lo, lreg_lo, rreg_lo);
1624 __ sbc(res_hi, lreg_hi, rreg_hi);
1625 break;
1626 default:
1627 ShouldNotReachHere();
1628 }
1629 } else {
1630 assert(right->is_constant(), "must be");
1631 assert((right->as_constant_ptr()->as_jlong() >> 32) == 0, "out of range");
1632 const jint c = (jint) right->as_constant_ptr()->as_jlong();
1633 if (res_lo == lreg_hi) {
1634 res_lo = Rtemp;
1635 }
1636 switch (code) {
1637 case lir_add:
1638 __ adds(res_lo, lreg_lo, c);
1639 __ adc(res_hi, lreg_hi, 0);
1640 break;
1641 case lir_sub:
1642 __ subs(res_lo, lreg_lo, c);
1643 __ sbc(res_hi, lreg_hi, 0);
1644 break;
1645 default:
1646 ShouldNotReachHere();
1647 }
1648 }
1649 move_regs(res_lo, dest->as_register_lo());
1650
1651 } else if (dest->is_single_fpu()) {
1652 assert(left->is_single_fpu(), "must be");
1653 assert(right->is_single_fpu(), "must be");
1654 const FloatRegister res = dest->as_float_reg();
1655 const FloatRegister lreg = left->as_float_reg();
1656 const FloatRegister rreg = right->as_float_reg();
1657 switch (code) {
1658 case lir_add: __ add_float(res, lreg, rreg); break;
1659 case lir_sub: __ sub_float(res, lreg, rreg); break;
1660 case lir_mul_strictfp: // fall through
1661 case lir_mul: __ mul_float(res, lreg, rreg); break;
1662 case lir_div_strictfp: // fall through
1663 case lir_div: __ div_float(res, lreg, rreg); break;
1664 default: ShouldNotReachHere();
1665 }
1666 } else if (dest->is_double_fpu()) {
1667 assert(left->is_double_fpu(), "must be");
1668 assert(right->is_double_fpu(), "must be");
1669 const FloatRegister res = dest->as_double_reg();
1670 const FloatRegister lreg = left->as_double_reg();
1671 const FloatRegister rreg = right->as_double_reg();
1672 switch (code) {
1673 case lir_add: __ add_double(res, lreg, rreg); break;
1674 case lir_sub: __ sub_double(res, lreg, rreg); break;
1675 case lir_mul_strictfp: // fall through
1676 case lir_mul: __ mul_double(res, lreg, rreg); break;
1677 case lir_div_strictfp: // fall through
1678 case lir_div: __ div_double(res, lreg, rreg); break;
1679 default: ShouldNotReachHere();
1680 }
1681 } else {
1682 ShouldNotReachHere();
1683 }
1684 }
1685
1686
1687 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1688 switch (code) {
1689 case lir_abs:
1690 __ abs_double(dest->as_double_reg(), value->as_double_reg());
1691 break;
1692 case lir_sqrt:
1693 __ sqrt_double(dest->as_double_reg(), value->as_double_reg());
1694 break;
1695 default:
1696 ShouldNotReachHere();
1697 }
1698 }
1699
1700
1701 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) {
1702 assert(dest->is_register(), "wrong items state");
1703 assert(left->is_register(), "wrong items state");
1704
1705 if (dest->is_single_cpu()) {
1706
1707 const Register res = dest->as_register();
1708 const Register lreg = left->as_register();
1709
1710 if (right->is_single_cpu()) {
1711 const Register rreg = right->as_register();
1712 switch (code) {
1713 case lir_logic_and: __ and_32(res, lreg, rreg); break;
1714 case lir_logic_or: __ orr_32(res, lreg, rreg); break;
1715 case lir_logic_xor: __ eor_32(res, lreg, rreg); break;
1716 default: ShouldNotReachHere();
1717 }
1718 } else {
1719 assert(right->is_constant(), "must be");
1720 const uint c = (uint)right->as_constant_ptr()->as_jint();
1721 switch (code) {
1722 case lir_logic_and: __ and_32(res, lreg, c); break;
1723 case lir_logic_or: __ orr_32(res, lreg, c); break;
1724 case lir_logic_xor: __ eor_32(res, lreg, c); break;
1725 default: ShouldNotReachHere();
1726 }
1727 }
1728 } else {
1729 assert(dest->is_double_cpu(), "should be");
1730 Register res_lo = dest->as_register_lo();
1731
1732 assert (dest->type() == T_LONG, "unexpected result type");
1733 assert (left->type() == T_LONG, "unexpected left type");
1734 assert (right->type() == T_LONG, "unexpected right type");
1735
1736 const Register res_hi = dest->as_register_hi();
1737 const Register lreg_lo = left->as_register_lo();
1738 const Register lreg_hi = left->as_register_hi();
1739
1740 if (right->is_register()) {
1741 const Register rreg_lo = right->as_register_lo();
1742 const Register rreg_hi = right->as_register_hi();
1743 if (res_lo == lreg_hi || res_lo == rreg_hi) {
1744 res_lo = Rtemp; // Temp register helps to avoid overlap between result and input
1745 }
1746 switch (code) {
1747 case lir_logic_and:
1748 __ andr(res_lo, lreg_lo, rreg_lo);
1749 __ andr(res_hi, lreg_hi, rreg_hi);
1750 break;
1751 case lir_logic_or:
1752 __ orr(res_lo, lreg_lo, rreg_lo);
1753 __ orr(res_hi, lreg_hi, rreg_hi);
1754 break;
1755 case lir_logic_xor:
1756 __ eor(res_lo, lreg_lo, rreg_lo);
1757 __ eor(res_hi, lreg_hi, rreg_hi);
1758 break;
1759 default:
1760 ShouldNotReachHere();
1761 }
1762 move_regs(res_lo, dest->as_register_lo());
1763 } else {
1764 assert(right->is_constant(), "must be");
1765 const jint c_lo = (jint) right->as_constant_ptr()->as_jlong();
1766 const jint c_hi = (jint) (right->as_constant_ptr()->as_jlong() >> 32);
1767 // Case for logic_or from do_ClassIDIntrinsic()
1768 if (c_hi == 0 && AsmOperand::is_rotated_imm(c_lo)) {
1769 switch (code) {
1770 case lir_logic_and:
1771 __ andr(res_lo, lreg_lo, c_lo);
1772 __ mov(res_hi, 0);
1773 break;
1774 case lir_logic_or:
1775 __ orr(res_lo, lreg_lo, c_lo);
1776 break;
1777 case lir_logic_xor:
1778 __ eor(res_lo, lreg_lo, c_lo);
1779 break;
1780 default:
1781 ShouldNotReachHere();
1782 }
1783 } else if (code == lir_logic_and &&
1784 c_hi == -1 &&
1785 (AsmOperand::is_rotated_imm(c_lo) ||
1786 AsmOperand::is_rotated_imm(~c_lo))) {
1787 // Another case which handles logic_and from do_ClassIDIntrinsic()
1788 if (AsmOperand::is_rotated_imm(c_lo)) {
1789 __ andr(res_lo, lreg_lo, c_lo);
1790 } else {
1791 __ bic(res_lo, lreg_lo, ~c_lo);
1792 }
1793 if (res_hi != lreg_hi) {
1794 __ mov(res_hi, lreg_hi);
1795 }
1796 } else {
1797 BAILOUT("64 bit constant cannot be inlined");
1798 }
1799 }
1800 }
1801 }
1802
1803
1804
1805 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1806 if (opr1->is_single_cpu()) {
1807 if (opr2->is_constant()) {
1808 switch (opr2->as_constant_ptr()->type()) {
1809 case T_INT: {
1810 const jint c = opr2->as_constant_ptr()->as_jint();
1811 if (Assembler::is_arith_imm_in_range(c)) {
1812 __ cmp_32(opr1->as_register(), c);
1813 } else if (Assembler::is_arith_imm_in_range(-c)) {
1814 __ cmn_32(opr1->as_register(), -c);
1815 } else {
1816 // This can happen when compiling lookupswitch
1817 __ mov_slow(Rtemp, c);
1818 __ cmp_32(opr1->as_register(), Rtemp);
1819 }
1820 break;
1821 }
1822 case T_OBJECT:
1823 assert(opr2->as_constant_ptr()->as_jobject() == NULL, "cannot handle otherwise");
1824 __ cmp(opr1->as_register(), 0);
1825 break;
1826 case T_METADATA:
1827 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "Only equality tests");
1828 assert(opr2->as_constant_ptr()->as_metadata() == NULL, "cannot handle otherwise");
1829 __ cmp(opr1->as_register(), 0);
1830 break;
1831 default:
1832 ShouldNotReachHere();
1833 }
1834 } else if (opr2->is_single_cpu()) {
1835 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1836 assert(opr2->type() == T_OBJECT || opr2->type() == T_ARRAY, "incompatibe type");
1837 __ cmpoop(opr1->as_register(), opr2->as_register());
1838 } else if (opr1->type() == T_METADATA || opr1->type() == T_ADDRESS) {
1839 assert(opr2->type() == T_METADATA || opr2->type() == T_ADDRESS, "incompatibe type");
1840 __ cmp(opr1->as_register(), opr2->as_register());
1841 } else {
1842 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY && opr2->type() != T_METADATA && opr2->type() != T_ADDRESS, "incompatibe type");
1843 __ cmp_32(opr1->as_register(), opr2->as_register());
1844 }
1845 } else {
1846 ShouldNotReachHere();
1847 }
1848 } else if (opr1->is_double_cpu()) {
1849 Register xlo = opr1->as_register_lo();
1850 Register xhi = opr1->as_register_hi();
1851 if (opr2->is_constant() && opr2->as_jlong() == 0) {
1852 assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "cannot handle otherwise");
1853 __ orrs(Rtemp, xlo, xhi);
1854 } else if (opr2->is_register()) {
1855 Register ylo = opr2->as_register_lo();
1856 Register yhi = opr2->as_register_hi();
1857 if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
1858 __ teq(xhi, yhi);
1859 __ teq(xlo, ylo, eq);
1860 } else {
1861 __ subs(xlo, xlo, ylo);
1862 __ sbcs(xhi, xhi, yhi);
1863 }
1864 } else {
1865 ShouldNotReachHere();
1866 }
1867 } else if (opr1->is_single_fpu()) {
1868 if (opr2->is_constant()) {
1869 assert(opr2->as_jfloat() == 0.0f, "cannot handle otherwise");
1870 __ cmp_zero_float(opr1->as_float_reg());
1871 } else {
1872 __ cmp_float(opr1->as_float_reg(), opr2->as_float_reg());
1873 }
1874 } else if (opr1->is_double_fpu()) {
1875 if (opr2->is_constant()) {
1876 assert(opr2->as_jdouble() == 0.0, "cannot handle otherwise");
1877 __ cmp_zero_double(opr1->as_double_reg());
1878 } else {
1879 __ cmp_double(opr1->as_double_reg(), opr2->as_double_reg());
1880 }
1881 } else {
1882 ShouldNotReachHere();
1883 }
1884 }
1885
1886 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
1887 const Register res = dst->as_register();
1888 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1889 comp_op(lir_cond_unknown, left, right, op);
1890 __ fmstat();
1891 if (code == lir_ucmp_fd2i) { // unordered is less
1892 __ mvn(res, 0, lt);
1893 __ mov(res, 1, ge);
1894 } else { // unordered is greater
1895 __ mov(res, 1, cs);
1896 __ mvn(res, 0, cc);
1897 }
1898 __ mov(res, 0, eq);
1899
1900 } else {
1901 assert(code == lir_cmp_l2i, "must be");
1902
1903 Label done;
1904 const Register xlo = left->as_register_lo();
1905 const Register xhi = left->as_register_hi();
1906 const Register ylo = right->as_register_lo();
1907 const Register yhi = right->as_register_hi();
1908 __ cmp(xhi, yhi);
1909 __ mov(res, 1, gt);
1910 __ mvn(res, 0, lt);
1911 __ b(done, ne);
1912 __ subs(res, xlo, ylo);
1913 __ mov(res, 1, hi);
1914 __ mvn(res, 0, lo);
1915 __ bind(done);
1916 }
1917 }
1918
1919
1920 void LIR_Assembler::align_call(LIR_Code code) {
1921 // Not needed
1922 }
1923
1924
1925 void LIR_Assembler::call(LIR_OpJavaCall *op, relocInfo::relocType rtype) {
1926 int ret_addr_offset = __ patchable_call(op->addr(), rtype);
1927 assert(ret_addr_offset == __ offset(), "embedded return address not allowed");
1928 add_call_info_here(op->info());
1929 }
1930
1931
1932 void LIR_Assembler::ic_call(LIR_OpJavaCall *op) {
1933 bool near_range = __ cache_fully_reachable();
1934 address oop_address = pc();
1935
1936 bool use_movw = VM_Version::supports_movw();
1937
1938 // Ricklass may contain something that is not a metadata pointer so
1939 // mov_metadata can't be used
1940 InlinedAddress value((address)Universe::non_oop_word());
1941 InlinedAddress addr(op->addr());
1942 if (use_movw) {
1943 __ movw(Ricklass, ((unsigned int)Universe::non_oop_word()) & 0xffff);
1944 __ movt(Ricklass, ((unsigned int)Universe::non_oop_word()) >> 16);
1945 } else {
1946 // No movw/movt, must be load a pc relative value but no
1947 // relocation so no metadata table to load from.
1948 // Use a b instruction rather than a bl, inline constant after the
1949 // branch, use a PC relative ldr to load the constant, arrange for
1950 // the call to return after the constant(s).
1951 __ ldr_literal(Ricklass, value);
1952 }
1953 __ relocate(virtual_call_Relocation::spec(oop_address));
1954 if (near_range && use_movw) {
1955 __ bl(op->addr());
1956 } else {
1957 Label call_return;
1958 __ adr(LR, call_return);
1959 if (near_range) {
1960 __ b(op->addr());
1961 } else {
1962 __ indirect_jump(addr, Rtemp);
1963 __ bind_literal(addr);
1964 }
1965 if (!use_movw) {
1966 __ bind_literal(value);
1967 }
1968 __ bind(call_return);
1969 }
1970 add_call_info(code_offset(), op->info());
1971 }
1972
1973
1974 /* Currently, vtable-dispatch is only enabled for sparc platforms */
1975 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
1976 ShouldNotReachHere();
1977 }
1978
1979 void LIR_Assembler::emit_static_call_stub() {
1980 address call_pc = __ pc();
1981 address stub = __ start_a_stub(call_stub_size());
1982 if (stub == NULL) {
1983 BAILOUT("static call stub overflow");
1984 }
1985
1986 DEBUG_ONLY(int offset = code_offset();)
1987
1988 InlinedMetadata metadata_literal(NULL);
1989 __ relocate(static_stub_Relocation::spec(call_pc));
1990 // If not a single instruction, NativeMovConstReg::next_instruction_address()
1991 // must jump over the whole following ldr_literal.
1992 // (See CompiledStaticCall::set_to_interpreted())
1993 #ifdef ASSERT
1994 address ldr_site = __ pc();
1995 #endif
1996 __ ldr_literal(Rmethod, metadata_literal);
1997 assert(nativeMovConstReg_at(ldr_site)->next_instruction_address() == __ pc(), "Fix ldr_literal or its parsing");
1998 bool near_range = __ cache_fully_reachable();
1999 InlinedAddress dest((address)-1);
2000 if (near_range) {
2001 address branch_site = __ pc();
2002 __ b(branch_site); // b to self maps to special NativeJump -1 destination
2003 } else {
2004 __ indirect_jump(dest, Rtemp);
2005 }
2006 __ bind_literal(metadata_literal); // includes spec_for_immediate reloc
2007 if (!near_range) {
2008 __ bind_literal(dest); // special NativeJump -1 destination
2009 }
2010
2011 assert(code_offset() - offset <= call_stub_size(), "overflow");
2012 __ end_a_stub();
2013 }
2014
2015 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2016 assert(exceptionOop->as_register() == Rexception_obj, "must match");
2017 assert(exceptionPC->as_register() == Rexception_pc, "must match");
2018 info->add_register_oop(exceptionOop);
2019
2020 Runtime1::StubID handle_id = compilation()->has_fpu_code() ?
2021 Runtime1::handle_exception_id :
2022 Runtime1::handle_exception_nofpu_id;
2023 Label return_address;
2024 __ adr(Rexception_pc, return_address);
2025 __ call(Runtime1::entry_for(handle_id), relocInfo::runtime_call_type);
2026 __ bind(return_address);
2027 add_call_info_here(info); // for exception handler
2028 }
2029
2030 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2031 assert(exceptionOop->as_register() == Rexception_obj, "must match");
2032 __ b(_unwind_handler_entry);
2033 }
2034
2035 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2036 AsmShift shift = lsl;
2037 switch (code) {
2038 case lir_shl: shift = lsl; break;
2039 case lir_shr: shift = asr; break;
2040 case lir_ushr: shift = lsr; break;
2041 default: ShouldNotReachHere();
2042 }
2043
2044 if (dest->is_single_cpu()) {
2045 __ andr(Rtemp, count->as_register(), 31);
2046 __ mov(dest->as_register(), AsmOperand(left->as_register(), shift, Rtemp));
2047 } else if (dest->is_double_cpu()) {
2048 Register dest_lo = dest->as_register_lo();
2049 Register dest_hi = dest->as_register_hi();
2050 Register src_lo = left->as_register_lo();
2051 Register src_hi = left->as_register_hi();
2052 Register Rcount = count->as_register();
2053 // Resolve possible register conflicts
2054 if (shift == lsl && dest_hi == src_lo) {
2055 dest_hi = Rtemp;
2056 } else if (shift != lsl && dest_lo == src_hi) {
2057 dest_lo = Rtemp;
2058 } else if (dest_lo == src_lo && dest_hi == src_hi) {
2059 dest_lo = Rtemp;
2060 } else if (dest_lo == Rcount || dest_hi == Rcount) {
2061 Rcount = Rtemp;
2062 }
2063 __ andr(Rcount, count->as_register(), 63);
2064 __ long_shift(dest_lo, dest_hi, src_lo, src_hi, shift, Rcount);
2065 move_regs(dest_lo, dest->as_register_lo());
2066 move_regs(dest_hi, dest->as_register_hi());
2067 } else {
2068 ShouldNotReachHere();
2069 }
2070 }
2071
2072
2073 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2074 AsmShift shift = lsl;
2075 switch (code) {
2076 case lir_shl: shift = lsl; break;
2077 case lir_shr: shift = asr; break;
2078 case lir_ushr: shift = lsr; break;
2079 default: ShouldNotReachHere();
2080 }
2081
2082 if (dest->is_single_cpu()) {
2083 count &= 31;
2084 if (count != 0) {
2085 __ mov(dest->as_register(), AsmOperand(left->as_register(), shift, count));
2086 } else {
2087 move_regs(left->as_register(), dest->as_register());
2088 }
2089 } else if (dest->is_double_cpu()) {
2090 count &= 63;
2091 if (count != 0) {
2092 Register dest_lo = dest->as_register_lo();
2093 Register dest_hi = dest->as_register_hi();
2094 Register src_lo = left->as_register_lo();
2095 Register src_hi = left->as_register_hi();
2096 // Resolve possible register conflicts
2097 if (shift == lsl && dest_hi == src_lo) {
2098 dest_hi = Rtemp;
2099 } else if (shift != lsl && dest_lo == src_hi) {
2100 dest_lo = Rtemp;
2101 }
2102 __ long_shift(dest_lo, dest_hi, src_lo, src_hi, shift, count);
2103 move_regs(dest_lo, dest->as_register_lo());
2104 move_regs(dest_hi, dest->as_register_hi());
2105 } else {
2106 __ long_move(dest->as_register_lo(), dest->as_register_hi(),
2107 left->as_register_lo(), left->as_register_hi());
2108 }
2109 } else {
2110 ShouldNotReachHere();
2111 }
2112 }
2113
2114
2115 // Saves 4 given registers in reserved argument area.
2116 void LIR_Assembler::save_in_reserved_area(Register r1, Register r2, Register r3, Register r4) {
2117 verify_reserved_argument_area_size(4);
2118 __ stmia(SP, RegisterSet(r1) | RegisterSet(r2) | RegisterSet(r3) | RegisterSet(r4));
2119 }
2120
2121 // Restores 4 given registers from reserved argument area.
2122 void LIR_Assembler::restore_from_reserved_area(Register r1, Register r2, Register r3, Register r4) {
2123 __ ldmia(SP, RegisterSet(r1) | RegisterSet(r2) | RegisterSet(r3) | RegisterSet(r4), no_writeback);
2124 }
2125
2126
2127 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2128 ciArrayKlass* default_type = op->expected_type();
2129 Register src = op->src()->as_register();
2130 Register src_pos = op->src_pos()->as_register();
2131 Register dst = op->dst()->as_register();
2132 Register dst_pos = op->dst_pos()->as_register();
2133 Register length = op->length()->as_register();
2134 Register tmp = op->tmp()->as_register();
2135 Register tmp2 = Rtemp;
2136
2137 assert(src == R0 && src_pos == R1 && dst == R2 && dst_pos == R3, "code assumption");
2138
2139 __ resolve(ACCESS_READ, src);
2140 __ resolve(ACCESS_WRITE, dst);
2141
2142 CodeStub* stub = op->stub();
2143
2144 int flags = op->flags();
2145 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2146 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2147
2148 // If we don't know anything or it's an object array, just go through the generic arraycopy
2149 if (default_type == NULL) {
2150
2151 // save arguments, because they will be killed by a runtime call
2152 save_in_reserved_area(R0, R1, R2, R3);
2153
2154 // pass length argument on SP[0]
2155 __ str(length, Address(SP, -2*wordSize, pre_indexed)); // 2 words for a proper stack alignment
2156
2157 address copyfunc_addr = StubRoutines::generic_arraycopy();
2158 assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2159 #ifndef PRODUCT
2160 if (PrintC1Statistics) {
2161 __ inc_counter((address)&Runtime1::_generic_arraycopystub_cnt, tmp, tmp2);
2162 }
2163 #endif // !PRODUCT
2164 // the stub is in the code cache so close enough
2165 __ call(copyfunc_addr, relocInfo::runtime_call_type);
2166
2167 __ add(SP, SP, 2*wordSize);
2168
2169 __ cbz_32(R0, *stub->continuation());
2170
2171 __ mvn_32(tmp, R0);
2172 restore_from_reserved_area(R0, R1, R2, R3); // load saved arguments in slow case only
2173 __ sub_32(length, length, tmp);
2174 __ add_32(src_pos, src_pos, tmp);
2175 __ add_32(dst_pos, dst_pos, tmp);
2176
2177 __ b(*stub->entry());
2178
2179 __ bind(*stub->continuation());
2180 return;
2181 }
2182
2183 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(),
2184 "must be true at this point");
2185 int elem_size = type2aelembytes(basic_type);
2186 int shift = exact_log2(elem_size);
2187
2188 // Check for NULL
2189 if (flags & LIR_OpArrayCopy::src_null_check) {
2190 if (flags & LIR_OpArrayCopy::dst_null_check) {
2191 __ cmp(src, 0);
2192 __ cond_cmp(dst, 0, ne); // make one instruction shorter if both checks are needed
2193 __ b(*stub->entry(), eq);
2194 } else {
2195 __ cbz(src, *stub->entry());
2196 }
2197 } else if (flags & LIR_OpArrayCopy::dst_null_check) {
2198 __ cbz(dst, *stub->entry());
2199 }
2200
2201 // If the compiler was not able to prove that exact type of the source or the destination
2202 // of the arraycopy is an array type, check at runtime if the source or the destination is
2203 // an instance type.
2204 if (flags & LIR_OpArrayCopy::type_check) {
2205 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2206 __ load_klass(tmp, dst);
2207 __ ldr_u32(tmp2, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2208 __ mov_slow(tmp, Klass::_lh_neutral_value);
2209 __ cmp_32(tmp2, tmp);
2210 __ b(*stub->entry(), ge);
2211 }
2212
2213 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2214 __ load_klass(tmp, src);
2215 __ ldr_u32(tmp2, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2216 __ mov_slow(tmp, Klass::_lh_neutral_value);
2217 __ cmp_32(tmp2, tmp);
2218 __ b(*stub->entry(), ge);
2219 }
2220 }
2221
2222 // Check if negative
2223 const int all_positive_checks = LIR_OpArrayCopy::src_pos_positive_check |
2224 LIR_OpArrayCopy::dst_pos_positive_check |
2225 LIR_OpArrayCopy::length_positive_check;
2226 switch (flags & all_positive_checks) {
2227 case LIR_OpArrayCopy::src_pos_positive_check:
2228 __ branch_if_negative_32(src_pos, *stub->entry());
2229 break;
2230 case LIR_OpArrayCopy::dst_pos_positive_check:
2231 __ branch_if_negative_32(dst_pos, *stub->entry());
2232 break;
2233 case LIR_OpArrayCopy::length_positive_check:
2234 __ branch_if_negative_32(length, *stub->entry());
2235 break;
2236 case LIR_OpArrayCopy::src_pos_positive_check | LIR_OpArrayCopy::dst_pos_positive_check:
2237 __ branch_if_any_negative_32(src_pos, dst_pos, tmp, *stub->entry());
2238 break;
2239 case LIR_OpArrayCopy::src_pos_positive_check | LIR_OpArrayCopy::length_positive_check:
2240 __ branch_if_any_negative_32(src_pos, length, tmp, *stub->entry());
2241 break;
2242 case LIR_OpArrayCopy::dst_pos_positive_check | LIR_OpArrayCopy::length_positive_check:
2243 __ branch_if_any_negative_32(dst_pos, length, tmp, *stub->entry());
2244 break;
2245 case all_positive_checks:
2246 __ branch_if_any_negative_32(src_pos, dst_pos, length, tmp, *stub->entry());
2247 break;
2248 default:
2249 assert((flags & all_positive_checks) == 0, "the last option");
2250 }
2251
2252 // Range checks
2253 if (flags & LIR_OpArrayCopy::src_range_check) {
2254 __ ldr_s32(tmp2, Address(src, arrayOopDesc::length_offset_in_bytes()));
2255 __ add_32(tmp, src_pos, length);
2256 __ cmp_32(tmp, tmp2);
2257 __ b(*stub->entry(), hi);
2258 }
2259 if (flags & LIR_OpArrayCopy::dst_range_check) {
2260 __ ldr_s32(tmp2, Address(dst, arrayOopDesc::length_offset_in_bytes()));
2261 __ add_32(tmp, dst_pos, length);
2262 __ cmp_32(tmp, tmp2);
2263 __ b(*stub->entry(), hi);
2264 }
2265
2266 // Check if src and dst are of the same type
2267 if (flags & LIR_OpArrayCopy::type_check) {
2268 // We don't know the array types are compatible
2269 if (basic_type != T_OBJECT) {
2270 // Simple test for basic type arrays
2271 if (UseCompressedClassPointers) {
2272 // We don't need decode because we just need to compare
2273 __ ldr_u32(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
2274 __ ldr_u32(tmp2, Address(dst, oopDesc::klass_offset_in_bytes()));
2275 __ cmp_32(tmp, tmp2);
2276 } else {
2277 __ load_klass(tmp, src);
2278 __ load_klass(tmp2, dst);
2279 __ cmp(tmp, tmp2);
2280 }
2281 __ b(*stub->entry(), ne);
2282 } else {
2283 // For object arrays, if src is a sub class of dst then we can
2284 // safely do the copy.
2285 Label cont, slow;
2286
2287 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2288
2289 __ load_klass(tmp, src);
2290 __ load_klass(tmp2, dst);
2291
2292 // We are at a call so all live registers are saved before we
2293 // get here
2294 assert_different_registers(tmp, tmp2, R6, altFP_7_11);
2295
2296 __ check_klass_subtype_fast_path(tmp, tmp2, R6, altFP_7_11, &cont, copyfunc_addr == NULL ? stub->entry() : &slow, NULL);
2297
2298 __ mov(R6, R0);
2299 __ mov(altFP_7_11, R1);
2300 __ mov(R0, tmp);
2301 __ mov(R1, tmp2);
2302 __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type); // does not blow any registers except R0, LR and Rtemp
2303 __ cmp_32(R0, 0);
2304 __ mov(R0, R6);
2305 __ mov(R1, altFP_7_11);
2306
2307 if (copyfunc_addr != NULL) { // use stub if available
2308 // src is not a sub class of dst so we have to do a
2309 // per-element check.
2310
2311 __ b(cont, ne);
2312
2313 __ bind(slow);
2314
2315 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2316 if ((flags & mask) != mask) {
2317 // Check that at least both of them object arrays.
2318 assert(flags & mask, "one of the two should be known to be an object array");
2319
2320 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2321 __ load_klass(tmp, src);
2322 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2323 __ load_klass(tmp, dst);
2324 }
2325 int lh_offset = in_bytes(Klass::layout_helper_offset());
2326
2327 __ ldr_u32(tmp2, Address(tmp, lh_offset));
2328
2329 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2330 __ mov_slow(tmp, objArray_lh);
2331 __ cmp_32(tmp, tmp2);
2332 __ b(*stub->entry(), ne);
2333 }
2334
2335 save_in_reserved_area(R0, R1, R2, R3);
2336
2337 Register src_ptr = R0;
2338 Register dst_ptr = R1;
2339 Register len = R2;
2340 Register chk_off = R3;
2341 Register super_k = tmp;
2342
2343 __ add(src_ptr, src, arrayOopDesc::base_offset_in_bytes(basic_type));
2344 __ add_ptr_scaled_int32(src_ptr, src_ptr, src_pos, shift);
2345
2346 __ add(dst_ptr, dst, arrayOopDesc::base_offset_in_bytes(basic_type));
2347 __ add_ptr_scaled_int32(dst_ptr, dst_ptr, dst_pos, shift);
2348 __ load_klass(tmp, dst);
2349
2350 int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2351 int sco_offset = in_bytes(Klass::super_check_offset_offset());
2352
2353 __ ldr(super_k, Address(tmp, ek_offset));
2354
2355 __ mov(len, length);
2356 __ ldr_u32(chk_off, Address(super_k, sco_offset));
2357 __ push(super_k);
2358
2359 __ call(copyfunc_addr, relocInfo::runtime_call_type);
2360
2361 #ifndef PRODUCT
2362 if (PrintC1Statistics) {
2363 Label failed;
2364 __ cbnz_32(R0, failed);
2365 __ inc_counter((address)&Runtime1::_arraycopy_checkcast_cnt, tmp, tmp2);
2366 __ bind(failed);
2367 }
2368 #endif // PRODUCT
2369
2370 __ add(SP, SP, wordSize); // Drop super_k argument
2371
2372 __ cbz_32(R0, *stub->continuation());
2373 __ mvn_32(tmp, R0);
2374
2375 // load saved arguments in slow case only
2376 restore_from_reserved_area(R0, R1, R2, R3);
2377
2378 __ sub_32(length, length, tmp);
2379 __ add_32(src_pos, src_pos, tmp);
2380 __ add_32(dst_pos, dst_pos, tmp);
2381
2382 #ifndef PRODUCT
2383 if (PrintC1Statistics) {
2384 __ inc_counter((address)&Runtime1::_arraycopy_checkcast_attempt_cnt, tmp, tmp2);
2385 }
2386 #endif
2387
2388 __ b(*stub->entry());
2389
2390 __ bind(cont);
2391 } else {
2392 __ b(*stub->entry(), eq);
2393 __ bind(cont);
2394 }
2395 }
2396 }
2397
2398 #ifndef PRODUCT
2399 if (PrintC1Statistics) {
2400 address counter = Runtime1::arraycopy_count_address(basic_type);
2401 __ inc_counter(counter, tmp, tmp2);
2402 }
2403 #endif // !PRODUCT
2404
2405 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2406 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2407 const char *name;
2408 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2409
2410 Register src_ptr = R0;
2411 Register dst_ptr = R1;
2412 Register len = R2;
2413
2414 __ add(src_ptr, src, arrayOopDesc::base_offset_in_bytes(basic_type));
2415 __ add_ptr_scaled_int32(src_ptr, src_ptr, src_pos, shift);
2416
2417 __ add(dst_ptr, dst, arrayOopDesc::base_offset_in_bytes(basic_type));
2418 __ add_ptr_scaled_int32(dst_ptr, dst_ptr, dst_pos, shift);
2419
2420 __ mov(len, length);
2421
2422 __ call(entry, relocInfo::runtime_call_type);
2423
2424 __ bind(*stub->continuation());
2425 }
2426
2427 #ifdef ASSERT
2428 // emit run-time assertion
2429 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2430 assert(op->code() == lir_assert, "must be");
2431
2432 if (op->in_opr1()->is_valid()) {
2433 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2434 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2435 } else {
2436 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2437 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2438 }
2439
2440 Label ok;
2441 if (op->condition() != lir_cond_always) {
2442 AsmCondition acond = al;
2443 switch (op->condition()) {
2444 case lir_cond_equal: acond = eq; break;
2445 case lir_cond_notEqual: acond = ne; break;
2446 case lir_cond_less: acond = lt; break;
2447 case lir_cond_lessEqual: acond = le; break;
2448 case lir_cond_greaterEqual: acond = ge; break;
2449 case lir_cond_greater: acond = gt; break;
2450 case lir_cond_aboveEqual: acond = hs; break;
2451 case lir_cond_belowEqual: acond = ls; break;
2452 default: ShouldNotReachHere();
2453 }
2454 __ b(ok, acond);
2455 }
2456 if (op->halt()) {
2457 const char* str = __ code_string(op->msg());
2458 __ stop(str);
2459 } else {
2460 breakpoint();
2461 }
2462 __ bind(ok);
2463 }
2464 #endif // ASSERT
2465
2466 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2467 fatal("CRC32 intrinsic is not implemented on this platform");
2468 }
2469
2470 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2471 Register obj = op->obj_opr()->as_pointer_register();
2472 Register hdr = op->hdr_opr()->as_pointer_register();
2473 Register lock = op->lock_opr()->as_pointer_register();
2474 Register tmp = op->scratch_opr()->is_illegal() ? noreg :
2475 op->scratch_opr()->as_pointer_register();
2476
2477 if (!UseFastLocking) {
2478 __ b(*op->stub()->entry());
2479 } else if (op->code() == lir_lock) {
2480 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2481 __ resolve(ACCESS_READ | ACCESS_WRITE, obj);
2482 int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2483 if (op->info() != NULL) {
2484 add_debug_info_for_null_check(null_check_offset, op->info());
2485 }
2486 } else if (op->code() == lir_unlock) {
2487 __ unlock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2488 } else {
2489 ShouldNotReachHere();
2490 }
2491 __ bind(*op->stub()->continuation());
2492 }
2493
2494
2495 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2496 ciMethod* method = op->profiled_method();
2497 int bci = op->profiled_bci();
2498 ciMethod* callee = op->profiled_callee();
2499
2500 // Update counter for all call types
2501 ciMethodData* md = method->method_data_or_null();
2502 assert(md != NULL, "Sanity");
2503 ciProfileData* data = md->bci_to_data(bci);
2504 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2505 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2506 Register mdo = op->mdo()->as_register();
2507 assert(op->tmp1()->is_register(), "tmp1 must be allocated");
2508 Register tmp1 = op->tmp1()->as_pointer_register();
2509 assert_different_registers(mdo, tmp1);
2510 __ mov_metadata(mdo, md->constant_encoding());
2511 int mdo_offset_bias = 0;
2512 int max_offset = 4096;
2513 if (md->byte_offset_of_slot(data, CounterData::count_offset()) + data->size_in_bytes() >= max_offset) {
2514 // The offset is large so bias the mdo by the base of the slot so
2515 // that the ldr can use an immediate offset to reference the slots of the data
2516 mdo_offset_bias = md->byte_offset_of_slot(data, CounterData::count_offset());
2517 __ mov_slow(tmp1, mdo_offset_bias);
2518 __ add(mdo, mdo, tmp1);
2519 }
2520
2521 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
2522 // Perform additional virtual call profiling for invokevirtual and
2523 // invokeinterface bytecodes
2524 if (op->should_profile_receiver_type()) {
2525 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2526 Register recv = op->recv()->as_register();
2527 assert_different_registers(mdo, tmp1, recv);
2528 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2529 ciKlass* known_klass = op->known_holder();
2530 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2531 // We know the type that will be seen at this call site; we can
2532 // statically update the MethodData* rather than needing to do
2533 // dynamic tests on the receiver type
2534
2535 // NOTE: we should probably put a lock around this search to
2536 // avoid collisions by concurrent compilations
2537 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2538 uint i;
2539 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2540 ciKlass* receiver = vc_data->receiver(i);
2541 if (known_klass->equals(receiver)) {
2542 Address data_addr(mdo, md->byte_offset_of_slot(data,
2543 VirtualCallData::receiver_count_offset(i)) -
2544 mdo_offset_bias);
2545 __ ldr(tmp1, data_addr);
2546 __ add(tmp1, tmp1, DataLayout::counter_increment);
2547 __ str(tmp1, data_addr);
2548 return;
2549 }
2550 }
2551
2552 // Receiver type not found in profile data; select an empty slot
2553
2554 // Note that this is less efficient than it should be because it
2555 // always does a write to the receiver part of the
2556 // VirtualCallData rather than just the first time
2557 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2558 ciKlass* receiver = vc_data->receiver(i);
2559 if (receiver == NULL) {
2560 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)) -
2561 mdo_offset_bias);
2562 __ mov_metadata(tmp1, known_klass->constant_encoding());
2563 __ str(tmp1, recv_addr);
2564 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)) -
2565 mdo_offset_bias);
2566 __ ldr(tmp1, data_addr);
2567 __ add(tmp1, tmp1, DataLayout::counter_increment);
2568 __ str(tmp1, data_addr);
2569 return;
2570 }
2571 }
2572 } else {
2573 __ load_klass(recv, recv);
2574 Label update_done;
2575 type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &update_done);
2576 // Receiver did not match any saved receiver and there is no empty row for it.
2577 // Increment total counter to indicate polymorphic case.
2578 __ ldr(tmp1, counter_addr);
2579 __ add(tmp1, tmp1, DataLayout::counter_increment);
2580 __ str(tmp1, counter_addr);
2581
2582 __ bind(update_done);
2583 }
2584 } else {
2585 // Static call
2586 __ ldr(tmp1, counter_addr);
2587 __ add(tmp1, tmp1, DataLayout::counter_increment);
2588 __ str(tmp1, counter_addr);
2589 }
2590 }
2591
2592 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2593 fatal("Type profiling not implemented on this platform");
2594 }
2595
2596 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2597 Unimplemented();
2598 }
2599
2600
2601 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2602 Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no);
2603 __ add_slow(dst->as_pointer_register(), mon_addr.base(), mon_addr.disp());
2604 }
2605
2606
2607 void LIR_Assembler::align_backward_branch_target() {
2608 // Some ARM processors do better with 8-byte branch target alignment
2609 __ align(8);
2610 }
2611
2612
2613 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2614 // tmp must be unused
2615 assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2616
2617 if (left->is_single_cpu()) {
2618 assert (dest->type() == T_INT, "unexpected result type");
2619 assert (left->type() == T_INT, "unexpected left type");
2620 __ neg_32(dest->as_register(), left->as_register());
2621 } else if (left->is_double_cpu()) {
2622 Register dest_lo = dest->as_register_lo();
2623 Register dest_hi = dest->as_register_hi();
2624 Register src_lo = left->as_register_lo();
2625 Register src_hi = left->as_register_hi();
2626 if (dest_lo == src_hi) {
2627 dest_lo = Rtemp;
2628 }
2629 __ rsbs(dest_lo, src_lo, 0);
2630 __ rsc(dest_hi, src_hi, 0);
2631 move_regs(dest_lo, dest->as_register_lo());
2632 } else if (left->is_single_fpu()) {
2633 __ neg_float(dest->as_float_reg(), left->as_float_reg());
2634 } else if (left->is_double_fpu()) {
2635 __ neg_double(dest->as_double_reg(), left->as_double_reg());
2636 } else {
2637 ShouldNotReachHere();
2638 }
2639 }
2640
2641
2642 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2643 assert(patch_code == lir_patch_none, "Patch code not supported");
2644 LIR_Address* addr = addr_opr->as_address_ptr();
2645 if (addr->index()->is_illegal()) {
2646 jint c = addr->disp();
2647 if (!Assembler::is_arith_imm_in_range(c)) {
2648 BAILOUT("illegal arithmetic operand");
2649 }
2650 __ add(dest->as_pointer_register(), addr->base()->as_pointer_register(), c);
2651 } else {
2652 assert(addr->disp() == 0, "cannot handle otherwise");
2653 __ add(dest->as_pointer_register(), addr->base()->as_pointer_register(),
2654 AsmOperand(addr->index()->as_pointer_register(), lsl, addr->scale()));
2655 }
2656 }
2657
2658
2659 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2660 assert(!tmp->is_valid(), "don't need temporary");
2661 __ call(dest);
2662 if (info != NULL) {
2663 add_call_info_here(info);
2664 }
2665 }
2666
2667
2668 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2669 assert(src->is_double_cpu() && dest->is_address() ||
2670 src->is_address() && dest->is_double_cpu(),
2671 "Simple move_op is called for all other cases");
2672
2673 int null_check_offset;
2674 if (dest->is_address()) {
2675 // Store
2676 const LIR_Address* addr = dest->as_address_ptr();
2677 const Register src_lo = src->as_register_lo();
2678 const Register src_hi = src->as_register_hi();
2679 assert(addr->index()->is_illegal() && addr->disp() == 0, "The address is simple already");
2680
2681 if (src_lo < src_hi) {
2682 null_check_offset = __ offset();
2683 __ stmia(addr->base()->as_register(), RegisterSet(src_lo) | RegisterSet(src_hi));
2684 } else {
2685 assert(src_lo < Rtemp, "Rtemp is higher than any allocatable register");
2686 __ mov(Rtemp, src_hi);
2687 null_check_offset = __ offset();
2688 __ stmia(addr->base()->as_register(), RegisterSet(src_lo) | RegisterSet(Rtemp));
2689 }
2690 } else {
2691 // Load
2692 const LIR_Address* addr = src->as_address_ptr();
2693 const Register dest_lo = dest->as_register_lo();
2694 const Register dest_hi = dest->as_register_hi();
2695 assert(addr->index()->is_illegal() && addr->disp() == 0, "The address is simple already");
2696
2697 null_check_offset = __ offset();
2698 if (dest_lo < dest_hi) {
2699 __ ldmia(addr->base()->as_register(), RegisterSet(dest_lo) | RegisterSet(dest_hi));
2700 } else {
2701 assert(dest_lo < Rtemp, "Rtemp is higher than any allocatable register");
2702 __ ldmia(addr->base()->as_register(), RegisterSet(dest_lo) | RegisterSet(Rtemp));
2703 __ mov(dest_hi, Rtemp);
2704 }
2705 }
2706
2707 if (info != NULL) {
2708 add_debug_info_for_null_check(null_check_offset, info);
2709 }
2710 }
2711
2712
2713 void LIR_Assembler::membar() {
2714 __ membar(MacroAssembler::StoreLoad, Rtemp);
2715 }
2716
2717 void LIR_Assembler::membar_acquire() {
2718 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::LoadLoad | MacroAssembler::LoadStore), Rtemp);
2719 }
2720
2721 void LIR_Assembler::membar_release() {
2722 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp);
2723 }
2724
2725 void LIR_Assembler::membar_loadload() {
2726 __ membar(MacroAssembler::LoadLoad, Rtemp);
2727 }
2728
2729 void LIR_Assembler::membar_storestore() {
2730 __ membar(MacroAssembler::StoreStore, Rtemp);
2731 }
2732
2733 void LIR_Assembler::membar_loadstore() {
2734 __ membar(MacroAssembler::LoadStore, Rtemp);
2735 }
2736
2737 void LIR_Assembler::membar_storeload() {
2738 __ membar(MacroAssembler::StoreLoad, Rtemp);
2739 }
2740
2741 void LIR_Assembler::on_spin_wait() {
2742 Unimplemented();
2743 }
2744
2745 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2746 // Not used on ARM
2747 Unimplemented();
2748 }
2749
2750 void LIR_Assembler::peephole(LIR_List* lir) {
2751 LIR_OpList* inst = lir->instructions_list();
2752 const int inst_length = inst->length();
2753 for (int i = 0; i < inst_length; i++) {
2754 LIR_Op* op = inst->at(i);
2755 switch (op->code()) {
2756 case lir_cmp: {
2757 // Replace:
2758 // cmp rX, y
2759 // cmove [EQ] y, z, rX
2760 // with
2761 // cmp rX, y
2762 // cmove [EQ] illegalOpr, z, rX
2763 //
2764 // or
2765 // cmp rX, y
2766 // cmove [NE] z, y, rX
2767 // with
2768 // cmp rX, y
2769 // cmove [NE] z, illegalOpr, rX
2770 //
2771 // moves from illegalOpr should be removed when converting LIR to native assembly
2772
2773 LIR_Op2* cmp = op->as_Op2();
2774 assert(cmp != NULL, "cmp LIR instruction is not an op2");
2775
2776 if (i + 1 < inst_length) {
2777 LIR_Op2* cmove = inst->at(i + 1)->as_Op2();
2778 if (cmove != NULL && cmove->code() == lir_cmove) {
2779 LIR_Opr cmove_res = cmove->result_opr();
2780 bool res_is_op1 = cmove_res == cmp->in_opr1();
2781 bool res_is_op2 = cmove_res == cmp->in_opr2();
2782 LIR_Opr cmp_res, cmp_arg;
2783 if (res_is_op1) {
2784 cmp_res = cmp->in_opr1();
2785 cmp_arg = cmp->in_opr2();
2786 } else if (res_is_op2) {
2787 cmp_res = cmp->in_opr2();
2788 cmp_arg = cmp->in_opr1();
2789 } else {
2790 cmp_res = LIR_OprFact::illegalOpr;
2791 cmp_arg = LIR_OprFact::illegalOpr;
2792 }
2793
2794 if (cmp_res != LIR_OprFact::illegalOpr) {
2795 LIR_Condition cond = cmove->condition();
2796 if (cond == lir_cond_equal && cmove->in_opr1() == cmp_arg) {
2797 cmove->set_in_opr1(LIR_OprFact::illegalOpr);
2798 } else if (cond == lir_cond_notEqual && cmove->in_opr2() == cmp_arg) {
2799 cmove->set_in_opr2(LIR_OprFact::illegalOpr);
2800 }
2801 }
2802 }
2803 }
2804 break;
2805 }
2806
2807 default:
2808 break;
2809 }
2810 }
2811 }
2812
2813 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
2814 assert(src->is_address(), "sanity");
2815 Address addr = as_Address(src->as_address_ptr());
2816
2817 if (code == lir_xchg) {
2818 } else {
2819 assert (!data->is_oop(), "xadd for oops");
2820 }
2821
2822 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreStore | MacroAssembler::LoadStore), Rtemp);
2823
2824 Label retry;
2825 __ bind(retry);
2826
2827 if (data->type() == T_INT || data->is_oop()) {
2828 Register dst = dest->as_register();
2829 Register new_val = noreg;
2830 __ ldrex(dst, addr);
2831 if (code == lir_xadd) {
2832 Register tmp_reg = tmp->as_register();
2833 if (data->is_constant()) {
2834 assert_different_registers(dst, tmp_reg);
2835 __ add_32(tmp_reg, dst, data->as_constant_ptr()->as_jint());
2836 } else {
2837 assert_different_registers(dst, tmp_reg, data->as_register());
2838 __ add_32(tmp_reg, dst, data->as_register());
2839 }
2840 new_val = tmp_reg;
2841 } else {
2842 if (UseCompressedOops && data->is_oop()) {
2843 new_val = tmp->as_pointer_register();
2844 } else {
2845 new_val = data->as_register();
2846 }
2847 assert_different_registers(dst, new_val);
2848 }
2849 __ strex(Rtemp, new_val, addr);
2850
2851 } else if (data->type() == T_LONG) {
2852 Register dst_lo = dest->as_register_lo();
2853 Register new_val_lo = noreg;
2854 Register dst_hi = dest->as_register_hi();
2855
2856 assert(dst_hi->encoding() == dst_lo->encoding() + 1, "non aligned register pair");
2857 assert((dst_lo->encoding() & 0x1) == 0, "misaligned register pair");
2858
2859 __ bind(retry);
2860 __ ldrexd(dst_lo, addr);
2861 if (code == lir_xadd) {
2862 Register tmp_lo = tmp->as_register_lo();
2863 Register tmp_hi = tmp->as_register_hi();
2864
2865 assert(tmp_hi->encoding() == tmp_lo->encoding() + 1, "non aligned register pair");
2866 assert((tmp_lo->encoding() & 0x1) == 0, "misaligned register pair");
2867
2868 if (data->is_constant()) {
2869 jlong c = data->as_constant_ptr()->as_jlong();
2870 assert((jlong)((jint)c) == c, "overflow");
2871 assert_different_registers(dst_lo, dst_hi, tmp_lo, tmp_hi);
2872 __ adds(tmp_lo, dst_lo, (jint)c);
2873 __ adc(tmp_hi, dst_hi, 0);
2874 } else {
2875 Register new_val_lo = data->as_register_lo();
2876 Register new_val_hi = data->as_register_hi();
2877 __ adds(tmp_lo, dst_lo, new_val_lo);
2878 __ adc(tmp_hi, dst_hi, new_val_hi);
2879 assert_different_registers(dst_lo, dst_hi, tmp_lo, tmp_hi, new_val_lo, new_val_hi);
2880 }
2881 new_val_lo = tmp_lo;
2882 } else {
2883 new_val_lo = data->as_register_lo();
2884 Register new_val_hi = data->as_register_hi();
2885
2886 assert_different_registers(dst_lo, dst_hi, new_val_lo, new_val_hi);
2887 assert(new_val_hi->encoding() == new_val_lo->encoding() + 1, "non aligned register pair");
2888 assert((new_val_lo->encoding() & 0x1) == 0, "misaligned register pair");
2889 }
2890 __ strexd(Rtemp, new_val_lo, addr);
2891 } else {
2892 ShouldNotReachHere();
2893 }
2894
2895 __ cbnz_32(Rtemp, retry);
2896 __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp);
2897
2898 }
2899
2900 #undef __