1 /*
2 * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "gc/shared/barrierSet.hpp"
37 #include "gc/shared/cardTableModRefBS.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "vmreg_aarch64.inline.hpp"
43
44
45
46 #ifndef PRODUCT
47 #define COMMENT(x) do { __ block_comment(x); } while (0)
48 #else
49 #define COMMENT(x)
50 #endif
51
52 NEEDS_CLEANUP // remove this definitions ?
53 const Register IC_Klass = rscratch2; // where the IC klass is cached
54 const Register SYNC_header = r0; // synchronization header
55 const Register SHIFT_count = r0; // where count for shift operations must be
56
57 #define __ _masm->
58
59
60 static void select_different_registers(Register preserve,
61 Register extra,
62 Register &tmp1,
63 Register &tmp2) {
64 if (tmp1 == preserve) {
65 assert_different_registers(tmp1, tmp2, extra);
66 tmp1 = extra;
67 } else if (tmp2 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp2 = extra;
70 }
71 assert_different_registers(preserve, tmp1, tmp2);
72 }
73
74
75
76 static void select_different_registers(Register preserve,
77 Register extra,
78 Register &tmp1,
79 Register &tmp2,
80 Register &tmp3) {
81 if (tmp1 == preserve) {
82 assert_different_registers(tmp1, tmp2, tmp3, extra);
83 tmp1 = extra;
84 } else if (tmp2 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp2 = extra;
87 } else if (tmp3 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp3 = extra;
90 }
91 assert_different_registers(preserve, tmp1, tmp2, tmp3);
92 }
93
94
95 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
96
97
98 LIR_Opr LIR_Assembler::receiverOpr() {
99 return FrameMap::receiver_opr;
100 }
101
102 LIR_Opr LIR_Assembler::osrBufferPointer() {
103 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
104 }
105
106 //--------------fpu register translations-----------------------
107
108
109 address LIR_Assembler::float_constant(float f) {
110 address const_addr = __ float_constant(f);
111 if (const_addr == NULL) {
112 bailout("const section overflow");
113 return __ code()->consts()->start();
114 } else {
115 return const_addr;
116 }
117 }
118
119
120 address LIR_Assembler::double_constant(double d) {
121 address const_addr = __ double_constant(d);
122 if (const_addr == NULL) {
123 bailout("const section overflow");
124 return __ code()->consts()->start();
125 } else {
126 return const_addr;
127 }
128 }
129
130 address LIR_Assembler::int_constant(jlong n) {
131 address const_addr = __ long_constant(n);
132 if (const_addr == NULL) {
133 bailout("const section overflow");
134 return __ code()->consts()->start();
135 } else {
136 return const_addr;
137 }
138 }
139
140 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
141
142 void LIR_Assembler::reset_FPU() { Unimplemented(); }
143
144 void LIR_Assembler::fpop() { Unimplemented(); }
145
146 void LIR_Assembler::fxch(int i) { Unimplemented(); }
147
148 void LIR_Assembler::fld(int i) { Unimplemented(); }
149
150 void LIR_Assembler::ffree(int i) { Unimplemented(); }
151
152 void LIR_Assembler::breakpoint() { Unimplemented(); }
153
154 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
155
156 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
157
158 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
159 //-------------------------------------------
160
161 static Register as_reg(LIR_Opr op) {
162 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
163 }
164
165 static jlong as_long(LIR_Opr data) {
166 jlong result;
167 switch (data->type()) {
168 case T_INT:
169 result = (data->as_jint());
170 break;
171 case T_LONG:
172 result = (data->as_jlong());
173 break;
174 default:
175 ShouldNotReachHere();
176 }
177 return result;
178 }
179
180 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
181 Register base = addr->base()->as_pointer_register();
182 LIR_Opr opr = addr->index();
183 if (opr->is_cpu_register()) {
184 Register index;
185 if (opr->is_single_cpu())
186 index = opr->as_register();
187 else
188 index = opr->as_register_lo();
189 assert(addr->disp() == 0, "must be");
190 switch(opr->type()) {
191 case T_INT:
192 return Address(base, index, Address::sxtw(addr->scale()));
193 case T_LONG:
194 return Address(base, index, Address::lsl(addr->scale()));
195 default:
196 ShouldNotReachHere();
197 }
198 } else {
199 intptr_t addr_offset = intptr_t(addr->disp());
200 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
201 return Address(base, addr_offset, Address::lsl(addr->scale()));
202 else {
203 __ mov(tmp, addr_offset);
204 return Address(base, tmp, Address::lsl(addr->scale()));
205 }
206 }
207 return Address();
208 }
209
210 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
211 ShouldNotReachHere();
212 return Address();
213 }
214
215 Address LIR_Assembler::as_Address(LIR_Address* addr) {
216 return as_Address(addr, rscratch1);
217 }
218
219 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
220 return as_Address(addr, rscratch1); // Ouch
221 // FIXME: This needs to be much more clever. See x86.
222 }
223
224
225 void LIR_Assembler::osr_entry() {
226 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
227 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
228 ValueStack* entry_state = osr_entry->state();
229 int number_of_locks = entry_state->locks_size();
230
231 // we jump here if osr happens with the interpreter
232 // state set up to continue at the beginning of the
233 // loop that triggered osr - in particular, we have
234 // the following registers setup:
235 //
236 // r2: osr buffer
237 //
238
239 // build frame
240 ciMethod* m = compilation()->method();
241 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
242
243 // OSR buffer is
244 //
245 // locals[nlocals-1..0]
246 // monitors[0..number_of_locks]
247 //
248 // locals is a direct copy of the interpreter frame so in the osr buffer
249 // so first slot in the local array is the last local from the interpreter
250 // and last slot is local[0] (receiver) from the interpreter
251 //
252 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
253 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
254 // in the interpreter frame (the method lock if a sync method)
255
256 // Initialize monitors in the compiled activation.
257 // r2: pointer to osr buffer
258 //
259 // All other registers are dead at this point and the locals will be
260 // copied into place by code emitted in the IR.
261
262 Register OSR_buf = osrBufferPointer()->as_pointer_register();
263 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
264 int monitor_offset = BytesPerWord * method()->max_locals() +
265 (2 * BytesPerWord) * (number_of_locks - 1);
266 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
267 // the OSR buffer using 2 word entries: first the lock and then
268 // the oop.
269 for (int i = 0; i < number_of_locks; i++) {
270 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
271 #ifdef ASSERT
272 // verify the interpreter's monitor has a non-null object
273 {
274 Label L;
275 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
276 __ cbnz(rscratch1, L);
277 __ stop("locked object is NULL");
278 __ bind(L);
279 }
280 #endif
281 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
282 __ str(r19, frame_map()->address_for_monitor_lock(i));
283 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
284 __ str(r19, frame_map()->address_for_monitor_object(i));
285 }
286 }
287 }
288
289
290 // inline cache check; done before the frame is built.
291 int LIR_Assembler::check_icache() {
292 Register receiver = FrameMap::receiver_opr->as_register();
293 Register ic_klass = IC_Klass;
294 int start_offset = __ offset();
295 __ inline_cache_check(receiver, ic_klass);
296
297 // if icache check fails, then jump to runtime routine
298 // Note: RECEIVER must still contain the receiver!
299 Label dont;
300 __ br(Assembler::EQ, dont);
301 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
302
303 // We align the verified entry point unless the method body
304 // (including its inline cache check) will fit in a single 64-byte
305 // icache line.
306 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
307 // force alignment after the cache check.
308 __ align(CodeEntryAlignment);
309 }
310
311 __ bind(dont);
312 return start_offset;
313 }
314
315
316 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
317 if (o == NULL) {
318 __ mov(reg, zr);
319 } else {
320 __ movoop(reg, o, /*immediate*/true);
321 }
322 }
323
324 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
325 address target = NULL;
326 relocInfo::relocType reloc_type = relocInfo::none;
327
328 switch (patching_id(info)) {
329 case PatchingStub::access_field_id:
330 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
331 reloc_type = relocInfo::section_word_type;
332 break;
333 case PatchingStub::load_klass_id:
334 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
335 reloc_type = relocInfo::metadata_type;
336 break;
337 case PatchingStub::load_mirror_id:
338 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
339 reloc_type = relocInfo::oop_type;
340 break;
341 case PatchingStub::load_appendix_id:
342 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
343 reloc_type = relocInfo::oop_type;
344 break;
345 default: ShouldNotReachHere();
346 }
347
348 __ far_call(RuntimeAddress(target));
349 add_call_info_here(info);
350 }
351
352 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
353 deoptimize_trap(info);
354 }
355
356
357 // This specifies the rsp decrement needed to build the frame
358 int LIR_Assembler::initial_frame_size_in_bytes() const {
359 // if rounding, must let FrameMap know!
360
361 // The frame_map records size in slots (32bit word)
362
363 // subtract two words to account for return address and link
364 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
365 }
366
367
368 int LIR_Assembler::emit_exception_handler() {
369 // if the last instruction is a call (typically to do a throw which
370 // is coming at the end after block reordering) the return address
371 // must still point into the code area in order to avoid assertion
372 // failures when searching for the corresponding bci => add a nop
373 // (was bug 5/14/1999 - gri)
374 __ nop();
375
376 // generate code for exception handler
377 address handler_base = __ start_a_stub(exception_handler_size);
378 if (handler_base == NULL) {
379 // not enough space left for the handler
380 bailout("exception handler overflow");
381 return -1;
382 }
383
384 int offset = code_offset();
385
386 // the exception oop and pc are in r0, and r3
387 // no other registers need to be preserved, so invalidate them
388 __ invalidate_registers(false, true, true, false, true, true);
389
390 // check that there is really an exception
391 __ verify_not_null_oop(r0);
392
393 // search an exception handler (r0: exception oop, r3: throwing pc)
394 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here();
395 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
396 __ end_a_stub();
397
398 return offset;
399 }
400
401
402 // Emit the code to remove the frame from the stack in the exception
403 // unwind path.
404 int LIR_Assembler::emit_unwind_handler() {
405 #ifndef PRODUCT
406 if (CommentedAssembly) {
407 _masm->block_comment("Unwind handler");
408 }
409 #endif
410
411 int offset = code_offset();
412
413 // Fetch the exception from TLS and clear out exception related thread state
414 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
415 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
417
418 __ bind(_unwind_handler_entry);
419 __ verify_not_null_oop(r0);
420 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
421 __ mov(r19, r0); // Preserve the exception
422 }
423
424 // Preform needed unlocking
425 MonitorExitStub* stub = NULL;
426 if (method()->is_synchronized()) {
427 monitor_address(0, FrameMap::r0_opr);
428 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
429 __ unlock_object(r5, r4, r0, *stub->entry());
430 __ bind(*stub->continuation());
431 }
432
433 if (compilation()->env()->dtrace_method_probes()) {
434 __ call_Unimplemented();
435 #if 0
436 __ movptr(Address(rsp, 0), rax);
437 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
438 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
439 #endif
440 }
441
442 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
443 __ mov(r0, r19); // Restore the exception
444 }
445
446 // remove the activation and dispatch to the unwind handler
447 __ block_comment("remove_frame and dispatch to the unwind handler");
448 __ remove_frame(initial_frame_size_in_bytes());
449 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
450
451 // Emit the slow path assembly
452 if (stub != NULL) {
453 stub->emit_code(this);
454 }
455
456 return offset;
457 }
458
459
460 int LIR_Assembler::emit_deopt_handler() {
461 // if the last instruction is a call (typically to do a throw which
462 // is coming at the end after block reordering) the return address
463 // must still point into the code area in order to avoid assertion
464 // failures when searching for the corresponding bci => add a nop
465 // (was bug 5/14/1999 - gri)
466 __ nop();
467
468 // generate code for exception handler
469 address handler_base = __ start_a_stub(deopt_handler_size);
470 if (handler_base == NULL) {
471 // not enough space left for the handler
472 bailout("deopt handler overflow");
473 return -1;
474 }
475
476 int offset = code_offset();
477
478 __ adr(lr, pc());
479 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
480 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
481 __ end_a_stub();
482
483 return offset;
484 }
485
486 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
487 _masm->code_section()->relocate(adr, relocInfo::poll_type);
488 int pc_offset = code_offset();
489 flush_debug_info(pc_offset);
490 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
491 if (info->exception_handlers() != NULL) {
492 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
493 }
494 }
495
496 // Rather than take a segfault when the polling page is protected,
497 // explicitly check for a safepoint in progress and if there is one,
498 // fake a call to the handler as if a segfault had been caught.
499 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
500 __ mov(rscratch1, SafepointSynchronize::address_of_state());
501 __ ldrb(rscratch1, Address(rscratch1));
502 Label nope, poll;
503 __ cbz(rscratch1, nope);
504 __ block_comment("safepoint");
505 __ enter();
506 __ push(0x3, sp); // r0 & r1
507 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
508 __ adr(r0, poll);
509 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
510 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
511 __ blrt(rscratch1, 1, 0, 1);
512 __ maybe_isb();
513 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
514 __ mov(rscratch1, r0);
515 __ pop(0x3, sp); // r0 & r1
516 __ leave();
517 __ br(rscratch1);
518 address polling_page(os::get_polling_page());
519 assert(os::is_poll_address(polling_page), "should be");
520 unsigned long off;
521 __ adrp(rscratch1, Address(polling_page, rtype), off);
522 __ bind(poll);
523 if (info)
524 add_debug_info_for_branch(info); // This isn't just debug info:
525 // it's the oop map
526 else
527 __ code_section()->relocate(pc(), rtype);
528 __ ldrw(zr, Address(rscratch1, off));
529 __ bind(nope);
530 }
531
532 void LIR_Assembler::return_op(LIR_Opr result) {
533 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
534 // Pop the stack before the safepoint code
535 __ remove_frame(initial_frame_size_in_bytes());
536 address polling_page(os::get_polling_page());
537 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
538 __ ret(lr);
539 }
540
541 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
542 address polling_page(os::get_polling_page());
543 guarantee(info != NULL, "Shouldn't be NULL");
544 assert(os::is_poll_address(polling_page), "should be");
545 unsigned long off;
546 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
547 assert(off == 0, "must be");
548 add_debug_info_for_branch(info); // This isn't just debug info:
549 // it's the oop map
550 __ read_polling_page(rscratch1, relocInfo::poll_type);
551 return __ offset();
552 }
553
554
555 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
556 if (from_reg == r31_sp)
557 from_reg = sp;
558 if (to_reg == r31_sp)
559 to_reg = sp;
560 __ mov(to_reg, from_reg);
561 }
562
563 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
564
565
566 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
567 assert(src->is_constant(), "should not call otherwise");
568 assert(dest->is_register(), "should not call otherwise");
569 LIR_Const* c = src->as_constant_ptr();
570
571 switch (c->type()) {
572 case T_INT: {
573 assert(patch_code == lir_patch_none, "no patching handled here");
574 __ movw(dest->as_register(), c->as_jint());
575 break;
576 }
577
578 case T_ADDRESS: {
579 assert(patch_code == lir_patch_none, "no patching handled here");
580 __ mov(dest->as_register(), c->as_jint());
581 break;
582 }
583
584 case T_LONG: {
585 assert(patch_code == lir_patch_none, "no patching handled here");
586 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
587 break;
588 }
589
590 case T_OBJECT: {
591 if (patch_code == lir_patch_none) {
592 jobject2reg(c->as_jobject(), dest->as_register());
593 } else {
594 jobject2reg_with_patching(dest->as_register(), info);
595 }
596 break;
597 }
598
599 case T_METADATA: {
600 if (patch_code != lir_patch_none) {
601 klass2reg_with_patching(dest->as_register(), info);
602 } else {
603 __ mov_metadata(dest->as_register(), c->as_metadata());
604 }
605 break;
606 }
607
608 case T_FLOAT: {
609 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
610 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
611 } else {
612 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
613 __ ldrs(dest->as_float_reg(), Address(rscratch1));
614 }
615 break;
616 }
617
618 case T_DOUBLE: {
619 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
620 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
621 } else {
622 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
623 __ ldrd(dest->as_double_reg(), Address(rscratch1));
624 }
625 break;
626 }
627
628 default:
629 ShouldNotReachHere();
630 }
631 }
632
633 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
634 LIR_Const* c = src->as_constant_ptr();
635 switch (c->type()) {
636 case T_OBJECT:
637 {
638 if (! c->as_jobject())
639 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
640 else {
641 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
642 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
643 }
644 }
645 break;
646 case T_ADDRESS:
647 {
648 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
649 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
650 }
651 case T_INT:
652 case T_FLOAT:
653 {
654 Register reg = zr;
655 if (c->as_jint_bits() == 0)
656 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
657 else {
658 __ movw(rscratch1, c->as_jint_bits());
659 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
660 }
661 }
662 break;
663 case T_LONG:
664 case T_DOUBLE:
665 {
666 Register reg = zr;
667 if (c->as_jlong_bits() == 0)
668 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
669 lo_word_offset_in_bytes));
670 else {
671 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
672 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
673 lo_word_offset_in_bytes));
674 }
675 }
676 break;
677 default:
678 ShouldNotReachHere();
679 }
680 }
681
682 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
683 assert(src->is_constant(), "should not call otherwise");
684 LIR_Const* c = src->as_constant_ptr();
685 LIR_Address* to_addr = dest->as_address_ptr();
686
687 void (Assembler::* insn)(Register Rt, const Address &adr);
688
689 switch (type) {
690 case T_ADDRESS:
691 assert(c->as_jint() == 0, "should be");
692 insn = &Assembler::str;
693 break;
694 case T_LONG:
695 assert(c->as_jlong() == 0, "should be");
696 insn = &Assembler::str;
697 break;
698 case T_INT:
699 assert(c->as_jint() == 0, "should be");
700 insn = &Assembler::strw;
701 break;
702 case T_OBJECT:
703 case T_ARRAY:
704 assert(c->as_jobject() == 0, "should be");
705 if (UseCompressedOops && !wide) {
706 insn = &Assembler::strw;
707 } else {
708 insn = &Assembler::str;
709 }
710 break;
711 case T_CHAR:
712 case T_SHORT:
713 assert(c->as_jint() == 0, "should be");
714 insn = &Assembler::strh;
715 break;
716 case T_BOOLEAN:
717 case T_BYTE:
718 assert(c->as_jint() == 0, "should be");
719 insn = &Assembler::strb;
720 break;
721 default:
722 ShouldNotReachHere();
723 }
724
725 if (info) add_debug_info_for_null_check_here(info);
726 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
727 }
728
729 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
730 assert(src->is_register(), "should not call otherwise");
731 assert(dest->is_register(), "should not call otherwise");
732
733 // move between cpu-registers
734 if (dest->is_single_cpu()) {
735 if (src->type() == T_LONG) {
736 // Can do LONG -> OBJECT
737 move_regs(src->as_register_lo(), dest->as_register());
738 return;
739 }
740 assert(src->is_single_cpu(), "must match");
741 if (src->type() == T_OBJECT) {
742 __ verify_oop(src->as_register());
743 }
744 move_regs(src->as_register(), dest->as_register());
745
746 } else if (dest->is_double_cpu()) {
747 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
748 // Surprising to me but we can see move of a long to t_object
749 __ verify_oop(src->as_register());
750 move_regs(src->as_register(), dest->as_register_lo());
751 return;
752 }
753 assert(src->is_double_cpu(), "must match");
754 Register f_lo = src->as_register_lo();
755 Register f_hi = src->as_register_hi();
756 Register t_lo = dest->as_register_lo();
757 Register t_hi = dest->as_register_hi();
758 assert(f_hi == f_lo, "must be same");
759 assert(t_hi == t_lo, "must be same");
760 move_regs(f_lo, t_lo);
761
762 } else if (dest->is_single_fpu()) {
763 __ fmovs(dest->as_float_reg(), src->as_float_reg());
764
765 } else if (dest->is_double_fpu()) {
766 __ fmovd(dest->as_double_reg(), src->as_double_reg());
767
768 } else {
769 ShouldNotReachHere();
770 }
771 }
772
773 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
774 if (src->is_single_cpu()) {
775 if (type == T_ARRAY || type == T_OBJECT) {
776 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
777 __ verify_oop(src->as_register());
778 } else if (type == T_METADATA || type == T_DOUBLE) {
779 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
780 } else {
781 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
782 }
783
784 } else if (src->is_double_cpu()) {
785 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
786 __ str(src->as_register_lo(), dest_addr_LO);
787
788 } else if (src->is_single_fpu()) {
789 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
790 __ strs(src->as_float_reg(), dest_addr);
791
792 } else if (src->is_double_fpu()) {
793 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
794 __ strd(src->as_double_reg(), dest_addr);
795
796 } else {
797 ShouldNotReachHere();
798 }
799
800 }
801
802
803 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
804 LIR_Address* to_addr = dest->as_address_ptr();
805 PatchingStub* patch = NULL;
806 Register compressed_src = rscratch1;
807
808 if (patch_code != lir_patch_none) {
809 deoptimize_trap(info);
810 return;
811 }
812
813 if (type == T_ARRAY || type == T_OBJECT) {
814 __ verify_oop(src->as_register());
815
816 if (UseCompressedOops && !wide) {
817 __ encode_heap_oop(compressed_src, src->as_register());
818 } else {
819 compressed_src = src->as_register();
820 }
821 }
822
823 int null_check_here = code_offset();
824 switch (type) {
825 case T_FLOAT: {
826 __ strs(src->as_float_reg(), as_Address(to_addr));
827 break;
828 }
829
830 case T_DOUBLE: {
831 __ strd(src->as_double_reg(), as_Address(to_addr));
832 break;
833 }
834
835 case T_ARRAY: // fall through
836 case T_OBJECT: // fall through
837 if (UseCompressedOops && !wide) {
838 __ strw(compressed_src, as_Address(to_addr, rscratch2));
839 } else {
840 __ str(compressed_src, as_Address(to_addr));
841 }
842 break;
843 case T_METADATA:
844 // We get here to store a method pointer to the stack to pass to
845 // a dtrace runtime call. This can't work on 64 bit with
846 // compressed klass ptrs: T_METADATA can be a compressed klass
847 // ptr or a 64 bit method pointer.
848 ShouldNotReachHere();
849 __ str(src->as_register(), as_Address(to_addr));
850 break;
851 case T_ADDRESS:
852 __ str(src->as_register(), as_Address(to_addr));
853 break;
854 case T_INT:
855 __ strw(src->as_register(), as_Address(to_addr));
856 break;
857
858 case T_LONG: {
859 __ str(src->as_register_lo(), as_Address_lo(to_addr));
860 break;
861 }
862
863 case T_BYTE: // fall through
864 case T_BOOLEAN: {
865 __ strb(src->as_register(), as_Address(to_addr));
866 break;
867 }
868
869 case T_CHAR: // fall through
870 case T_SHORT:
871 __ strh(src->as_register(), as_Address(to_addr));
872 break;
873
874 default:
875 ShouldNotReachHere();
876 }
877 if (info != NULL) {
878 add_debug_info_for_null_check(null_check_here, info);
879 }
880 }
881
882
883 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
884 assert(src->is_stack(), "should not call otherwise");
885 assert(dest->is_register(), "should not call otherwise");
886
887 if (dest->is_single_cpu()) {
888 if (type == T_ARRAY || type == T_OBJECT) {
889 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
890 __ verify_oop(dest->as_register());
891 } else if (type == T_METADATA) {
892 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
893 } else {
894 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
895 }
896
897 } else if (dest->is_double_cpu()) {
898 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
899 __ ldr(dest->as_register_lo(), src_addr_LO);
900
901 } else if (dest->is_single_fpu()) {
902 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
903 __ ldrs(dest->as_float_reg(), src_addr);
904
905 } else if (dest->is_double_fpu()) {
906 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
907 __ ldrd(dest->as_double_reg(), src_addr);
908
909 } else {
910 ShouldNotReachHere();
911 }
912 }
913
914
915 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
916 address target = NULL;
917 relocInfo::relocType reloc_type = relocInfo::none;
918
919 switch (patching_id(info)) {
920 case PatchingStub::access_field_id:
921 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
922 reloc_type = relocInfo::section_word_type;
923 break;
924 case PatchingStub::load_klass_id:
925 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
926 reloc_type = relocInfo::metadata_type;
927 break;
928 case PatchingStub::load_mirror_id:
929 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
930 reloc_type = relocInfo::oop_type;
931 break;
932 case PatchingStub::load_appendix_id:
933 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
934 reloc_type = relocInfo::oop_type;
935 break;
936 default: ShouldNotReachHere();
937 }
938
939 __ far_call(RuntimeAddress(target));
940 add_call_info_here(info);
941 }
942
943 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
944
945 LIR_Opr temp;
946 if (type == T_LONG || type == T_DOUBLE)
947 temp = FrameMap::rscratch1_long_opr;
948 else
949 temp = FrameMap::rscratch1_opr;
950
951 stack2reg(src, temp, src->type());
952 reg2stack(temp, dest, dest->type(), false);
953 }
954
955
956 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
957 LIR_Address* addr = src->as_address_ptr();
958 LIR_Address* from_addr = src->as_address_ptr();
959
960 if (addr->base()->type() == T_OBJECT) {
961 __ verify_oop(addr->base()->as_pointer_register());
962 }
963
964 if (patch_code != lir_patch_none) {
965 deoptimize_trap(info);
966 return;
967 }
968
969 if (info != NULL) {
970 add_debug_info_for_null_check_here(info);
971 }
972 int null_check_here = code_offset();
973 switch (type) {
974 case T_FLOAT: {
975 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
976 break;
977 }
978
979 case T_DOUBLE: {
980 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
981 break;
982 }
983
984 case T_ARRAY: // fall through
985 case T_OBJECT: // fall through
986 if (UseCompressedOops && !wide) {
987 __ ldrw(dest->as_register(), as_Address(from_addr));
988 } else {
989 __ ldr(dest->as_register(), as_Address(from_addr));
990 }
991 break;
992 case T_METADATA:
993 // We get here to store a method pointer to the stack to pass to
994 // a dtrace runtime call. This can't work on 64 bit with
995 // compressed klass ptrs: T_METADATA can be a compressed klass
996 // ptr or a 64 bit method pointer.
997 ShouldNotReachHere();
998 __ ldr(dest->as_register(), as_Address(from_addr));
999 break;
1000 case T_ADDRESS:
1001 // FIXME: OMG this is a horrible kludge. Any offset from an
1002 // address that matches klass_offset_in_bytes() will be loaded
1003 // as a word, not a long.
1004 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1005 __ ldrw(dest->as_register(), as_Address(from_addr));
1006 } else {
1007 __ ldr(dest->as_register(), as_Address(from_addr));
1008 }
1009 break;
1010 case T_INT:
1011 __ ldrw(dest->as_register(), as_Address(from_addr));
1012 break;
1013
1014 case T_LONG: {
1015 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1016 break;
1017 }
1018
1019 case T_BYTE:
1020 __ ldrsb(dest->as_register(), as_Address(from_addr));
1021 break;
1022 case T_BOOLEAN: {
1023 __ ldrb(dest->as_register(), as_Address(from_addr));
1024 break;
1025 }
1026
1027 case T_CHAR:
1028 __ ldrh(dest->as_register(), as_Address(from_addr));
1029 break;
1030 case T_SHORT:
1031 __ ldrsh(dest->as_register(), as_Address(from_addr));
1032 break;
1033
1034 default:
1035 ShouldNotReachHere();
1036 }
1037
1038 if (type == T_ARRAY || type == T_OBJECT) {
1039 if (UseCompressedOops && !wide) {
1040 __ decode_heap_oop(dest->as_register());
1041 }
1042 __ verify_oop(dest->as_register());
1043 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1044 if (UseCompressedClassPointers) {
1045 __ decode_klass_not_null(dest->as_register());
1046 }
1047 }
1048 }
1049
1050
1051 int LIR_Assembler::array_element_size(BasicType type) const {
1052 int elem_size = type2aelembytes(type);
1053 return exact_log2(elem_size);
1054 }
1055
1056 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1057 Register Rdividend = op->in_opr1()->as_register();
1058 Register Rdivisor = op->in_opr2()->as_register();
1059 Register Rscratch = op->in_opr3()->as_register();
1060 Register Rresult = op->result_opr()->as_register();
1061 int divisor = -1;
1062
1063 /*
1064 TODO: For some reason, using the Rscratch that gets passed in is
1065 not possible because the register allocator does not see the tmp reg
1066 as used, and assignes it the same register as Rdividend. We use rscratch1
1067 instead.
1068
1069 assert(Rdividend != Rscratch, "");
1070 assert(Rdivisor != Rscratch, "");
1071 */
1072
1073 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1074 // convert division by a power of two into some shifts and logical operations
1075 }
1076
1077 if (op->code() == lir_irem) {
1078 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1079 } else if (op->code() == lir_idiv) {
1080 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1081 } else
1082 ShouldNotReachHere();
1083 }
1084
1085 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1086 #ifdef ASSERT
1087 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1088 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1089 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1090 #endif
1091
1092 if (op->cond() == lir_cond_always) {
1093 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1094 __ b(*(op->label()));
1095 } else {
1096 Assembler::Condition acond;
1097 if (op->code() == lir_cond_float_branch) {
1098 bool is_unordered = (op->ublock() == op->block());
1099 // Assembler::EQ does not permit unordered branches, so we add
1100 // another branch here. Likewise, Assembler::NE does not permit
1101 // ordered branches.
1102 if (is_unordered && op->cond() == lir_cond_equal
1103 || !is_unordered && op->cond() == lir_cond_notEqual)
1104 __ br(Assembler::VS, *(op->ublock()->label()));
1105 switch(op->cond()) {
1106 case lir_cond_equal: acond = Assembler::EQ; break;
1107 case lir_cond_notEqual: acond = Assembler::NE; break;
1108 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1109 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1110 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1111 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1112 default: ShouldNotReachHere();
1113 }
1114 } else {
1115 switch (op->cond()) {
1116 case lir_cond_equal: acond = Assembler::EQ; break;
1117 case lir_cond_notEqual: acond = Assembler::NE; break;
1118 case lir_cond_less: acond = Assembler::LT; break;
1119 case lir_cond_lessEqual: acond = Assembler::LE; break;
1120 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1121 case lir_cond_greater: acond = Assembler::GT; break;
1122 case lir_cond_belowEqual: acond = Assembler::LS; break;
1123 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1124 default: ShouldNotReachHere();
1125 }
1126 }
1127 __ br(acond,*(op->label()));
1128 }
1129 }
1130
1131
1132
1133 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1134 LIR_Opr src = op->in_opr();
1135 LIR_Opr dest = op->result_opr();
1136
1137 switch (op->bytecode()) {
1138 case Bytecodes::_i2f:
1139 {
1140 __ scvtfws(dest->as_float_reg(), src->as_register());
1141 break;
1142 }
1143 case Bytecodes::_i2d:
1144 {
1145 __ scvtfwd(dest->as_double_reg(), src->as_register());
1146 break;
1147 }
1148 case Bytecodes::_l2d:
1149 {
1150 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1151 break;
1152 }
1153 case Bytecodes::_l2f:
1154 {
1155 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1156 break;
1157 }
1158 case Bytecodes::_f2d:
1159 {
1160 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1161 break;
1162 }
1163 case Bytecodes::_d2f:
1164 {
1165 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1166 break;
1167 }
1168 case Bytecodes::_i2c:
1169 {
1170 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1171 break;
1172 }
1173 case Bytecodes::_i2l:
1174 {
1175 __ sxtw(dest->as_register_lo(), src->as_register());
1176 break;
1177 }
1178 case Bytecodes::_i2s:
1179 {
1180 __ sxth(dest->as_register(), src->as_register());
1181 break;
1182 }
1183 case Bytecodes::_i2b:
1184 {
1185 __ sxtb(dest->as_register(), src->as_register());
1186 break;
1187 }
1188 case Bytecodes::_l2i:
1189 {
1190 _masm->block_comment("FIXME: This could be a no-op");
1191 __ uxtw(dest->as_register(), src->as_register_lo());
1192 break;
1193 }
1194 case Bytecodes::_d2l:
1195 {
1196 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1197 break;
1198 }
1199 case Bytecodes::_f2i:
1200 {
1201 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1202 break;
1203 }
1204 case Bytecodes::_f2l:
1205 {
1206 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1207 break;
1208 }
1209 case Bytecodes::_d2i:
1210 {
1211 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1212 break;
1213 }
1214 default: ShouldNotReachHere();
1215 }
1216 }
1217
1218 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1219 if (op->init_check()) {
1220 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1221 InstanceKlass::init_state_offset()));
1222 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1223 add_debug_info_for_null_check_here(op->stub()->info());
1224 __ br(Assembler::NE, *op->stub()->entry());
1225 }
1226 __ allocate_object(op->obj()->as_register(),
1227 op->tmp1()->as_register(),
1228 op->tmp2()->as_register(),
1229 op->header_size(),
1230 op->object_size(),
1231 op->klass()->as_register(),
1232 *op->stub()->entry());
1233 __ bind(*op->stub()->continuation());
1234 }
1235
1236 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1237 Register len = op->len()->as_register();
1238 __ uxtw(len, len);
1239
1240 if (UseSlowPath ||
1241 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1242 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1243 __ b(*op->stub()->entry());
1244 } else {
1245 Register tmp1 = op->tmp1()->as_register();
1246 Register tmp2 = op->tmp2()->as_register();
1247 Register tmp3 = op->tmp3()->as_register();
1248 if (len == tmp1) {
1249 tmp1 = tmp3;
1250 } else if (len == tmp2) {
1251 tmp2 = tmp3;
1252 } else if (len == tmp3) {
1253 // everything is ok
1254 } else {
1255 __ mov(tmp3, len);
1256 }
1257 __ allocate_array(op->obj()->as_register(),
1258 len,
1259 tmp1,
1260 tmp2,
1261 arrayOopDesc::header_size(op->type()),
1262 array_element_size(op->type()),
1263 op->klass()->as_register(),
1264 *op->stub()->entry());
1265 }
1266 __ bind(*op->stub()->continuation());
1267 }
1268
1269 void LIR_Assembler::type_profile_helper(Register mdo,
1270 ciMethodData *md, ciProfileData *data,
1271 Register recv, Label* update_done) {
1272 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1273 Label next_test;
1274 // See if the receiver is receiver[n].
1275 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1276 __ ldr(rscratch1, Address(rscratch2));
1277 __ cmp(recv, rscratch1);
1278 __ br(Assembler::NE, next_test);
1279 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1280 __ addptr(data_addr, DataLayout::counter_increment);
1281 __ b(*update_done);
1282 __ bind(next_test);
1283 }
1284
1285 // Didn't find receiver; find next empty slot and fill it in
1286 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1287 Label next_test;
1288 __ lea(rscratch2,
1289 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1290 Address recv_addr(rscratch2);
1291 __ ldr(rscratch1, recv_addr);
1292 __ cbnz(rscratch1, next_test);
1293 __ str(recv, recv_addr);
1294 __ mov(rscratch1, DataLayout::counter_increment);
1295 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1296 __ str(rscratch1, Address(rscratch2));
1297 __ b(*update_done);
1298 __ bind(next_test);
1299 }
1300 }
1301
1302 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1303 // we always need a stub for the failure case.
1304 CodeStub* stub = op->stub();
1305 Register obj = op->object()->as_register();
1306 Register k_RInfo = op->tmp1()->as_register();
1307 Register klass_RInfo = op->tmp2()->as_register();
1308 Register dst = op->result_opr()->as_register();
1309 ciKlass* k = op->klass();
1310 Register Rtmp1 = noreg;
1311
1312 // check if it needs to be profiled
1313 ciMethodData* md;
1314 ciProfileData* data;
1315
1316 if (op->should_profile()) {
1317 ciMethod* method = op->profiled_method();
1318 assert(method != NULL, "Should have method");
1319 int bci = op->profiled_bci();
1320 md = method->method_data_or_null();
1321 assert(md != NULL, "Sanity");
1322 data = md->bci_to_data(bci);
1323 assert(data != NULL, "need data for type check");
1324 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1325 }
1326 Label profile_cast_success, profile_cast_failure;
1327 Label *success_target = op->should_profile() ? &profile_cast_success : success;
1328 Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1329
1330 if (obj == k_RInfo) {
1331 k_RInfo = dst;
1332 } else if (obj == klass_RInfo) {
1333 klass_RInfo = dst;
1334 }
1335 if (k->is_loaded() && !UseCompressedClassPointers) {
1336 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1337 } else {
1338 Rtmp1 = op->tmp3()->as_register();
1339 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1340 }
1341
1342 assert_different_registers(obj, k_RInfo, klass_RInfo);
1343
1344 if (op->should_profile()) {
1345 Label not_null;
1346 __ cbnz(obj, not_null);
1347 // Object is null; update MDO and exit
1348 Register mdo = klass_RInfo;
1349 __ mov_metadata(mdo, md->constant_encoding());
1350 Address data_addr
1351 = __ form_address(rscratch2, mdo,
1352 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1353 LogBytesPerWord);
1354 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1355 __ ldr(rscratch1, data_addr);
1356 __ orr(rscratch1, rscratch1, header_bits);
1357 __ str(rscratch1, data_addr);
1358 __ b(*obj_is_null);
1359 __ bind(not_null);
1360 } else {
1361 __ cbz(obj, *obj_is_null);
1362 }
1363
1364 if (!k->is_loaded()) {
1365 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1366 } else {
1367 __ mov_metadata(k_RInfo, k->constant_encoding());
1368 }
1369 __ verify_oop(obj);
1370
1371 if (op->fast_check()) {
1372 // get object class
1373 // not a safepoint as obj null check happens earlier
1374 __ load_klass(rscratch1, obj);
1375 __ cmp( rscratch1, k_RInfo);
1376
1377 __ br(Assembler::NE, *failure_target);
1378 // successful cast, fall through to profile or jump
1379 } else {
1380 // get object class
1381 // not a safepoint as obj null check happens earlier
1382 __ load_klass(klass_RInfo, obj);
1383 if (k->is_loaded()) {
1384 // See if we get an immediate positive hit
1385 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1386 __ cmp(k_RInfo, rscratch1);
1387 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1388 __ br(Assembler::NE, *failure_target);
1389 // successful cast, fall through to profile or jump
1390 } else {
1391 // See if we get an immediate positive hit
1392 __ br(Assembler::EQ, *success_target);
1393 // check for self
1394 __ cmp(klass_RInfo, k_RInfo);
1395 __ br(Assembler::EQ, *success_target);
1396
1397 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1398 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1399 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1400 // result is a boolean
1401 __ cbzw(klass_RInfo, *failure_target);
1402 // successful cast, fall through to profile or jump
1403 }
1404 } else {
1405 // perform the fast part of the checking logic
1406 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1407 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1408 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1409 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1410 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1411 // result is a boolean
1412 __ cbz(k_RInfo, *failure_target);
1413 // successful cast, fall through to profile or jump
1414 }
1415 }
1416 if (op->should_profile()) {
1417 Register mdo = klass_RInfo, recv = k_RInfo;
1418 __ bind(profile_cast_success);
1419 __ mov_metadata(mdo, md->constant_encoding());
1420 __ load_klass(recv, obj);
1421 Label update_done;
1422 type_profile_helper(mdo, md, data, recv, success);
1423 __ b(*success);
1424
1425 __ bind(profile_cast_failure);
1426 __ mov_metadata(mdo, md->constant_encoding());
1427 Address counter_addr
1428 = __ form_address(rscratch2, mdo,
1429 md->byte_offset_of_slot(data, CounterData::count_offset()),
1430 LogBytesPerWord);
1431 __ ldr(rscratch1, counter_addr);
1432 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1433 __ str(rscratch1, counter_addr);
1434 __ b(*failure);
1435 }
1436 __ b(*success);
1437 }
1438
1439
1440 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1441 LIR_Code code = op->code();
1442 if (code == lir_store_check) {
1443 Register value = op->object()->as_register();
1444 Register array = op->array()->as_register();
1445 Register k_RInfo = op->tmp1()->as_register();
1446 Register klass_RInfo = op->tmp2()->as_register();
1447 Register Rtmp1 = op->tmp3()->as_register();
1448
1449 CodeStub* stub = op->stub();
1450
1451 // check if it needs to be profiled
1452 ciMethodData* md;
1453 ciProfileData* data;
1454
1455 if (op->should_profile()) {
1456 ciMethod* method = op->profiled_method();
1457 assert(method != NULL, "Should have method");
1458 int bci = op->profiled_bci();
1459 md = method->method_data_or_null();
1460 assert(md != NULL, "Sanity");
1461 data = md->bci_to_data(bci);
1462 assert(data != NULL, "need data for type check");
1463 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1464 }
1465 Label profile_cast_success, profile_cast_failure, done;
1466 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1467 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1468
1469 if (op->should_profile()) {
1470 Label not_null;
1471 __ cbnz(value, not_null);
1472 // Object is null; update MDO and exit
1473 Register mdo = klass_RInfo;
1474 __ mov_metadata(mdo, md->constant_encoding());
1475 Address data_addr
1476 = __ form_address(rscratch2, mdo,
1477 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1478 LogBytesPerInt);
1479 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1480 __ ldrw(rscratch1, data_addr);
1481 __ orrw(rscratch1, rscratch1, header_bits);
1482 __ strw(rscratch1, data_addr);
1483 __ b(done);
1484 __ bind(not_null);
1485 } else {
1486 __ cbz(value, done);
1487 }
1488
1489 add_debug_info_for_null_check_here(op->info_for_exception());
1490 __ load_klass(k_RInfo, array);
1491 __ load_klass(klass_RInfo, value);
1492
1493 // get instance klass (it's already uncompressed)
1494 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1495 // perform the fast part of the checking logic
1496 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1497 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1498 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1499 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1500 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1501 // result is a boolean
1502 __ cbzw(k_RInfo, *failure_target);
1503 // fall through to the success case
1504
1505 if (op->should_profile()) {
1506 Register mdo = klass_RInfo, recv = k_RInfo;
1507 __ bind(profile_cast_success);
1508 __ mov_metadata(mdo, md->constant_encoding());
1509 __ load_klass(recv, value);
1510 Label update_done;
1511 type_profile_helper(mdo, md, data, recv, &done);
1512 __ b(done);
1513
1514 __ bind(profile_cast_failure);
1515 __ mov_metadata(mdo, md->constant_encoding());
1516 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1517 __ lea(rscratch2, counter_addr);
1518 __ ldr(rscratch1, Address(rscratch2));
1519 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1520 __ str(rscratch1, Address(rscratch2));
1521 __ b(*stub->entry());
1522 }
1523
1524 __ bind(done);
1525 } else if (code == lir_checkcast) {
1526 Register obj = op->object()->as_register();
1527 Register dst = op->result_opr()->as_register();
1528 Label success;
1529 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1530 __ bind(success);
1531 if (dst != obj) {
1532 __ mov(dst, obj);
1533 }
1534 } else if (code == lir_instanceof) {
1535 Register obj = op->object()->as_register();
1536 Register dst = op->result_opr()->as_register();
1537 Label success, failure, done;
1538 emit_typecheck_helper(op, &success, &failure, &failure);
1539 __ bind(failure);
1540 __ mov(dst, zr);
1541 __ b(done);
1542 __ bind(success);
1543 __ mov(dst, 1);
1544 __ bind(done);
1545 } else {
1546 ShouldNotReachHere();
1547 }
1548 }
1549
1550 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1551 Label retry_load, nope;
1552 // flush and load exclusive from the memory location
1553 // and fail if it is not what we expect
1554 __ bind(retry_load);
1555 __ ldaxrw(rscratch1, addr);
1556 __ cmpw(rscratch1, cmpval);
1557 __ cset(rscratch1, Assembler::NE);
1558 __ br(Assembler::NE, nope);
1559 // if we store+flush with no intervening write rscratch1 wil be zero
1560 __ stlxrw(rscratch1, newval, addr);
1561 // retry so we only ever return after a load fails to compare
1562 // ensures we don't return a stale value after a failed write.
1563 __ cbnzw(rscratch1, retry_load);
1564 __ bind(nope);
1565 __ membar(__ AnyAny);
1566 }
1567
1568 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1569 Label retry_load, nope;
1570 // flush and load exclusive from the memory location
1571 // and fail if it is not what we expect
1572 __ bind(retry_load);
1573 __ ldaxr(rscratch1, addr);
1574 __ cmp(rscratch1, cmpval);
1575 __ cset(rscratch1, Assembler::NE);
1576 __ br(Assembler::NE, nope);
1577 // if we store+flush with no intervening write rscratch1 wil be zero
1578 __ stlxr(rscratch1, newval, addr);
1579 // retry so we only ever return after a load fails to compare
1580 // ensures we don't return a stale value after a failed write.
1581 __ cbnz(rscratch1, retry_load);
1582 __ bind(nope);
1583 __ membar(__ AnyAny);
1584 }
1585
1586
1587 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1588 assert(VM_Version::supports_cx8(), "wrong machine");
1589 Register addr = as_reg(op->addr());
1590 Register newval = as_reg(op->new_value());
1591 Register cmpval = as_reg(op->cmp_value());
1592 Label succeed, fail, around;
1593
1594 if (op->code() == lir_cas_obj) {
1595 if (UseCompressedOops) {
1596 Register t1 = op->tmp1()->as_register();
1597 assert(op->tmp1()->is_valid(), "must be");
1598 __ encode_heap_oop(t1, cmpval);
1599 cmpval = t1;
1600 __ encode_heap_oop(rscratch2, newval);
1601 newval = rscratch2;
1602 casw(addr, newval, cmpval);
1603 } else {
1604 casl(addr, newval, cmpval);
1605 }
1606 } else if (op->code() == lir_cas_int) {
1607 casw(addr, newval, cmpval);
1608 } else {
1609 casl(addr, newval, cmpval);
1610 }
1611 }
1612
1613
1614 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1615
1616 Assembler::Condition acond, ncond;
1617 switch (condition) {
1618 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1619 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1620 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1621 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1622 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1623 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1624 case lir_cond_belowEqual: Unimplemented(); break;
1625 case lir_cond_aboveEqual: Unimplemented(); break;
1626 default: ShouldNotReachHere();
1627 }
1628
1629 assert(result->is_single_cpu() || result->is_double_cpu(),
1630 "expect single register for result");
1631 if (opr1->is_constant() && opr2->is_constant()
1632 && opr1->type() == T_INT && opr2->type() == T_INT) {
1633 jint val1 = opr1->as_jint();
1634 jint val2 = opr2->as_jint();
1635 if (val1 == 0 && val2 == 1) {
1636 __ cset(result->as_register(), ncond);
1637 return;
1638 } else if (val1 == 1 && val2 == 0) {
1639 __ cset(result->as_register(), acond);
1640 return;
1641 }
1642 }
1643
1644 if (opr1->is_constant() && opr2->is_constant()
1645 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1646 jlong val1 = opr1->as_jlong();
1647 jlong val2 = opr2->as_jlong();
1648 if (val1 == 0 && val2 == 1) {
1649 __ cset(result->as_register_lo(), ncond);
1650 return;
1651 } else if (val1 == 1 && val2 == 0) {
1652 __ cset(result->as_register_lo(), acond);
1653 return;
1654 }
1655 }
1656
1657 if (opr1->is_stack()) {
1658 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1659 opr1 = FrameMap::rscratch1_opr;
1660 } else if (opr1->is_constant()) {
1661 LIR_Opr tmp
1662 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1663 const2reg(opr1, tmp, lir_patch_none, NULL);
1664 opr1 = tmp;
1665 }
1666
1667 if (opr2->is_stack()) {
1668 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1669 opr2 = FrameMap::rscratch2_opr;
1670 } else if (opr2->is_constant()) {
1671 LIR_Opr tmp
1672 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1673 const2reg(opr2, tmp, lir_patch_none, NULL);
1674 opr2 = tmp;
1675 }
1676
1677 if (result->type() == T_LONG)
1678 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1679 else
1680 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1681 }
1682
1683 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1684 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1685
1686 if (left->is_single_cpu()) {
1687 Register lreg = left->as_register();
1688 Register dreg = as_reg(dest);
1689
1690 if (right->is_single_cpu()) {
1691 // cpu register - cpu register
1692
1693 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1694 "should be");
1695 Register rreg = right->as_register();
1696 switch (code) {
1697 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1698 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1699 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1700 default: ShouldNotReachHere();
1701 }
1702
1703 } else if (right->is_double_cpu()) {
1704 Register rreg = right->as_register_lo();
1705 // single_cpu + double_cpu: can happen with obj+long
1706 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1707 switch (code) {
1708 case lir_add: __ add(dreg, lreg, rreg); break;
1709 case lir_sub: __ sub(dreg, lreg, rreg); break;
1710 default: ShouldNotReachHere();
1711 }
1712 } else if (right->is_constant()) {
1713 // cpu register - constant
1714 jlong c;
1715
1716 // FIXME. This is fugly: we really need to factor all this logic.
1717 switch(right->type()) {
1718 case T_LONG:
1719 c = right->as_constant_ptr()->as_jlong();
1720 break;
1721 case T_INT:
1722 case T_ADDRESS:
1723 c = right->as_constant_ptr()->as_jint();
1724 break;
1725 default:
1726 ShouldNotReachHere();
1727 break;
1728 }
1729
1730 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1731 if (c == 0 && dreg == lreg) {
1732 COMMENT("effective nop elided");
1733 return;
1734 }
1735 switch(left->type()) {
1736 case T_INT:
1737 switch (code) {
1738 case lir_add: __ addw(dreg, lreg, c); break;
1739 case lir_sub: __ subw(dreg, lreg, c); break;
1740 default: ShouldNotReachHere();
1741 }
1742 break;
1743 case T_OBJECT:
1744 case T_ADDRESS:
1745 switch (code) {
1746 case lir_add: __ add(dreg, lreg, c); break;
1747 case lir_sub: __ sub(dreg, lreg, c); break;
1748 default: ShouldNotReachHere();
1749 }
1750 break;
1751 ShouldNotReachHere();
1752 }
1753 } else {
1754 ShouldNotReachHere();
1755 }
1756
1757 } else if (left->is_double_cpu()) {
1758 Register lreg_lo = left->as_register_lo();
1759
1760 if (right->is_double_cpu()) {
1761 // cpu register - cpu register
1762 Register rreg_lo = right->as_register_lo();
1763 switch (code) {
1764 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1765 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1766 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1767 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1768 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1769 default:
1770 ShouldNotReachHere();
1771 }
1772
1773 } else if (right->is_constant()) {
1774 jlong c = right->as_constant_ptr()->as_jlong_bits();
1775 Register dreg = as_reg(dest);
1776 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1777 if (c == 0 && dreg == lreg_lo) {
1778 COMMENT("effective nop elided");
1779 return;
1780 }
1781 switch (code) {
1782 case lir_add: __ add(dreg, lreg_lo, c); break;
1783 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1784 default:
1785 ShouldNotReachHere();
1786 }
1787 } else {
1788 ShouldNotReachHere();
1789 }
1790 } else if (left->is_single_fpu()) {
1791 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1792 switch (code) {
1793 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1794 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1795 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1796 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1797 default:
1798 ShouldNotReachHere();
1799 }
1800 } else if (left->is_double_fpu()) {
1801 if (right->is_double_fpu()) {
1802 // cpu register - cpu register
1803 switch (code) {
1804 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1805 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1806 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1807 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1808 default:
1809 ShouldNotReachHere();
1810 }
1811 } else {
1812 if (right->is_constant()) {
1813 ShouldNotReachHere();
1814 }
1815 ShouldNotReachHere();
1816 }
1817 } else if (left->is_single_stack() || left->is_address()) {
1818 assert(left == dest, "left and dest must be equal");
1819 ShouldNotReachHere();
1820 } else {
1821 ShouldNotReachHere();
1822 }
1823 }
1824
1825 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1826
1827
1828 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1829 switch(code) {
1830 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1831 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1832 default : ShouldNotReachHere();
1833 }
1834 }
1835
1836 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1837
1838 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1839 Register Rleft = left->is_single_cpu() ? left->as_register() :
1840 left->as_register_lo();
1841 if (dst->is_single_cpu()) {
1842 Register Rdst = dst->as_register();
1843 if (right->is_constant()) {
1844 switch (code) {
1845 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1846 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1847 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1848 default: ShouldNotReachHere(); break;
1849 }
1850 } else {
1851 Register Rright = right->is_single_cpu() ? right->as_register() :
1852 right->as_register_lo();
1853 switch (code) {
1854 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1855 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1856 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1857 default: ShouldNotReachHere(); break;
1858 }
1859 }
1860 } else {
1861 Register Rdst = dst->as_register_lo();
1862 if (right->is_constant()) {
1863 switch (code) {
1864 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1865 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1866 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1867 default: ShouldNotReachHere(); break;
1868 }
1869 } else {
1870 Register Rright = right->is_single_cpu() ? right->as_register() :
1871 right->as_register_lo();
1872 switch (code) {
1873 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1874 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1875 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1876 default: ShouldNotReachHere(); break;
1877 }
1878 }
1879 }
1880 }
1881
1882
1883
1884 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1885
1886
1887 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1888 if (opr1->is_constant() && opr2->is_single_cpu()) {
1889 // tableswitch
1890 Register reg = as_reg(opr2);
1891 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1892 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1893 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1894 Register reg1 = as_reg(opr1);
1895 if (opr2->is_single_cpu()) {
1896 // cpu register - cpu register
1897 Register reg2 = opr2->as_register();
1898 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1899 __ cmp(reg1, reg2);
1900 } else {
1901 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1902 __ cmpw(reg1, reg2);
1903 }
1904 return;
1905 }
1906 if (opr2->is_double_cpu()) {
1907 // cpu register - cpu register
1908 Register reg2 = opr2->as_register_lo();
1909 __ cmp(reg1, reg2);
1910 return;
1911 }
1912
1913 if (opr2->is_constant()) {
1914 jlong imm;
1915 switch(opr2->type()) {
1916 case T_LONG:
1917 imm = opr2->as_constant_ptr()->as_jlong();
1918 break;
1919 case T_INT:
1920 case T_ADDRESS:
1921 imm = opr2->as_constant_ptr()->as_jint();
1922 break;
1923 case T_OBJECT:
1924 case T_ARRAY:
1925 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1926 break;
1927 default:
1928 ShouldNotReachHere();
1929 break;
1930 }
1931
1932 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1933 if (type2aelembytes(opr1->type()) <= 4)
1934 __ cmpw(reg1, imm);
1935 else
1936 __ cmp(reg1, imm);
1937 return;
1938 } else {
1939 __ mov(rscratch1, imm);
1940 if (type2aelembytes(opr1->type()) <= 4)
1941 __ cmpw(reg1, rscratch1);
1942 else
1943 __ cmp(reg1, rscratch1);
1944 return;
1945 }
1946 } else
1947 ShouldNotReachHere();
1948 } else if (opr1->is_single_fpu()) {
1949 FloatRegister reg1 = opr1->as_float_reg();
1950 assert(opr2->is_single_fpu(), "expect single float register");
1951 FloatRegister reg2 = opr2->as_float_reg();
1952 __ fcmps(reg1, reg2);
1953 } else if (opr1->is_double_fpu()) {
1954 FloatRegister reg1 = opr1->as_double_reg();
1955 assert(opr2->is_double_fpu(), "expect double float register");
1956 FloatRegister reg2 = opr2->as_double_reg();
1957 __ fcmpd(reg1, reg2);
1958 } else {
1959 ShouldNotReachHere();
1960 }
1961 }
1962
1963 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1964 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1965 bool is_unordered_less = (code == lir_ucmp_fd2i);
1966 if (left->is_single_fpu()) {
1967 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1968 } else if (left->is_double_fpu()) {
1969 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1970 } else {
1971 ShouldNotReachHere();
1972 }
1973 } else if (code == lir_cmp_l2i) {
1974 Label done;
1975 __ cmp(left->as_register_lo(), right->as_register_lo());
1976 __ mov(dst->as_register(), (u_int64_t)-1L);
1977 __ br(Assembler::LT, done);
1978 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1979 __ bind(done);
1980 } else {
1981 ShouldNotReachHere();
1982 }
1983 }
1984
1985
1986 void LIR_Assembler::align_call(LIR_Code code) { }
1987
1988
1989 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1990 address call = __ trampoline_call(Address(op->addr(), rtype));
1991 if (call == NULL) {
1992 bailout("trampoline stub overflow");
1993 return;
1994 }
1995 add_call_info(code_offset(), op->info());
1996 }
1997
1998
1999 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2000 address call = __ ic_call(op->addr());
2001 if (call == NULL) {
2002 bailout("trampoline stub overflow");
2003 return;
2004 }
2005 add_call_info(code_offset(), op->info());
2006 }
2007
2008
2009 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2010 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2011 ShouldNotReachHere();
2012 }
2013
2014
2015 void LIR_Assembler::emit_static_call_stub() {
2016 address call_pc = __ pc();
2017 address stub = __ start_a_stub(call_stub_size);
2018 if (stub == NULL) {
2019 bailout("static call stub overflow");
2020 return;
2021 }
2022
2023 int start = __ offset();
2024
2025 __ relocate(static_stub_Relocation::spec(call_pc));
2026 __ mov_metadata(rmethod, (Metadata*)NULL);
2027 __ movptr(rscratch1, 0);
2028 __ br(rscratch1);
2029
2030 assert(__ offset() - start <= call_stub_size, "stub too big");
2031 __ end_a_stub();
2032 }
2033
2034
2035 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2036 assert(exceptionOop->as_register() == r0, "must match");
2037 assert(exceptionPC->as_register() == r3, "must match");
2038
2039 // exception object is not added to oop map by LinearScan
2040 // (LinearScan assumes that no oops are in fixed registers)
2041 info->add_register_oop(exceptionOop);
2042 Runtime1::StubID unwind_id;
2043
2044 // get current pc information
2045 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2046 int pc_for_athrow_offset = __ offset();
2047 InternalAddress pc_for_athrow(__ pc());
2048 __ adr(exceptionPC->as_register(), pc_for_athrow);
2049 add_call_info(pc_for_athrow_offset, info); // for exception handler
2050
2051 __ verify_not_null_oop(r0);
2052 // search an exception handler (r0: exception oop, r3: throwing pc)
2053 if (compilation()->has_fpu_code()) {
2054 unwind_id = Runtime1::handle_exception_id;
2055 } else {
2056 unwind_id = Runtime1::handle_exception_nofpu_id;
2057 }
2058 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2059
2060 // FIXME: enough room for two byte trap ????
2061 __ nop();
2062 }
2063
2064
2065 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2066 assert(exceptionOop->as_register() == r0, "must match");
2067
2068 __ b(_unwind_handler_entry);
2069 }
2070
2071
2072 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2073 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2074 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2075
2076 switch (left->type()) {
2077 case T_INT: {
2078 switch (code) {
2079 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2080 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2081 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2082 default:
2083 ShouldNotReachHere();
2084 break;
2085 }
2086 break;
2087 case T_LONG:
2088 case T_ADDRESS:
2089 case T_OBJECT:
2090 switch (code) {
2091 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2092 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2093 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2094 default:
2095 ShouldNotReachHere();
2096 break;
2097 }
2098 break;
2099 default:
2100 ShouldNotReachHere();
2101 break;
2102 }
2103 }
2104 }
2105
2106
2107 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2108 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2109 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2110
2111 switch (left->type()) {
2112 case T_INT: {
2113 switch (code) {
2114 case lir_shl: __ lslw (dreg, lreg, count); break;
2115 case lir_shr: __ asrw (dreg, lreg, count); break;
2116 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2117 default:
2118 ShouldNotReachHere();
2119 break;
2120 }
2121 break;
2122 case T_LONG:
2123 case T_ADDRESS:
2124 case T_OBJECT:
2125 switch (code) {
2126 case lir_shl: __ lsl (dreg, lreg, count); break;
2127 case lir_shr: __ asr (dreg, lreg, count); break;
2128 case lir_ushr: __ lsr (dreg, lreg, count); break;
2129 default:
2130 ShouldNotReachHere();
2131 break;
2132 }
2133 break;
2134 default:
2135 ShouldNotReachHere();
2136 break;
2137 }
2138 }
2139 }
2140
2141
2142 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2143 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2144 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2145 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2146 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2147 }
2148
2149
2150 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2151 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2152 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2153 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2154 __ mov (rscratch1, c);
2155 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2156 }
2157
2158
2159 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2160 ShouldNotReachHere();
2161 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2162 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2163 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2164 __ lea(rscratch1, __ constant_oop_address(o));
2165 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2166 }
2167
2168
2169 // This code replaces a call to arraycopy; no exception may
2170 // be thrown in this code, they must be thrown in the System.arraycopy
2171 // activation frame; we could save some checks if this would not be the case
2172 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2173 ciArrayKlass* default_type = op->expected_type();
2174 Register src = op->src()->as_register();
2175 Register dst = op->dst()->as_register();
2176 Register src_pos = op->src_pos()->as_register();
2177 Register dst_pos = op->dst_pos()->as_register();
2178 Register length = op->length()->as_register();
2179 Register tmp = op->tmp()->as_register();
2180
2181 CodeStub* stub = op->stub();
2182 int flags = op->flags();
2183 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2184 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2185
2186 // if we don't know anything, just go through the generic arraycopy
2187 if (default_type == NULL // || basic_type == T_OBJECT
2188 ) {
2189 Label done;
2190 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2191
2192 // Save the arguments in case the generic arraycopy fails and we
2193 // have to fall back to the JNI stub
2194 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2195 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2196 __ str(src, Address(sp, 4*BytesPerWord));
2197
2198 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2199 address copyfunc_addr = StubRoutines::generic_arraycopy();
2200
2201 // The arguments are in java calling convention so we shift them
2202 // to C convention
2203 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2204 __ mov(c_rarg0, j_rarg0);
2205 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2206 __ mov(c_rarg1, j_rarg1);
2207 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2208 __ mov(c_rarg2, j_rarg2);
2209 assert_different_registers(c_rarg3, j_rarg4);
2210 __ mov(c_rarg3, j_rarg3);
2211 __ mov(c_rarg4, j_rarg4);
2212 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2213 __ mov(rscratch1, RuntimeAddress(C_entry));
2214 __ blrt(rscratch1, 5, 0, 1);
2215 } else {
2216 #ifndef PRODUCT
2217 if (PrintC1Statistics) {
2218 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2219 }
2220 #endif
2221 __ far_call(RuntimeAddress(copyfunc_addr));
2222 }
2223
2224 __ cbz(r0, *stub->continuation());
2225
2226 // Reload values from the stack so they are where the stub
2227 // expects them.
2228 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2229 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2230 __ ldr(src, Address(sp, 4*BytesPerWord));
2231
2232 if (copyfunc_addr != NULL) {
2233 // r0 is -1^K where K == partial copied count
2234 __ eonw(rscratch1, r0, 0);
2235 // adjust length down and src/end pos up by partial copied count
2236 __ subw(length, length, rscratch1);
2237 __ addw(src_pos, src_pos, rscratch1);
2238 __ addw(dst_pos, dst_pos, rscratch1);
2239 }
2240 __ b(*stub->entry());
2241
2242 __ bind(*stub->continuation());
2243 return;
2244 }
2245
2246 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2247
2248 int elem_size = type2aelembytes(basic_type);
2249 int shift_amount;
2250 int scale = exact_log2(elem_size);
2251
2252 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2253 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2254 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2255 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2256
2257 // test for NULL
2258 if (flags & LIR_OpArrayCopy::src_null_check) {
2259 __ cbz(src, *stub->entry());
2260 }
2261 if (flags & LIR_OpArrayCopy::dst_null_check) {
2262 __ cbz(dst, *stub->entry());
2263 }
2264
2265 // check if negative
2266 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2267 __ cmpw(src_pos, 0);
2268 __ br(Assembler::LT, *stub->entry());
2269 }
2270 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2271 __ cmpw(dst_pos, 0);
2272 __ br(Assembler::LT, *stub->entry());
2273 }
2274
2275 if (flags & LIR_OpArrayCopy::length_positive_check) {
2276 __ cmpw(length, 0);
2277 __ br(Assembler::LT, *stub->entry());
2278 }
2279
2280 if (flags & LIR_OpArrayCopy::src_range_check) {
2281 __ addw(tmp, src_pos, length);
2282 __ ldrw(rscratch1, src_length_addr);
2283 __ cmpw(tmp, rscratch1);
2284 __ br(Assembler::HI, *stub->entry());
2285 }
2286 if (flags & LIR_OpArrayCopy::dst_range_check) {
2287 __ addw(tmp, dst_pos, length);
2288 __ ldrw(rscratch1, dst_length_addr);
2289 __ cmpw(tmp, rscratch1);
2290 __ br(Assembler::HI, *stub->entry());
2291 }
2292
2293 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2294 // length_positive_check if the source of our length operand is an
2295 // arraylength. However, that arraylength might be zero, and the
2296 // stub that we're about to call contains an assertion that count !=
2297 // 0 . So we make this check purely in order not to trigger an
2298 // assertion failure.
2299 __ cbzw(length, *stub->continuation());
2300
2301 if (flags & LIR_OpArrayCopy::type_check) {
2302 // We don't know the array types are compatible
2303 if (basic_type != T_OBJECT) {
2304 // Simple test for basic type arrays
2305 if (UseCompressedClassPointers) {
2306 __ ldrw(tmp, src_klass_addr);
2307 __ ldrw(rscratch1, dst_klass_addr);
2308 __ cmpw(tmp, rscratch1);
2309 } else {
2310 __ ldr(tmp, src_klass_addr);
2311 __ ldr(rscratch1, dst_klass_addr);
2312 __ cmp(tmp, rscratch1);
2313 }
2314 __ br(Assembler::NE, *stub->entry());
2315 } else {
2316 // For object arrays, if src is a sub class of dst then we can
2317 // safely do the copy.
2318 Label cont, slow;
2319
2320 #define PUSH(r1, r2) \
2321 stp(r1, r2, __ pre(sp, -2 * wordSize));
2322
2323 #define POP(r1, r2) \
2324 ldp(r1, r2, __ post(sp, 2 * wordSize));
2325
2326 __ PUSH(src, dst);
2327
2328 __ load_klass(src, src);
2329 __ load_klass(dst, dst);
2330
2331 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2332
2333 __ PUSH(src, dst);
2334 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2335 __ POP(src, dst);
2336
2337 __ cbnz(src, cont);
2338
2339 __ bind(slow);
2340 __ POP(src, dst);
2341
2342 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2343 if (copyfunc_addr != NULL) { // use stub if available
2344 // src is not a sub class of dst so we have to do a
2345 // per-element check.
2346
2347 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2348 if ((flags & mask) != mask) {
2349 // Check that at least both of them object arrays.
2350 assert(flags & mask, "one of the two should be known to be an object array");
2351
2352 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2353 __ load_klass(tmp, src);
2354 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2355 __ load_klass(tmp, dst);
2356 }
2357 int lh_offset = in_bytes(Klass::layout_helper_offset());
2358 Address klass_lh_addr(tmp, lh_offset);
2359 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2360 __ ldrw(rscratch1, klass_lh_addr);
2361 __ mov(rscratch2, objArray_lh);
2362 __ eorw(rscratch1, rscratch1, rscratch2);
2363 __ cbnzw(rscratch1, *stub->entry());
2364 }
2365
2366 // Spill because stubs can use any register they like and it's
2367 // easier to restore just those that we care about.
2368 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2369 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2370 __ str(src, Address(sp, 4*BytesPerWord));
2371
2372 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2373 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2374 assert_different_registers(c_rarg0, dst, dst_pos, length);
2375 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2376 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2377 assert_different_registers(c_rarg1, dst, length);
2378 __ uxtw(c_rarg2, length);
2379 assert_different_registers(c_rarg2, dst);
2380
2381 __ load_klass(c_rarg4, dst);
2382 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2383 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2384 __ far_call(RuntimeAddress(copyfunc_addr));
2385
2386 #ifndef PRODUCT
2387 if (PrintC1Statistics) {
2388 Label failed;
2389 __ cbnz(r0, failed);
2390 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2391 __ bind(failed);
2392 }
2393 #endif
2394
2395 __ cbz(r0, *stub->continuation());
2396
2397 #ifndef PRODUCT
2398 if (PrintC1Statistics) {
2399 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2400 }
2401 #endif
2402 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2403
2404 // Restore previously spilled arguments
2405 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2406 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2407 __ ldr(src, Address(sp, 4*BytesPerWord));
2408
2409 // return value is -1^K where K is partial copied count
2410 __ eonw(rscratch1, r0, zr);
2411 // adjust length down and src/end pos up by partial copied count
2412 __ subw(length, length, rscratch1);
2413 __ addw(src_pos, src_pos, rscratch1);
2414 __ addw(dst_pos, dst_pos, rscratch1);
2415 }
2416
2417 __ b(*stub->entry());
2418
2419 __ bind(cont);
2420 __ POP(src, dst);
2421 }
2422 }
2423
2424 #ifdef ASSERT
2425 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2426 // Sanity check the known type with the incoming class. For the
2427 // primitive case the types must match exactly with src.klass and
2428 // dst.klass each exactly matching the default type. For the
2429 // object array case, if no type check is needed then either the
2430 // dst type is exactly the expected type and the src type is a
2431 // subtype which we can't check or src is the same array as dst
2432 // but not necessarily exactly of type default_type.
2433 Label known_ok, halt;
2434 __ mov_metadata(tmp, default_type->constant_encoding());
2435 if (UseCompressedClassPointers) {
2436 __ encode_klass_not_null(tmp);
2437 }
2438
2439 if (basic_type != T_OBJECT) {
2440
2441 if (UseCompressedClassPointers) {
2442 __ ldrw(rscratch1, dst_klass_addr);
2443 __ cmpw(tmp, rscratch1);
2444 } else {
2445 __ ldr(rscratch1, dst_klass_addr);
2446 __ cmp(tmp, rscratch1);
2447 }
2448 __ br(Assembler::NE, halt);
2449 if (UseCompressedClassPointers) {
2450 __ ldrw(rscratch1, src_klass_addr);
2451 __ cmpw(tmp, rscratch1);
2452 } else {
2453 __ ldr(rscratch1, src_klass_addr);
2454 __ cmp(tmp, rscratch1);
2455 }
2456 __ br(Assembler::EQ, known_ok);
2457 } else {
2458 if (UseCompressedClassPointers) {
2459 __ ldrw(rscratch1, dst_klass_addr);
2460 __ cmpw(tmp, rscratch1);
2461 } else {
2462 __ ldr(rscratch1, dst_klass_addr);
2463 __ cmp(tmp, rscratch1);
2464 }
2465 __ br(Assembler::EQ, known_ok);
2466 __ cmp(src, dst);
2467 __ br(Assembler::EQ, known_ok);
2468 }
2469 __ bind(halt);
2470 __ stop("incorrect type information in arraycopy");
2471 __ bind(known_ok);
2472 }
2473 #endif
2474
2475 #ifndef PRODUCT
2476 if (PrintC1Statistics) {
2477 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2478 }
2479 #endif
2480
2481 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2482 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2483 assert_different_registers(c_rarg0, dst, dst_pos, length);
2484 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2485 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2486 assert_different_registers(c_rarg1, dst, length);
2487 __ uxtw(c_rarg2, length);
2488 assert_different_registers(c_rarg2, dst);
2489
2490 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2491 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2492 const char *name;
2493 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2494
2495 CodeBlob *cb = CodeCache::find_blob(entry);
2496 if (cb) {
2497 __ far_call(RuntimeAddress(entry));
2498 } else {
2499 __ call_VM_leaf(entry, 3);
2500 }
2501
2502 __ bind(*stub->continuation());
2503 }
2504
2505
2506
2507
2508 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2509 Register obj = op->obj_opr()->as_register(); // may not be an oop
2510 Register hdr = op->hdr_opr()->as_register();
2511 Register lock = op->lock_opr()->as_register();
2512 if (!UseFastLocking) {
2513 __ b(*op->stub()->entry());
2514 } else if (op->code() == lir_lock) {
2515 Register scratch = noreg;
2516 if (UseBiasedLocking) {
2517 scratch = op->scratch_opr()->as_register();
2518 }
2519 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2520 // add debug info for NullPointerException only if one is possible
2521 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2522 if (op->info() != NULL) {
2523 add_debug_info_for_null_check(null_check_offset, op->info());
2524 }
2525 // done
2526 } else if (op->code() == lir_unlock) {
2527 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2528 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2529 } else {
2530 Unimplemented();
2531 }
2532 __ bind(*op->stub()->continuation());
2533 }
2534
2535
2536 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2537 ciMethod* method = op->profiled_method();
2538 int bci = op->profiled_bci();
2539 ciMethod* callee = op->profiled_callee();
2540
2541 // Update counter for all call types
2542 ciMethodData* md = method->method_data_or_null();
2543 assert(md != NULL, "Sanity");
2544 ciProfileData* data = md->bci_to_data(bci);
2545 assert(data->is_CounterData(), "need CounterData for calls");
2546 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2547 Register mdo = op->mdo()->as_register();
2548 __ mov_metadata(mdo, md->constant_encoding());
2549 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2550 Bytecodes::Code bc = method->java_code_at_bci(bci);
2551 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2552 // Perform additional virtual call profiling for invokevirtual and
2553 // invokeinterface bytecodes
2554 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2555 !callee_is_static && // required for optimized MH invokes
2556 C1ProfileVirtualCalls) {
2557 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2558 Register recv = op->recv()->as_register();
2559 assert_different_registers(mdo, recv);
2560 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2561 ciKlass* known_klass = op->known_holder();
2562 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2563 // We know the type that will be seen at this call site; we can
2564 // statically update the MethodData* rather than needing to do
2565 // dynamic tests on the receiver type
2566
2567 // NOTE: we should probably put a lock around this search to
2568 // avoid collisions by concurrent compilations
2569 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2570 uint i;
2571 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2572 ciKlass* receiver = vc_data->receiver(i);
2573 if (known_klass->equals(receiver)) {
2574 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2575 __ addptr(data_addr, DataLayout::counter_increment);
2576 return;
2577 }
2578 }
2579
2580 // Receiver type not found in profile data; select an empty slot
2581
2582 // Note that this is less efficient than it should be because it
2583 // always does a write to the receiver part of the
2584 // VirtualCallData rather than just the first time
2585 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2586 ciKlass* receiver = vc_data->receiver(i);
2587 if (receiver == NULL) {
2588 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2589 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2590 __ lea(rscratch2, recv_addr);
2591 __ str(rscratch1, Address(rscratch2));
2592 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2593 __ addptr(data_addr, DataLayout::counter_increment);
2594 return;
2595 }
2596 }
2597 } else {
2598 __ load_klass(recv, recv);
2599 Label update_done;
2600 type_profile_helper(mdo, md, data, recv, &update_done);
2601 // Receiver did not match any saved receiver and there is no empty row for it.
2602 // Increment total counter to indicate polymorphic case.
2603 __ addptr(counter_addr, DataLayout::counter_increment);
2604
2605 __ bind(update_done);
2606 }
2607 } else {
2608 // Static call
2609 __ addptr(counter_addr, DataLayout::counter_increment);
2610 }
2611 }
2612
2613
2614 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2615 Unimplemented();
2616 }
2617
2618
2619 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2620 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2621 }
2622
2623 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2624 assert(op->crc()->is_single_cpu(), "crc must be register");
2625 assert(op->val()->is_single_cpu(), "byte value must be register");
2626 assert(op->result_opr()->is_single_cpu(), "result must be register");
2627 Register crc = op->crc()->as_register();
2628 Register val = op->val()->as_register();
2629 Register res = op->result_opr()->as_register();
2630
2631 assert_different_registers(val, crc, res);
2632 unsigned long offset;
2633 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2634 if (offset) __ add(res, res, offset);
2635
2636 __ ornw(crc, zr, crc); // ~crc
2637 __ update_byte_crc32(crc, val, res);
2638 __ ornw(res, zr, crc); // ~crc
2639 }
2640
2641 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2642 COMMENT("emit_profile_type {");
2643 Register obj = op->obj()->as_register();
2644 Register tmp = op->tmp()->as_pointer_register();
2645 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2646 ciKlass* exact_klass = op->exact_klass();
2647 intptr_t current_klass = op->current_klass();
2648 bool not_null = op->not_null();
2649 bool no_conflict = op->no_conflict();
2650
2651 Label update, next, none;
2652
2653 bool do_null = !not_null;
2654 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2655 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2656
2657 assert(do_null || do_update, "why are we here?");
2658 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2659 assert(mdo_addr.base() != rscratch1, "wrong register");
2660
2661 __ verify_oop(obj);
2662
2663 if (tmp != obj) {
2664 __ mov(tmp, obj);
2665 }
2666 if (do_null) {
2667 __ cbnz(tmp, update);
2668 if (!TypeEntries::was_null_seen(current_klass)) {
2669 __ ldr(rscratch2, mdo_addr);
2670 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2671 __ str(rscratch2, mdo_addr);
2672 }
2673 if (do_update) {
2674 #ifndef ASSERT
2675 __ b(next);
2676 }
2677 #else
2678 __ b(next);
2679 }
2680 } else {
2681 __ cbnz(tmp, update);
2682 __ stop("unexpected null obj");
2683 #endif
2684 }
2685
2686 __ bind(update);
2687
2688 if (do_update) {
2689 #ifdef ASSERT
2690 if (exact_klass != NULL) {
2691 Label ok;
2692 __ load_klass(tmp, tmp);
2693 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2694 __ eor(rscratch1, tmp, rscratch1);
2695 __ cbz(rscratch1, ok);
2696 __ stop("exact klass and actual klass differ");
2697 __ bind(ok);
2698 }
2699 #endif
2700 if (!no_conflict) {
2701 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2702 if (exact_klass != NULL) {
2703 __ mov_metadata(tmp, exact_klass->constant_encoding());
2704 } else {
2705 __ load_klass(tmp, tmp);
2706 }
2707
2708 __ ldr(rscratch2, mdo_addr);
2709 __ eor(tmp, tmp, rscratch2);
2710 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2711 // klass seen before, nothing to do. The unknown bit may have been
2712 // set already but no need to check.
2713 __ cbz(rscratch1, next);
2714
2715 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2716 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2717
2718 if (TypeEntries::is_type_none(current_klass)) {
2719 __ cbz(rscratch2, none);
2720 __ cmp(rscratch2, TypeEntries::null_seen);
2721 __ br(Assembler::EQ, none);
2722 // There is a chance that the checks above (re-reading profiling
2723 // data from memory) fail if another thread has just set the
2724 // profiling to this obj's klass
2725 __ dmb(Assembler::ISHLD);
2726 __ ldr(rscratch2, mdo_addr);
2727 __ eor(tmp, tmp, rscratch2);
2728 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2729 __ cbz(rscratch1, next);
2730 }
2731 } else {
2732 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2733 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2734
2735 __ ldr(tmp, mdo_addr);
2736 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2737 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2738 }
2739
2740 // different than before. Cannot keep accurate profile.
2741 __ ldr(rscratch2, mdo_addr);
2742 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2743 __ str(rscratch2, mdo_addr);
2744
2745 if (TypeEntries::is_type_none(current_klass)) {
2746 __ b(next);
2747
2748 __ bind(none);
2749 // first time here. Set profile type.
2750 __ str(tmp, mdo_addr);
2751 }
2752 } else {
2753 // There's a single possible klass at this profile point
2754 assert(exact_klass != NULL, "should be");
2755 if (TypeEntries::is_type_none(current_klass)) {
2756 __ mov_metadata(tmp, exact_klass->constant_encoding());
2757 __ ldr(rscratch2, mdo_addr);
2758 __ eor(tmp, tmp, rscratch2);
2759 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2760 __ cbz(rscratch1, next);
2761 #ifdef ASSERT
2762 {
2763 Label ok;
2764 __ ldr(rscratch1, mdo_addr);
2765 __ cbz(rscratch1, ok);
2766 __ cmp(rscratch1, TypeEntries::null_seen);
2767 __ br(Assembler::EQ, ok);
2768 // may have been set by another thread
2769 __ dmb(Assembler::ISHLD);
2770 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2771 __ ldr(rscratch2, mdo_addr);
2772 __ eor(rscratch2, rscratch1, rscratch2);
2773 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2774 __ cbz(rscratch2, ok);
2775
2776 __ stop("unexpected profiling mismatch");
2777 __ bind(ok);
2778 }
2779 #endif
2780 // first time here. Set profile type.
2781 __ ldr(tmp, mdo_addr);
2782 } else {
2783 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2784 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2785
2786 __ ldr(tmp, mdo_addr);
2787 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2788 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2789
2790 __ orr(tmp, tmp, TypeEntries::type_unknown);
2791 __ str(tmp, mdo_addr);
2792 // FIXME: Write barrier needed here?
2793 }
2794 }
2795
2796 __ bind(next);
2797 }
2798 COMMENT("} emit_profile_type");
2799 }
2800
2801
2802 void LIR_Assembler::align_backward_branch_target() {
2803 }
2804
2805
2806 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2807 if (left->is_single_cpu()) {
2808 assert(dest->is_single_cpu(), "expect single result reg");
2809 __ negw(dest->as_register(), left->as_register());
2810 } else if (left->is_double_cpu()) {
2811 assert(dest->is_double_cpu(), "expect double result reg");
2812 __ neg(dest->as_register_lo(), left->as_register_lo());
2813 } else if (left->is_single_fpu()) {
2814 assert(dest->is_single_fpu(), "expect single float result reg");
2815 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2816 } else {
2817 assert(left->is_double_fpu(), "expect double float operand reg");
2818 assert(dest->is_double_fpu(), "expect double float result reg");
2819 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2820 }
2821 }
2822
2823
2824 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2825 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2826 }
2827
2828
2829 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2830 assert(!tmp->is_valid(), "don't need temporary");
2831
2832 CodeBlob *cb = CodeCache::find_blob(dest);
2833 if (cb) {
2834 __ far_call(RuntimeAddress(dest));
2835 } else {
2836 __ mov(rscratch1, RuntimeAddress(dest));
2837 int len = args->length();
2838 int type = 0;
2839 if (! result->is_illegal()) {
2840 switch (result->type()) {
2841 case T_VOID:
2842 type = 0;
2843 break;
2844 case T_INT:
2845 case T_LONG:
2846 case T_OBJECT:
2847 type = 1;
2848 break;
2849 case T_FLOAT:
2850 type = 2;
2851 break;
2852 case T_DOUBLE:
2853 type = 3;
2854 break;
2855 default:
2856 ShouldNotReachHere();
2857 break;
2858 }
2859 }
2860 int num_gpargs = 0;
2861 int num_fpargs = 0;
2862 for (int i = 0; i < args->length(); i++) {
2863 LIR_Opr arg = args->at(i);
2864 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2865 num_fpargs++;
2866 } else {
2867 num_gpargs++;
2868 }
2869 }
2870 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2871 }
2872
2873 if (info != NULL) {
2874 add_call_info_here(info);
2875 }
2876 __ maybe_isb();
2877 }
2878
2879 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2880 if (dest->is_address() || src->is_address()) {
2881 move_op(src, dest, type, lir_patch_none, info,
2882 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2883 } else {
2884 ShouldNotReachHere();
2885 }
2886 }
2887
2888 #ifdef ASSERT
2889 // emit run-time assertion
2890 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2891 assert(op->code() == lir_assert, "must be");
2892
2893 if (op->in_opr1()->is_valid()) {
2894 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2895 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2896 } else {
2897 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2898 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2899 }
2900
2901 Label ok;
2902 if (op->condition() != lir_cond_always) {
2903 Assembler::Condition acond = Assembler::AL;
2904 switch (op->condition()) {
2905 case lir_cond_equal: acond = Assembler::EQ; break;
2906 case lir_cond_notEqual: acond = Assembler::NE; break;
2907 case lir_cond_less: acond = Assembler::LT; break;
2908 case lir_cond_lessEqual: acond = Assembler::LE; break;
2909 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2910 case lir_cond_greater: acond = Assembler::GT; break;
2911 case lir_cond_belowEqual: acond = Assembler::LS; break;
2912 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2913 default: ShouldNotReachHere();
2914 }
2915 __ br(acond, ok);
2916 }
2917 if (op->halt()) {
2918 const char* str = __ code_string(op->msg());
2919 __ stop(str);
2920 } else {
2921 breakpoint();
2922 }
2923 __ bind(ok);
2924 }
2925 #endif
2926
2927 #ifndef PRODUCT
2928 #define COMMENT(x) do { __ block_comment(x); } while (0)
2929 #else
2930 #define COMMENT(x)
2931 #endif
2932
2933 void LIR_Assembler::membar() {
2934 COMMENT("membar");
2935 __ membar(MacroAssembler::AnyAny);
2936 }
2937
2938 void LIR_Assembler::membar_acquire() {
2939 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2940 }
2941
2942 void LIR_Assembler::membar_release() {
2943 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2944 }
2945
2946 void LIR_Assembler::membar_loadload() {
2947 __ membar(Assembler::LoadLoad);
2948 }
2949
2950 void LIR_Assembler::membar_storestore() {
2951 __ membar(MacroAssembler::StoreStore);
2952 }
2953
2954 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2955
2956 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2957
2958 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2959 __ mov(result_reg->as_register(), rthread);
2960 }
2961
2962
2963 void LIR_Assembler::peephole(LIR_List *lir) {
2964 #if 0
2965 if (tableswitch_count >= max_tableswitches)
2966 return;
2967
2968 /*
2969 This finite-state automaton recognizes sequences of compare-and-
2970 branch instructions. We will turn them into a tableswitch. You
2971 could argue that C1 really shouldn't be doing this sort of
2972 optimization, but without it the code is really horrible.
2973 */
2974
2975 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2976 int first_key, last_key = -2147483648;
2977 int next_key = 0;
2978 int start_insn = -1;
2979 int last_insn = -1;
2980 Register reg = noreg;
2981 LIR_Opr reg_opr;
2982 state = start_s;
2983
2984 LIR_OpList* inst = lir->instructions_list();
2985 for (int i = 0; i < inst->length(); i++) {
2986 LIR_Op* op = inst->at(i);
2987 switch (state) {
2988 case start_s:
2989 first_key = -1;
2990 start_insn = i;
2991 switch (op->code()) {
2992 case lir_cmp:
2993 LIR_Opr opr1 = op->as_Op2()->in_opr1();
2994 LIR_Opr opr2 = op->as_Op2()->in_opr2();
2995 if (opr1->is_cpu_register() && opr1->is_single_cpu()
2996 && opr2->is_constant()
2997 && opr2->type() == T_INT) {
2998 reg_opr = opr1;
2999 reg = opr1->as_register();
3000 first_key = opr2->as_constant_ptr()->as_jint();
3001 next_key = first_key + 1;
3002 state = cmp_s;
3003 goto next_state;
3004 }
3005 break;
3006 }
3007 break;
3008 case cmp_s:
3009 switch (op->code()) {
3010 case lir_branch:
3011 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3012 state = beq_s;
3013 last_insn = i;
3014 goto next_state;
3015 }
3016 }
3017 state = start_s;
3018 break;
3019 case beq_s:
3020 switch (op->code()) {
3021 case lir_cmp: {
3022 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3023 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3024 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3025 && opr1->as_register() == reg
3026 && opr2->is_constant()
3027 && opr2->type() == T_INT
3028 && opr2->as_constant_ptr()->as_jint() == next_key) {
3029 last_key = next_key;
3030 next_key++;
3031 state = cmp_s;
3032 goto next_state;
3033 }
3034 }
3035 }
3036 last_key = next_key;
3037 state = start_s;
3038 break;
3039 default:
3040 assert(false, "impossible state");
3041 }
3042 if (state == start_s) {
3043 if (first_key < last_key - 5L && reg != noreg) {
3044 {
3045 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3046 // reg->encoding(),
3047 // start_insn, first_key, last_key);
3048 // for (int i = 0; i < inst->length(); i++) {
3049 // inst->at(i)->print();
3050 // tty->print("\n");
3051 // }
3052 // tty->print("\n");
3053 }
3054
3055 struct tableswitch *sw = &switches[tableswitch_count];
3056 sw->_insn_index = start_insn, sw->_first_key = first_key,
3057 sw->_last_key = last_key, sw->_reg = reg;
3058 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3059 {
3060 // Insert the new table of branches
3061 int offset = last_insn;
3062 for (int n = first_key; n < last_key; n++) {
3063 inst->insert_before
3064 (last_insn + 1,
3065 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3066 inst->at(offset)->as_OpBranch()->label()));
3067 offset -= 2, i++;
3068 }
3069 }
3070 // Delete all the old compare-and-branch instructions
3071 for (int n = first_key; n < last_key; n++) {
3072 inst->remove_at(start_insn);
3073 inst->remove_at(start_insn);
3074 }
3075 // Insert the tableswitch instruction
3076 inst->insert_before(start_insn,
3077 new LIR_Op2(lir_cmp, lir_cond_always,
3078 LIR_OprFact::intConst(tableswitch_count),
3079 reg_opr));
3080 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3081 tableswitch_count++;
3082 }
3083 reg = noreg;
3084 last_key = -2147483648;
3085 }
3086 next_state:
3087 ;
3088 }
3089 #endif
3090 }
3091
3092 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3093 Address addr = as_Address(src->as_address_ptr(), noreg);
3094 BasicType type = src->type();
3095 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3096
3097 void (MacroAssembler::* lda)(Register Rd, Register Ra);
3098 void (MacroAssembler::* add)(Register Rd, Register Rn, RegisterOrConstant increment);
3099 void (MacroAssembler::* stl)(Register Rs, Register Rt, Register Rn);
3100
3101 switch(type) {
3102 case T_INT:
3103 lda = &MacroAssembler::ldaxrw;
3104 add = &MacroAssembler::addw;
3105 stl = &MacroAssembler::stlxrw;
3106 break;
3107 case T_LONG:
3108 lda = &MacroAssembler::ldaxr;
3109 add = &MacroAssembler::add;
3110 stl = &MacroAssembler::stlxr;
3111 break;
3112 case T_OBJECT:
3113 case T_ARRAY:
3114 if (UseCompressedOops) {
3115 lda = &MacroAssembler::ldaxrw;
3116 add = &MacroAssembler::addw;
3117 stl = &MacroAssembler::stlxrw;
3118 } else {
3119 lda = &MacroAssembler::ldaxr;
3120 add = &MacroAssembler::add;
3121 stl = &MacroAssembler::stlxr;
3122 }
3123 break;
3124 default:
3125 ShouldNotReachHere();
3126 }
3127
3128 switch (code) {
3129 case lir_xadd:
3130 {
3131 RegisterOrConstant inc;
3132 Register tmp = as_reg(tmp_op);
3133 Register dst = as_reg(dest);
3134 if (data->is_constant()) {
3135 inc = RegisterOrConstant(as_long(data));
3136 assert_different_registers(dst, addr.base(), tmp,
3137 rscratch1, rscratch2);
3138 } else {
3139 inc = RegisterOrConstant(as_reg(data));
3140 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3141 rscratch1, rscratch2);
3142 }
3143 Label again;
3144 __ lea(tmp, addr);
3145 __ bind(again);
3146 (_masm->*lda)(dst, tmp);
3147 (_masm->*add)(rscratch1, dst, inc);
3148 (_masm->*stl)(rscratch2, rscratch1, tmp);
3149 __ cbnzw(rscratch2, again);
3150 break;
3151 }
3152 case lir_xchg:
3153 {
3154 Register tmp = tmp_op->as_register();
3155 Register obj = as_reg(data);
3156 Register dst = as_reg(dest);
3157 if (is_oop && UseCompressedOops) {
3158 __ encode_heap_oop(obj);
3159 }
3160 assert_different_registers(obj, addr.base(), tmp, rscratch2, dst);
3161 Label again;
3162 __ lea(tmp, addr);
3163 __ bind(again);
3164 (_masm->*lda)(dst, tmp);
3165 (_masm->*stl)(rscratch2, obj, tmp);
3166 __ cbnzw(rscratch2, again);
3167 if (is_oop && UseCompressedOops) {
3168 __ decode_heap_oop(dst);
3169 }
3170 }
3171 break;
3172 default:
3173 ShouldNotReachHere();
3174 }
3175 __ membar(__ AnyAny);
3176 }
3177
3178 #undef __