1 /*
2 * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "gc_interface/collectedHeap.hpp"
37 #include "memory/barrierSet.hpp"
38 #include "memory/cardTableModRefBS.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "vmreg_aarch64.inline.hpp"
43
44
45
46 #ifndef PRODUCT
47 #define COMMENT(x) do { __ block_comment(x); } while (0)
48 #else
49 #define COMMENT(x)
50 #endif
51
52 NEEDS_CLEANUP // remove this definitions ?
53 const Register IC_Klass = rscratch2; // where the IC klass is cached
54 const Register SYNC_header = r0; // synchronization header
55 const Register SHIFT_count = r0; // where count for shift operations must be
56
57 #define __ _masm->
58
59
60 static void select_different_registers(Register preserve,
61 Register extra,
62 Register &tmp1,
63 Register &tmp2) {
64 if (tmp1 == preserve) {
65 assert_different_registers(tmp1, tmp2, extra);
66 tmp1 = extra;
67 } else if (tmp2 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp2 = extra;
70 }
71 assert_different_registers(preserve, tmp1, tmp2);
72 }
73
74
75
76 static void select_different_registers(Register preserve,
77 Register extra,
78 Register &tmp1,
79 Register &tmp2,
80 Register &tmp3) {
81 if (tmp1 == preserve) {
82 assert_different_registers(tmp1, tmp2, tmp3, extra);
83 tmp1 = extra;
84 } else if (tmp2 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp2 = extra;
87 } else if (tmp3 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp3 = extra;
90 }
91 assert_different_registers(preserve, tmp1, tmp2, tmp3);
92 }
93
94
95 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
96
97
98 LIR_Opr LIR_Assembler::receiverOpr() {
99 return FrameMap::receiver_opr;
100 }
101
102 LIR_Opr LIR_Assembler::osrBufferPointer() {
103 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
104 }
105
106 //--------------fpu register translations-----------------------
107
108
109 address LIR_Assembler::float_constant(float f) {
110 address const_addr = __ float_constant(f);
111 if (const_addr == NULL) {
112 bailout("const section overflow");
113 return __ code()->consts()->start();
114 } else {
115 return const_addr;
116 }
117 }
118
119
120 address LIR_Assembler::double_constant(double d) {
121 address const_addr = __ double_constant(d);
122 if (const_addr == NULL) {
123 bailout("const section overflow");
124 return __ code()->consts()->start();
125 } else {
126 return const_addr;
127 }
128 }
129
130 address LIR_Assembler::int_constant(jlong n) {
131 address const_addr = __ long_constant(n);
132 if (const_addr == NULL) {
133 bailout("const section overflow");
134 return __ code()->consts()->start();
135 } else {
136 return const_addr;
137 }
138 }
139
140 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
141
142 void LIR_Assembler::reset_FPU() { Unimplemented(); }
143
144 void LIR_Assembler::fpop() { Unimplemented(); }
145
146 void LIR_Assembler::fxch(int i) { Unimplemented(); }
147
148 void LIR_Assembler::fld(int i) { Unimplemented(); }
149
150 void LIR_Assembler::ffree(int i) { Unimplemented(); }
151
152 void LIR_Assembler::breakpoint() { Unimplemented(); }
153
154 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
155
156 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
157
158 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
159 //-------------------------------------------
160
161 static Register as_reg(LIR_Opr op) {
162 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
163 }
164
165 static jlong as_long(LIR_Opr data) {
166 jlong result;
167 switch (data->type()) {
168 case T_INT:
169 result = (data->as_jint());
170 break;
171 case T_LONG:
172 result = (data->as_jlong());
173 break;
174 default:
175 ShouldNotReachHere();
176 }
177 return result;
178 }
179
180 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
181 Register base = addr->base()->as_pointer_register();
182 LIR_Opr opr = addr->index();
183 if (opr->is_cpu_register()) {
184 Register index;
185 if (opr->is_single_cpu())
186 index = opr->as_register();
187 else
188 index = opr->as_register_lo();
189 assert(addr->disp() == 0, "must be");
190 switch(opr->type()) {
191 case T_INT:
192 return Address(base, index, Address::sxtw(addr->scale()));
193 case T_LONG:
194 return Address(base, index, Address::lsl(addr->scale()));
195 default:
196 ShouldNotReachHere();
197 }
198 } else {
199 intptr_t addr_offset = intptr_t(addr->disp());
200 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
201 return Address(base, addr_offset, Address::lsl(addr->scale()));
202 else {
203 __ mov(tmp, addr_offset);
204 return Address(base, tmp, Address::lsl(addr->scale()));
205 }
206 }
207 return Address();
208 }
209
210 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
211 ShouldNotReachHere();
212 return Address();
213 }
214
215 Address LIR_Assembler::as_Address(LIR_Address* addr) {
216 return as_Address(addr, rscratch1);
217 }
218
219 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
220 return as_Address(addr, rscratch1); // Ouch
221 // FIXME: This needs to be much more clever. See x86.
222 }
223
224
225 void LIR_Assembler::osr_entry() {
226 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
227 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
228 ValueStack* entry_state = osr_entry->state();
229 int number_of_locks = entry_state->locks_size();
230
231 // we jump here if osr happens with the interpreter
232 // state set up to continue at the beginning of the
233 // loop that triggered osr - in particular, we have
234 // the following registers setup:
235 //
236 // r2: osr buffer
237 //
238
239 // build frame
240 ciMethod* m = compilation()->method();
241 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
242
243 // OSR buffer is
244 //
245 // locals[nlocals-1..0]
246 // monitors[0..number_of_locks]
247 //
248 // locals is a direct copy of the interpreter frame so in the osr buffer
249 // so first slot in the local array is the last local from the interpreter
250 // and last slot is local[0] (receiver) from the interpreter
251 //
252 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
253 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
254 // in the interpreter frame (the method lock if a sync method)
255
256 // Initialize monitors in the compiled activation.
257 // r2: pointer to osr buffer
258 //
259 // All other registers are dead at this point and the locals will be
260 // copied into place by code emitted in the IR.
261
262 Register OSR_buf = osrBufferPointer()->as_pointer_register();
263 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
264 int monitor_offset = BytesPerWord * method()->max_locals() +
265 (2 * BytesPerWord) * (number_of_locks - 1);
266 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
267 // the OSR buffer using 2 word entries: first the lock and then
268 // the oop.
269 for (int i = 0; i < number_of_locks; i++) {
270 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
271 #ifdef ASSERT
272 // verify the interpreter's monitor has a non-null object
273 {
274 Label L;
275 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
276 __ cbnz(rscratch1, L);
277 __ stop("locked object is NULL");
278 __ bind(L);
279 }
280 #endif
281 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
282 __ str(r19, frame_map()->address_for_monitor_lock(i));
283 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
284 __ str(r19, frame_map()->address_for_monitor_object(i));
285 }
286 }
287 }
288
289
290 // inline cache check; done before the frame is built.
291 int LIR_Assembler::check_icache() {
292 Register receiver = FrameMap::receiver_opr->as_register();
293 Register ic_klass = IC_Klass;
294 int start_offset = __ offset();
295 __ inline_cache_check(receiver, ic_klass);
296
297 // if icache check fails, then jump to runtime routine
298 // Note: RECEIVER must still contain the receiver!
299 Label dont;
300 __ br(Assembler::EQ, dont);
301 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
302
303 // We align the verified entry point unless the method body
304 // (including its inline cache check) will fit in a single 64-byte
305 // icache line.
306 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
307 // force alignment after the cache check.
308 __ align(CodeEntryAlignment);
309 }
310
311 __ bind(dont);
312 return start_offset;
313 }
314
315
316 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
317 if (o == NULL) {
318 __ mov(reg, zr);
319 } else {
320 __ movoop(reg, o, /*immediate*/true);
321 }
322 }
323
324 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
325 address target = NULL;
326 relocInfo::relocType reloc_type = relocInfo::none;
327
328 switch (patching_id(info)) {
329 case PatchingStub::access_field_id:
330 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
331 reloc_type = relocInfo::section_word_type;
332 break;
333 case PatchingStub::load_klass_id:
334 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
335 reloc_type = relocInfo::metadata_type;
336 break;
337 case PatchingStub::load_mirror_id:
338 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
339 reloc_type = relocInfo::oop_type;
340 break;
341 case PatchingStub::load_appendix_id:
342 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
343 reloc_type = relocInfo::oop_type;
344 break;
345 default: ShouldNotReachHere();
346 }
347
348 __ far_call(RuntimeAddress(target));
349 add_call_info_here(info);
350 }
351
352 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
353 deoptimize_trap(info);
354 }
355
356
357 // This specifies the rsp decrement needed to build the frame
358 int LIR_Assembler::initial_frame_size_in_bytes() const {
359 // if rounding, must let FrameMap know!
360
361 // The frame_map records size in slots (32bit word)
362
363 // subtract two words to account for return address and link
364 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
365 }
366
367
368 int LIR_Assembler::emit_exception_handler() {
369 // if the last instruction is a call (typically to do a throw which
370 // is coming at the end after block reordering) the return address
371 // must still point into the code area in order to avoid assertion
372 // failures when searching for the corresponding bci => add a nop
373 // (was bug 5/14/1999 - gri)
374 __ nop();
375
376 // generate code for exception handler
377 address handler_base = __ start_a_stub(exception_handler_size);
378 if (handler_base == NULL) {
379 // not enough space left for the handler
380 bailout("exception handler overflow");
381 return -1;
382 }
383
384 int offset = code_offset();
385
386 // the exception oop and pc are in r0, and r3
387 // no other registers need to be preserved, so invalidate them
388 __ invalidate_registers(false, true, true, false, true, true);
389
390 // check that there is really an exception
391 __ verify_not_null_oop(r0);
392
393 // search an exception handler (r0: exception oop, r3: throwing pc)
394 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here();
395 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
396 __ end_a_stub();
397
398 return offset;
399 }
400
401
402 // Emit the code to remove the frame from the stack in the exception
403 // unwind path.
404 int LIR_Assembler::emit_unwind_handler() {
405 #ifndef PRODUCT
406 if (CommentedAssembly) {
407 _masm->block_comment("Unwind handler");
408 }
409 #endif
410
411 int offset = code_offset();
412
413 // Fetch the exception from TLS and clear out exception related thread state
414 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
415 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
417
418 __ bind(_unwind_handler_entry);
419 __ verify_not_null_oop(r0);
420 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
421 __ mov(r19, r0); // Preserve the exception
422 }
423
424 // Preform needed unlocking
425 MonitorExitStub* stub = NULL;
426 if (method()->is_synchronized()) {
427 monitor_address(0, FrameMap::r0_opr);
428 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
429 __ unlock_object(r5, r4, r0, *stub->entry());
430 __ bind(*stub->continuation());
431 }
432
433 if (compilation()->env()->dtrace_method_probes()) {
434 __ call_Unimplemented();
435 #if 0
436 __ movptr(Address(rsp, 0), rax);
437 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
438 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
439 #endif
440 }
441
442 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
443 __ mov(r0, r19); // Restore the exception
444 }
445
446 // remove the activation and dispatch to the unwind handler
447 __ block_comment("remove_frame and dispatch to the unwind handler");
448 __ remove_frame(initial_frame_size_in_bytes());
449 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
450
451 // Emit the slow path assembly
452 if (stub != NULL) {
453 stub->emit_code(this);
454 }
455
456 return offset;
457 }
458
459
460 int LIR_Assembler::emit_deopt_handler() {
461 // if the last instruction is a call (typically to do a throw which
462 // is coming at the end after block reordering) the return address
463 // must still point into the code area in order to avoid assertion
464 // failures when searching for the corresponding bci => add a nop
465 // (was bug 5/14/1999 - gri)
466 __ nop();
467
468 // generate code for exception handler
469 address handler_base = __ start_a_stub(deopt_handler_size);
470 if (handler_base == NULL) {
471 // not enough space left for the handler
472 bailout("deopt handler overflow");
473 return -1;
474 }
475
476 int offset = code_offset();
477
478 __ adr(lr, pc());
479 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
480 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
481 __ end_a_stub();
482
483 return offset;
484 }
485
486
487 // This is the fast version of java.lang.String.compare; it has not
488 // OSR-entry and therefore, we generate a slow version for OSR's
489 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
490 __ mov(r2, (address)__FUNCTION__);
491 __ call_Unimplemented();
492 }
493
494
495 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
496 _masm->code_section()->relocate(adr, relocInfo::poll_type);
497 int pc_offset = code_offset();
498 flush_debug_info(pc_offset);
499 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
500 if (info->exception_handlers() != NULL) {
501 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
502 }
503 }
504
505 // Rather than take a segfault when the polling page is protected,
506 // explicitly check for a safepoint in progress and if there is one,
507 // fake a call to the handler as if a segfault had been caught.
508 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
509 __ mov(rscratch1, SafepointSynchronize::address_of_state());
510 __ ldrb(rscratch1, Address(rscratch1));
511 Label nope, poll;
512 __ cbz(rscratch1, nope);
513 __ block_comment("safepoint");
514 __ enter();
515 __ push(0x3, sp); // r0 & r1
516 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
517 __ adr(r0, poll);
518 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
519 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
520 __ blrt(rscratch1, 1, 0, 1);
521 __ maybe_isb();
522 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
523 __ mov(rscratch1, r0);
524 __ pop(0x3, sp); // r0 & r1
525 __ leave();
526 __ br(rscratch1);
527 address polling_page(os::get_polling_page());
528 assert(os::is_poll_address(polling_page), "should be");
529 unsigned long off;
530 __ adrp(rscratch1, Address(polling_page, rtype), off);
531 __ bind(poll);
532 if (info)
533 add_debug_info_for_branch(info); // This isn't just debug info:
534 // it's the oop map
535 else
536 __ code_section()->relocate(pc(), rtype);
537 __ ldrw(zr, Address(rscratch1, off));
538 __ bind(nope);
539 }
540
541 void LIR_Assembler::return_op(LIR_Opr result) {
542 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
543 // Pop the stack before the safepoint code
544 __ remove_frame(initial_frame_size_in_bytes());
545 address polling_page(os::get_polling_page());
546 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
547 __ ret(lr);
548 }
549
550 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
551 address polling_page(os::get_polling_page());
552 guarantee(info != NULL, "Shouldn't be NULL");
553 assert(os::is_poll_address(polling_page), "should be");
554 unsigned long off;
555 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
556 assert(off == 0, "must be");
557 add_debug_info_for_branch(info); // This isn't just debug info:
558 // it's the oop map
559 __ read_polling_page(rscratch1, relocInfo::poll_type);
560 return __ offset();
561 }
562
563
564 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
565 if (from_reg == r31_sp)
566 from_reg = sp;
567 if (to_reg == r31_sp)
568 to_reg = sp;
569 __ mov(to_reg, from_reg);
570 }
571
572 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
573
574
575 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
576 assert(src->is_constant(), "should not call otherwise");
577 assert(dest->is_register(), "should not call otherwise");
578 LIR_Const* c = src->as_constant_ptr();
579
580 switch (c->type()) {
581 case T_INT: {
582 assert(patch_code == lir_patch_none, "no patching handled here");
583 __ movw(dest->as_register(), c->as_jint());
584 break;
585 }
586
587 case T_ADDRESS: {
588 assert(patch_code == lir_patch_none, "no patching handled here");
589 __ mov(dest->as_register(), c->as_jint());
590 break;
591 }
592
593 case T_LONG: {
594 assert(patch_code == lir_patch_none, "no patching handled here");
595 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
596 break;
597 }
598
599 case T_OBJECT: {
600 if (patch_code == lir_patch_none) {
601 jobject2reg(c->as_jobject(), dest->as_register());
602 } else {
603 jobject2reg_with_patching(dest->as_register(), info);
604 }
605 break;
606 }
607
608 case T_METADATA: {
609 if (patch_code != lir_patch_none) {
610 klass2reg_with_patching(dest->as_register(), info);
611 } else {
612 __ mov_metadata(dest->as_register(), c->as_metadata());
613 }
614 break;
615 }
616
617 case T_FLOAT: {
618 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
619 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
620 } else {
621 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
622 __ ldrs(dest->as_float_reg(), Address(rscratch1));
623 }
624 break;
625 }
626
627 case T_DOUBLE: {
628 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
629 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
630 } else {
631 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
632 __ ldrd(dest->as_double_reg(), Address(rscratch1));
633 }
634 break;
635 }
636
637 default:
638 ShouldNotReachHere();
639 }
640 }
641
642 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
643 LIR_Const* c = src->as_constant_ptr();
644 switch (c->type()) {
645 case T_OBJECT:
646 {
647 if (! c->as_jobject())
648 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
649 else {
650 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
651 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
652 }
653 }
654 break;
655 case T_ADDRESS:
656 {
657 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
658 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
659 }
660 case T_INT:
661 case T_FLOAT:
662 {
663 Register reg = zr;
664 if (c->as_jint_bits() == 0)
665 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
666 else {
667 __ movw(rscratch1, c->as_jint_bits());
668 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
669 }
670 }
671 break;
672 case T_LONG:
673 case T_DOUBLE:
674 {
675 Register reg = zr;
676 if (c->as_jlong_bits() == 0)
677 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
678 lo_word_offset_in_bytes));
679 else {
680 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
681 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
682 lo_word_offset_in_bytes));
683 }
684 }
685 break;
686 default:
687 ShouldNotReachHere();
688 }
689 }
690
691 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
692 assert(src->is_constant(), "should not call otherwise");
693 LIR_Const* c = src->as_constant_ptr();
694 LIR_Address* to_addr = dest->as_address_ptr();
695
696 void (Assembler::* insn)(Register Rt, const Address &adr);
697
698 switch (type) {
699 case T_ADDRESS:
700 assert(c->as_jint() == 0, "should be");
701 insn = &Assembler::str;
702 break;
703 case T_LONG:
704 assert(c->as_jlong() == 0, "should be");
705 insn = &Assembler::str;
706 break;
707 case T_INT:
708 assert(c->as_jint() == 0, "should be");
709 insn = &Assembler::strw;
710 break;
711 case T_OBJECT:
712 case T_ARRAY:
713 assert(c->as_jobject() == 0, "should be");
714 if (UseCompressedOops && !wide) {
715 insn = &Assembler::strw;
716 } else {
717 insn = &Assembler::str;
718 }
719 break;
720 case T_CHAR:
721 case T_SHORT:
722 assert(c->as_jint() == 0, "should be");
723 insn = &Assembler::strh;
724 break;
725 case T_BOOLEAN:
726 case T_BYTE:
727 assert(c->as_jint() == 0, "should be");
728 insn = &Assembler::strb;
729 break;
730 default:
731 ShouldNotReachHere();
732 }
733
734 if (info) add_debug_info_for_null_check_here(info);
735 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
736 }
737
738 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
739 assert(src->is_register(), "should not call otherwise");
740 assert(dest->is_register(), "should not call otherwise");
741
742 // move between cpu-registers
743 if (dest->is_single_cpu()) {
744 if (src->type() == T_LONG) {
745 // Can do LONG -> OBJECT
746 move_regs(src->as_register_lo(), dest->as_register());
747 return;
748 }
749 assert(src->is_single_cpu(), "must match");
750 if (src->type() == T_OBJECT) {
751 __ verify_oop(src->as_register());
752 }
753 move_regs(src->as_register(), dest->as_register());
754
755 } else if (dest->is_double_cpu()) {
756 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
757 // Surprising to me but we can see move of a long to t_object
758 __ verify_oop(src->as_register());
759 move_regs(src->as_register(), dest->as_register_lo());
760 return;
761 }
762 assert(src->is_double_cpu(), "must match");
763 Register f_lo = src->as_register_lo();
764 Register f_hi = src->as_register_hi();
765 Register t_lo = dest->as_register_lo();
766 Register t_hi = dest->as_register_hi();
767 assert(f_hi == f_lo, "must be same");
768 assert(t_hi == t_lo, "must be same");
769 move_regs(f_lo, t_lo);
770
771 } else if (dest->is_single_fpu()) {
772 __ fmovs(dest->as_float_reg(), src->as_float_reg());
773
774 } else if (dest->is_double_fpu()) {
775 __ fmovd(dest->as_double_reg(), src->as_double_reg());
776
777 } else {
778 ShouldNotReachHere();
779 }
780 }
781
782 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
783 if (src->is_single_cpu()) {
784 if (type == T_ARRAY || type == T_OBJECT) {
785 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
786 __ verify_oop(src->as_register());
787 } else if (type == T_METADATA || type == T_DOUBLE) {
788 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
789 } else {
790 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
791 }
792
793 } else if (src->is_double_cpu()) {
794 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
795 __ str(src->as_register_lo(), dest_addr_LO);
796
797 } else if (src->is_single_fpu()) {
798 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
799 __ strs(src->as_float_reg(), dest_addr);
800
801 } else if (src->is_double_fpu()) {
802 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
803 __ strd(src->as_double_reg(), dest_addr);
804
805 } else {
806 ShouldNotReachHere();
807 }
808
809 }
810
811
812 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
813 LIR_Address* to_addr = dest->as_address_ptr();
814 PatchingStub* patch = NULL;
815 Register compressed_src = rscratch1;
816
817 if (patch_code != lir_patch_none) {
818 deoptimize_trap(info);
819 return;
820 }
821
822 if (type == T_ARRAY || type == T_OBJECT) {
823 __ verify_oop(src->as_register());
824
825 if (UseCompressedOops && !wide) {
826 __ encode_heap_oop(compressed_src, src->as_register());
827 } else {
828 compressed_src = src->as_register();
829 }
830 }
831
832 int null_check_here = code_offset();
833 switch (type) {
834 case T_FLOAT: {
835 __ strs(src->as_float_reg(), as_Address(to_addr));
836 break;
837 }
838
839 case T_DOUBLE: {
840 __ strd(src->as_double_reg(), as_Address(to_addr));
841 break;
842 }
843
844 case T_ARRAY: // fall through
845 case T_OBJECT: // fall through
846 if (UseCompressedOops && !wide) {
847 __ strw(compressed_src, as_Address(to_addr, rscratch2));
848 } else {
849 __ str(compressed_src, as_Address(to_addr));
850 }
851 break;
852 case T_METADATA:
853 // We get here to store a method pointer to the stack to pass to
854 // a dtrace runtime call. This can't work on 64 bit with
855 // compressed klass ptrs: T_METADATA can be a compressed klass
856 // ptr or a 64 bit method pointer.
857 ShouldNotReachHere();
858 __ str(src->as_register(), as_Address(to_addr));
859 break;
860 case T_ADDRESS:
861 __ str(src->as_register(), as_Address(to_addr));
862 break;
863 case T_INT:
864 __ strw(src->as_register(), as_Address(to_addr));
865 break;
866
867 case T_LONG: {
868 __ str(src->as_register_lo(), as_Address_lo(to_addr));
869 break;
870 }
871
872 case T_BYTE: // fall through
873 case T_BOOLEAN: {
874 __ strb(src->as_register(), as_Address(to_addr));
875 break;
876 }
877
878 case T_CHAR: // fall through
879 case T_SHORT:
880 __ strh(src->as_register(), as_Address(to_addr));
881 break;
882
883 default:
884 ShouldNotReachHere();
885 }
886 if (info != NULL) {
887 add_debug_info_for_null_check(null_check_here, info);
888 }
889 }
890
891
892 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
893 assert(src->is_stack(), "should not call otherwise");
894 assert(dest->is_register(), "should not call otherwise");
895
896 if (dest->is_single_cpu()) {
897 if (type == T_ARRAY || type == T_OBJECT) {
898 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
899 __ verify_oop(dest->as_register());
900 } else if (type == T_METADATA) {
901 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
902 } else {
903 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
904 }
905
906 } else if (dest->is_double_cpu()) {
907 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
908 __ ldr(dest->as_register_lo(), src_addr_LO);
909
910 } else if (dest->is_single_fpu()) {
911 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
912 __ ldrs(dest->as_float_reg(), src_addr);
913
914 } else if (dest->is_double_fpu()) {
915 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
916 __ ldrd(dest->as_double_reg(), src_addr);
917
918 } else {
919 ShouldNotReachHere();
920 }
921 }
922
923
924 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
925 address target = NULL;
926 relocInfo::relocType reloc_type = relocInfo::none;
927
928 switch (patching_id(info)) {
929 case PatchingStub::access_field_id:
930 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
931 reloc_type = relocInfo::section_word_type;
932 break;
933 case PatchingStub::load_klass_id:
934 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
935 reloc_type = relocInfo::metadata_type;
936 break;
937 case PatchingStub::load_mirror_id:
938 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
939 reloc_type = relocInfo::oop_type;
940 break;
941 case PatchingStub::load_appendix_id:
942 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
943 reloc_type = relocInfo::oop_type;
944 break;
945 default: ShouldNotReachHere();
946 }
947
948 __ far_call(RuntimeAddress(target));
949 add_call_info_here(info);
950 }
951
952 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
953
954 LIR_Opr temp;
955 if (type == T_LONG || type == T_DOUBLE)
956 temp = FrameMap::rscratch1_long_opr;
957 else
958 temp = FrameMap::rscratch1_opr;
959
960 stack2reg(src, temp, src->type());
961 reg2stack(temp, dest, dest->type(), false);
962 }
963
964
965 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
966 LIR_Address* addr = src->as_address_ptr();
967 LIR_Address* from_addr = src->as_address_ptr();
968
969 if (addr->base()->type() == T_OBJECT) {
970 __ verify_oop(addr->base()->as_pointer_register());
971 }
972
973 if (patch_code != lir_patch_none) {
974 deoptimize_trap(info);
975 return;
976 }
977
978 if (info != NULL) {
979 add_debug_info_for_null_check_here(info);
980 }
981 int null_check_here = code_offset();
982 switch (type) {
983 case T_FLOAT: {
984 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
985 break;
986 }
987
988 case T_DOUBLE: {
989 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
990 break;
991 }
992
993 case T_ARRAY: // fall through
994 case T_OBJECT: // fall through
995 if (UseCompressedOops && !wide) {
996 __ ldrw(dest->as_register(), as_Address(from_addr));
997 } else {
998 __ ldr(dest->as_register(), as_Address(from_addr));
999 }
1000 break;
1001 case T_METADATA:
1002 // We get here to store a method pointer to the stack to pass to
1003 // a dtrace runtime call. This can't work on 64 bit with
1004 // compressed klass ptrs: T_METADATA can be a compressed klass
1005 // ptr or a 64 bit method pointer.
1006 ShouldNotReachHere();
1007 __ ldr(dest->as_register(), as_Address(from_addr));
1008 break;
1009 case T_ADDRESS:
1010 // FIXME: OMG this is a horrible kludge. Any offset from an
1011 // address that matches klass_offset_in_bytes() will be loaded
1012 // as a word, not a long.
1013 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1014 __ ldrw(dest->as_register(), as_Address(from_addr));
1015 } else {
1016 __ ldr(dest->as_register(), as_Address(from_addr));
1017 }
1018 break;
1019 case T_INT:
1020 __ ldrw(dest->as_register(), as_Address(from_addr));
1021 break;
1022
1023 case T_LONG: {
1024 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1025 break;
1026 }
1027
1028 case T_BYTE:
1029 __ ldrsb(dest->as_register(), as_Address(from_addr));
1030 break;
1031 case T_BOOLEAN: {
1032 __ ldrb(dest->as_register(), as_Address(from_addr));
1033 break;
1034 }
1035
1036 case T_CHAR:
1037 __ ldrh(dest->as_register(), as_Address(from_addr));
1038 break;
1039 case T_SHORT:
1040 __ ldrsh(dest->as_register(), as_Address(from_addr));
1041 break;
1042
1043 default:
1044 ShouldNotReachHere();
1045 }
1046
1047 if (type == T_ARRAY || type == T_OBJECT) {
1048 if (UseCompressedOops && !wide) {
1049 __ decode_heap_oop(dest->as_register());
1050 }
1051 __ verify_oop(dest->as_register());
1052 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1053 if (UseCompressedClassPointers) {
1054 __ decode_klass_not_null(dest->as_register());
1055 }
1056 }
1057 }
1058
1059
1060 void LIR_Assembler::prefetchr(LIR_Opr src) { Unimplemented(); }
1061
1062
1063 void LIR_Assembler::prefetchw(LIR_Opr src) { Unimplemented(); }
1064
1065
1066 int LIR_Assembler::array_element_size(BasicType type) const {
1067 int elem_size = type2aelembytes(type);
1068 return exact_log2(elem_size);
1069 }
1070
1071 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1072 Register Rdividend = op->in_opr1()->as_register();
1073 Register Rdivisor = op->in_opr2()->as_register();
1074 Register Rscratch = op->in_opr3()->as_register();
1075 Register Rresult = op->result_opr()->as_register();
1076 int divisor = -1;
1077
1078 /*
1079 TODO: For some reason, using the Rscratch that gets passed in is
1080 not possible because the register allocator does not see the tmp reg
1081 as used, and assignes it the same register as Rdividend. We use rscratch1
1082 instead.
1083
1084 assert(Rdividend != Rscratch, "");
1085 assert(Rdivisor != Rscratch, "");
1086 */
1087
1088 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1089 // convert division by a power of two into some shifts and logical operations
1090 }
1091
1092 if (op->code() == lir_irem) {
1093 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1094 } else if (op->code() == lir_idiv) {
1095 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1096 } else
1097 ShouldNotReachHere();
1098 }
1099
1100 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1101 #ifdef ASSERT
1102 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1103 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1104 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1105 #endif
1106
1107 if (op->cond() == lir_cond_always) {
1108 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1109 __ b(*(op->label()));
1110 } else {
1111 Assembler::Condition acond;
1112 if (op->code() == lir_cond_float_branch) {
1113 bool is_unordered = (op->ublock() == op->block());
1114 // Assembler::EQ does not permit unordered branches, so we add
1115 // another branch here. Likewise, Assembler::NE does not permit
1116 // ordered branches.
1117 if (is_unordered && op->cond() == lir_cond_equal
1118 || !is_unordered && op->cond() == lir_cond_notEqual)
1119 __ br(Assembler::VS, *(op->ublock()->label()));
1120 switch(op->cond()) {
1121 case lir_cond_equal: acond = Assembler::EQ; break;
1122 case lir_cond_notEqual: acond = Assembler::NE; break;
1123 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1124 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1125 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1126 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1127 default: ShouldNotReachHere();
1128 }
1129 } else {
1130 switch (op->cond()) {
1131 case lir_cond_equal: acond = Assembler::EQ; break;
1132 case lir_cond_notEqual: acond = Assembler::NE; break;
1133 case lir_cond_less: acond = Assembler::LT; break;
1134 case lir_cond_lessEqual: acond = Assembler::LE; break;
1135 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1136 case lir_cond_greater: acond = Assembler::GT; break;
1137 case lir_cond_belowEqual: acond = Assembler::LS; break;
1138 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1139 default: ShouldNotReachHere();
1140 }
1141 }
1142 __ br(acond,*(op->label()));
1143 }
1144 }
1145
1146
1147
1148 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1149 LIR_Opr src = op->in_opr();
1150 LIR_Opr dest = op->result_opr();
1151
1152 switch (op->bytecode()) {
1153 case Bytecodes::_i2f:
1154 {
1155 __ scvtfws(dest->as_float_reg(), src->as_register());
1156 break;
1157 }
1158 case Bytecodes::_i2d:
1159 {
1160 __ scvtfwd(dest->as_double_reg(), src->as_register());
1161 break;
1162 }
1163 case Bytecodes::_l2d:
1164 {
1165 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1166 break;
1167 }
1168 case Bytecodes::_l2f:
1169 {
1170 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1171 break;
1172 }
1173 case Bytecodes::_f2d:
1174 {
1175 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1176 break;
1177 }
1178 case Bytecodes::_d2f:
1179 {
1180 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1181 break;
1182 }
1183 case Bytecodes::_i2c:
1184 {
1185 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1186 break;
1187 }
1188 case Bytecodes::_i2l:
1189 {
1190 __ sxtw(dest->as_register_lo(), src->as_register());
1191 break;
1192 }
1193 case Bytecodes::_i2s:
1194 {
1195 __ sxth(dest->as_register(), src->as_register());
1196 break;
1197 }
1198 case Bytecodes::_i2b:
1199 {
1200 __ sxtb(dest->as_register(), src->as_register());
1201 break;
1202 }
1203 case Bytecodes::_l2i:
1204 {
1205 _masm->block_comment("FIXME: This could be a no-op");
1206 __ uxtw(dest->as_register(), src->as_register_lo());
1207 break;
1208 }
1209 case Bytecodes::_d2l:
1210 {
1211 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1212 break;
1213 }
1214 case Bytecodes::_f2i:
1215 {
1216 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1217 break;
1218 }
1219 case Bytecodes::_f2l:
1220 {
1221 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1222 break;
1223 }
1224 case Bytecodes::_d2i:
1225 {
1226 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1227 break;
1228 }
1229 default: ShouldNotReachHere();
1230 }
1231 }
1232
1233 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1234 if (op->init_check()) {
1235 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1236 InstanceKlass::init_state_offset()));
1237 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1238 add_debug_info_for_null_check_here(op->stub()->info());
1239 __ br(Assembler::NE, *op->stub()->entry());
1240 }
1241 __ allocate_object(op->obj()->as_register(),
1242 op->tmp1()->as_register(),
1243 op->tmp2()->as_register(),
1244 op->header_size(),
1245 op->object_size(),
1246 op->klass()->as_register(),
1247 *op->stub()->entry());
1248 __ bind(*op->stub()->continuation());
1249 }
1250
1251 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1252 Register len = op->len()->as_register();
1253 __ uxtw(len, len);
1254
1255 if (UseSlowPath ||
1256 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1257 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1258 __ b(*op->stub()->entry());
1259 } else {
1260 Register tmp1 = op->tmp1()->as_register();
1261 Register tmp2 = op->tmp2()->as_register();
1262 Register tmp3 = op->tmp3()->as_register();
1263 if (len == tmp1) {
1264 tmp1 = tmp3;
1265 } else if (len == tmp2) {
1266 tmp2 = tmp3;
1267 } else if (len == tmp3) {
1268 // everything is ok
1269 } else {
1270 __ mov(tmp3, len);
1271 }
1272 __ allocate_array(op->obj()->as_register(),
1273 len,
1274 tmp1,
1275 tmp2,
1276 arrayOopDesc::header_size(op->type()),
1277 array_element_size(op->type()),
1278 op->klass()->as_register(),
1279 *op->stub()->entry());
1280 }
1281 __ bind(*op->stub()->continuation());
1282 }
1283
1284 void LIR_Assembler::type_profile_helper(Register mdo,
1285 ciMethodData *md, ciProfileData *data,
1286 Register recv, Label* update_done) {
1287 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1288 Label next_test;
1289 // See if the receiver is receiver[n].
1290 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1291 __ ldr(rscratch1, Address(rscratch2));
1292 __ cmp(recv, rscratch1);
1293 __ br(Assembler::NE, next_test);
1294 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1295 __ addptr(data_addr, DataLayout::counter_increment);
1296 __ b(*update_done);
1297 __ bind(next_test);
1298 }
1299
1300 // Didn't find receiver; find next empty slot and fill it in
1301 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1302 Label next_test;
1303 __ lea(rscratch2,
1304 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1305 Address recv_addr(rscratch2);
1306 __ ldr(rscratch1, recv_addr);
1307 __ cbnz(rscratch1, next_test);
1308 __ str(recv, recv_addr);
1309 __ mov(rscratch1, DataLayout::counter_increment);
1310 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1311 __ str(rscratch1, Address(rscratch2));
1312 __ b(*update_done);
1313 __ bind(next_test);
1314 }
1315 }
1316
1317 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1318 // we always need a stub for the failure case.
1319 CodeStub* stub = op->stub();
1320 Register obj = op->object()->as_register();
1321 Register k_RInfo = op->tmp1()->as_register();
1322 Register klass_RInfo = op->tmp2()->as_register();
1323 Register dst = op->result_opr()->as_register();
1324 ciKlass* k = op->klass();
1325 Register Rtmp1 = noreg;
1326
1327 // check if it needs to be profiled
1328 ciMethodData* md;
1329 ciProfileData* data;
1330
1331 if (op->should_profile()) {
1332 ciMethod* method = op->profiled_method();
1333 assert(method != NULL, "Should have method");
1334 int bci = op->profiled_bci();
1335 md = method->method_data_or_null();
1336 assert(md != NULL, "Sanity");
1337 data = md->bci_to_data(bci);
1338 assert(data != NULL, "need data for type check");
1339 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1340 }
1341 Label profile_cast_success, profile_cast_failure;
1342 Label *success_target = op->should_profile() ? &profile_cast_success : success;
1343 Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1344
1345 if (obj == k_RInfo) {
1346 k_RInfo = dst;
1347 } else if (obj == klass_RInfo) {
1348 klass_RInfo = dst;
1349 }
1350 if (k->is_loaded() && !UseCompressedClassPointers) {
1351 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1352 } else {
1353 Rtmp1 = op->tmp3()->as_register();
1354 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1355 }
1356
1357 assert_different_registers(obj, k_RInfo, klass_RInfo);
1358
1359 if (op->should_profile()) {
1360 Label not_null;
1361 __ cbnz(obj, not_null);
1362 // Object is null; update MDO and exit
1363 Register mdo = klass_RInfo;
1364 __ mov_metadata(mdo, md->constant_encoding());
1365 Address data_addr
1366 = __ form_address(rscratch2, mdo,
1367 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1368 LogBytesPerWord);
1369 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1370 __ ldr(rscratch1, data_addr);
1371 __ orr(rscratch1, rscratch1, header_bits);
1372 __ str(rscratch1, data_addr);
1373 __ b(*obj_is_null);
1374 __ bind(not_null);
1375 } else {
1376 __ cbz(obj, *obj_is_null);
1377 }
1378
1379 if (!k->is_loaded()) {
1380 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1381 } else {
1382 __ mov_metadata(k_RInfo, k->constant_encoding());
1383 }
1384 __ verify_oop(obj);
1385
1386 if (op->fast_check()) {
1387 // get object class
1388 // not a safepoint as obj null check happens earlier
1389 __ load_klass(rscratch1, obj);
1390 __ cmp( rscratch1, k_RInfo);
1391
1392 __ br(Assembler::NE, *failure_target);
1393 // successful cast, fall through to profile or jump
1394 } else {
1395 // get object class
1396 // not a safepoint as obj null check happens earlier
1397 __ load_klass(klass_RInfo, obj);
1398 if (k->is_loaded()) {
1399 // See if we get an immediate positive hit
1400 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1401 __ cmp(k_RInfo, rscratch1);
1402 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1403 __ br(Assembler::NE, *failure_target);
1404 // successful cast, fall through to profile or jump
1405 } else {
1406 // See if we get an immediate positive hit
1407 __ br(Assembler::EQ, *success_target);
1408 // check for self
1409 __ cmp(klass_RInfo, k_RInfo);
1410 __ br(Assembler::EQ, *success_target);
1411
1412 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1413 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1414 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1415 // result is a boolean
1416 __ cbzw(klass_RInfo, *failure_target);
1417 // successful cast, fall through to profile or jump
1418 }
1419 } else {
1420 // perform the fast part of the checking logic
1421 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1422 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1423 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1424 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1425 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1426 // result is a boolean
1427 __ cbz(k_RInfo, *failure_target);
1428 // successful cast, fall through to profile or jump
1429 }
1430 }
1431 if (op->should_profile()) {
1432 Register mdo = klass_RInfo, recv = k_RInfo;
1433 __ bind(profile_cast_success);
1434 __ mov_metadata(mdo, md->constant_encoding());
1435 __ load_klass(recv, obj);
1436 Label update_done;
1437 type_profile_helper(mdo, md, data, recv, success);
1438 __ b(*success);
1439
1440 __ bind(profile_cast_failure);
1441 __ mov_metadata(mdo, md->constant_encoding());
1442 Address counter_addr
1443 = __ form_address(rscratch2, mdo,
1444 md->byte_offset_of_slot(data, CounterData::count_offset()),
1445 LogBytesPerWord);
1446 __ ldr(rscratch1, counter_addr);
1447 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1448 __ str(rscratch1, counter_addr);
1449 __ b(*failure);
1450 }
1451 __ b(*success);
1452 }
1453
1454
1455 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1456 LIR_Code code = op->code();
1457 if (code == lir_store_check) {
1458 Register value = op->object()->as_register();
1459 Register array = op->array()->as_register();
1460 Register k_RInfo = op->tmp1()->as_register();
1461 Register klass_RInfo = op->tmp2()->as_register();
1462 Register Rtmp1 = op->tmp3()->as_register();
1463
1464 CodeStub* stub = op->stub();
1465
1466 // check if it needs to be profiled
1467 ciMethodData* md;
1468 ciProfileData* data;
1469
1470 if (op->should_profile()) {
1471 ciMethod* method = op->profiled_method();
1472 assert(method != NULL, "Should have method");
1473 int bci = op->profiled_bci();
1474 md = method->method_data_or_null();
1475 assert(md != NULL, "Sanity");
1476 data = md->bci_to_data(bci);
1477 assert(data != NULL, "need data for type check");
1478 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1479 }
1480 Label profile_cast_success, profile_cast_failure, done;
1481 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1482 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1483
1484 if (op->should_profile()) {
1485 Label not_null;
1486 __ cbnz(value, not_null);
1487 // Object is null; update MDO and exit
1488 Register mdo = klass_RInfo;
1489 __ mov_metadata(mdo, md->constant_encoding());
1490 Address data_addr
1491 = __ form_address(rscratch2, mdo,
1492 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1493 LogBytesPerInt);
1494 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1495 __ ldrw(rscratch1, data_addr);
1496 __ orrw(rscratch1, rscratch1, header_bits);
1497 __ strw(rscratch1, data_addr);
1498 __ b(done);
1499 __ bind(not_null);
1500 } else {
1501 __ cbz(value, done);
1502 }
1503
1504 add_debug_info_for_null_check_here(op->info_for_exception());
1505 __ load_klass(k_RInfo, array);
1506 __ load_klass(klass_RInfo, value);
1507
1508 // get instance klass (it's already uncompressed)
1509 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1510 // perform the fast part of the checking logic
1511 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1512 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1513 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1514 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1515 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1516 // result is a boolean
1517 __ cbzw(k_RInfo, *failure_target);
1518 // fall through to the success case
1519
1520 if (op->should_profile()) {
1521 Register mdo = klass_RInfo, recv = k_RInfo;
1522 __ bind(profile_cast_success);
1523 __ mov_metadata(mdo, md->constant_encoding());
1524 __ load_klass(recv, value);
1525 Label update_done;
1526 type_profile_helper(mdo, md, data, recv, &done);
1527 __ b(done);
1528
1529 __ bind(profile_cast_failure);
1530 __ mov_metadata(mdo, md->constant_encoding());
1531 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1532 __ lea(rscratch2, counter_addr);
1533 __ ldr(rscratch1, Address(rscratch2));
1534 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1535 __ str(rscratch1, Address(rscratch2));
1536 __ b(*stub->entry());
1537 }
1538
1539 __ bind(done);
1540 } else if (code == lir_checkcast) {
1541 Register obj = op->object()->as_register();
1542 Register dst = op->result_opr()->as_register();
1543 Label success;
1544 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1545 __ bind(success);
1546 if (dst != obj) {
1547 __ mov(dst, obj);
1548 }
1549 } else if (code == lir_instanceof) {
1550 Register obj = op->object()->as_register();
1551 Register dst = op->result_opr()->as_register();
1552 Label success, failure, done;
1553 emit_typecheck_helper(op, &success, &failure, &failure);
1554 __ bind(failure);
1555 __ mov(dst, zr);
1556 __ b(done);
1557 __ bind(success);
1558 __ mov(dst, 1);
1559 __ bind(done);
1560 } else {
1561 ShouldNotReachHere();
1562 }
1563 }
1564
1565 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1566 Label retry_load, nope;
1567 // flush and load exclusive from the memory location
1568 // and fail if it is not what we expect
1569 __ bind(retry_load);
1570 __ ldaxrw(rscratch1, addr);
1571 __ cmpw(rscratch1, cmpval);
1572 __ cset(rscratch1, Assembler::NE);
1573 __ br(Assembler::NE, nope);
1574 // if we store+flush with no intervening write rscratch1 wil be zero
1575 __ stlxrw(rscratch1, newval, addr);
1576 // retry so we only ever return after a load fails to compare
1577 // ensures we don't return a stale value after a failed write.
1578 __ cbnzw(rscratch1, retry_load);
1579 __ bind(nope);
1580 __ membar(__ AnyAny);
1581 }
1582
1583 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1584 Label retry_load, nope;
1585 // flush and load exclusive from the memory location
1586 // and fail if it is not what we expect
1587 __ bind(retry_load);
1588 __ ldaxr(rscratch1, addr);
1589 __ cmp(rscratch1, cmpval);
1590 __ cset(rscratch1, Assembler::NE);
1591 __ br(Assembler::NE, nope);
1592 // if we store+flush with no intervening write rscratch1 wil be zero
1593 __ stlxr(rscratch1, newval, addr);
1594 // retry so we only ever return after a load fails to compare
1595 // ensures we don't return a stale value after a failed write.
1596 __ cbnz(rscratch1, retry_load);
1597 __ bind(nope);
1598 __ membar(__ AnyAny);
1599 }
1600
1601
1602 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1603 assert(VM_Version::supports_cx8(), "wrong machine");
1604 Register addr = as_reg(op->addr());
1605 Register newval = as_reg(op->new_value());
1606 Register cmpval = as_reg(op->cmp_value());
1607 Label succeed, fail, around;
1608
1609 if (op->code() == lir_cas_obj) {
1610 if (UseCompressedOops) {
1611 Register t1 = op->tmp1()->as_register();
1612 assert(op->tmp1()->is_valid(), "must be");
1613 __ encode_heap_oop(t1, cmpval);
1614 cmpval = t1;
1615 __ encode_heap_oop(rscratch2, newval);
1616 newval = rscratch2;
1617 casw(addr, newval, cmpval);
1618 } else {
1619 casl(addr, newval, cmpval);
1620 }
1621 } else if (op->code() == lir_cas_int) {
1622 casw(addr, newval, cmpval);
1623 } else {
1624 casl(addr, newval, cmpval);
1625 }
1626 }
1627
1628
1629 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1630
1631 Assembler::Condition acond, ncond;
1632 switch (condition) {
1633 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1634 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1635 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1636 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1637 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1638 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1639 case lir_cond_belowEqual: Unimplemented(); break;
1640 case lir_cond_aboveEqual: Unimplemented(); break;
1641 default: ShouldNotReachHere();
1642 }
1643
1644 assert(result->is_single_cpu() || result->is_double_cpu(),
1645 "expect single register for result");
1646 if (opr1->is_constant() && opr2->is_constant()
1647 && opr1->type() == T_INT && opr2->type() == T_INT) {
1648 jint val1 = opr1->as_jint();
1649 jint val2 = opr2->as_jint();
1650 if (val1 == 0 && val2 == 1) {
1651 __ cset(result->as_register(), ncond);
1652 return;
1653 } else if (val1 == 1 && val2 == 0) {
1654 __ cset(result->as_register(), acond);
1655 return;
1656 }
1657 }
1658
1659 if (opr1->is_constant() && opr2->is_constant()
1660 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1661 jlong val1 = opr1->as_jlong();
1662 jlong val2 = opr2->as_jlong();
1663 if (val1 == 0 && val2 == 1) {
1664 __ cset(result->as_register_lo(), ncond);
1665 return;
1666 } else if (val1 == 1 && val2 == 0) {
1667 __ cset(result->as_register_lo(), acond);
1668 return;
1669 }
1670 }
1671
1672 if (opr1->is_stack()) {
1673 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1674 opr1 = FrameMap::rscratch1_opr;
1675 } else if (opr1->is_constant()) {
1676 LIR_Opr tmp
1677 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1678 const2reg(opr1, tmp, lir_patch_none, NULL);
1679 opr1 = tmp;
1680 }
1681
1682 if (opr2->is_stack()) {
1683 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1684 opr2 = FrameMap::rscratch2_opr;
1685 } else if (opr2->is_constant()) {
1686 LIR_Opr tmp
1687 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1688 const2reg(opr2, tmp, lir_patch_none, NULL);
1689 opr2 = tmp;
1690 }
1691
1692 if (result->type() == T_LONG)
1693 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1694 else
1695 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1696 }
1697
1698 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1699 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1700
1701 if (left->is_single_cpu()) {
1702 Register lreg = left->as_register();
1703 Register dreg = as_reg(dest);
1704
1705 if (right->is_single_cpu()) {
1706 // cpu register - cpu register
1707
1708 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1709 "should be");
1710 Register rreg = right->as_register();
1711 switch (code) {
1712 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1713 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1714 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1715 default: ShouldNotReachHere();
1716 }
1717
1718 } else if (right->is_double_cpu()) {
1719 Register rreg = right->as_register_lo();
1720 // single_cpu + double_cpu: can happen with obj+long
1721 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1722 switch (code) {
1723 case lir_add: __ add(dreg, lreg, rreg); break;
1724 case lir_sub: __ sub(dreg, lreg, rreg); break;
1725 default: ShouldNotReachHere();
1726 }
1727 } else if (right->is_constant()) {
1728 // cpu register - constant
1729 jlong c;
1730
1731 // FIXME. This is fugly: we really need to factor all this logic.
1732 switch(right->type()) {
1733 case T_LONG:
1734 c = right->as_constant_ptr()->as_jlong();
1735 break;
1736 case T_INT:
1737 case T_ADDRESS:
1738 c = right->as_constant_ptr()->as_jint();
1739 break;
1740 default:
1741 ShouldNotReachHere();
1742 break;
1743 }
1744
1745 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1746 if (c == 0 && dreg == lreg) {
1747 COMMENT("effective nop elided");
1748 return;
1749 }
1750 switch(left->type()) {
1751 case T_INT:
1752 switch (code) {
1753 case lir_add: __ addw(dreg, lreg, c); break;
1754 case lir_sub: __ subw(dreg, lreg, c); break;
1755 default: ShouldNotReachHere();
1756 }
1757 break;
1758 case T_OBJECT:
1759 case T_ADDRESS:
1760 switch (code) {
1761 case lir_add: __ add(dreg, lreg, c); break;
1762 case lir_sub: __ sub(dreg, lreg, c); break;
1763 default: ShouldNotReachHere();
1764 }
1765 break;
1766 ShouldNotReachHere();
1767 }
1768 } else {
1769 ShouldNotReachHere();
1770 }
1771
1772 } else if (left->is_double_cpu()) {
1773 Register lreg_lo = left->as_register_lo();
1774
1775 if (right->is_double_cpu()) {
1776 // cpu register - cpu register
1777 Register rreg_lo = right->as_register_lo();
1778 switch (code) {
1779 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1780 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1781 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1782 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1783 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1784 default:
1785 ShouldNotReachHere();
1786 }
1787
1788 } else if (right->is_constant()) {
1789 jlong c = right->as_constant_ptr()->as_jlong_bits();
1790 Register dreg = as_reg(dest);
1791 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1792 if (c == 0 && dreg == lreg_lo) {
1793 COMMENT("effective nop elided");
1794 return;
1795 }
1796 switch (code) {
1797 case lir_add: __ add(dreg, lreg_lo, c); break;
1798 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1799 default:
1800 ShouldNotReachHere();
1801 }
1802 } else {
1803 ShouldNotReachHere();
1804 }
1805 } else if (left->is_single_fpu()) {
1806 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1807 switch (code) {
1808 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1809 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1810 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1811 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1812 default:
1813 ShouldNotReachHere();
1814 }
1815 } else if (left->is_double_fpu()) {
1816 if (right->is_double_fpu()) {
1817 // cpu register - cpu register
1818 switch (code) {
1819 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1820 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1821 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1822 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1823 default:
1824 ShouldNotReachHere();
1825 }
1826 } else {
1827 if (right->is_constant()) {
1828 ShouldNotReachHere();
1829 }
1830 ShouldNotReachHere();
1831 }
1832 } else if (left->is_single_stack() || left->is_address()) {
1833 assert(left == dest, "left and dest must be equal");
1834 ShouldNotReachHere();
1835 } else {
1836 ShouldNotReachHere();
1837 }
1838 }
1839
1840 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1841
1842
1843 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1844 switch(code) {
1845 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1846 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1847 default : ShouldNotReachHere();
1848 }
1849 }
1850
1851 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1852
1853 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1854 Register Rleft = left->is_single_cpu() ? left->as_register() :
1855 left->as_register_lo();
1856 if (dst->is_single_cpu()) {
1857 Register Rdst = dst->as_register();
1858 if (right->is_constant()) {
1859 switch (code) {
1860 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1861 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1862 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1863 default: ShouldNotReachHere(); break;
1864 }
1865 } else {
1866 Register Rright = right->is_single_cpu() ? right->as_register() :
1867 right->as_register_lo();
1868 switch (code) {
1869 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1870 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1871 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1872 default: ShouldNotReachHere(); break;
1873 }
1874 }
1875 } else {
1876 Register Rdst = dst->as_register_lo();
1877 if (right->is_constant()) {
1878 switch (code) {
1879 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1880 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1881 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1882 default: ShouldNotReachHere(); break;
1883 }
1884 } else {
1885 Register Rright = right->is_single_cpu() ? right->as_register() :
1886 right->as_register_lo();
1887 switch (code) {
1888 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1889 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1890 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1891 default: ShouldNotReachHere(); break;
1892 }
1893 }
1894 }
1895 }
1896
1897
1898
1899 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1900
1901
1902 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1903 if (opr1->is_constant() && opr2->is_single_cpu()) {
1904 // tableswitch
1905 Register reg = as_reg(opr2);
1906 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1907 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1908 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1909 Register reg1 = as_reg(opr1);
1910 if (opr2->is_single_cpu()) {
1911 // cpu register - cpu register
1912 Register reg2 = opr2->as_register();
1913 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1914 __ cmp(reg1, reg2);
1915 } else {
1916 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1917 __ cmpw(reg1, reg2);
1918 }
1919 return;
1920 }
1921 if (opr2->is_double_cpu()) {
1922 // cpu register - cpu register
1923 Register reg2 = opr2->as_register_lo();
1924 __ cmp(reg1, reg2);
1925 return;
1926 }
1927
1928 if (opr2->is_constant()) {
1929 jlong imm;
1930 switch(opr2->type()) {
1931 case T_LONG:
1932 imm = opr2->as_constant_ptr()->as_jlong();
1933 break;
1934 case T_INT:
1935 case T_ADDRESS:
1936 imm = opr2->as_constant_ptr()->as_jint();
1937 break;
1938 case T_OBJECT:
1939 case T_ARRAY:
1940 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1941 break;
1942 default:
1943 ShouldNotReachHere();
1944 break;
1945 }
1946
1947 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1948 if (type2aelembytes(opr1->type()) <= 4)
1949 __ cmpw(reg1, imm);
1950 else
1951 __ cmp(reg1, imm);
1952 return;
1953 } else {
1954 __ mov(rscratch1, imm);
1955 if (type2aelembytes(opr1->type()) <= 4)
1956 __ cmpw(reg1, rscratch1);
1957 else
1958 __ cmp(reg1, rscratch1);
1959 return;
1960 }
1961 } else
1962 ShouldNotReachHere();
1963 } else if (opr1->is_single_fpu()) {
1964 FloatRegister reg1 = opr1->as_float_reg();
1965 assert(opr2->is_single_fpu(), "expect single float register");
1966 FloatRegister reg2 = opr2->as_float_reg();
1967 __ fcmps(reg1, reg2);
1968 } else if (opr1->is_double_fpu()) {
1969 FloatRegister reg1 = opr1->as_double_reg();
1970 assert(opr2->is_double_fpu(), "expect double float register");
1971 FloatRegister reg2 = opr2->as_double_reg();
1972 __ fcmpd(reg1, reg2);
1973 } else {
1974 ShouldNotReachHere();
1975 }
1976 }
1977
1978 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1979 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1980 bool is_unordered_less = (code == lir_ucmp_fd2i);
1981 if (left->is_single_fpu()) {
1982 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1983 } else if (left->is_double_fpu()) {
1984 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1985 } else {
1986 ShouldNotReachHere();
1987 }
1988 } else if (code == lir_cmp_l2i) {
1989 Label done;
1990 __ cmp(left->as_register_lo(), right->as_register_lo());
1991 __ mov(dst->as_register(), (u_int64_t)-1L);
1992 __ br(Assembler::LT, done);
1993 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1994 __ bind(done);
1995 } else {
1996 ShouldNotReachHere();
1997 }
1998 }
1999
2000
2001 void LIR_Assembler::align_call(LIR_Code code) { }
2002
2003
2004 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2005 __ trampoline_call(Address(op->addr(), rtype));
2006 add_call_info(code_offset(), op->info());
2007 }
2008
2009
2010 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2011 __ ic_call(op->addr());
2012 add_call_info(code_offset(), op->info());
2013 }
2014
2015
2016 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2017 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2018 ShouldNotReachHere();
2019 }
2020
2021
2022 void LIR_Assembler::emit_static_call_stub() {
2023 address call_pc = __ pc();
2024 address stub = __ start_a_stub(call_stub_size);
2025 if (stub == NULL) {
2026 bailout("static call stub overflow");
2027 return;
2028 }
2029
2030 int start = __ offset();
2031
2032 __ relocate(static_stub_Relocation::spec(call_pc));
2033 __ mov_metadata(rmethod, (Metadata*)NULL);
2034 __ movptr(rscratch1, 0);
2035 __ br(rscratch1);
2036
2037 assert(__ offset() - start <= call_stub_size, "stub too big");
2038 __ end_a_stub();
2039 }
2040
2041
2042 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2043 assert(exceptionOop->as_register() == r0, "must match");
2044 assert(exceptionPC->as_register() == r3, "must match");
2045
2046 // exception object is not added to oop map by LinearScan
2047 // (LinearScan assumes that no oops are in fixed registers)
2048 info->add_register_oop(exceptionOop);
2049 Runtime1::StubID unwind_id;
2050
2051 // get current pc information
2052 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2053 int pc_for_athrow_offset = __ offset();
2054 InternalAddress pc_for_athrow(__ pc());
2055 __ adr(exceptionPC->as_register(), pc_for_athrow);
2056 add_call_info(pc_for_athrow_offset, info); // for exception handler
2057
2058 __ verify_not_null_oop(r0);
2059 // search an exception handler (r0: exception oop, r3: throwing pc)
2060 if (compilation()->has_fpu_code()) {
2061 unwind_id = Runtime1::handle_exception_id;
2062 } else {
2063 unwind_id = Runtime1::handle_exception_nofpu_id;
2064 }
2065 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2066
2067 // FIXME: enough room for two byte trap ????
2068 __ nop();
2069 }
2070
2071
2072 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2073 assert(exceptionOop->as_register() == r0, "must match");
2074
2075 __ b(_unwind_handler_entry);
2076 }
2077
2078
2079 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2080 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2081 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2082
2083 switch (left->type()) {
2084 case T_INT: {
2085 switch (code) {
2086 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2087 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2088 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2089 default:
2090 ShouldNotReachHere();
2091 break;
2092 }
2093 break;
2094 case T_LONG:
2095 case T_ADDRESS:
2096 case T_OBJECT:
2097 switch (code) {
2098 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2099 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2100 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2101 default:
2102 ShouldNotReachHere();
2103 break;
2104 }
2105 break;
2106 default:
2107 ShouldNotReachHere();
2108 break;
2109 }
2110 }
2111 }
2112
2113
2114 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2115 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2116 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2117
2118 switch (left->type()) {
2119 case T_INT: {
2120 switch (code) {
2121 case lir_shl: __ lslw (dreg, lreg, count); break;
2122 case lir_shr: __ asrw (dreg, lreg, count); break;
2123 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2124 default:
2125 ShouldNotReachHere();
2126 break;
2127 }
2128 break;
2129 case T_LONG:
2130 case T_ADDRESS:
2131 case T_OBJECT:
2132 switch (code) {
2133 case lir_shl: __ lsl (dreg, lreg, count); break;
2134 case lir_shr: __ asr (dreg, lreg, count); break;
2135 case lir_ushr: __ lsr (dreg, lreg, count); break;
2136 default:
2137 ShouldNotReachHere();
2138 break;
2139 }
2140 break;
2141 default:
2142 ShouldNotReachHere();
2143 break;
2144 }
2145 }
2146 }
2147
2148
2149 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2150 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2151 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2152 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2153 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2154 }
2155
2156
2157 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2158 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2159 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2160 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2161 __ mov (rscratch1, c);
2162 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2163 }
2164
2165
2166 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2167 ShouldNotReachHere();
2168 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2169 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2170 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2171 __ lea(rscratch1, __ constant_oop_address(o));
2172 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2173 }
2174
2175
2176 // This code replaces a call to arraycopy; no exception may
2177 // be thrown in this code, they must be thrown in the System.arraycopy
2178 // activation frame; we could save some checks if this would not be the case
2179 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2180 ciArrayKlass* default_type = op->expected_type();
2181 Register src = op->src()->as_register();
2182 Register dst = op->dst()->as_register();
2183 Register src_pos = op->src_pos()->as_register();
2184 Register dst_pos = op->dst_pos()->as_register();
2185 Register length = op->length()->as_register();
2186 Register tmp = op->tmp()->as_register();
2187
2188 CodeStub* stub = op->stub();
2189 int flags = op->flags();
2190 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2191 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2192
2193 // if we don't know anything, just go through the generic arraycopy
2194 if (default_type == NULL // || basic_type == T_OBJECT
2195 ) {
2196 Label done;
2197 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2198
2199 // Save the arguments in case the generic arraycopy fails and we
2200 // have to fall back to the JNI stub
2201 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2202 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2203 __ str(src, Address(sp, 4*BytesPerWord));
2204
2205 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2206 address copyfunc_addr = StubRoutines::generic_arraycopy();
2207
2208 // The arguments are in java calling convention so we shift them
2209 // to C convention
2210 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2211 __ mov(c_rarg0, j_rarg0);
2212 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2213 __ mov(c_rarg1, j_rarg1);
2214 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2215 __ mov(c_rarg2, j_rarg2);
2216 assert_different_registers(c_rarg3, j_rarg4);
2217 __ mov(c_rarg3, j_rarg3);
2218 __ mov(c_rarg4, j_rarg4);
2219 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2220 __ mov(rscratch1, RuntimeAddress(C_entry));
2221 __ blrt(rscratch1, 5, 0, 1);
2222 } else {
2223 #ifndef PRODUCT
2224 if (PrintC1Statistics) {
2225 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2226 }
2227 #endif
2228 __ far_call(RuntimeAddress(copyfunc_addr));
2229 }
2230
2231 __ cbz(r0, *stub->continuation());
2232
2233 // Reload values from the stack so they are where the stub
2234 // expects them.
2235 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2236 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2237 __ ldr(src, Address(sp, 4*BytesPerWord));
2238
2239 if (copyfunc_addr != NULL) {
2240 // r0 is -1^K where K == partial copied count
2241 __ eonw(rscratch1, r0, 0);
2242 // adjust length down and src/end pos up by partial copied count
2243 __ subw(length, length, rscratch1);
2244 __ addw(src_pos, src_pos, rscratch1);
2245 __ addw(dst_pos, dst_pos, rscratch1);
2246 }
2247 __ b(*stub->entry());
2248
2249 __ bind(*stub->continuation());
2250 return;
2251 }
2252
2253 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2254
2255 int elem_size = type2aelembytes(basic_type);
2256 int shift_amount;
2257 int scale = exact_log2(elem_size);
2258
2259 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2260 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2261 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2262 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2263
2264 // test for NULL
2265 if (flags & LIR_OpArrayCopy::src_null_check) {
2266 __ cbz(src, *stub->entry());
2267 }
2268 if (flags & LIR_OpArrayCopy::dst_null_check) {
2269 __ cbz(dst, *stub->entry());
2270 }
2271
2272 // check if negative
2273 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2274 __ cmpw(src_pos, 0);
2275 __ br(Assembler::LT, *stub->entry());
2276 }
2277 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2278 __ cmpw(dst_pos, 0);
2279 __ br(Assembler::LT, *stub->entry());
2280 }
2281
2282 if (flags & LIR_OpArrayCopy::length_positive_check) {
2283 __ cmpw(length, 0);
2284 __ br(Assembler::LT, *stub->entry());
2285 }
2286
2287 if (flags & LIR_OpArrayCopy::src_range_check) {
2288 __ addw(tmp, src_pos, length);
2289 __ ldrw(rscratch1, src_length_addr);
2290 __ cmpw(tmp, rscratch1);
2291 __ br(Assembler::HI, *stub->entry());
2292 }
2293 if (flags & LIR_OpArrayCopy::dst_range_check) {
2294 __ addw(tmp, dst_pos, length);
2295 __ ldrw(rscratch1, dst_length_addr);
2296 __ cmpw(tmp, rscratch1);
2297 __ br(Assembler::HI, *stub->entry());
2298 }
2299
2300 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2301 // length_positive_check if the source of our length operand is an
2302 // arraylength. However, that arraylength might be zero, and the
2303 // stub that we're about to call contains an assertion that count !=
2304 // 0 . So we make this check purely in order not to trigger an
2305 // assertion failure.
2306 __ cbzw(length, *stub->continuation());
2307
2308 if (flags & LIR_OpArrayCopy::type_check) {
2309 // We don't know the array types are compatible
2310 if (basic_type != T_OBJECT) {
2311 // Simple test for basic type arrays
2312 if (UseCompressedClassPointers) {
2313 __ ldrw(tmp, src_klass_addr);
2314 __ ldrw(rscratch1, dst_klass_addr);
2315 __ cmpw(tmp, rscratch1);
2316 } else {
2317 __ ldr(tmp, src_klass_addr);
2318 __ ldr(rscratch1, dst_klass_addr);
2319 __ cmp(tmp, rscratch1);
2320 }
2321 __ br(Assembler::NE, *stub->entry());
2322 } else {
2323 // For object arrays, if src is a sub class of dst then we can
2324 // safely do the copy.
2325 Label cont, slow;
2326
2327 #define PUSH(r1, r2) \
2328 stp(r1, r2, __ pre(sp, -2 * wordSize));
2329
2330 #define POP(r1, r2) \
2331 ldp(r1, r2, __ post(sp, 2 * wordSize));
2332
2333 __ PUSH(src, dst);
2334
2335 __ load_klass(src, src);
2336 __ load_klass(dst, dst);
2337
2338 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2339
2340 __ PUSH(src, dst);
2341 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2342 __ POP(src, dst);
2343
2344 __ cbnz(src, cont);
2345
2346 __ bind(slow);
2347 __ POP(src, dst);
2348
2349 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2350 if (copyfunc_addr != NULL) { // use stub if available
2351 // src is not a sub class of dst so we have to do a
2352 // per-element check.
2353
2354 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2355 if ((flags & mask) != mask) {
2356 // Check that at least both of them object arrays.
2357 assert(flags & mask, "one of the two should be known to be an object array");
2358
2359 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2360 __ load_klass(tmp, src);
2361 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2362 __ load_klass(tmp, dst);
2363 }
2364 int lh_offset = in_bytes(Klass::layout_helper_offset());
2365 Address klass_lh_addr(tmp, lh_offset);
2366 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2367 __ ldrw(rscratch1, klass_lh_addr);
2368 __ mov(rscratch2, objArray_lh);
2369 __ eorw(rscratch1, rscratch1, rscratch2);
2370 __ cbnzw(rscratch1, *stub->entry());
2371 }
2372
2373 // Spill because stubs can use any register they like and it's
2374 // easier to restore just those that we care about.
2375 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2376 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2377 __ str(src, Address(sp, 4*BytesPerWord));
2378
2379 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2380 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2381 assert_different_registers(c_rarg0, dst, dst_pos, length);
2382 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2383 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2384 assert_different_registers(c_rarg1, dst, length);
2385 __ uxtw(c_rarg2, length);
2386 assert_different_registers(c_rarg2, dst);
2387
2388 __ load_klass(c_rarg4, dst);
2389 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2390 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2391 __ far_call(RuntimeAddress(copyfunc_addr));
2392
2393 #ifndef PRODUCT
2394 if (PrintC1Statistics) {
2395 Label failed;
2396 __ cbnz(r0, failed);
2397 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2398 __ bind(failed);
2399 }
2400 #endif
2401
2402 __ cbz(r0, *stub->continuation());
2403
2404 #ifndef PRODUCT
2405 if (PrintC1Statistics) {
2406 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2407 }
2408 #endif
2409 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2410
2411 // Restore previously spilled arguments
2412 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2413 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2414 __ ldr(src, Address(sp, 4*BytesPerWord));
2415
2416 // return value is -1^K where K is partial copied count
2417 __ eonw(rscratch1, r0, zr);
2418 // adjust length down and src/end pos up by partial copied count
2419 __ subw(length, length, rscratch1);
2420 __ addw(src_pos, src_pos, rscratch1);
2421 __ addw(dst_pos, dst_pos, rscratch1);
2422 }
2423
2424 __ b(*stub->entry());
2425
2426 __ bind(cont);
2427 __ POP(src, dst);
2428 }
2429 }
2430
2431 #ifdef ASSERT
2432 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2433 // Sanity check the known type with the incoming class. For the
2434 // primitive case the types must match exactly with src.klass and
2435 // dst.klass each exactly matching the default type. For the
2436 // object array case, if no type check is needed then either the
2437 // dst type is exactly the expected type and the src type is a
2438 // subtype which we can't check or src is the same array as dst
2439 // but not necessarily exactly of type default_type.
2440 Label known_ok, halt;
2441 __ mov_metadata(tmp, default_type->constant_encoding());
2442 if (UseCompressedClassPointers) {
2443 __ encode_klass_not_null(tmp);
2444 }
2445
2446 if (basic_type != T_OBJECT) {
2447
2448 if (UseCompressedClassPointers) {
2449 __ ldrw(rscratch1, dst_klass_addr);
2450 __ cmpw(tmp, rscratch1);
2451 } else {
2452 __ ldr(rscratch1, dst_klass_addr);
2453 __ cmp(tmp, rscratch1);
2454 }
2455 __ br(Assembler::NE, halt);
2456 if (UseCompressedClassPointers) {
2457 __ ldrw(rscratch1, src_klass_addr);
2458 __ cmpw(tmp, rscratch1);
2459 } else {
2460 __ ldr(rscratch1, src_klass_addr);
2461 __ cmp(tmp, rscratch1);
2462 }
2463 __ br(Assembler::EQ, known_ok);
2464 } else {
2465 if (UseCompressedClassPointers) {
2466 __ ldrw(rscratch1, dst_klass_addr);
2467 __ cmpw(tmp, rscratch1);
2468 } else {
2469 __ ldr(rscratch1, dst_klass_addr);
2470 __ cmp(tmp, rscratch1);
2471 }
2472 __ br(Assembler::EQ, known_ok);
2473 __ cmp(src, dst);
2474 __ br(Assembler::EQ, known_ok);
2475 }
2476 __ bind(halt);
2477 __ stop("incorrect type information in arraycopy");
2478 __ bind(known_ok);
2479 }
2480 #endif
2481
2482 #ifndef PRODUCT
2483 if (PrintC1Statistics) {
2484 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2485 }
2486 #endif
2487
2488 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2489 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2490 assert_different_registers(c_rarg0, dst, dst_pos, length);
2491 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2492 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2493 assert_different_registers(c_rarg1, dst, length);
2494 __ uxtw(c_rarg2, length);
2495 assert_different_registers(c_rarg2, dst);
2496
2497 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2498 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2499 const char *name;
2500 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2501
2502 CodeBlob *cb = CodeCache::find_blob(entry);
2503 if (cb) {
2504 __ far_call(RuntimeAddress(entry));
2505 } else {
2506 __ call_VM_leaf(entry, 3);
2507 }
2508
2509 __ bind(*stub->continuation());
2510 }
2511
2512
2513
2514
2515 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2516 Register obj = op->obj_opr()->as_register(); // may not be an oop
2517 Register hdr = op->hdr_opr()->as_register();
2518 Register lock = op->lock_opr()->as_register();
2519 if (!UseFastLocking) {
2520 __ b(*op->stub()->entry());
2521 } else if (op->code() == lir_lock) {
2522 Register scratch = noreg;
2523 if (UseBiasedLocking) {
2524 scratch = op->scratch_opr()->as_register();
2525 }
2526 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2527 // add debug info for NullPointerException only if one is possible
2528 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2529 if (op->info() != NULL) {
2530 add_debug_info_for_null_check(null_check_offset, op->info());
2531 }
2532 // done
2533 } else if (op->code() == lir_unlock) {
2534 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2535 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2536 } else {
2537 Unimplemented();
2538 }
2539 __ bind(*op->stub()->continuation());
2540 }
2541
2542
2543 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2544 ciMethod* method = op->profiled_method();
2545 int bci = op->profiled_bci();
2546 ciMethod* callee = op->profiled_callee();
2547
2548 // Update counter for all call types
2549 ciMethodData* md = method->method_data_or_null();
2550 assert(md != NULL, "Sanity");
2551 ciProfileData* data = md->bci_to_data(bci);
2552 assert(data->is_CounterData(), "need CounterData for calls");
2553 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2554 Register mdo = op->mdo()->as_register();
2555 __ mov_metadata(mdo, md->constant_encoding());
2556 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2557 Bytecodes::Code bc = method->java_code_at_bci(bci);
2558 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2559 // Perform additional virtual call profiling for invokevirtual and
2560 // invokeinterface bytecodes
2561 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2562 !callee_is_static && // required for optimized MH invokes
2563 C1ProfileVirtualCalls) {
2564 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2565 Register recv = op->recv()->as_register();
2566 assert_different_registers(mdo, recv);
2567 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2568 ciKlass* known_klass = op->known_holder();
2569 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2570 // We know the type that will be seen at this call site; we can
2571 // statically update the MethodData* rather than needing to do
2572 // dynamic tests on the receiver type
2573
2574 // NOTE: we should probably put a lock around this search to
2575 // avoid collisions by concurrent compilations
2576 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2577 uint i;
2578 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2579 ciKlass* receiver = vc_data->receiver(i);
2580 if (known_klass->equals(receiver)) {
2581 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2582 __ addptr(data_addr, DataLayout::counter_increment);
2583 return;
2584 }
2585 }
2586
2587 // Receiver type not found in profile data; select an empty slot
2588
2589 // Note that this is less efficient than it should be because it
2590 // always does a write to the receiver part of the
2591 // VirtualCallData rather than just the first time
2592 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2593 ciKlass* receiver = vc_data->receiver(i);
2594 if (receiver == NULL) {
2595 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2596 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2597 __ lea(rscratch2, recv_addr);
2598 __ str(rscratch1, Address(rscratch2));
2599 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2600 __ addptr(data_addr, DataLayout::counter_increment);
2601 return;
2602 }
2603 }
2604 } else {
2605 __ load_klass(recv, recv);
2606 Label update_done;
2607 type_profile_helper(mdo, md, data, recv, &update_done);
2608 // Receiver did not match any saved receiver and there is no empty row for it.
2609 // Increment total counter to indicate polymorphic case.
2610 __ addptr(counter_addr, DataLayout::counter_increment);
2611
2612 __ bind(update_done);
2613 }
2614 } else {
2615 // Static call
2616 __ addptr(counter_addr, DataLayout::counter_increment);
2617 }
2618 }
2619
2620
2621 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2622 Unimplemented();
2623 }
2624
2625
2626 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2627 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2628 }
2629
2630 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2631 assert(op->crc()->is_single_cpu(), "crc must be register");
2632 assert(op->val()->is_single_cpu(), "byte value must be register");
2633 assert(op->result_opr()->is_single_cpu(), "result must be register");
2634 Register crc = op->crc()->as_register();
2635 Register val = op->val()->as_register();
2636 Register res = op->result_opr()->as_register();
2637
2638 assert_different_registers(val, crc, res);
2639 unsigned long offset;
2640 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2641 if (offset) __ add(res, res, offset);
2642
2643 __ ornw(crc, zr, crc); // ~crc
2644 __ update_byte_crc32(crc, val, res);
2645 __ ornw(res, zr, crc); // ~crc
2646 }
2647
2648 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2649 COMMENT("emit_profile_type {");
2650 Register obj = op->obj()->as_register();
2651 Register tmp = op->tmp()->as_pointer_register();
2652 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2653 ciKlass* exact_klass = op->exact_klass();
2654 intptr_t current_klass = op->current_klass();
2655 bool not_null = op->not_null();
2656 bool no_conflict = op->no_conflict();
2657
2658 Label update, next, none;
2659
2660 bool do_null = !not_null;
2661 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2662 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2663
2664 assert(do_null || do_update, "why are we here?");
2665 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2666 assert(mdo_addr.base() != rscratch1, "wrong register");
2667
2668 __ verify_oop(obj);
2669
2670 if (tmp != obj) {
2671 __ mov(tmp, obj);
2672 }
2673 if (do_null) {
2674 __ cbnz(tmp, update);
2675 if (!TypeEntries::was_null_seen(current_klass)) {
2676 __ ldr(rscratch2, mdo_addr);
2677 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2678 __ str(rscratch2, mdo_addr);
2679 }
2680 if (do_update) {
2681 #ifndef ASSERT
2682 __ b(next);
2683 }
2684 #else
2685 __ b(next);
2686 }
2687 } else {
2688 __ cbnz(tmp, update);
2689 __ stop("unexpected null obj");
2690 #endif
2691 }
2692
2693 __ bind(update);
2694
2695 if (do_update) {
2696 #ifdef ASSERT
2697 if (exact_klass != NULL) {
2698 Label ok;
2699 __ load_klass(tmp, tmp);
2700 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2701 __ eor(rscratch1, tmp, rscratch1);
2702 __ cbz(rscratch1, ok);
2703 __ stop("exact klass and actual klass differ");
2704 __ bind(ok);
2705 }
2706 #endif
2707 if (!no_conflict) {
2708 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2709 if (exact_klass != NULL) {
2710 __ mov_metadata(tmp, exact_klass->constant_encoding());
2711 } else {
2712 __ load_klass(tmp, tmp);
2713 }
2714
2715 __ ldr(rscratch2, mdo_addr);
2716 __ eor(tmp, tmp, rscratch2);
2717 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2718 // klass seen before, nothing to do. The unknown bit may have been
2719 // set already but no need to check.
2720 __ cbz(rscratch1, next);
2721
2722 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2723 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2724
2725 if (TypeEntries::is_type_none(current_klass)) {
2726 __ cbz(rscratch2, none);
2727 __ cmp(rscratch2, TypeEntries::null_seen);
2728 __ br(Assembler::EQ, none);
2729 // There is a chance that the checks above (re-reading profiling
2730 // data from memory) fail if another thread has just set the
2731 // profiling to this obj's klass
2732 __ dmb(Assembler::ISHLD);
2733 __ ldr(rscratch2, mdo_addr);
2734 __ eor(tmp, tmp, rscratch2);
2735 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2736 __ cbz(rscratch1, next);
2737 }
2738 } else {
2739 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2740 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2741
2742 __ ldr(tmp, mdo_addr);
2743 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2744 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2745 }
2746
2747 // different than before. Cannot keep accurate profile.
2748 __ ldr(rscratch2, mdo_addr);
2749 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2750 __ str(rscratch2, mdo_addr);
2751
2752 if (TypeEntries::is_type_none(current_klass)) {
2753 __ b(next);
2754
2755 __ bind(none);
2756 // first time here. Set profile type.
2757 __ str(tmp, mdo_addr);
2758 }
2759 } else {
2760 // There's a single possible klass at this profile point
2761 assert(exact_klass != NULL, "should be");
2762 if (TypeEntries::is_type_none(current_klass)) {
2763 __ mov_metadata(tmp, exact_klass->constant_encoding());
2764 __ ldr(rscratch2, mdo_addr);
2765 __ eor(tmp, tmp, rscratch2);
2766 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2767 __ cbz(rscratch1, next);
2768 #ifdef ASSERT
2769 {
2770 Label ok;
2771 __ ldr(rscratch1, mdo_addr);
2772 __ cbz(rscratch1, ok);
2773 __ cmp(rscratch1, TypeEntries::null_seen);
2774 __ br(Assembler::EQ, ok);
2775 // may have been set by another thread
2776 __ dmb(Assembler::ISHLD);
2777 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2778 __ ldr(rscratch2, mdo_addr);
2779 __ eor(rscratch2, rscratch1, rscratch2);
2780 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2781 __ cbz(rscratch2, ok);
2782
2783 __ stop("unexpected profiling mismatch");
2784 __ bind(ok);
2785 __ pop(tmp);
2786 }
2787 #endif
2788 // first time here. Set profile type.
2789 __ ldr(tmp, mdo_addr);
2790 } else {
2791 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2792 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2793
2794 __ ldr(tmp, mdo_addr);
2795 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2796 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2797
2798 __ orr(tmp, tmp, TypeEntries::type_unknown);
2799 __ str(tmp, mdo_addr);
2800 // FIXME: Write barrier needed here?
2801 }
2802 }
2803
2804 __ bind(next);
2805 }
2806 COMMENT("} emit_profile_type");
2807 }
2808
2809
2810 void LIR_Assembler::align_backward_branch_target() {
2811 }
2812
2813
2814 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2815 if (left->is_single_cpu()) {
2816 assert(dest->is_single_cpu(), "expect single result reg");
2817 __ negw(dest->as_register(), left->as_register());
2818 } else if (left->is_double_cpu()) {
2819 assert(dest->is_double_cpu(), "expect double result reg");
2820 __ neg(dest->as_register_lo(), left->as_register_lo());
2821 } else if (left->is_single_fpu()) {
2822 assert(dest->is_single_fpu(), "expect single float result reg");
2823 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2824 } else {
2825 assert(left->is_double_fpu(), "expect double float operand reg");
2826 assert(dest->is_double_fpu(), "expect double float result reg");
2827 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2828 }
2829 }
2830
2831
2832 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2833 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2834 }
2835
2836
2837 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2838 assert(!tmp->is_valid(), "don't need temporary");
2839
2840 CodeBlob *cb = CodeCache::find_blob(dest);
2841 if (cb) {
2842 __ far_call(RuntimeAddress(dest));
2843 } else {
2844 __ mov(rscratch1, RuntimeAddress(dest));
2845 int len = args->length();
2846 int type = 0;
2847 if (! result->is_illegal()) {
2848 switch (result->type()) {
2849 case T_VOID:
2850 type = 0;
2851 break;
2852 case T_INT:
2853 case T_LONG:
2854 case T_OBJECT:
2855 type = 1;
2856 break;
2857 case T_FLOAT:
2858 type = 2;
2859 break;
2860 case T_DOUBLE:
2861 type = 3;
2862 break;
2863 default:
2864 ShouldNotReachHere();
2865 break;
2866 }
2867 }
2868 int num_gpargs = 0;
2869 int num_fpargs = 0;
2870 for (int i = 0; i < args->length(); i++) {
2871 LIR_Opr arg = args->at(i);
2872 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2873 num_fpargs++;
2874 } else {
2875 num_gpargs++;
2876 }
2877 }
2878 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2879 }
2880
2881 if (info != NULL) {
2882 add_call_info_here(info);
2883 }
2884 __ maybe_isb();
2885 }
2886
2887 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2888 if (dest->is_address() || src->is_address()) {
2889 move_op(src, dest, type, lir_patch_none, info,
2890 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2891 } else {
2892 ShouldNotReachHere();
2893 }
2894 }
2895
2896 #ifdef ASSERT
2897 // emit run-time assertion
2898 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2899 assert(op->code() == lir_assert, "must be");
2900
2901 if (op->in_opr1()->is_valid()) {
2902 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2903 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2904 } else {
2905 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2906 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2907 }
2908
2909 Label ok;
2910 if (op->condition() != lir_cond_always) {
2911 Assembler::Condition acond = Assembler::AL;
2912 switch (op->condition()) {
2913 case lir_cond_equal: acond = Assembler::EQ; break;
2914 case lir_cond_notEqual: acond = Assembler::NE; break;
2915 case lir_cond_less: acond = Assembler::LT; break;
2916 case lir_cond_lessEqual: acond = Assembler::LE; break;
2917 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2918 case lir_cond_greater: acond = Assembler::GT; break;
2919 case lir_cond_belowEqual: acond = Assembler::LS; break;
2920 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2921 default: ShouldNotReachHere();
2922 }
2923 __ br(acond, ok);
2924 }
2925 if (op->halt()) {
2926 const char* str = __ code_string(op->msg());
2927 __ stop(str);
2928 } else {
2929 breakpoint();
2930 }
2931 __ bind(ok);
2932 }
2933 #endif
2934
2935 #ifndef PRODUCT
2936 #define COMMENT(x) do { __ block_comment(x); } while (0)
2937 #else
2938 #define COMMENT(x)
2939 #endif
2940
2941 void LIR_Assembler::membar() {
2942 COMMENT("membar");
2943 __ membar(MacroAssembler::AnyAny);
2944 }
2945
2946 void LIR_Assembler::membar_acquire() {
2947 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2948 }
2949
2950 void LIR_Assembler::membar_release() {
2951 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2952 }
2953
2954 void LIR_Assembler::membar_loadload() {
2955 __ membar(Assembler::LoadLoad);
2956 }
2957
2958 void LIR_Assembler::membar_storestore() {
2959 __ membar(MacroAssembler::StoreStore);
2960 }
2961
2962 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2963
2964 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2965
2966 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2967 __ mov(result_reg->as_register(), rthread);
2968 }
2969
2970
2971 void LIR_Assembler::peephole(LIR_List *lir) {
2972 #if 0
2973 if (tableswitch_count >= max_tableswitches)
2974 return;
2975
2976 /*
2977 This finite-state automaton recognizes sequences of compare-and-
2978 branch instructions. We will turn them into a tableswitch. You
2979 could argue that C1 really shouldn't be doing this sort of
2980 optimization, but without it the code is really horrible.
2981 */
2982
2983 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2984 int first_key, last_key = -2147483648;
2985 int next_key = 0;
2986 int start_insn = -1;
2987 int last_insn = -1;
2988 Register reg = noreg;
2989 LIR_Opr reg_opr;
2990 state = start_s;
2991
2992 LIR_OpList* inst = lir->instructions_list();
2993 for (int i = 0; i < inst->length(); i++) {
2994 LIR_Op* op = inst->at(i);
2995 switch (state) {
2996 case start_s:
2997 first_key = -1;
2998 start_insn = i;
2999 switch (op->code()) {
3000 case lir_cmp:
3001 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3002 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3003 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3004 && opr2->is_constant()
3005 && opr2->type() == T_INT) {
3006 reg_opr = opr1;
3007 reg = opr1->as_register();
3008 first_key = opr2->as_constant_ptr()->as_jint();
3009 next_key = first_key + 1;
3010 state = cmp_s;
3011 goto next_state;
3012 }
3013 break;
3014 }
3015 break;
3016 case cmp_s:
3017 switch (op->code()) {
3018 case lir_branch:
3019 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3020 state = beq_s;
3021 last_insn = i;
3022 goto next_state;
3023 }
3024 }
3025 state = start_s;
3026 break;
3027 case beq_s:
3028 switch (op->code()) {
3029 case lir_cmp: {
3030 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3031 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3032 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3033 && opr1->as_register() == reg
3034 && opr2->is_constant()
3035 && opr2->type() == T_INT
3036 && opr2->as_constant_ptr()->as_jint() == next_key) {
3037 last_key = next_key;
3038 next_key++;
3039 state = cmp_s;
3040 goto next_state;
3041 }
3042 }
3043 }
3044 last_key = next_key;
3045 state = start_s;
3046 break;
3047 default:
3048 assert(false, "impossible state");
3049 }
3050 if (state == start_s) {
3051 if (first_key < last_key - 5L && reg != noreg) {
3052 {
3053 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3054 // reg->encoding(),
3055 // start_insn, first_key, last_key);
3056 // for (int i = 0; i < inst->length(); i++) {
3057 // inst->at(i)->print();
3058 // tty->print("\n");
3059 // }
3060 // tty->print("\n");
3061 }
3062
3063 struct tableswitch *sw = &switches[tableswitch_count];
3064 sw->_insn_index = start_insn, sw->_first_key = first_key,
3065 sw->_last_key = last_key, sw->_reg = reg;
3066 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3067 {
3068 // Insert the new table of branches
3069 int offset = last_insn;
3070 for (int n = first_key; n < last_key; n++) {
3071 inst->insert_before
3072 (last_insn + 1,
3073 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3074 inst->at(offset)->as_OpBranch()->label()));
3075 offset -= 2, i++;
3076 }
3077 }
3078 // Delete all the old compare-and-branch instructions
3079 for (int n = first_key; n < last_key; n++) {
3080 inst->remove_at(start_insn);
3081 inst->remove_at(start_insn);
3082 }
3083 // Insert the tableswitch instruction
3084 inst->insert_before(start_insn,
3085 new LIR_Op2(lir_cmp, lir_cond_always,
3086 LIR_OprFact::intConst(tableswitch_count),
3087 reg_opr));
3088 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3089 tableswitch_count++;
3090 }
3091 reg = noreg;
3092 last_key = -2147483648;
3093 }
3094 next_state:
3095 ;
3096 }
3097 #endif
3098 }
3099
3100 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3101 Address addr = as_Address(src->as_address_ptr(), noreg);
3102 BasicType type = src->type();
3103 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3104
3105 void (MacroAssembler::* lda)(Register Rd, Register Ra);
3106 void (MacroAssembler::* add)(Register Rd, Register Rn, RegisterOrConstant increment);
3107 void (MacroAssembler::* stl)(Register Rs, Register Rt, Register Rn);
3108
3109 switch(type) {
3110 case T_INT:
3111 lda = &MacroAssembler::ldaxrw;
3112 add = &MacroAssembler::addw;
3113 stl = &MacroAssembler::stlxrw;
3114 break;
3115 case T_LONG:
3116 lda = &MacroAssembler::ldaxr;
3117 add = &MacroAssembler::add;
3118 stl = &MacroAssembler::stlxr;
3119 break;
3120 case T_OBJECT:
3121 case T_ARRAY:
3122 if (UseCompressedOops) {
3123 lda = &MacroAssembler::ldaxrw;
3124 add = &MacroAssembler::addw;
3125 stl = &MacroAssembler::stlxrw;
3126 } else {
3127 lda = &MacroAssembler::ldaxr;
3128 add = &MacroAssembler::add;
3129 stl = &MacroAssembler::stlxr;
3130 }
3131 break;
3132 default:
3133 ShouldNotReachHere();
3134 }
3135
3136 switch (code) {
3137 case lir_xadd:
3138 {
3139 RegisterOrConstant inc;
3140 Register tmp = as_reg(tmp_op);
3141 Register dst = as_reg(dest);
3142 if (data->is_constant()) {
3143 inc = RegisterOrConstant(as_long(data));
3144 assert_different_registers(dst, addr.base(), tmp,
3145 rscratch1, rscratch2);
3146 } else {
3147 inc = RegisterOrConstant(as_reg(data));
3148 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3149 rscratch1, rscratch2);
3150 }
3151 Label again;
3152 __ lea(tmp, addr);
3153 __ bind(again);
3154 (_masm->*lda)(dst, tmp);
3155 (_masm->*add)(rscratch1, dst, inc);
3156 (_masm->*stl)(rscratch2, rscratch1, tmp);
3157 __ cbnzw(rscratch2, again);
3158 break;
3159 }
3160 case lir_xchg:
3161 {
3162 Register tmp = tmp_op->as_register();
3163 Register obj = as_reg(data);
3164 Register dst = as_reg(dest);
3165 if (is_oop && UseCompressedOops) {
3166 __ encode_heap_oop(obj);
3167 }
3168 assert_different_registers(obj, addr.base(), tmp, rscratch2, dst);
3169 Label again;
3170 __ lea(tmp, addr);
3171 __ bind(again);
3172 (_masm->*lda)(dst, tmp);
3173 (_masm->*stl)(rscratch2, obj, tmp);
3174 __ cbnzw(rscratch2, again);
3175 if (is_oop && UseCompressedOops) {
3176 __ decode_heap_oop(dst);
3177 }
3178 }
3179 break;
3180 default:
3181 ShouldNotReachHere();
3182 }
3183 __ membar(__ AnyAny);
3184 }
3185
3186 #undef __