1 /*
2 * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "gc/shared/barrierSet.hpp"
37 #include "gc/shared/cardTableModRefBS.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "vmreg_aarch64.inline.hpp"
43
44
45
46 #ifndef PRODUCT
47 #define COMMENT(x) do { __ block_comment(x); } while (0)
48 #else
49 #define COMMENT(x)
50 #endif
51
52 NEEDS_CLEANUP // remove this definitions ?
53 const Register IC_Klass = rscratch2; // where the IC klass is cached
54 const Register SYNC_header = r0; // synchronization header
55 const Register SHIFT_count = r0; // where count for shift operations must be
56
57 #define __ _masm->
58
59
60 static void select_different_registers(Register preserve,
61 Register extra,
62 Register &tmp1,
63 Register &tmp2) {
64 if (tmp1 == preserve) {
65 assert_different_registers(tmp1, tmp2, extra);
66 tmp1 = extra;
67 } else if (tmp2 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp2 = extra;
70 }
71 assert_different_registers(preserve, tmp1, tmp2);
72 }
73
74
75
76 static void select_different_registers(Register preserve,
77 Register extra,
78 Register &tmp1,
79 Register &tmp2,
80 Register &tmp3) {
81 if (tmp1 == preserve) {
82 assert_different_registers(tmp1, tmp2, tmp3, extra);
83 tmp1 = extra;
84 } else if (tmp2 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp2 = extra;
87 } else if (tmp3 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp3 = extra;
90 }
91 assert_different_registers(preserve, tmp1, tmp2, tmp3);
92 }
93
94
95 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
96
97
98 LIR_Opr LIR_Assembler::receiverOpr() {
99 return FrameMap::receiver_opr;
100 }
101
102 LIR_Opr LIR_Assembler::osrBufferPointer() {
103 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
104 }
105
106 //--------------fpu register translations-----------------------
107
108
109 address LIR_Assembler::float_constant(float f) {
110 address const_addr = __ float_constant(f);
111 if (const_addr == NULL) {
112 bailout("const section overflow");
113 return __ code()->consts()->start();
114 } else {
115 return const_addr;
116 }
117 }
118
119
120 address LIR_Assembler::double_constant(double d) {
121 address const_addr = __ double_constant(d);
122 if (const_addr == NULL) {
123 bailout("const section overflow");
124 return __ code()->consts()->start();
125 } else {
126 return const_addr;
127 }
128 }
129
130 address LIR_Assembler::int_constant(jlong n) {
131 address const_addr = __ long_constant(n);
132 if (const_addr == NULL) {
133 bailout("const section overflow");
134 return __ code()->consts()->start();
135 } else {
136 return const_addr;
137 }
138 }
139
140 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
141
142 void LIR_Assembler::reset_FPU() { Unimplemented(); }
143
144 void LIR_Assembler::fpop() { Unimplemented(); }
145
146 void LIR_Assembler::fxch(int i) { Unimplemented(); }
147
148 void LIR_Assembler::fld(int i) { Unimplemented(); }
149
150 void LIR_Assembler::ffree(int i) { Unimplemented(); }
151
152 void LIR_Assembler::breakpoint() { Unimplemented(); }
153
154 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
155
156 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
157
158 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
159 //-------------------------------------------
160
161 static Register as_reg(LIR_Opr op) {
162 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
163 }
164
165 static jlong as_long(LIR_Opr data) {
166 jlong result;
167 switch (data->type()) {
168 case T_INT:
169 result = (data->as_jint());
170 break;
171 case T_LONG:
172 result = (data->as_jlong());
173 break;
174 default:
175 ShouldNotReachHere();
176 }
177 return result;
178 }
179
180 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
181 Register base = addr->base()->as_pointer_register();
182 LIR_Opr opr = addr->index();
183 if (opr->is_cpu_register()) {
184 Register index;
185 if (opr->is_single_cpu())
186 index = opr->as_register();
187 else
188 index = opr->as_register_lo();
189 assert(addr->disp() == 0, "must be");
190 switch(opr->type()) {
191 case T_INT:
192 return Address(base, index, Address::sxtw(addr->scale()));
193 case T_LONG:
194 return Address(base, index, Address::lsl(addr->scale()));
195 default:
196 ShouldNotReachHere();
197 }
198 } else {
199 intptr_t addr_offset = intptr_t(addr->disp());
200 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
201 return Address(base, addr_offset, Address::lsl(addr->scale()));
202 else {
203 __ mov(tmp, addr_offset);
204 return Address(base, tmp, Address::lsl(addr->scale()));
205 }
206 }
207 return Address();
208 }
209
210 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
211 ShouldNotReachHere();
212 return Address();
213 }
214
215 Address LIR_Assembler::as_Address(LIR_Address* addr) {
216 return as_Address(addr, rscratch1);
217 }
218
219 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
220 return as_Address(addr, rscratch1); // Ouch
221 // FIXME: This needs to be much more clever. See x86.
222 }
223
224
225 void LIR_Assembler::osr_entry() {
226 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
227 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
228 ValueStack* entry_state = osr_entry->state();
229 int number_of_locks = entry_state->locks_size();
230
231 // we jump here if osr happens with the interpreter
232 // state set up to continue at the beginning of the
233 // loop that triggered osr - in particular, we have
234 // the following registers setup:
235 //
236 // r2: osr buffer
237 //
238
239 // build frame
240 ciMethod* m = compilation()->method();
241 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
242
243 // OSR buffer is
244 //
245 // locals[nlocals-1..0]
246 // monitors[0..number_of_locks]
247 //
248 // locals is a direct copy of the interpreter frame so in the osr buffer
249 // so first slot in the local array is the last local from the interpreter
250 // and last slot is local[0] (receiver) from the interpreter
251 //
252 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
253 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
254 // in the interpreter frame (the method lock if a sync method)
255
256 // Initialize monitors in the compiled activation.
257 // r2: pointer to osr buffer
258 //
259 // All other registers are dead at this point and the locals will be
260 // copied into place by code emitted in the IR.
261
262 Register OSR_buf = osrBufferPointer()->as_pointer_register();
263 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
264 int monitor_offset = BytesPerWord * method()->max_locals() +
265 (2 * BytesPerWord) * (number_of_locks - 1);
266 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
267 // the OSR buffer using 2 word entries: first the lock and then
268 // the oop.
269 for (int i = 0; i < number_of_locks; i++) {
270 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
271 #ifdef ASSERT
272 // verify the interpreter's monitor has a non-null object
273 {
274 Label L;
275 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
276 __ cbnz(rscratch1, L);
277 __ stop("locked object is NULL");
278 __ bind(L);
279 }
280 #endif
281 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
282 __ str(r19, frame_map()->address_for_monitor_lock(i));
283 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
284 __ str(r19, frame_map()->address_for_monitor_object(i));
285 }
286 }
287 }
288
289
290 // inline cache check; done before the frame is built.
291 int LIR_Assembler::check_icache() {
292 Register receiver = FrameMap::receiver_opr->as_register();
293 Register ic_klass = IC_Klass;
294 int start_offset = __ offset();
295 __ inline_cache_check(receiver, ic_klass);
296
297 // if icache check fails, then jump to runtime routine
298 // Note: RECEIVER must still contain the receiver!
299 Label dont;
300 __ br(Assembler::EQ, dont);
301 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
302
303 // We align the verified entry point unless the method body
304 // (including its inline cache check) will fit in a single 64-byte
305 // icache line.
306 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
307 // force alignment after the cache check.
308 __ align(CodeEntryAlignment);
309 }
310
311 __ bind(dont);
312 return start_offset;
313 }
314
315
316 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
317 if (o == NULL) {
318 __ mov(reg, zr);
319 } else {
320 __ movoop(reg, o, /*immediate*/true);
321 }
322 }
323
324 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
325 address target = NULL;
326 relocInfo::relocType reloc_type = relocInfo::none;
327
328 switch (patching_id(info)) {
329 case PatchingStub::access_field_id:
330 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
331 reloc_type = relocInfo::section_word_type;
332 break;
333 case PatchingStub::load_klass_id:
334 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
335 reloc_type = relocInfo::metadata_type;
336 break;
337 case PatchingStub::load_mirror_id:
338 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
339 reloc_type = relocInfo::oop_type;
340 break;
341 case PatchingStub::load_appendix_id:
342 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
343 reloc_type = relocInfo::oop_type;
344 break;
345 default: ShouldNotReachHere();
346 }
347
348 __ far_call(RuntimeAddress(target));
349 add_call_info_here(info);
350 }
351
352 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
353 deoptimize_trap(info);
354 }
355
356
357 // This specifies the rsp decrement needed to build the frame
358 int LIR_Assembler::initial_frame_size_in_bytes() const {
359 // if rounding, must let FrameMap know!
360
361 // The frame_map records size in slots (32bit word)
362
363 // subtract two words to account for return address and link
364 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
365 }
366
367
368 int LIR_Assembler::emit_exception_handler() {
369 // if the last instruction is a call (typically to do a throw which
370 // is coming at the end after block reordering) the return address
371 // must still point into the code area in order to avoid assertion
372 // failures when searching for the corresponding bci => add a nop
373 // (was bug 5/14/1999 - gri)
374 __ nop();
375
376 // generate code for exception handler
377 address handler_base = __ start_a_stub(exception_handler_size);
378 if (handler_base == NULL) {
379 // not enough space left for the handler
380 bailout("exception handler overflow");
381 return -1;
382 }
383
384 int offset = code_offset();
385
386 // the exception oop and pc are in r0, and r3
387 // no other registers need to be preserved, so invalidate them
388 __ invalidate_registers(false, true, true, false, true, true);
389
390 // check that there is really an exception
391 __ verify_not_null_oop(r0);
392
393 // search an exception handler (r0: exception oop, r3: throwing pc)
394 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here();
395 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
396 __ end_a_stub();
397
398 return offset;
399 }
400
401
402 // Emit the code to remove the frame from the stack in the exception
403 // unwind path.
404 int LIR_Assembler::emit_unwind_handler() {
405 #ifndef PRODUCT
406 if (CommentedAssembly) {
407 _masm->block_comment("Unwind handler");
408 }
409 #endif
410
411 int offset = code_offset();
412
413 // Fetch the exception from TLS and clear out exception related thread state
414 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
415 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
417
418 __ bind(_unwind_handler_entry);
419 __ verify_not_null_oop(r0);
420 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
421 __ mov(r19, r0); // Preserve the exception
422 }
423
424 // Preform needed unlocking
425 MonitorExitStub* stub = NULL;
426 if (method()->is_synchronized()) {
427 monitor_address(0, FrameMap::r0_opr);
428 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
429 __ unlock_object(r5, r4, r0, *stub->entry());
430 __ bind(*stub->continuation());
431 }
432
433 if (compilation()->env()->dtrace_method_probes()) {
434 __ call_Unimplemented();
435 #if 0
436 __ movptr(Address(rsp, 0), rax);
437 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
438 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
439 #endif
440 }
441
442 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
443 __ mov(r0, r19); // Restore the exception
444 }
445
446 // remove the activation and dispatch to the unwind handler
447 __ block_comment("remove_frame and dispatch to the unwind handler");
448 __ remove_frame(initial_frame_size_in_bytes());
449 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
450
451 // Emit the slow path assembly
452 if (stub != NULL) {
453 stub->emit_code(this);
454 }
455
456 return offset;
457 }
458
459
460 int LIR_Assembler::emit_deopt_handler() {
461 // if the last instruction is a call (typically to do a throw which
462 // is coming at the end after block reordering) the return address
463 // must still point into the code area in order to avoid assertion
464 // failures when searching for the corresponding bci => add a nop
465 // (was bug 5/14/1999 - gri)
466 __ nop();
467
468 // generate code for exception handler
469 address handler_base = __ start_a_stub(deopt_handler_size);
470 if (handler_base == NULL) {
471 // not enough space left for the handler
472 bailout("deopt handler overflow");
473 return -1;
474 }
475
476 int offset = code_offset();
477
478 __ adr(lr, pc());
479 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
480 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
481 __ end_a_stub();
482
483 return offset;
484 }
485
486
487 // This is the fast version of java.lang.String.compare; it has not
488 // OSR-entry and therefore, we generate a slow version for OSR's
489 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
490 __ mov(r2, (address)__FUNCTION__);
491 __ call_Unimplemented();
492 }
493
494
495 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
496 _masm->code_section()->relocate(adr, relocInfo::poll_type);
497 int pc_offset = code_offset();
498 flush_debug_info(pc_offset);
499 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
500 if (info->exception_handlers() != NULL) {
501 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
502 }
503 }
504
505 // Rather than take a segfault when the polling page is protected,
506 // explicitly check for a safepoint in progress and if there is one,
507 // fake a call to the handler as if a segfault had been caught.
508 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
509 __ mov(rscratch1, SafepointSynchronize::address_of_state());
510 __ ldrb(rscratch1, Address(rscratch1));
511 Label nope, poll;
512 __ cbz(rscratch1, nope);
513 __ block_comment("safepoint");
514 __ enter();
515 __ push(0x3, sp); // r0 & r1
516 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
517 __ adr(r0, poll);
518 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
519 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
520 __ blrt(rscratch1, 1, 0, 1);
521 __ maybe_isb();
522 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
523 __ mov(rscratch1, r0);
524 __ pop(0x3, sp); // r0 & r1
525 __ leave();
526 __ br(rscratch1);
527 address polling_page(os::get_polling_page());
528 assert(os::is_poll_address(polling_page), "should be");
529 unsigned long off;
530 __ adrp(rscratch1, Address(polling_page, rtype), off);
531 __ bind(poll);
532 if (info)
533 add_debug_info_for_branch(info); // This isn't just debug info:
534 // it's the oop map
535 else
536 __ code_section()->relocate(pc(), rtype);
537 __ ldrw(zr, Address(rscratch1, off));
538 __ bind(nope);
539 }
540
541 void LIR_Assembler::return_op(LIR_Opr result) {
542 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
543 // Pop the stack before the safepoint code
544 __ remove_frame(initial_frame_size_in_bytes());
545 address polling_page(os::get_polling_page());
546 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
547 __ ret(lr);
548 }
549
550 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
551 address polling_page(os::get_polling_page());
552 guarantee(info != NULL, "Shouldn't be NULL");
553 assert(os::is_poll_address(polling_page), "should be");
554 unsigned long off;
555 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
556 assert(off == 0, "must be");
557 add_debug_info_for_branch(info); // This isn't just debug info:
558 // it's the oop map
559 __ read_polling_page(rscratch1, relocInfo::poll_type);
560 return __ offset();
561 }
562
563
564 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
565 if (from_reg == r31_sp)
566 from_reg = sp;
567 if (to_reg == r31_sp)
568 to_reg = sp;
569 __ mov(to_reg, from_reg);
570 }
571
572 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
573
574
575 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
576 assert(src->is_constant(), "should not call otherwise");
577 assert(dest->is_register(), "should not call otherwise");
578 LIR_Const* c = src->as_constant_ptr();
579
580 switch (c->type()) {
581 case T_INT: {
582 assert(patch_code == lir_patch_none, "no patching handled here");
583 __ movw(dest->as_register(), c->as_jint());
584 break;
585 }
586
587 case T_ADDRESS: {
588 assert(patch_code == lir_patch_none, "no patching handled here");
589 __ mov(dest->as_register(), c->as_jint());
590 break;
591 }
592
593 case T_LONG: {
594 assert(patch_code == lir_patch_none, "no patching handled here");
595 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
596 break;
597 }
598
599 case T_OBJECT: {
600 if (patch_code == lir_patch_none) {
601 jobject2reg(c->as_jobject(), dest->as_register());
602 } else {
603 jobject2reg_with_patching(dest->as_register(), info);
604 }
605 break;
606 }
607
608 case T_METADATA: {
609 if (patch_code != lir_patch_none) {
610 klass2reg_with_patching(dest->as_register(), info);
611 } else {
612 __ mov_metadata(dest->as_register(), c->as_metadata());
613 }
614 break;
615 }
616
617 case T_FLOAT: {
618 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
619 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
620 } else {
621 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
622 __ ldrs(dest->as_float_reg(), Address(rscratch1));
623 }
624 break;
625 }
626
627 case T_DOUBLE: {
628 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
629 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
630 } else {
631 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
632 __ ldrd(dest->as_double_reg(), Address(rscratch1));
633 }
634 break;
635 }
636
637 default:
638 ShouldNotReachHere();
639 }
640 }
641
642 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
643 LIR_Const* c = src->as_constant_ptr();
644 switch (c->type()) {
645 case T_OBJECT:
646 {
647 if (! c->as_jobject())
648 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
649 else {
650 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
651 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
652 }
653 }
654 break;
655 case T_ADDRESS:
656 {
657 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
658 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
659 }
660 case T_INT:
661 case T_FLOAT:
662 {
663 Register reg = zr;
664 if (c->as_jint_bits() == 0)
665 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
666 else {
667 __ movw(rscratch1, c->as_jint_bits());
668 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
669 }
670 }
671 break;
672 case T_LONG:
673 case T_DOUBLE:
674 {
675 Register reg = zr;
676 if (c->as_jlong_bits() == 0)
677 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
678 lo_word_offset_in_bytes));
679 else {
680 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
681 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
682 lo_word_offset_in_bytes));
683 }
684 }
685 break;
686 default:
687 ShouldNotReachHere();
688 }
689 }
690
691 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
692 assert(src->is_constant(), "should not call otherwise");
693 LIR_Const* c = src->as_constant_ptr();
694 LIR_Address* to_addr = dest->as_address_ptr();
695
696 void (Assembler::* insn)(Register Rt, const Address &adr);
697
698 switch (type) {
699 case T_ADDRESS:
700 assert(c->as_jint() == 0, "should be");
701 insn = &Assembler::str;
702 break;
703 case T_LONG:
704 assert(c->as_jlong() == 0, "should be");
705 insn = &Assembler::str;
706 break;
707 case T_INT:
708 assert(c->as_jint() == 0, "should be");
709 insn = &Assembler::strw;
710 break;
711 case T_OBJECT:
712 case T_ARRAY:
713 assert(c->as_jobject() == 0, "should be");
714 if (UseCompressedOops && !wide) {
715 insn = &Assembler::strw;
716 } else {
717 insn = &Assembler::str;
718 }
719 break;
720 case T_CHAR:
721 case T_SHORT:
722 assert(c->as_jint() == 0, "should be");
723 insn = &Assembler::strh;
724 break;
725 case T_BOOLEAN:
726 case T_BYTE:
727 assert(c->as_jint() == 0, "should be");
728 insn = &Assembler::strb;
729 break;
730 default:
731 ShouldNotReachHere();
732 }
733
734 if (info) add_debug_info_for_null_check_here(info);
735 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
736 }
737
738 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
739 assert(src->is_register(), "should not call otherwise");
740 assert(dest->is_register(), "should not call otherwise");
741
742 // move between cpu-registers
743 if (dest->is_single_cpu()) {
744 if (src->type() == T_LONG) {
745 // Can do LONG -> OBJECT
746 move_regs(src->as_register_lo(), dest->as_register());
747 return;
748 }
749 assert(src->is_single_cpu(), "must match");
750 if (src->type() == T_OBJECT) {
751 __ verify_oop(src->as_register());
752 }
753 move_regs(src->as_register(), dest->as_register());
754
755 } else if (dest->is_double_cpu()) {
756 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
757 // Surprising to me but we can see move of a long to t_object
758 __ verify_oop(src->as_register());
759 move_regs(src->as_register(), dest->as_register_lo());
760 return;
761 }
762 assert(src->is_double_cpu(), "must match");
763 Register f_lo = src->as_register_lo();
764 Register f_hi = src->as_register_hi();
765 Register t_lo = dest->as_register_lo();
766 Register t_hi = dest->as_register_hi();
767 assert(f_hi == f_lo, "must be same");
768 assert(t_hi == t_lo, "must be same");
769 move_regs(f_lo, t_lo);
770
771 } else if (dest->is_single_fpu()) {
772 __ fmovs(dest->as_float_reg(), src->as_float_reg());
773
774 } else if (dest->is_double_fpu()) {
775 __ fmovd(dest->as_double_reg(), src->as_double_reg());
776
777 } else {
778 ShouldNotReachHere();
779 }
780 }
781
782 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
783 if (src->is_single_cpu()) {
784 if (type == T_ARRAY || type == T_OBJECT) {
785 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
786 __ verify_oop(src->as_register());
787 } else if (type == T_METADATA || type == T_DOUBLE) {
788 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
789 } else {
790 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
791 }
792
793 } else if (src->is_double_cpu()) {
794 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
795 __ str(src->as_register_lo(), dest_addr_LO);
796
797 } else if (src->is_single_fpu()) {
798 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
799 __ strs(src->as_float_reg(), dest_addr);
800
801 } else if (src->is_double_fpu()) {
802 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
803 __ strd(src->as_double_reg(), dest_addr);
804
805 } else {
806 ShouldNotReachHere();
807 }
808
809 }
810
811
812 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
813 LIR_Address* to_addr = dest->as_address_ptr();
814 PatchingStub* patch = NULL;
815 Register compressed_src = rscratch1;
816
817 if (patch_code != lir_patch_none) {
818 deoptimize_trap(info);
819 return;
820 }
821
822 if (type == T_ARRAY || type == T_OBJECT) {
823 __ verify_oop(src->as_register());
824
825 if (UseCompressedOops && !wide) {
826 __ encode_heap_oop(compressed_src, src->as_register());
827 } else {
828 compressed_src = src->as_register();
829 }
830 }
831
832 int null_check_here = code_offset();
833 switch (type) {
834 case T_FLOAT: {
835 __ strs(src->as_float_reg(), as_Address(to_addr));
836 break;
837 }
838
839 case T_DOUBLE: {
840 __ strd(src->as_double_reg(), as_Address(to_addr));
841 break;
842 }
843
844 case T_ARRAY: // fall through
845 case T_OBJECT: // fall through
846 if (UseCompressedOops && !wide) {
847 __ strw(compressed_src, as_Address(to_addr, rscratch2));
848 } else {
849 __ str(compressed_src, as_Address(to_addr));
850 }
851 break;
852 case T_METADATA:
853 // We get here to store a method pointer to the stack to pass to
854 // a dtrace runtime call. This can't work on 64 bit with
855 // compressed klass ptrs: T_METADATA can be a compressed klass
856 // ptr or a 64 bit method pointer.
857 ShouldNotReachHere();
858 __ str(src->as_register(), as_Address(to_addr));
859 break;
860 case T_ADDRESS:
861 __ str(src->as_register(), as_Address(to_addr));
862 break;
863 case T_INT:
864 __ strw(src->as_register(), as_Address(to_addr));
865 break;
866
867 case T_LONG: {
868 __ str(src->as_register_lo(), as_Address_lo(to_addr));
869 break;
870 }
871
872 case T_BYTE: // fall through
873 case T_BOOLEAN: {
874 __ strb(src->as_register(), as_Address(to_addr));
875 break;
876 }
877
878 case T_CHAR: // fall through
879 case T_SHORT:
880 __ strh(src->as_register(), as_Address(to_addr));
881 break;
882
883 default:
884 ShouldNotReachHere();
885 }
886 if (info != NULL) {
887 add_debug_info_for_null_check(null_check_here, info);
888 }
889 }
890
891
892 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
893 assert(src->is_stack(), "should not call otherwise");
894 assert(dest->is_register(), "should not call otherwise");
895
896 if (dest->is_single_cpu()) {
897 if (type == T_ARRAY || type == T_OBJECT) {
898 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
899 __ verify_oop(dest->as_register());
900 } else if (type == T_METADATA) {
901 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
902 } else {
903 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
904 }
905
906 } else if (dest->is_double_cpu()) {
907 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
908 __ ldr(dest->as_register_lo(), src_addr_LO);
909
910 } else if (dest->is_single_fpu()) {
911 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
912 __ ldrs(dest->as_float_reg(), src_addr);
913
914 } else if (dest->is_double_fpu()) {
915 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
916 __ ldrd(dest->as_double_reg(), src_addr);
917
918 } else {
919 ShouldNotReachHere();
920 }
921 }
922
923
924 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
925 address target = NULL;
926 relocInfo::relocType reloc_type = relocInfo::none;
927
928 switch (patching_id(info)) {
929 case PatchingStub::access_field_id:
930 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
931 reloc_type = relocInfo::section_word_type;
932 break;
933 case PatchingStub::load_klass_id:
934 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
935 reloc_type = relocInfo::metadata_type;
936 break;
937 case PatchingStub::load_mirror_id:
938 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
939 reloc_type = relocInfo::oop_type;
940 break;
941 case PatchingStub::load_appendix_id:
942 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
943 reloc_type = relocInfo::oop_type;
944 break;
945 default: ShouldNotReachHere();
946 }
947
948 __ far_call(RuntimeAddress(target));
949 add_call_info_here(info);
950 }
951
952 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
953
954 LIR_Opr temp;
955 if (type == T_LONG || type == T_DOUBLE)
956 temp = FrameMap::rscratch1_long_opr;
957 else
958 temp = FrameMap::rscratch1_opr;
959
960 stack2reg(src, temp, src->type());
961 reg2stack(temp, dest, dest->type(), false);
962 }
963
964
965 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
966 LIR_Address* addr = src->as_address_ptr();
967 LIR_Address* from_addr = src->as_address_ptr();
968
969 if (addr->base()->type() == T_OBJECT) {
970 __ verify_oop(addr->base()->as_pointer_register());
971 }
972
973 if (patch_code != lir_patch_none) {
974 deoptimize_trap(info);
975 return;
976 }
977
978 if (info != NULL) {
979 add_debug_info_for_null_check_here(info);
980 }
981 int null_check_here = code_offset();
982 switch (type) {
983 case T_FLOAT: {
984 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
985 break;
986 }
987
988 case T_DOUBLE: {
989 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
990 break;
991 }
992
993 case T_ARRAY: // fall through
994 case T_OBJECT: // fall through
995 if (UseCompressedOops && !wide) {
996 __ ldrw(dest->as_register(), as_Address(from_addr));
997 } else {
998 __ ldr(dest->as_register(), as_Address(from_addr));
999 }
1000 break;
1001 case T_METADATA:
1002 // We get here to store a method pointer to the stack to pass to
1003 // a dtrace runtime call. This can't work on 64 bit with
1004 // compressed klass ptrs: T_METADATA can be a compressed klass
1005 // ptr or a 64 bit method pointer.
1006 ShouldNotReachHere();
1007 __ ldr(dest->as_register(), as_Address(from_addr));
1008 break;
1009 case T_ADDRESS:
1010 // FIXME: OMG this is a horrible kludge. Any offset from an
1011 // address that matches klass_offset_in_bytes() will be loaded
1012 // as a word, not a long.
1013 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1014 __ ldrw(dest->as_register(), as_Address(from_addr));
1015 } else {
1016 __ ldr(dest->as_register(), as_Address(from_addr));
1017 }
1018 break;
1019 case T_INT:
1020 __ ldrw(dest->as_register(), as_Address(from_addr));
1021 break;
1022
1023 case T_LONG: {
1024 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1025 break;
1026 }
1027
1028 case T_BYTE:
1029 __ ldrsb(dest->as_register(), as_Address(from_addr));
1030 break;
1031 case T_BOOLEAN: {
1032 __ ldrb(dest->as_register(), as_Address(from_addr));
1033 break;
1034 }
1035
1036 case T_CHAR:
1037 __ ldrh(dest->as_register(), as_Address(from_addr));
1038 break;
1039 case T_SHORT:
1040 __ ldrsh(dest->as_register(), as_Address(from_addr));
1041 break;
1042
1043 default:
1044 ShouldNotReachHere();
1045 }
1046
1047 if (type == T_ARRAY || type == T_OBJECT) {
1048 if (UseCompressedOops && !wide) {
1049 __ decode_heap_oop(dest->as_register());
1050 }
1051 __ verify_oop(dest->as_register());
1052 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1053 if (UseCompressedClassPointers) {
1054 __ decode_klass_not_null(dest->as_register());
1055 }
1056 }
1057 }
1058
1059
1060 int LIR_Assembler::array_element_size(BasicType type) const {
1061 int elem_size = type2aelembytes(type);
1062 return exact_log2(elem_size);
1063 }
1064
1065 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1066 Register Rdividend = op->in_opr1()->as_register();
1067 Register Rdivisor = op->in_opr2()->as_register();
1068 Register Rscratch = op->in_opr3()->as_register();
1069 Register Rresult = op->result_opr()->as_register();
1070 int divisor = -1;
1071
1072 /*
1073 TODO: For some reason, using the Rscratch that gets passed in is
1074 not possible because the register allocator does not see the tmp reg
1075 as used, and assignes it the same register as Rdividend. We use rscratch1
1076 instead.
1077
1078 assert(Rdividend != Rscratch, "");
1079 assert(Rdivisor != Rscratch, "");
1080 */
1081
1082 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1083 // convert division by a power of two into some shifts and logical operations
1084 }
1085
1086 if (op->code() == lir_irem) {
1087 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1088 } else if (op->code() == lir_idiv) {
1089 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1090 } else
1091 ShouldNotReachHere();
1092 }
1093
1094 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1095 #ifdef ASSERT
1096 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1097 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1098 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1099 #endif
1100
1101 if (op->cond() == lir_cond_always) {
1102 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1103 __ b(*(op->label()));
1104 } else {
1105 Assembler::Condition acond;
1106 if (op->code() == lir_cond_float_branch) {
1107 bool is_unordered = (op->ublock() == op->block());
1108 // Assembler::EQ does not permit unordered branches, so we add
1109 // another branch here. Likewise, Assembler::NE does not permit
1110 // ordered branches.
1111 if (is_unordered && op->cond() == lir_cond_equal
1112 || !is_unordered && op->cond() == lir_cond_notEqual)
1113 __ br(Assembler::VS, *(op->ublock()->label()));
1114 switch(op->cond()) {
1115 case lir_cond_equal: acond = Assembler::EQ; break;
1116 case lir_cond_notEqual: acond = Assembler::NE; break;
1117 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1118 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1119 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1120 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1121 default: ShouldNotReachHere();
1122 }
1123 } else {
1124 switch (op->cond()) {
1125 case lir_cond_equal: acond = Assembler::EQ; break;
1126 case lir_cond_notEqual: acond = Assembler::NE; break;
1127 case lir_cond_less: acond = Assembler::LT; break;
1128 case lir_cond_lessEqual: acond = Assembler::LE; break;
1129 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1130 case lir_cond_greater: acond = Assembler::GT; break;
1131 case lir_cond_belowEqual: acond = Assembler::LS; break;
1132 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1133 default: ShouldNotReachHere();
1134 }
1135 }
1136 __ br(acond,*(op->label()));
1137 }
1138 }
1139
1140
1141
1142 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1143 LIR_Opr src = op->in_opr();
1144 LIR_Opr dest = op->result_opr();
1145
1146 switch (op->bytecode()) {
1147 case Bytecodes::_i2f:
1148 {
1149 __ scvtfws(dest->as_float_reg(), src->as_register());
1150 break;
1151 }
1152 case Bytecodes::_i2d:
1153 {
1154 __ scvtfwd(dest->as_double_reg(), src->as_register());
1155 break;
1156 }
1157 case Bytecodes::_l2d:
1158 {
1159 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1160 break;
1161 }
1162 case Bytecodes::_l2f:
1163 {
1164 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1165 break;
1166 }
1167 case Bytecodes::_f2d:
1168 {
1169 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1170 break;
1171 }
1172 case Bytecodes::_d2f:
1173 {
1174 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1175 break;
1176 }
1177 case Bytecodes::_i2c:
1178 {
1179 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1180 break;
1181 }
1182 case Bytecodes::_i2l:
1183 {
1184 __ sxtw(dest->as_register_lo(), src->as_register());
1185 break;
1186 }
1187 case Bytecodes::_i2s:
1188 {
1189 __ sxth(dest->as_register(), src->as_register());
1190 break;
1191 }
1192 case Bytecodes::_i2b:
1193 {
1194 __ sxtb(dest->as_register(), src->as_register());
1195 break;
1196 }
1197 case Bytecodes::_l2i:
1198 {
1199 _masm->block_comment("FIXME: This could be a no-op");
1200 __ uxtw(dest->as_register(), src->as_register_lo());
1201 break;
1202 }
1203 case Bytecodes::_d2l:
1204 {
1205 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1206 break;
1207 }
1208 case Bytecodes::_f2i:
1209 {
1210 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1211 break;
1212 }
1213 case Bytecodes::_f2l:
1214 {
1215 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1216 break;
1217 }
1218 case Bytecodes::_d2i:
1219 {
1220 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1221 break;
1222 }
1223 default: ShouldNotReachHere();
1224 }
1225 }
1226
1227 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1228 if (op->init_check()) {
1229 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1230 InstanceKlass::init_state_offset()));
1231 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1232 add_debug_info_for_null_check_here(op->stub()->info());
1233 __ br(Assembler::NE, *op->stub()->entry());
1234 }
1235 __ allocate_object(op->obj()->as_register(),
1236 op->tmp1()->as_register(),
1237 op->tmp2()->as_register(),
1238 op->header_size(),
1239 op->object_size(),
1240 op->klass()->as_register(),
1241 *op->stub()->entry());
1242 __ bind(*op->stub()->continuation());
1243 }
1244
1245 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1246 Register len = op->len()->as_register();
1247 __ uxtw(len, len);
1248
1249 if (UseSlowPath ||
1250 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1251 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1252 __ b(*op->stub()->entry());
1253 } else {
1254 Register tmp1 = op->tmp1()->as_register();
1255 Register tmp2 = op->tmp2()->as_register();
1256 Register tmp3 = op->tmp3()->as_register();
1257 if (len == tmp1) {
1258 tmp1 = tmp3;
1259 } else if (len == tmp2) {
1260 tmp2 = tmp3;
1261 } else if (len == tmp3) {
1262 // everything is ok
1263 } else {
1264 __ mov(tmp3, len);
1265 }
1266 __ allocate_array(op->obj()->as_register(),
1267 len,
1268 tmp1,
1269 tmp2,
1270 arrayOopDesc::header_size(op->type()),
1271 array_element_size(op->type()),
1272 op->klass()->as_register(),
1273 *op->stub()->entry());
1274 }
1275 __ bind(*op->stub()->continuation());
1276 }
1277
1278 void LIR_Assembler::type_profile_helper(Register mdo,
1279 ciMethodData *md, ciProfileData *data,
1280 Register recv, Label* update_done) {
1281 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1282 Label next_test;
1283 // See if the receiver is receiver[n].
1284 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1285 __ ldr(rscratch1, Address(rscratch2));
1286 __ cmp(recv, rscratch1);
1287 __ br(Assembler::NE, next_test);
1288 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1289 __ addptr(data_addr, DataLayout::counter_increment);
1290 __ b(*update_done);
1291 __ bind(next_test);
1292 }
1293
1294 // Didn't find receiver; find next empty slot and fill it in
1295 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1296 Label next_test;
1297 __ lea(rscratch2,
1298 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1299 Address recv_addr(rscratch2);
1300 __ ldr(rscratch1, recv_addr);
1301 __ cbnz(rscratch1, next_test);
1302 __ str(recv, recv_addr);
1303 __ mov(rscratch1, DataLayout::counter_increment);
1304 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1305 __ str(rscratch1, Address(rscratch2));
1306 __ b(*update_done);
1307 __ bind(next_test);
1308 }
1309 }
1310
1311 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1312 // we always need a stub for the failure case.
1313 CodeStub* stub = op->stub();
1314 Register obj = op->object()->as_register();
1315 Register k_RInfo = op->tmp1()->as_register();
1316 Register klass_RInfo = op->tmp2()->as_register();
1317 Register dst = op->result_opr()->as_register();
1318 ciKlass* k = op->klass();
1319 Register Rtmp1 = noreg;
1320
1321 // check if it needs to be profiled
1322 ciMethodData* md;
1323 ciProfileData* data;
1324
1325 if (op->should_profile()) {
1326 ciMethod* method = op->profiled_method();
1327 assert(method != NULL, "Should have method");
1328 int bci = op->profiled_bci();
1329 md = method->method_data_or_null();
1330 assert(md != NULL, "Sanity");
1331 data = md->bci_to_data(bci);
1332 assert(data != NULL, "need data for type check");
1333 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1334 }
1335 Label profile_cast_success, profile_cast_failure;
1336 Label *success_target = op->should_profile() ? &profile_cast_success : success;
1337 Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1338
1339 if (obj == k_RInfo) {
1340 k_RInfo = dst;
1341 } else if (obj == klass_RInfo) {
1342 klass_RInfo = dst;
1343 }
1344 if (k->is_loaded() && !UseCompressedClassPointers) {
1345 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1346 } else {
1347 Rtmp1 = op->tmp3()->as_register();
1348 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1349 }
1350
1351 assert_different_registers(obj, k_RInfo, klass_RInfo);
1352
1353 if (op->should_profile()) {
1354 Label not_null;
1355 __ cbnz(obj, not_null);
1356 // Object is null; update MDO and exit
1357 Register mdo = klass_RInfo;
1358 __ mov_metadata(mdo, md->constant_encoding());
1359 Address data_addr
1360 = __ form_address(rscratch2, mdo,
1361 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1362 LogBytesPerWord);
1363 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1364 __ ldr(rscratch1, data_addr);
1365 __ orr(rscratch1, rscratch1, header_bits);
1366 __ str(rscratch1, data_addr);
1367 __ b(*obj_is_null);
1368 __ bind(not_null);
1369 } else {
1370 __ cbz(obj, *obj_is_null);
1371 }
1372
1373 if (!k->is_loaded()) {
1374 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1375 } else {
1376 __ mov_metadata(k_RInfo, k->constant_encoding());
1377 }
1378 __ verify_oop(obj);
1379
1380 if (op->fast_check()) {
1381 // get object class
1382 // not a safepoint as obj null check happens earlier
1383 __ load_klass(rscratch1, obj);
1384 __ cmp( rscratch1, k_RInfo);
1385
1386 __ br(Assembler::NE, *failure_target);
1387 // successful cast, fall through to profile or jump
1388 } else {
1389 // get object class
1390 // not a safepoint as obj null check happens earlier
1391 __ load_klass(klass_RInfo, obj);
1392 if (k->is_loaded()) {
1393 // See if we get an immediate positive hit
1394 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1395 __ cmp(k_RInfo, rscratch1);
1396 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1397 __ br(Assembler::NE, *failure_target);
1398 // successful cast, fall through to profile or jump
1399 } else {
1400 // See if we get an immediate positive hit
1401 __ br(Assembler::EQ, *success_target);
1402 // check for self
1403 __ cmp(klass_RInfo, k_RInfo);
1404 __ br(Assembler::EQ, *success_target);
1405
1406 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1407 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1408 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1409 // result is a boolean
1410 __ cbzw(klass_RInfo, *failure_target);
1411 // successful cast, fall through to profile or jump
1412 }
1413 } else {
1414 // perform the fast part of the checking logic
1415 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1416 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1417 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1418 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1419 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1420 // result is a boolean
1421 __ cbz(k_RInfo, *failure_target);
1422 // successful cast, fall through to profile or jump
1423 }
1424 }
1425 if (op->should_profile()) {
1426 Register mdo = klass_RInfo, recv = k_RInfo;
1427 __ bind(profile_cast_success);
1428 __ mov_metadata(mdo, md->constant_encoding());
1429 __ load_klass(recv, obj);
1430 Label update_done;
1431 type_profile_helper(mdo, md, data, recv, success);
1432 __ b(*success);
1433
1434 __ bind(profile_cast_failure);
1435 __ mov_metadata(mdo, md->constant_encoding());
1436 Address counter_addr
1437 = __ form_address(rscratch2, mdo,
1438 md->byte_offset_of_slot(data, CounterData::count_offset()),
1439 LogBytesPerWord);
1440 __ ldr(rscratch1, counter_addr);
1441 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1442 __ str(rscratch1, counter_addr);
1443 __ b(*failure);
1444 }
1445 __ b(*success);
1446 }
1447
1448
1449 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1450 LIR_Code code = op->code();
1451 if (code == lir_store_check) {
1452 Register value = op->object()->as_register();
1453 Register array = op->array()->as_register();
1454 Register k_RInfo = op->tmp1()->as_register();
1455 Register klass_RInfo = op->tmp2()->as_register();
1456 Register Rtmp1 = op->tmp3()->as_register();
1457
1458 CodeStub* stub = op->stub();
1459
1460 // check if it needs to be profiled
1461 ciMethodData* md;
1462 ciProfileData* data;
1463
1464 if (op->should_profile()) {
1465 ciMethod* method = op->profiled_method();
1466 assert(method != NULL, "Should have method");
1467 int bci = op->profiled_bci();
1468 md = method->method_data_or_null();
1469 assert(md != NULL, "Sanity");
1470 data = md->bci_to_data(bci);
1471 assert(data != NULL, "need data for type check");
1472 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1473 }
1474 Label profile_cast_success, profile_cast_failure, done;
1475 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1476 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1477
1478 if (op->should_profile()) {
1479 Label not_null;
1480 __ cbnz(value, not_null);
1481 // Object is null; update MDO and exit
1482 Register mdo = klass_RInfo;
1483 __ mov_metadata(mdo, md->constant_encoding());
1484 Address data_addr
1485 = __ form_address(rscratch2, mdo,
1486 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1487 LogBytesPerInt);
1488 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1489 __ ldrw(rscratch1, data_addr);
1490 __ orrw(rscratch1, rscratch1, header_bits);
1491 __ strw(rscratch1, data_addr);
1492 __ b(done);
1493 __ bind(not_null);
1494 } else {
1495 __ cbz(value, done);
1496 }
1497
1498 add_debug_info_for_null_check_here(op->info_for_exception());
1499 __ load_klass(k_RInfo, array);
1500 __ load_klass(klass_RInfo, value);
1501
1502 // get instance klass (it's already uncompressed)
1503 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1504 // perform the fast part of the checking logic
1505 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1506 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1507 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1508 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1509 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1510 // result is a boolean
1511 __ cbzw(k_RInfo, *failure_target);
1512 // fall through to the success case
1513
1514 if (op->should_profile()) {
1515 Register mdo = klass_RInfo, recv = k_RInfo;
1516 __ bind(profile_cast_success);
1517 __ mov_metadata(mdo, md->constant_encoding());
1518 __ load_klass(recv, value);
1519 Label update_done;
1520 type_profile_helper(mdo, md, data, recv, &done);
1521 __ b(done);
1522
1523 __ bind(profile_cast_failure);
1524 __ mov_metadata(mdo, md->constant_encoding());
1525 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1526 __ lea(rscratch2, counter_addr);
1527 __ ldr(rscratch1, Address(rscratch2));
1528 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1529 __ str(rscratch1, Address(rscratch2));
1530 __ b(*stub->entry());
1531 }
1532
1533 __ bind(done);
1534 } else if (code == lir_checkcast) {
1535 Register obj = op->object()->as_register();
1536 Register dst = op->result_opr()->as_register();
1537 Label success;
1538 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1539 __ bind(success);
1540 if (dst != obj) {
1541 __ mov(dst, obj);
1542 }
1543 } else if (code == lir_instanceof) {
1544 Register obj = op->object()->as_register();
1545 Register dst = op->result_opr()->as_register();
1546 Label success, failure, done;
1547 emit_typecheck_helper(op, &success, &failure, &failure);
1548 __ bind(failure);
1549 __ mov(dst, zr);
1550 __ b(done);
1551 __ bind(success);
1552 __ mov(dst, 1);
1553 __ bind(done);
1554 } else {
1555 ShouldNotReachHere();
1556 }
1557 }
1558
1559 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1560 Label retry_load, nope;
1561 // flush and load exclusive from the memory location
1562 // and fail if it is not what we expect
1563 __ bind(retry_load);
1564 __ ldaxrw(rscratch1, addr);
1565 __ cmpw(rscratch1, cmpval);
1566 __ cset(rscratch1, Assembler::NE);
1567 __ br(Assembler::NE, nope);
1568 // if we store+flush with no intervening write rscratch1 wil be zero
1569 __ stlxrw(rscratch1, newval, addr);
1570 // retry so we only ever return after a load fails to compare
1571 // ensures we don't return a stale value after a failed write.
1572 __ cbnzw(rscratch1, retry_load);
1573 __ bind(nope);
1574 __ membar(__ AnyAny);
1575 }
1576
1577 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1578 Label retry_load, nope;
1579 // flush and load exclusive from the memory location
1580 // and fail if it is not what we expect
1581 __ bind(retry_load);
1582 __ ldaxr(rscratch1, addr);
1583 __ cmp(rscratch1, cmpval);
1584 __ cset(rscratch1, Assembler::NE);
1585 __ br(Assembler::NE, nope);
1586 // if we store+flush with no intervening write rscratch1 wil be zero
1587 __ stlxr(rscratch1, newval, addr);
1588 // retry so we only ever return after a load fails to compare
1589 // ensures we don't return a stale value after a failed write.
1590 __ cbnz(rscratch1, retry_load);
1591 __ bind(nope);
1592 __ membar(__ AnyAny);
1593 }
1594
1595
1596 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1597 assert(VM_Version::supports_cx8(), "wrong machine");
1598 Register addr = as_reg(op->addr());
1599 Register newval = as_reg(op->new_value());
1600 Register cmpval = as_reg(op->cmp_value());
1601 Label succeed, fail, around;
1602
1603 if (op->code() == lir_cas_obj) {
1604 if (UseCompressedOops) {
1605 Register t1 = op->tmp1()->as_register();
1606 assert(op->tmp1()->is_valid(), "must be");
1607 __ encode_heap_oop(t1, cmpval);
1608 cmpval = t1;
1609 __ encode_heap_oop(rscratch2, newval);
1610 newval = rscratch2;
1611 casw(addr, newval, cmpval);
1612 } else {
1613 casl(addr, newval, cmpval);
1614 }
1615 } else if (op->code() == lir_cas_int) {
1616 casw(addr, newval, cmpval);
1617 } else {
1618 casl(addr, newval, cmpval);
1619 }
1620 }
1621
1622
1623 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1624
1625 Assembler::Condition acond, ncond;
1626 switch (condition) {
1627 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1628 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1629 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1630 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1631 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1632 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1633 case lir_cond_belowEqual: Unimplemented(); break;
1634 case lir_cond_aboveEqual: Unimplemented(); break;
1635 default: ShouldNotReachHere();
1636 }
1637
1638 assert(result->is_single_cpu() || result->is_double_cpu(),
1639 "expect single register for result");
1640 if (opr1->is_constant() && opr2->is_constant()
1641 && opr1->type() == T_INT && opr2->type() == T_INT) {
1642 jint val1 = opr1->as_jint();
1643 jint val2 = opr2->as_jint();
1644 if (val1 == 0 && val2 == 1) {
1645 __ cset(result->as_register(), ncond);
1646 return;
1647 } else if (val1 == 1 && val2 == 0) {
1648 __ cset(result->as_register(), acond);
1649 return;
1650 }
1651 }
1652
1653 if (opr1->is_constant() && opr2->is_constant()
1654 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1655 jlong val1 = opr1->as_jlong();
1656 jlong val2 = opr2->as_jlong();
1657 if (val1 == 0 && val2 == 1) {
1658 __ cset(result->as_register_lo(), ncond);
1659 return;
1660 } else if (val1 == 1 && val2 == 0) {
1661 __ cset(result->as_register_lo(), acond);
1662 return;
1663 }
1664 }
1665
1666 if (opr1->is_stack()) {
1667 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1668 opr1 = FrameMap::rscratch1_opr;
1669 } else if (opr1->is_constant()) {
1670 LIR_Opr tmp
1671 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1672 const2reg(opr1, tmp, lir_patch_none, NULL);
1673 opr1 = tmp;
1674 }
1675
1676 if (opr2->is_stack()) {
1677 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1678 opr2 = FrameMap::rscratch2_opr;
1679 } else if (opr2->is_constant()) {
1680 LIR_Opr tmp
1681 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1682 const2reg(opr2, tmp, lir_patch_none, NULL);
1683 opr2 = tmp;
1684 }
1685
1686 if (result->type() == T_LONG)
1687 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1688 else
1689 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1690 }
1691
1692 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1693 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1694
1695 if (left->is_single_cpu()) {
1696 Register lreg = left->as_register();
1697 Register dreg = as_reg(dest);
1698
1699 if (right->is_single_cpu()) {
1700 // cpu register - cpu register
1701
1702 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1703 "should be");
1704 Register rreg = right->as_register();
1705 switch (code) {
1706 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1707 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1708 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1709 default: ShouldNotReachHere();
1710 }
1711
1712 } else if (right->is_double_cpu()) {
1713 Register rreg = right->as_register_lo();
1714 // single_cpu + double_cpu: can happen with obj+long
1715 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1716 switch (code) {
1717 case lir_add: __ add(dreg, lreg, rreg); break;
1718 case lir_sub: __ sub(dreg, lreg, rreg); break;
1719 default: ShouldNotReachHere();
1720 }
1721 } else if (right->is_constant()) {
1722 // cpu register - constant
1723 jlong c;
1724
1725 // FIXME. This is fugly: we really need to factor all this logic.
1726 switch(right->type()) {
1727 case T_LONG:
1728 c = right->as_constant_ptr()->as_jlong();
1729 break;
1730 case T_INT:
1731 case T_ADDRESS:
1732 c = right->as_constant_ptr()->as_jint();
1733 break;
1734 default:
1735 ShouldNotReachHere();
1736 break;
1737 }
1738
1739 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1740 if (c == 0 && dreg == lreg) {
1741 COMMENT("effective nop elided");
1742 return;
1743 }
1744 switch(left->type()) {
1745 case T_INT:
1746 switch (code) {
1747 case lir_add: __ addw(dreg, lreg, c); break;
1748 case lir_sub: __ subw(dreg, lreg, c); break;
1749 default: ShouldNotReachHere();
1750 }
1751 break;
1752 case T_OBJECT:
1753 case T_ADDRESS:
1754 switch (code) {
1755 case lir_add: __ add(dreg, lreg, c); break;
1756 case lir_sub: __ sub(dreg, lreg, c); break;
1757 default: ShouldNotReachHere();
1758 }
1759 break;
1760 ShouldNotReachHere();
1761 }
1762 } else {
1763 ShouldNotReachHere();
1764 }
1765
1766 } else if (left->is_double_cpu()) {
1767 Register lreg_lo = left->as_register_lo();
1768
1769 if (right->is_double_cpu()) {
1770 // cpu register - cpu register
1771 Register rreg_lo = right->as_register_lo();
1772 switch (code) {
1773 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1774 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1775 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1776 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1777 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1778 default:
1779 ShouldNotReachHere();
1780 }
1781
1782 } else if (right->is_constant()) {
1783 jlong c = right->as_constant_ptr()->as_jlong_bits();
1784 Register dreg = as_reg(dest);
1785 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1786 if (c == 0 && dreg == lreg_lo) {
1787 COMMENT("effective nop elided");
1788 return;
1789 }
1790 switch (code) {
1791 case lir_add: __ add(dreg, lreg_lo, c); break;
1792 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1793 default:
1794 ShouldNotReachHere();
1795 }
1796 } else {
1797 ShouldNotReachHere();
1798 }
1799 } else if (left->is_single_fpu()) {
1800 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1801 switch (code) {
1802 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1803 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1804 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1805 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1806 default:
1807 ShouldNotReachHere();
1808 }
1809 } else if (left->is_double_fpu()) {
1810 if (right->is_double_fpu()) {
1811 // cpu register - cpu register
1812 switch (code) {
1813 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1814 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1815 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1816 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1817 default:
1818 ShouldNotReachHere();
1819 }
1820 } else {
1821 if (right->is_constant()) {
1822 ShouldNotReachHere();
1823 }
1824 ShouldNotReachHere();
1825 }
1826 } else if (left->is_single_stack() || left->is_address()) {
1827 assert(left == dest, "left and dest must be equal");
1828 ShouldNotReachHere();
1829 } else {
1830 ShouldNotReachHere();
1831 }
1832 }
1833
1834 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1835
1836
1837 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1838 switch(code) {
1839 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1840 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1841 default : ShouldNotReachHere();
1842 }
1843 }
1844
1845 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1846
1847 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1848 Register Rleft = left->is_single_cpu() ? left->as_register() :
1849 left->as_register_lo();
1850 if (dst->is_single_cpu()) {
1851 Register Rdst = dst->as_register();
1852 if (right->is_constant()) {
1853 switch (code) {
1854 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1855 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1856 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1857 default: ShouldNotReachHere(); break;
1858 }
1859 } else {
1860 Register Rright = right->is_single_cpu() ? right->as_register() :
1861 right->as_register_lo();
1862 switch (code) {
1863 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1864 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1865 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1866 default: ShouldNotReachHere(); break;
1867 }
1868 }
1869 } else {
1870 Register Rdst = dst->as_register_lo();
1871 if (right->is_constant()) {
1872 switch (code) {
1873 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1874 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1875 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1876 default: ShouldNotReachHere(); break;
1877 }
1878 } else {
1879 Register Rright = right->is_single_cpu() ? right->as_register() :
1880 right->as_register_lo();
1881 switch (code) {
1882 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1883 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1884 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1885 default: ShouldNotReachHere(); break;
1886 }
1887 }
1888 }
1889 }
1890
1891
1892
1893 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1894
1895
1896 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1897 if (opr1->is_constant() && opr2->is_single_cpu()) {
1898 // tableswitch
1899 Register reg = as_reg(opr2);
1900 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1901 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1902 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1903 Register reg1 = as_reg(opr1);
1904 if (opr2->is_single_cpu()) {
1905 // cpu register - cpu register
1906 Register reg2 = opr2->as_register();
1907 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1908 __ cmp(reg1, reg2);
1909 } else {
1910 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1911 __ cmpw(reg1, reg2);
1912 }
1913 return;
1914 }
1915 if (opr2->is_double_cpu()) {
1916 // cpu register - cpu register
1917 Register reg2 = opr2->as_register_lo();
1918 __ cmp(reg1, reg2);
1919 return;
1920 }
1921
1922 if (opr2->is_constant()) {
1923 jlong imm;
1924 switch(opr2->type()) {
1925 case T_LONG:
1926 imm = opr2->as_constant_ptr()->as_jlong();
1927 break;
1928 case T_INT:
1929 case T_ADDRESS:
1930 imm = opr2->as_constant_ptr()->as_jint();
1931 break;
1932 case T_OBJECT:
1933 case T_ARRAY:
1934 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1935 break;
1936 default:
1937 ShouldNotReachHere();
1938 break;
1939 }
1940
1941 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1942 if (type2aelembytes(opr1->type()) <= 4)
1943 __ cmpw(reg1, imm);
1944 else
1945 __ cmp(reg1, imm);
1946 return;
1947 } else {
1948 __ mov(rscratch1, imm);
1949 if (type2aelembytes(opr1->type()) <= 4)
1950 __ cmpw(reg1, rscratch1);
1951 else
1952 __ cmp(reg1, rscratch1);
1953 return;
1954 }
1955 } else
1956 ShouldNotReachHere();
1957 } else if (opr1->is_single_fpu()) {
1958 FloatRegister reg1 = opr1->as_float_reg();
1959 assert(opr2->is_single_fpu(), "expect single float register");
1960 FloatRegister reg2 = opr2->as_float_reg();
1961 __ fcmps(reg1, reg2);
1962 } else if (opr1->is_double_fpu()) {
1963 FloatRegister reg1 = opr1->as_double_reg();
1964 assert(opr2->is_double_fpu(), "expect double float register");
1965 FloatRegister reg2 = opr2->as_double_reg();
1966 __ fcmpd(reg1, reg2);
1967 } else {
1968 ShouldNotReachHere();
1969 }
1970 }
1971
1972 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1973 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1974 bool is_unordered_less = (code == lir_ucmp_fd2i);
1975 if (left->is_single_fpu()) {
1976 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1977 } else if (left->is_double_fpu()) {
1978 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1979 } else {
1980 ShouldNotReachHere();
1981 }
1982 } else if (code == lir_cmp_l2i) {
1983 Label done;
1984 __ cmp(left->as_register_lo(), right->as_register_lo());
1985 __ mov(dst->as_register(), (u_int64_t)-1L);
1986 __ br(Assembler::LT, done);
1987 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1988 __ bind(done);
1989 } else {
1990 ShouldNotReachHere();
1991 }
1992 }
1993
1994
1995 void LIR_Assembler::align_call(LIR_Code code) { }
1996
1997
1998 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1999 address call = __ trampoline_call(Address(op->addr(), rtype));
2000 if (call == NULL) {
2001 bailout("trampoline stub overflow");
2002 return;
2003 }
2004 add_call_info(code_offset(), op->info());
2005 }
2006
2007
2008 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2009 address call = __ ic_call(op->addr());
2010 if (call == NULL) {
2011 bailout("trampoline stub overflow");
2012 return;
2013 }
2014 add_call_info(code_offset(), op->info());
2015 }
2016
2017
2018 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2019 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2020 ShouldNotReachHere();
2021 }
2022
2023
2024 void LIR_Assembler::emit_static_call_stub() {
2025 address call_pc = __ pc();
2026 address stub = __ start_a_stub(call_stub_size);
2027 if (stub == NULL) {
2028 bailout("static call stub overflow");
2029 return;
2030 }
2031
2032 int start = __ offset();
2033
2034 __ relocate(static_stub_Relocation::spec(call_pc));
2035 __ mov_metadata(rmethod, (Metadata*)NULL);
2036 __ movptr(rscratch1, 0);
2037 __ br(rscratch1);
2038
2039 assert(__ offset() - start <= call_stub_size, "stub too big");
2040 __ end_a_stub();
2041 }
2042
2043
2044 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2045 assert(exceptionOop->as_register() == r0, "must match");
2046 assert(exceptionPC->as_register() == r3, "must match");
2047
2048 // exception object is not added to oop map by LinearScan
2049 // (LinearScan assumes that no oops are in fixed registers)
2050 info->add_register_oop(exceptionOop);
2051 Runtime1::StubID unwind_id;
2052
2053 // get current pc information
2054 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2055 int pc_for_athrow_offset = __ offset();
2056 InternalAddress pc_for_athrow(__ pc());
2057 __ adr(exceptionPC->as_register(), pc_for_athrow);
2058 add_call_info(pc_for_athrow_offset, info); // for exception handler
2059
2060 __ verify_not_null_oop(r0);
2061 // search an exception handler (r0: exception oop, r3: throwing pc)
2062 if (compilation()->has_fpu_code()) {
2063 unwind_id = Runtime1::handle_exception_id;
2064 } else {
2065 unwind_id = Runtime1::handle_exception_nofpu_id;
2066 }
2067 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2068
2069 // FIXME: enough room for two byte trap ????
2070 __ nop();
2071 }
2072
2073
2074 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2075 assert(exceptionOop->as_register() == r0, "must match");
2076
2077 __ b(_unwind_handler_entry);
2078 }
2079
2080
2081 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2082 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2083 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2084
2085 switch (left->type()) {
2086 case T_INT: {
2087 switch (code) {
2088 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2089 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2090 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2091 default:
2092 ShouldNotReachHere();
2093 break;
2094 }
2095 break;
2096 case T_LONG:
2097 case T_ADDRESS:
2098 case T_OBJECT:
2099 switch (code) {
2100 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2101 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2102 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2103 default:
2104 ShouldNotReachHere();
2105 break;
2106 }
2107 break;
2108 default:
2109 ShouldNotReachHere();
2110 break;
2111 }
2112 }
2113 }
2114
2115
2116 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2117 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2118 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2119
2120 switch (left->type()) {
2121 case T_INT: {
2122 switch (code) {
2123 case lir_shl: __ lslw (dreg, lreg, count); break;
2124 case lir_shr: __ asrw (dreg, lreg, count); break;
2125 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2126 default:
2127 ShouldNotReachHere();
2128 break;
2129 }
2130 break;
2131 case T_LONG:
2132 case T_ADDRESS:
2133 case T_OBJECT:
2134 switch (code) {
2135 case lir_shl: __ lsl (dreg, lreg, count); break;
2136 case lir_shr: __ asr (dreg, lreg, count); break;
2137 case lir_ushr: __ lsr (dreg, lreg, count); break;
2138 default:
2139 ShouldNotReachHere();
2140 break;
2141 }
2142 break;
2143 default:
2144 ShouldNotReachHere();
2145 break;
2146 }
2147 }
2148 }
2149
2150
2151 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2152 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2153 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2154 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2155 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2156 }
2157
2158
2159 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2160 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2161 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2162 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2163 __ mov (rscratch1, c);
2164 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2165 }
2166
2167
2168 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2169 ShouldNotReachHere();
2170 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2171 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2172 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2173 __ lea(rscratch1, __ constant_oop_address(o));
2174 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2175 }
2176
2177
2178 // This code replaces a call to arraycopy; no exception may
2179 // be thrown in this code, they must be thrown in the System.arraycopy
2180 // activation frame; we could save some checks if this would not be the case
2181 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2182 ciArrayKlass* default_type = op->expected_type();
2183 Register src = op->src()->as_register();
2184 Register dst = op->dst()->as_register();
2185 Register src_pos = op->src_pos()->as_register();
2186 Register dst_pos = op->dst_pos()->as_register();
2187 Register length = op->length()->as_register();
2188 Register tmp = op->tmp()->as_register();
2189
2190 CodeStub* stub = op->stub();
2191 int flags = op->flags();
2192 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2193 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2194
2195 // if we don't know anything, just go through the generic arraycopy
2196 if (default_type == NULL // || basic_type == T_OBJECT
2197 ) {
2198 Label done;
2199 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2200
2201 // Save the arguments in case the generic arraycopy fails and we
2202 // have to fall back to the JNI stub
2203 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2204 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2205 __ str(src, Address(sp, 4*BytesPerWord));
2206
2207 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2208 address copyfunc_addr = StubRoutines::generic_arraycopy();
2209
2210 // The arguments are in java calling convention so we shift them
2211 // to C convention
2212 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2213 __ mov(c_rarg0, j_rarg0);
2214 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2215 __ mov(c_rarg1, j_rarg1);
2216 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2217 __ mov(c_rarg2, j_rarg2);
2218 assert_different_registers(c_rarg3, j_rarg4);
2219 __ mov(c_rarg3, j_rarg3);
2220 __ mov(c_rarg4, j_rarg4);
2221 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2222 __ mov(rscratch1, RuntimeAddress(C_entry));
2223 __ blrt(rscratch1, 5, 0, 1);
2224 } else {
2225 #ifndef PRODUCT
2226 if (PrintC1Statistics) {
2227 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2228 }
2229 #endif
2230 __ far_call(RuntimeAddress(copyfunc_addr));
2231 }
2232
2233 __ cbz(r0, *stub->continuation());
2234
2235 // Reload values from the stack so they are where the stub
2236 // expects them.
2237 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2238 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2239 __ ldr(src, Address(sp, 4*BytesPerWord));
2240
2241 if (copyfunc_addr != NULL) {
2242 // r0 is -1^K where K == partial copied count
2243 __ eonw(rscratch1, r0, 0);
2244 // adjust length down and src/end pos up by partial copied count
2245 __ subw(length, length, rscratch1);
2246 __ addw(src_pos, src_pos, rscratch1);
2247 __ addw(dst_pos, dst_pos, rscratch1);
2248 }
2249 __ b(*stub->entry());
2250
2251 __ bind(*stub->continuation());
2252 return;
2253 }
2254
2255 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2256
2257 int elem_size = type2aelembytes(basic_type);
2258 int shift_amount;
2259 int scale = exact_log2(elem_size);
2260
2261 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2262 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2263 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2264 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2265
2266 // test for NULL
2267 if (flags & LIR_OpArrayCopy::src_null_check) {
2268 __ cbz(src, *stub->entry());
2269 }
2270 if (flags & LIR_OpArrayCopy::dst_null_check) {
2271 __ cbz(dst, *stub->entry());
2272 }
2273
2274 // check if negative
2275 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2276 __ cmpw(src_pos, 0);
2277 __ br(Assembler::LT, *stub->entry());
2278 }
2279 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2280 __ cmpw(dst_pos, 0);
2281 __ br(Assembler::LT, *stub->entry());
2282 }
2283
2284 if (flags & LIR_OpArrayCopy::length_positive_check) {
2285 __ cmpw(length, 0);
2286 __ br(Assembler::LT, *stub->entry());
2287 }
2288
2289 if (flags & LIR_OpArrayCopy::src_range_check) {
2290 __ addw(tmp, src_pos, length);
2291 __ ldrw(rscratch1, src_length_addr);
2292 __ cmpw(tmp, rscratch1);
2293 __ br(Assembler::HI, *stub->entry());
2294 }
2295 if (flags & LIR_OpArrayCopy::dst_range_check) {
2296 __ addw(tmp, dst_pos, length);
2297 __ ldrw(rscratch1, dst_length_addr);
2298 __ cmpw(tmp, rscratch1);
2299 __ br(Assembler::HI, *stub->entry());
2300 }
2301
2302 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2303 // length_positive_check if the source of our length operand is an
2304 // arraylength. However, that arraylength might be zero, and the
2305 // stub that we're about to call contains an assertion that count !=
2306 // 0 . So we make this check purely in order not to trigger an
2307 // assertion failure.
2308 __ cbzw(length, *stub->continuation());
2309
2310 if (flags & LIR_OpArrayCopy::type_check) {
2311 // We don't know the array types are compatible
2312 if (basic_type != T_OBJECT) {
2313 // Simple test for basic type arrays
2314 if (UseCompressedClassPointers) {
2315 __ ldrw(tmp, src_klass_addr);
2316 __ ldrw(rscratch1, dst_klass_addr);
2317 __ cmpw(tmp, rscratch1);
2318 } else {
2319 __ ldr(tmp, src_klass_addr);
2320 __ ldr(rscratch1, dst_klass_addr);
2321 __ cmp(tmp, rscratch1);
2322 }
2323 __ br(Assembler::NE, *stub->entry());
2324 } else {
2325 // For object arrays, if src is a sub class of dst then we can
2326 // safely do the copy.
2327 Label cont, slow;
2328
2329 #define PUSH(r1, r2) \
2330 stp(r1, r2, __ pre(sp, -2 * wordSize));
2331
2332 #define POP(r1, r2) \
2333 ldp(r1, r2, __ post(sp, 2 * wordSize));
2334
2335 __ PUSH(src, dst);
2336
2337 __ load_klass(src, src);
2338 __ load_klass(dst, dst);
2339
2340 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2341
2342 __ PUSH(src, dst);
2343 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2344 __ POP(src, dst);
2345
2346 __ cbnz(src, cont);
2347
2348 __ bind(slow);
2349 __ POP(src, dst);
2350
2351 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2352 if (copyfunc_addr != NULL) { // use stub if available
2353 // src is not a sub class of dst so we have to do a
2354 // per-element check.
2355
2356 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2357 if ((flags & mask) != mask) {
2358 // Check that at least both of them object arrays.
2359 assert(flags & mask, "one of the two should be known to be an object array");
2360
2361 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2362 __ load_klass(tmp, src);
2363 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2364 __ load_klass(tmp, dst);
2365 }
2366 int lh_offset = in_bytes(Klass::layout_helper_offset());
2367 Address klass_lh_addr(tmp, lh_offset);
2368 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2369 __ ldrw(rscratch1, klass_lh_addr);
2370 __ mov(rscratch2, objArray_lh);
2371 __ eorw(rscratch1, rscratch1, rscratch2);
2372 __ cbnzw(rscratch1, *stub->entry());
2373 }
2374
2375 // Spill because stubs can use any register they like and it's
2376 // easier to restore just those that we care about.
2377 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2378 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2379 __ str(src, Address(sp, 4*BytesPerWord));
2380
2381 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2382 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2383 assert_different_registers(c_rarg0, dst, dst_pos, length);
2384 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2385 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2386 assert_different_registers(c_rarg1, dst, length);
2387 __ uxtw(c_rarg2, length);
2388 assert_different_registers(c_rarg2, dst);
2389
2390 __ load_klass(c_rarg4, dst);
2391 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2392 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2393 __ far_call(RuntimeAddress(copyfunc_addr));
2394
2395 #ifndef PRODUCT
2396 if (PrintC1Statistics) {
2397 Label failed;
2398 __ cbnz(r0, failed);
2399 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2400 __ bind(failed);
2401 }
2402 #endif
2403
2404 __ cbz(r0, *stub->continuation());
2405
2406 #ifndef PRODUCT
2407 if (PrintC1Statistics) {
2408 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2409 }
2410 #endif
2411 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2412
2413 // Restore previously spilled arguments
2414 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2415 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2416 __ ldr(src, Address(sp, 4*BytesPerWord));
2417
2418 // return value is -1^K where K is partial copied count
2419 __ eonw(rscratch1, r0, zr);
2420 // adjust length down and src/end pos up by partial copied count
2421 __ subw(length, length, rscratch1);
2422 __ addw(src_pos, src_pos, rscratch1);
2423 __ addw(dst_pos, dst_pos, rscratch1);
2424 }
2425
2426 __ b(*stub->entry());
2427
2428 __ bind(cont);
2429 __ POP(src, dst);
2430 }
2431 }
2432
2433 #ifdef ASSERT
2434 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2435 // Sanity check the known type with the incoming class. For the
2436 // primitive case the types must match exactly with src.klass and
2437 // dst.klass each exactly matching the default type. For the
2438 // object array case, if no type check is needed then either the
2439 // dst type is exactly the expected type and the src type is a
2440 // subtype which we can't check or src is the same array as dst
2441 // but not necessarily exactly of type default_type.
2442 Label known_ok, halt;
2443 __ mov_metadata(tmp, default_type->constant_encoding());
2444 if (UseCompressedClassPointers) {
2445 __ encode_klass_not_null(tmp);
2446 }
2447
2448 if (basic_type != T_OBJECT) {
2449
2450 if (UseCompressedClassPointers) {
2451 __ ldrw(rscratch1, dst_klass_addr);
2452 __ cmpw(tmp, rscratch1);
2453 } else {
2454 __ ldr(rscratch1, dst_klass_addr);
2455 __ cmp(tmp, rscratch1);
2456 }
2457 __ br(Assembler::NE, halt);
2458 if (UseCompressedClassPointers) {
2459 __ ldrw(rscratch1, src_klass_addr);
2460 __ cmpw(tmp, rscratch1);
2461 } else {
2462 __ ldr(rscratch1, src_klass_addr);
2463 __ cmp(tmp, rscratch1);
2464 }
2465 __ br(Assembler::EQ, known_ok);
2466 } else {
2467 if (UseCompressedClassPointers) {
2468 __ ldrw(rscratch1, dst_klass_addr);
2469 __ cmpw(tmp, rscratch1);
2470 } else {
2471 __ ldr(rscratch1, dst_klass_addr);
2472 __ cmp(tmp, rscratch1);
2473 }
2474 __ br(Assembler::EQ, known_ok);
2475 __ cmp(src, dst);
2476 __ br(Assembler::EQ, known_ok);
2477 }
2478 __ bind(halt);
2479 __ stop("incorrect type information in arraycopy");
2480 __ bind(known_ok);
2481 }
2482 #endif
2483
2484 #ifndef PRODUCT
2485 if (PrintC1Statistics) {
2486 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2487 }
2488 #endif
2489
2490 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2491 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2492 assert_different_registers(c_rarg0, dst, dst_pos, length);
2493 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2494 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2495 assert_different_registers(c_rarg1, dst, length);
2496 __ uxtw(c_rarg2, length);
2497 assert_different_registers(c_rarg2, dst);
2498
2499 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2500 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2501 const char *name;
2502 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2503
2504 CodeBlob *cb = CodeCache::find_blob(entry);
2505 if (cb) {
2506 __ far_call(RuntimeAddress(entry));
2507 } else {
2508 __ call_VM_leaf(entry, 3);
2509 }
2510
2511 __ bind(*stub->continuation());
2512 }
2513
2514
2515
2516
2517 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2518 Register obj = op->obj_opr()->as_register(); // may not be an oop
2519 Register hdr = op->hdr_opr()->as_register();
2520 Register lock = op->lock_opr()->as_register();
2521 if (!UseFastLocking) {
2522 __ b(*op->stub()->entry());
2523 } else if (op->code() == lir_lock) {
2524 Register scratch = noreg;
2525 if (UseBiasedLocking) {
2526 scratch = op->scratch_opr()->as_register();
2527 }
2528 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2529 // add debug info for NullPointerException only if one is possible
2530 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2531 if (op->info() != NULL) {
2532 add_debug_info_for_null_check(null_check_offset, op->info());
2533 }
2534 // done
2535 } else if (op->code() == lir_unlock) {
2536 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2537 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2538 } else {
2539 Unimplemented();
2540 }
2541 __ bind(*op->stub()->continuation());
2542 }
2543
2544
2545 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2546 ciMethod* method = op->profiled_method();
2547 int bci = op->profiled_bci();
2548 ciMethod* callee = op->profiled_callee();
2549
2550 // Update counter for all call types
2551 ciMethodData* md = method->method_data_or_null();
2552 assert(md != NULL, "Sanity");
2553 ciProfileData* data = md->bci_to_data(bci);
2554 assert(data->is_CounterData(), "need CounterData for calls");
2555 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2556 Register mdo = op->mdo()->as_register();
2557 __ mov_metadata(mdo, md->constant_encoding());
2558 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2559 Bytecodes::Code bc = method->java_code_at_bci(bci);
2560 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2561 // Perform additional virtual call profiling for invokevirtual and
2562 // invokeinterface bytecodes
2563 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2564 !callee_is_static && // required for optimized MH invokes
2565 C1ProfileVirtualCalls) {
2566 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2567 Register recv = op->recv()->as_register();
2568 assert_different_registers(mdo, recv);
2569 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2570 ciKlass* known_klass = op->known_holder();
2571 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2572 // We know the type that will be seen at this call site; we can
2573 // statically update the MethodData* rather than needing to do
2574 // dynamic tests on the receiver type
2575
2576 // NOTE: we should probably put a lock around this search to
2577 // avoid collisions by concurrent compilations
2578 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2579 uint i;
2580 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2581 ciKlass* receiver = vc_data->receiver(i);
2582 if (known_klass->equals(receiver)) {
2583 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2584 __ addptr(data_addr, DataLayout::counter_increment);
2585 return;
2586 }
2587 }
2588
2589 // Receiver type not found in profile data; select an empty slot
2590
2591 // Note that this is less efficient than it should be because it
2592 // always does a write to the receiver part of the
2593 // VirtualCallData rather than just the first time
2594 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2595 ciKlass* receiver = vc_data->receiver(i);
2596 if (receiver == NULL) {
2597 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2598 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2599 __ lea(rscratch2, recv_addr);
2600 __ str(rscratch1, Address(rscratch2));
2601 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2602 __ addptr(data_addr, DataLayout::counter_increment);
2603 return;
2604 }
2605 }
2606 } else {
2607 __ load_klass(recv, recv);
2608 Label update_done;
2609 type_profile_helper(mdo, md, data, recv, &update_done);
2610 // Receiver did not match any saved receiver and there is no empty row for it.
2611 // Increment total counter to indicate polymorphic case.
2612 __ addptr(counter_addr, DataLayout::counter_increment);
2613
2614 __ bind(update_done);
2615 }
2616 } else {
2617 // Static call
2618 __ addptr(counter_addr, DataLayout::counter_increment);
2619 }
2620 }
2621
2622
2623 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2624 Unimplemented();
2625 }
2626
2627
2628 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2629 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2630 }
2631
2632 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2633 assert(op->crc()->is_single_cpu(), "crc must be register");
2634 assert(op->val()->is_single_cpu(), "byte value must be register");
2635 assert(op->result_opr()->is_single_cpu(), "result must be register");
2636 Register crc = op->crc()->as_register();
2637 Register val = op->val()->as_register();
2638 Register res = op->result_opr()->as_register();
2639
2640 assert_different_registers(val, crc, res);
2641 unsigned long offset;
2642 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2643 if (offset) __ add(res, res, offset);
2644
2645 __ ornw(crc, zr, crc); // ~crc
2646 __ update_byte_crc32(crc, val, res);
2647 __ ornw(res, zr, crc); // ~crc
2648 }
2649
2650 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2651 COMMENT("emit_profile_type {");
2652 Register obj = op->obj()->as_register();
2653 Register tmp = op->tmp()->as_pointer_register();
2654 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2655 ciKlass* exact_klass = op->exact_klass();
2656 intptr_t current_klass = op->current_klass();
2657 bool not_null = op->not_null();
2658 bool no_conflict = op->no_conflict();
2659
2660 Label update, next, none;
2661
2662 bool do_null = !not_null;
2663 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2664 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2665
2666 assert(do_null || do_update, "why are we here?");
2667 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2668 assert(mdo_addr.base() != rscratch1, "wrong register");
2669
2670 __ verify_oop(obj);
2671
2672 if (tmp != obj) {
2673 __ mov(tmp, obj);
2674 }
2675 if (do_null) {
2676 __ cbnz(tmp, update);
2677 if (!TypeEntries::was_null_seen(current_klass)) {
2678 __ ldr(rscratch2, mdo_addr);
2679 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2680 __ str(rscratch2, mdo_addr);
2681 }
2682 if (do_update) {
2683 #ifndef ASSERT
2684 __ b(next);
2685 }
2686 #else
2687 __ b(next);
2688 }
2689 } else {
2690 __ cbnz(tmp, update);
2691 __ stop("unexpected null obj");
2692 #endif
2693 }
2694
2695 __ bind(update);
2696
2697 if (do_update) {
2698 #ifdef ASSERT
2699 if (exact_klass != NULL) {
2700 Label ok;
2701 __ load_klass(tmp, tmp);
2702 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2703 __ eor(rscratch1, tmp, rscratch1);
2704 __ cbz(rscratch1, ok);
2705 __ stop("exact klass and actual klass differ");
2706 __ bind(ok);
2707 }
2708 #endif
2709 if (!no_conflict) {
2710 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2711 if (exact_klass != NULL) {
2712 __ mov_metadata(tmp, exact_klass->constant_encoding());
2713 } else {
2714 __ load_klass(tmp, tmp);
2715 }
2716
2717 __ ldr(rscratch2, mdo_addr);
2718 __ eor(tmp, tmp, rscratch2);
2719 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2720 // klass seen before, nothing to do. The unknown bit may have been
2721 // set already but no need to check.
2722 __ cbz(rscratch1, next);
2723
2724 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2725 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2726
2727 if (TypeEntries::is_type_none(current_klass)) {
2728 __ cbz(rscratch2, none);
2729 __ cmp(rscratch2, TypeEntries::null_seen);
2730 __ br(Assembler::EQ, none);
2731 // There is a chance that the checks above (re-reading profiling
2732 // data from memory) fail if another thread has just set the
2733 // profiling to this obj's klass
2734 __ dmb(Assembler::ISHLD);
2735 __ ldr(rscratch2, mdo_addr);
2736 __ eor(tmp, tmp, rscratch2);
2737 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2738 __ cbz(rscratch1, next);
2739 }
2740 } else {
2741 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2742 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2743
2744 __ ldr(tmp, mdo_addr);
2745 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2746 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2747 }
2748
2749 // different than before. Cannot keep accurate profile.
2750 __ ldr(rscratch2, mdo_addr);
2751 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2752 __ str(rscratch2, mdo_addr);
2753
2754 if (TypeEntries::is_type_none(current_klass)) {
2755 __ b(next);
2756
2757 __ bind(none);
2758 // first time here. Set profile type.
2759 __ str(tmp, mdo_addr);
2760 }
2761 } else {
2762 // There's a single possible klass at this profile point
2763 assert(exact_klass != NULL, "should be");
2764 if (TypeEntries::is_type_none(current_klass)) {
2765 __ mov_metadata(tmp, exact_klass->constant_encoding());
2766 __ ldr(rscratch2, mdo_addr);
2767 __ eor(tmp, tmp, rscratch2);
2768 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2769 __ cbz(rscratch1, next);
2770 #ifdef ASSERT
2771 {
2772 Label ok;
2773 __ ldr(rscratch1, mdo_addr);
2774 __ cbz(rscratch1, ok);
2775 __ cmp(rscratch1, TypeEntries::null_seen);
2776 __ br(Assembler::EQ, ok);
2777 // may have been set by another thread
2778 __ dmb(Assembler::ISHLD);
2779 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2780 __ ldr(rscratch2, mdo_addr);
2781 __ eor(rscratch2, rscratch1, rscratch2);
2782 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2783 __ cbz(rscratch2, ok);
2784
2785 __ stop("unexpected profiling mismatch");
2786 __ bind(ok);
2787 }
2788 #endif
2789 // first time here. Set profile type.
2790 __ ldr(tmp, mdo_addr);
2791 } else {
2792 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2793 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2794
2795 __ ldr(tmp, mdo_addr);
2796 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2797 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2798
2799 __ orr(tmp, tmp, TypeEntries::type_unknown);
2800 __ str(tmp, mdo_addr);
2801 // FIXME: Write barrier needed here?
2802 }
2803 }
2804
2805 __ bind(next);
2806 }
2807 COMMENT("} emit_profile_type");
2808 }
2809
2810
2811 void LIR_Assembler::align_backward_branch_target() {
2812 }
2813
2814
2815 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2816 if (left->is_single_cpu()) {
2817 assert(dest->is_single_cpu(), "expect single result reg");
2818 __ negw(dest->as_register(), left->as_register());
2819 } else if (left->is_double_cpu()) {
2820 assert(dest->is_double_cpu(), "expect double result reg");
2821 __ neg(dest->as_register_lo(), left->as_register_lo());
2822 } else if (left->is_single_fpu()) {
2823 assert(dest->is_single_fpu(), "expect single float result reg");
2824 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2825 } else {
2826 assert(left->is_double_fpu(), "expect double float operand reg");
2827 assert(dest->is_double_fpu(), "expect double float result reg");
2828 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2829 }
2830 }
2831
2832
2833 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2834 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2835 }
2836
2837
2838 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2839 assert(!tmp->is_valid(), "don't need temporary");
2840
2841 CodeBlob *cb = CodeCache::find_blob(dest);
2842 if (cb) {
2843 __ far_call(RuntimeAddress(dest));
2844 } else {
2845 __ mov(rscratch1, RuntimeAddress(dest));
2846 int len = args->length();
2847 int type = 0;
2848 if (! result->is_illegal()) {
2849 switch (result->type()) {
2850 case T_VOID:
2851 type = 0;
2852 break;
2853 case T_INT:
2854 case T_LONG:
2855 case T_OBJECT:
2856 type = 1;
2857 break;
2858 case T_FLOAT:
2859 type = 2;
2860 break;
2861 case T_DOUBLE:
2862 type = 3;
2863 break;
2864 default:
2865 ShouldNotReachHere();
2866 break;
2867 }
2868 }
2869 int num_gpargs = 0;
2870 int num_fpargs = 0;
2871 for (int i = 0; i < args->length(); i++) {
2872 LIR_Opr arg = args->at(i);
2873 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2874 num_fpargs++;
2875 } else {
2876 num_gpargs++;
2877 }
2878 }
2879 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2880 }
2881
2882 if (info != NULL) {
2883 add_call_info_here(info);
2884 }
2885 __ maybe_isb();
2886 }
2887
2888 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2889 if (dest->is_address() || src->is_address()) {
2890 move_op(src, dest, type, lir_patch_none, info,
2891 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2892 } else {
2893 ShouldNotReachHere();
2894 }
2895 }
2896
2897 #ifdef ASSERT
2898 // emit run-time assertion
2899 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2900 assert(op->code() == lir_assert, "must be");
2901
2902 if (op->in_opr1()->is_valid()) {
2903 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2904 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2905 } else {
2906 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2907 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2908 }
2909
2910 Label ok;
2911 if (op->condition() != lir_cond_always) {
2912 Assembler::Condition acond = Assembler::AL;
2913 switch (op->condition()) {
2914 case lir_cond_equal: acond = Assembler::EQ; break;
2915 case lir_cond_notEqual: acond = Assembler::NE; break;
2916 case lir_cond_less: acond = Assembler::LT; break;
2917 case lir_cond_lessEqual: acond = Assembler::LE; break;
2918 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2919 case lir_cond_greater: acond = Assembler::GT; break;
2920 case lir_cond_belowEqual: acond = Assembler::LS; break;
2921 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2922 default: ShouldNotReachHere();
2923 }
2924 __ br(acond, ok);
2925 }
2926 if (op->halt()) {
2927 const char* str = __ code_string(op->msg());
2928 __ stop(str);
2929 } else {
2930 breakpoint();
2931 }
2932 __ bind(ok);
2933 }
2934 #endif
2935
2936 #ifndef PRODUCT
2937 #define COMMENT(x) do { __ block_comment(x); } while (0)
2938 #else
2939 #define COMMENT(x)
2940 #endif
2941
2942 void LIR_Assembler::membar() {
2943 COMMENT("membar");
2944 __ membar(MacroAssembler::AnyAny);
2945 }
2946
2947 void LIR_Assembler::membar_acquire() {
2948 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2949 }
2950
2951 void LIR_Assembler::membar_release() {
2952 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2953 }
2954
2955 void LIR_Assembler::membar_loadload() {
2956 __ membar(Assembler::LoadLoad);
2957 }
2958
2959 void LIR_Assembler::membar_storestore() {
2960 __ membar(MacroAssembler::StoreStore);
2961 }
2962
2963 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2964
2965 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2966
2967 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2968 __ mov(result_reg->as_register(), rthread);
2969 }
2970
2971
2972 void LIR_Assembler::peephole(LIR_List *lir) {
2973 #if 0
2974 if (tableswitch_count >= max_tableswitches)
2975 return;
2976
2977 /*
2978 This finite-state automaton recognizes sequences of compare-and-
2979 branch instructions. We will turn them into a tableswitch. You
2980 could argue that C1 really shouldn't be doing this sort of
2981 optimization, but without it the code is really horrible.
2982 */
2983
2984 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2985 int first_key, last_key = -2147483648;
2986 int next_key = 0;
2987 int start_insn = -1;
2988 int last_insn = -1;
2989 Register reg = noreg;
2990 LIR_Opr reg_opr;
2991 state = start_s;
2992
2993 LIR_OpList* inst = lir->instructions_list();
2994 for (int i = 0; i < inst->length(); i++) {
2995 LIR_Op* op = inst->at(i);
2996 switch (state) {
2997 case start_s:
2998 first_key = -1;
2999 start_insn = i;
3000 switch (op->code()) {
3001 case lir_cmp:
3002 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3003 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3004 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3005 && opr2->is_constant()
3006 && opr2->type() == T_INT) {
3007 reg_opr = opr1;
3008 reg = opr1->as_register();
3009 first_key = opr2->as_constant_ptr()->as_jint();
3010 next_key = first_key + 1;
3011 state = cmp_s;
3012 goto next_state;
3013 }
3014 break;
3015 }
3016 break;
3017 case cmp_s:
3018 switch (op->code()) {
3019 case lir_branch:
3020 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3021 state = beq_s;
3022 last_insn = i;
3023 goto next_state;
3024 }
3025 }
3026 state = start_s;
3027 break;
3028 case beq_s:
3029 switch (op->code()) {
3030 case lir_cmp: {
3031 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3032 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3033 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3034 && opr1->as_register() == reg
3035 && opr2->is_constant()
3036 && opr2->type() == T_INT
3037 && opr2->as_constant_ptr()->as_jint() == next_key) {
3038 last_key = next_key;
3039 next_key++;
3040 state = cmp_s;
3041 goto next_state;
3042 }
3043 }
3044 }
3045 last_key = next_key;
3046 state = start_s;
3047 break;
3048 default:
3049 assert(false, "impossible state");
3050 }
3051 if (state == start_s) {
3052 if (first_key < last_key - 5L && reg != noreg) {
3053 {
3054 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3055 // reg->encoding(),
3056 // start_insn, first_key, last_key);
3057 // for (int i = 0; i < inst->length(); i++) {
3058 // inst->at(i)->print();
3059 // tty->print("\n");
3060 // }
3061 // tty->print("\n");
3062 }
3063
3064 struct tableswitch *sw = &switches[tableswitch_count];
3065 sw->_insn_index = start_insn, sw->_first_key = first_key,
3066 sw->_last_key = last_key, sw->_reg = reg;
3067 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3068 {
3069 // Insert the new table of branches
3070 int offset = last_insn;
3071 for (int n = first_key; n < last_key; n++) {
3072 inst->insert_before
3073 (last_insn + 1,
3074 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3075 inst->at(offset)->as_OpBranch()->label()));
3076 offset -= 2, i++;
3077 }
3078 }
3079 // Delete all the old compare-and-branch instructions
3080 for (int n = first_key; n < last_key; n++) {
3081 inst->remove_at(start_insn);
3082 inst->remove_at(start_insn);
3083 }
3084 // Insert the tableswitch instruction
3085 inst->insert_before(start_insn,
3086 new LIR_Op2(lir_cmp, lir_cond_always,
3087 LIR_OprFact::intConst(tableswitch_count),
3088 reg_opr));
3089 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3090 tableswitch_count++;
3091 }
3092 reg = noreg;
3093 last_key = -2147483648;
3094 }
3095 next_state:
3096 ;
3097 }
3098 #endif
3099 }
3100
3101 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3102 Address addr = as_Address(src->as_address_ptr(), noreg);
3103 BasicType type = src->type();
3104 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3105
3106 void (MacroAssembler::* lda)(Register Rd, Register Ra);
3107 void (MacroAssembler::* add)(Register Rd, Register Rn, RegisterOrConstant increment);
3108 void (MacroAssembler::* stl)(Register Rs, Register Rt, Register Rn);
3109
3110 switch(type) {
3111 case T_INT:
3112 lda = &MacroAssembler::ldaxrw;
3113 add = &MacroAssembler::addw;
3114 stl = &MacroAssembler::stlxrw;
3115 break;
3116 case T_LONG:
3117 lda = &MacroAssembler::ldaxr;
3118 add = &MacroAssembler::add;
3119 stl = &MacroAssembler::stlxr;
3120 break;
3121 case T_OBJECT:
3122 case T_ARRAY:
3123 if (UseCompressedOops) {
3124 lda = &MacroAssembler::ldaxrw;
3125 add = &MacroAssembler::addw;
3126 stl = &MacroAssembler::stlxrw;
3127 } else {
3128 lda = &MacroAssembler::ldaxr;
3129 add = &MacroAssembler::add;
3130 stl = &MacroAssembler::stlxr;
3131 }
3132 break;
3133 default:
3134 ShouldNotReachHere();
3135 }
3136
3137 switch (code) {
3138 case lir_xadd:
3139 {
3140 RegisterOrConstant inc;
3141 Register tmp = as_reg(tmp_op);
3142 Register dst = as_reg(dest);
3143 if (data->is_constant()) {
3144 inc = RegisterOrConstant(as_long(data));
3145 assert_different_registers(dst, addr.base(), tmp,
3146 rscratch1, rscratch2);
3147 } else {
3148 inc = RegisterOrConstant(as_reg(data));
3149 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3150 rscratch1, rscratch2);
3151 }
3152 Label again;
3153 __ lea(tmp, addr);
3154 __ bind(again);
3155 (_masm->*lda)(dst, tmp);
3156 (_masm->*add)(rscratch1, dst, inc);
3157 (_masm->*stl)(rscratch2, rscratch1, tmp);
3158 __ cbnzw(rscratch2, again);
3159 break;
3160 }
3161 case lir_xchg:
3162 {
3163 Register tmp = tmp_op->as_register();
3164 Register obj = as_reg(data);
3165 Register dst = as_reg(dest);
3166 if (is_oop && UseCompressedOops) {
3167 __ encode_heap_oop(obj);
3168 }
3169 assert_different_registers(obj, addr.base(), tmp, rscratch2, dst);
3170 Label again;
3171 __ lea(tmp, addr);
3172 __ bind(again);
3173 (_masm->*lda)(dst, tmp);
3174 (_masm->*stl)(rscratch2, obj, tmp);
3175 __ cbnzw(rscratch2, again);
3176 if (is_oop && UseCompressedOops) {
3177 __ decode_heap_oop(dst);
3178 }
3179 }
3180 break;
3181 default:
3182 ShouldNotReachHere();
3183 }
3184 __ membar(__ AnyAny);
3185 }
3186
3187 #undef __