1 /*
2 * Copyright (c) 2000, 2012, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "gc_interface/collectedHeap.hpp"
37 #include "memory/barrierSet.hpp"
38 #include "memory/cardTableModRefBS.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "vmreg_aarch64.inline.hpp"
43
44
45
46 #ifndef PRODUCT
47 #define COMMENT(x) do { __ block_comment(x); } while (0)
48 #else
49 #define COMMENT(x)
50 #endif
51
52 NEEDS_CLEANUP // remove this definitions ?
53 const Register IC_Klass = rscratch2; // where the IC klass is cached
54 const Register SYNC_header = r0; // synchronization header
55 const Register SHIFT_count = r0; // where count for shift operations must be
56
57 #define __ _masm->
58
59
60 static void select_different_registers(Register preserve,
61 Register extra,
62 Register &tmp1,
63 Register &tmp2) {
64 if (tmp1 == preserve) {
65 assert_different_registers(tmp1, tmp2, extra);
66 tmp1 = extra;
67 } else if (tmp2 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp2 = extra;
70 }
71 assert_different_registers(preserve, tmp1, tmp2);
72 }
73
74
75
76 static void select_different_registers(Register preserve,
77 Register extra,
78 Register &tmp1,
79 Register &tmp2,
80 Register &tmp3) {
81 if (tmp1 == preserve) {
82 assert_different_registers(tmp1, tmp2, tmp3, extra);
83 tmp1 = extra;
84 } else if (tmp2 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp2 = extra;
87 } else if (tmp3 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp3 = extra;
90 }
91 assert_different_registers(preserve, tmp1, tmp2, tmp3);
92 }
93
94
95 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
96
97
98 LIR_Opr LIR_Assembler::receiverOpr() {
99 return FrameMap::receiver_opr;
100 }
101
102 LIR_Opr LIR_Assembler::osrBufferPointer() {
103 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
104 }
105
106 //--------------fpu register translations-----------------------
107
108
109 address LIR_Assembler::float_constant(float f) {
110 address const_addr = __ float_constant(f);
111 if (const_addr == NULL) {
112 bailout("const section overflow");
113 return __ code()->consts()->start();
114 } else {
115 return const_addr;
116 }
117 }
118
119
120 address LIR_Assembler::double_constant(double d) {
121 address const_addr = __ double_constant(d);
122 if (const_addr == NULL) {
123 bailout("const section overflow");
124 return __ code()->consts()->start();
125 } else {
126 return const_addr;
127 }
128 }
129
130 address LIR_Assembler::int_constant(jlong n) {
131 address const_addr = __ long_constant(n);
132 if (const_addr == NULL) {
133 bailout("const section overflow");
134 return __ code()->consts()->start();
135 } else {
136 return const_addr;
137 }
138 }
139
140 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
141
142 void LIR_Assembler::reset_FPU() { Unimplemented(); }
143
144 void LIR_Assembler::fpop() { Unimplemented(); }
145
146 void LIR_Assembler::fxch(int i) { Unimplemented(); }
147
148 void LIR_Assembler::fld(int i) { Unimplemented(); }
149
150 void LIR_Assembler::ffree(int i) { Unimplemented(); }
151
152 void LIR_Assembler::breakpoint() { Unimplemented(); }
153
154 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
155
156 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
157
158 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
159 //-------------------------------------------
160
161 static Register as_reg(LIR_Opr op) {
162 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
163 }
164
165 static jlong as_long(LIR_Opr data) {
166 jlong result;
167 switch (data->type()) {
168 case T_INT:
169 result = (data->as_jint());
170 break;
171 case T_LONG:
172 result = (data->as_jlong());
173 break;
174 default:
175 ShouldNotReachHere();
176 }
177 return result;
178 }
179
180 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
181 Register base = addr->base()->as_pointer_register();
182 LIR_Opr opr = addr->index();
183 if (opr->is_cpu_register()) {
184 Register index;
185 if (opr->is_single_cpu())
186 index = opr->as_register();
187 else
188 index = opr->as_register_lo();
189 assert(addr->disp() == 0, "must be");
190 switch(opr->type()) {
191 case T_INT:
192 return Address(base, index, Address::sxtw(addr->scale()));
193 case T_LONG:
194 return Address(base, index, Address::lsl(addr->scale()));
195 default:
196 ShouldNotReachHere();
197 }
198 } else {
199 intptr_t addr_offset = intptr_t(addr->disp());
200 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
201 return Address(base, addr_offset, Address::lsl(addr->scale()));
202 else {
203 __ mov(tmp, addr_offset);
204 return Address(base, tmp, Address::lsl(addr->scale()));
205 }
206 }
207 return Address();
208 }
209
210 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
211 ShouldNotReachHere();
212 return Address();
213 }
214
215 Address LIR_Assembler::as_Address(LIR_Address* addr) {
216 return as_Address(addr, rscratch1);
217 }
218
219 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
220 return as_Address(addr, rscratch1); // Ouch
221 // FIXME: This needs to be much more clever. See x86.
222 }
223
224
225 void LIR_Assembler::osr_entry() {
226 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
227 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
228 ValueStack* entry_state = osr_entry->state();
229 int number_of_locks = entry_state->locks_size();
230
231 // we jump here if osr happens with the interpreter
232 // state set up to continue at the beginning of the
233 // loop that triggered osr - in particular, we have
234 // the following registers setup:
235 //
236 // r2: osr buffer
237 //
238
239 // build frame
240 ciMethod* m = compilation()->method();
241 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
242
243 // OSR buffer is
244 //
245 // locals[nlocals-1..0]
246 // monitors[0..number_of_locks]
247 //
248 // locals is a direct copy of the interpreter frame so in the osr buffer
249 // so first slot in the local array is the last local from the interpreter
250 // and last slot is local[0] (receiver) from the interpreter
251 //
252 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
253 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
254 // in the interpreter frame (the method lock if a sync method)
255
256 // Initialize monitors in the compiled activation.
257 // r2: pointer to osr buffer
258 //
259 // All other registers are dead at this point and the locals will be
260 // copied into place by code emitted in the IR.
261
262 Register OSR_buf = osrBufferPointer()->as_pointer_register();
263 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
264 int monitor_offset = BytesPerWord * method()->max_locals() +
265 (2 * BytesPerWord) * (number_of_locks - 1);
266 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
267 // the OSR buffer using 2 word entries: first the lock and then
268 // the oop.
269 for (int i = 0; i < number_of_locks; i++) {
270 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
271 #ifdef ASSERT
272 // verify the interpreter's monitor has a non-null object
273 {
274 Label L;
275 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
276 __ cbnz(rscratch1, L);
277 __ stop("locked object is NULL");
278 __ bind(L);
279 }
280 #endif
281 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
282 __ str(r19, frame_map()->address_for_monitor_lock(i));
283 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
284 __ str(r19, frame_map()->address_for_monitor_object(i));
285 }
286 }
287 }
288
289
290 // inline cache check; done before the frame is built.
291 int LIR_Assembler::check_icache() {
292 Register receiver = FrameMap::receiver_opr->as_register();
293 Register ic_klass = IC_Klass;
294 int start_offset = __ offset();
295 __ inline_cache_check(receiver, ic_klass);
296
297 // if icache check fails, then jump to runtime routine
298 // Note: RECEIVER must still contain the receiver!
299 Label dont;
300 __ br(Assembler::EQ, dont);
301 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
302
303 // We align the verified entry point unless the method body
304 // (including its inline cache check) will fit in a single 64-byte
305 // icache line.
306 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
307 // force alignment after the cache check.
308 __ align(CodeEntryAlignment);
309 }
310
311 __ bind(dont);
312 return start_offset;
313 }
314
315
316 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
317 if (o == NULL) {
318 __ mov(reg, zr);
319 } else {
320 __ movoop(reg, o, /*immediate*/true);
321 }
322 }
323
324 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
325 address target = NULL;
326 relocInfo::relocType reloc_type = relocInfo::none;
327
328 switch (patching_id(info)) {
329 case PatchingStub::access_field_id:
330 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
331 reloc_type = relocInfo::section_word_type;
332 break;
333 case PatchingStub::load_klass_id:
334 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
335 reloc_type = relocInfo::metadata_type;
336 break;
337 case PatchingStub::load_mirror_id:
338 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
339 reloc_type = relocInfo::oop_type;
340 break;
341 case PatchingStub::load_appendix_id:
342 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
343 reloc_type = relocInfo::oop_type;
344 break;
345 default: ShouldNotReachHere();
346 }
347
348 __ far_call(RuntimeAddress(target));
349 add_call_info_here(info);
350 }
351
352 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
353 deoptimize_trap(info);
354 }
355
356
357 // This specifies the rsp decrement needed to build the frame
358 int LIR_Assembler::initial_frame_size_in_bytes() const {
359 // if rounding, must let FrameMap know!
360
361 // The frame_map records size in slots (32bit word)
362
363 // subtract two words to account for return address and link
364 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
365 }
366
367
368 int LIR_Assembler::emit_exception_handler() {
369 // if the last instruction is a call (typically to do a throw which
370 // is coming at the end after block reordering) the return address
371 // must still point into the code area in order to avoid assertion
372 // failures when searching for the corresponding bci => add a nop
373 // (was bug 5/14/1999 - gri)
374 __ nop();
375
376 // generate code for exception handler
377 address handler_base = __ start_a_stub(exception_handler_size);
378 if (handler_base == NULL) {
379 // not enough space left for the handler
380 bailout("exception handler overflow");
381 return -1;
382 }
383
384 int offset = code_offset();
385
386 // the exception oop and pc are in r0, and r3
387 // no other registers need to be preserved, so invalidate them
388 __ invalidate_registers(false, true, true, false, true, true);
389
390 // check that there is really an exception
391 __ verify_not_null_oop(r0);
392
393 // search an exception handler (r0: exception oop, r3: throwing pc)
394 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here();
395 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
396 __ end_a_stub();
397
398 return offset;
399 }
400
401
402 // Emit the code to remove the frame from the stack in the exception
403 // unwind path.
404 int LIR_Assembler::emit_unwind_handler() {
405 #ifndef PRODUCT
406 if (CommentedAssembly) {
407 _masm->block_comment("Unwind handler");
408 }
409 #endif
410
411 int offset = code_offset();
412
413 // Fetch the exception from TLS and clear out exception related thread state
414 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
415 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
417
418 __ bind(_unwind_handler_entry);
419 __ verify_not_null_oop(r0);
420 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
421 __ mov(r19, r0); // Preserve the exception
422 }
423
424 // Preform needed unlocking
425 MonitorExitStub* stub = NULL;
426 if (method()->is_synchronized()) {
427 monitor_address(0, FrameMap::r0_opr);
428 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
429 __ unlock_object(r5, r4, r0, *stub->entry());
430 __ bind(*stub->continuation());
431 }
432
433 if (compilation()->env()->dtrace_method_probes()) {
434 __ call_Unimplemented();
435 #if 0
436 __ movptr(Address(rsp, 0), rax);
437 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
438 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
439 #endif
440 }
441
442 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
443 __ mov(r0, r19); // Restore the exception
444 }
445
446 // remove the activation and dispatch to the unwind handler
447 __ block_comment("remove_frame and dispatch to the unwind handler");
448 __ remove_frame(initial_frame_size_in_bytes());
449 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
450
451 // Emit the slow path assembly
452 if (stub != NULL) {
453 stub->emit_code(this);
454 }
455
456 return offset;
457 }
458
459
460 int LIR_Assembler::emit_deopt_handler() {
461 // if the last instruction is a call (typically to do a throw which
462 // is coming at the end after block reordering) the return address
463 // must still point into the code area in order to avoid assertion
464 // failures when searching for the corresponding bci => add a nop
465 // (was bug 5/14/1999 - gri)
466 __ nop();
467
468 // generate code for exception handler
469 address handler_base = __ start_a_stub(deopt_handler_size);
470 if (handler_base == NULL) {
471 // not enough space left for the handler
472 bailout("deopt handler overflow");
473 return -1;
474 }
475
476 int offset = code_offset();
477
478 __ adr(lr, pc());
479 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
480 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
481 __ end_a_stub();
482
483 return offset;
484 }
485
486
487 // This is the fast version of java.lang.String.compare; it has not
488 // OSR-entry and therefore, we generate a slow version for OSR's
489 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
490 __ mov(r2, (address)__FUNCTION__);
491 __ call_Unimplemented();
492 }
493
494
495 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
496 _masm->code_section()->relocate(adr, relocInfo::poll_type);
497 int pc_offset = code_offset();
498 flush_debug_info(pc_offset);
499 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
500 if (info->exception_handlers() != NULL) {
501 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
502 }
503 }
504
505 // Rather than take a segfault when the polling page is protected,
506 // explicitly check for a safepoint in progress and if there is one,
507 // fake a call to the handler as if a segfault had been caught.
508 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
509 __ mov(rscratch1, SafepointSynchronize::address_of_state());
510 __ ldrb(rscratch1, Address(rscratch1));
511 Label nope, poll;
512 __ cbz(rscratch1, nope);
513 __ block_comment("safepoint");
514 __ enter();
515 __ push(0x3, sp); // r0 & r1
516 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
517 __ adr(r0, poll);
518 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
519 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
520 __ blrt(rscratch1, 1, 0, 1);
521 __ maybe_isb();
522 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
523 __ mov(rscratch1, r0);
524 __ pop(0x3, sp); // r0 & r1
525 __ leave();
526 __ br(rscratch1);
527 address polling_page(os::get_polling_page());
528 assert(os::is_poll_address(polling_page), "should be");
529 unsigned long off;
530 __ adrp(rscratch1, Address(polling_page, rtype), off);
531 __ bind(poll);
532 if (info)
533 add_debug_info_for_branch(info); // This isn't just debug info:
534 // it's the oop map
535 else
536 __ code_section()->relocate(pc(), rtype);
537 __ ldrw(zr, Address(rscratch1, off));
538 __ bind(nope);
539 }
540
541 void LIR_Assembler::return_op(LIR_Opr result) {
542 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
543 // Pop the stack before the safepoint code
544 __ remove_frame(initial_frame_size_in_bytes());
545 address polling_page(os::get_polling_page());
546 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
547 __ ret(lr);
548 }
549
550 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
551 address polling_page(os::get_polling_page());
552 guarantee(info != NULL, "Shouldn't be NULL");
553 assert(os::is_poll_address(polling_page), "should be");
554 unsigned long off;
555 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
556 assert(off == 0, "must be");
557 add_debug_info_for_branch(info); // This isn't just debug info:
558 // it's the oop map
559 __ read_polling_page(rscratch1, relocInfo::poll_type);
560 return __ offset();
561 }
562
563
564 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
565 if (from_reg == r31_sp)
566 from_reg = sp;
567 if (to_reg == r31_sp)
568 to_reg = sp;
569 __ mov(to_reg, from_reg);
570 }
571
572 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
573
574
575 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
576 assert(src->is_constant(), "should not call otherwise");
577 assert(dest->is_register(), "should not call otherwise");
578 LIR_Const* c = src->as_constant_ptr();
579
580 switch (c->type()) {
581 case T_INT: {
582 assert(patch_code == lir_patch_none, "no patching handled here");
583 __ movw(dest->as_register(), c->as_jint());
584 break;
585 }
586
587 case T_ADDRESS: {
588 assert(patch_code == lir_patch_none, "no patching handled here");
589 __ mov(dest->as_register(), c->as_jint());
590 break;
591 }
592
593 case T_LONG: {
594 assert(patch_code == lir_patch_none, "no patching handled here");
595 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
596 break;
597 }
598
599 case T_OBJECT: {
600 if (patch_code == lir_patch_none) {
601 jobject2reg(c->as_jobject(), dest->as_register());
602 } else {
603 jobject2reg_with_patching(dest->as_register(), info);
604 }
605 break;
606 }
607
608 case T_METADATA: {
609 if (patch_code != lir_patch_none) {
610 klass2reg_with_patching(dest->as_register(), info);
611 } else {
612 __ mov_metadata(dest->as_register(), c->as_metadata());
613 }
614 break;
615 }
616
617 case T_FLOAT: {
618 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
619 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
620 } else {
621 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
622 __ ldrs(dest->as_float_reg(), Address(rscratch1));
623 }
624 break;
625 }
626
627 case T_DOUBLE: {
628 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
629 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
630 } else {
631 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
632 __ ldrd(dest->as_double_reg(), Address(rscratch1));
633 }
634 break;
635 }
636
637 default:
638 ShouldNotReachHere();
639 }
640 }
641
642 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
643 LIR_Const* c = src->as_constant_ptr();
644 switch (c->type()) {
645 case T_OBJECT:
646 {
647 if (! c->as_jobject())
648 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
649 else {
650 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
651 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
652 }
653 }
654 break;
655 case T_ADDRESS:
656 {
657 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
658 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
659 }
660 case T_INT:
661 case T_FLOAT:
662 {
663 Register reg = zr;
664 if (c->as_jint_bits() == 0)
665 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
666 else {
667 __ movw(rscratch1, c->as_jint_bits());
668 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
669 }
670 }
671 break;
672 case T_LONG:
673 case T_DOUBLE:
674 {
675 Register reg = zr;
676 if (c->as_jlong_bits() == 0)
677 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
678 lo_word_offset_in_bytes));
679 else {
680 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
681 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
682 lo_word_offset_in_bytes));
683 }
684 }
685 break;
686 default:
687 ShouldNotReachHere();
688 }
689 }
690
691 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
692 assert(src->is_constant(), "should not call otherwise");
693 LIR_Const* c = src->as_constant_ptr();
694 LIR_Address* to_addr = dest->as_address_ptr();
695
696 void (Assembler::* insn)(Register Rt, const Address &adr);
697
698 switch (type) {
699 case T_ADDRESS:
700 assert(c->as_jint() == 0, "should be");
701 insn = &Assembler::str;
702 break;
703 case T_LONG:
704 assert(c->as_jlong() == 0, "should be");
705 insn = &Assembler::str;
706 break;
707 case T_INT:
708 assert(c->as_jint() == 0, "should be");
709 insn = &Assembler::strw;
710 break;
711 case T_OBJECT:
712 case T_ARRAY:
713 assert(c->as_jobject() == 0, "should be");
714 if (UseCompressedOops && !wide) {
715 insn = &Assembler::strw;
716 } else {
717 insn = &Assembler::str;
718 }
719 break;
720 case T_CHAR:
721 case T_SHORT:
722 assert(c->as_jint() == 0, "should be");
723 insn = &Assembler::strh;
724 break;
725 case T_BOOLEAN:
726 case T_BYTE:
727 assert(c->as_jint() == 0, "should be");
728 insn = &Assembler::strb;
729 break;
730 default:
731 ShouldNotReachHere();
732 }
733
734 if (info) add_debug_info_for_null_check_here(info);
735 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
736 }
737
738 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
739 assert(src->is_register(), "should not call otherwise");
740 assert(dest->is_register(), "should not call otherwise");
741
742 // move between cpu-registers
743 if (dest->is_single_cpu()) {
744 if (src->type() == T_LONG) {
745 // Can do LONG -> OBJECT
746 move_regs(src->as_register_lo(), dest->as_register());
747 return;
748 }
749 assert(src->is_single_cpu(), "must match");
750 if (src->type() == T_OBJECT) {
751 __ verify_oop(src->as_register());
752 }
753 move_regs(src->as_register(), dest->as_register());
754
755 } else if (dest->is_double_cpu()) {
756 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
757 // Surprising to me but we can see move of a long to t_object
758 __ verify_oop(src->as_register());
759 move_regs(src->as_register(), dest->as_register_lo());
760 return;
761 }
762 assert(src->is_double_cpu(), "must match");
763 Register f_lo = src->as_register_lo();
764 Register f_hi = src->as_register_hi();
765 Register t_lo = dest->as_register_lo();
766 Register t_hi = dest->as_register_hi();
767 assert(f_hi == f_lo, "must be same");
768 assert(t_hi == t_lo, "must be same");
769 move_regs(f_lo, t_lo);
770
771 } else if (dest->is_single_fpu()) {
772 __ fmovs(dest->as_float_reg(), src->as_float_reg());
773
774 } else if (dest->is_double_fpu()) {
775 __ fmovd(dest->as_double_reg(), src->as_double_reg());
776
777 } else {
778 ShouldNotReachHere();
779 }
780 }
781
782 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
783 if (src->is_single_cpu()) {
784 if (type == T_ARRAY || type == T_OBJECT) {
785 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
786 __ verify_oop(src->as_register());
787 } else if (type == T_METADATA || type == T_DOUBLE) {
788 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
789 } else {
790 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
791 }
792
793 } else if (src->is_double_cpu()) {
794 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
795 __ str(src->as_register_lo(), dest_addr_LO);
796
797 } else if (src->is_single_fpu()) {
798 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
799 __ strs(src->as_float_reg(), dest_addr);
800
801 } else if (src->is_double_fpu()) {
802 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
803 __ strd(src->as_double_reg(), dest_addr);
804
805 } else {
806 ShouldNotReachHere();
807 }
808
809 }
810
811
812 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
813 LIR_Address* to_addr = dest->as_address_ptr();
814 PatchingStub* patch = NULL;
815 Register compressed_src = rscratch1;
816
817 if (patch_code != lir_patch_none) {
818 deoptimize_trap(info);
819 return;
820 }
821
822 if (type == T_ARRAY || type == T_OBJECT) {
823 __ verify_oop(src->as_register());
824
825 if (UseCompressedOops && !wide) {
826 __ encode_heap_oop(compressed_src, src->as_register());
827 } else {
828 compressed_src = src->as_register();
829 }
830 }
831
832 int null_check_here = code_offset();
833 switch (type) {
834 case T_FLOAT: {
835 __ strs(src->as_float_reg(), as_Address(to_addr));
836 break;
837 }
838
839 case T_DOUBLE: {
840 __ strd(src->as_double_reg(), as_Address(to_addr));
841 break;
842 }
843
844 case T_ARRAY: // fall through
845 case T_OBJECT: // fall through
846 if (UseCompressedOops && !wide) {
847 __ strw(compressed_src, as_Address(to_addr, rscratch2));
848 } else {
849 __ str(compressed_src, as_Address(to_addr));
850 }
851 break;
852 case T_METADATA:
853 // We get here to store a method pointer to the stack to pass to
854 // a dtrace runtime call. This can't work on 64 bit with
855 // compressed klass ptrs: T_METADATA can be a compressed klass
856 // ptr or a 64 bit method pointer.
857 ShouldNotReachHere();
858 __ str(src->as_register(), as_Address(to_addr));
859 break;
860 case T_ADDRESS:
861 __ str(src->as_register(), as_Address(to_addr));
862 break;
863 case T_INT:
864 __ strw(src->as_register(), as_Address(to_addr));
865 break;
866
867 case T_LONG: {
868 __ str(src->as_register_lo(), as_Address_lo(to_addr));
869 break;
870 }
871
872 case T_BYTE: // fall through
873 case T_BOOLEAN: {
874 __ strb(src->as_register(), as_Address(to_addr));
875 break;
876 }
877
878 case T_CHAR: // fall through
879 case T_SHORT:
880 __ strh(src->as_register(), as_Address(to_addr));
881 break;
882
883 default:
884 ShouldNotReachHere();
885 }
886 if (info != NULL) {
887 add_debug_info_for_null_check(null_check_here, info);
888 }
889 }
890
891
892 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
893 assert(src->is_stack(), "should not call otherwise");
894 assert(dest->is_register(), "should not call otherwise");
895
896 if (dest->is_single_cpu()) {
897 if (type == T_ARRAY || type == T_OBJECT) {
898 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
899 __ verify_oop(dest->as_register());
900 } else if (type == T_METADATA) {
901 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
902 } else {
903 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
904 }
905
906 } else if (dest->is_double_cpu()) {
907 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
908 __ ldr(dest->as_register_lo(), src_addr_LO);
909
910 } else if (dest->is_single_fpu()) {
911 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
912 __ ldrs(dest->as_float_reg(), src_addr);
913
914 } else if (dest->is_double_fpu()) {
915 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
916 __ ldrd(dest->as_double_reg(), src_addr);
917
918 } else {
919 ShouldNotReachHere();
920 }
921 }
922
923
924 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
925 address target = NULL;
926 relocInfo::relocType reloc_type = relocInfo::none;
927
928 switch (patching_id(info)) {
929 case PatchingStub::access_field_id:
930 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
931 reloc_type = relocInfo::section_word_type;
932 break;
933 case PatchingStub::load_klass_id:
934 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
935 reloc_type = relocInfo::metadata_type;
936 break;
937 case PatchingStub::load_mirror_id:
938 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
939 reloc_type = relocInfo::oop_type;
940 break;
941 case PatchingStub::load_appendix_id:
942 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
943 reloc_type = relocInfo::oop_type;
944 break;
945 default: ShouldNotReachHere();
946 }
947
948 __ far_call(RuntimeAddress(target));
949 add_call_info_here(info);
950 }
951
952 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
953
954 LIR_Opr temp;
955 if (type == T_LONG || type == T_DOUBLE)
956 temp = FrameMap::rscratch1_long_opr;
957 else
958 temp = FrameMap::rscratch1_opr;
959
960 stack2reg(src, temp, src->type());
961 reg2stack(temp, dest, dest->type(), false);
962 }
963
964
965 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
966 LIR_Address* addr = src->as_address_ptr();
967 LIR_Address* from_addr = src->as_address_ptr();
968
969 if (addr->base()->type() == T_OBJECT) {
970 __ verify_oop(addr->base()->as_pointer_register());
971 }
972
973 if (patch_code != lir_patch_none) {
974 deoptimize_trap(info);
975 return;
976 }
977
978 if (info != NULL) {
979 add_debug_info_for_null_check_here(info);
980 }
981 int null_check_here = code_offset();
982 switch (type) {
983 case T_FLOAT: {
984 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
985 break;
986 }
987
988 case T_DOUBLE: {
989 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
990 break;
991 }
992
993 case T_ARRAY: // fall through
994 case T_OBJECT: // fall through
995 if (UseCompressedOops && !wide) {
996 __ ldrw(dest->as_register(), as_Address(from_addr));
997 } else {
998 __ ldr(dest->as_register(), as_Address(from_addr));
999 }
1000 break;
1001 case T_METADATA:
1002 // We get here to store a method pointer to the stack to pass to
1003 // a dtrace runtime call. This can't work on 64 bit with
1004 // compressed klass ptrs: T_METADATA can be a compressed klass
1005 // ptr or a 64 bit method pointer.
1006 ShouldNotReachHere();
1007 __ ldr(dest->as_register(), as_Address(from_addr));
1008 break;
1009 case T_ADDRESS:
1010 // FIXME: OMG this is a horrible kludge. Any offset from an
1011 // address that matches klass_offset_in_bytes() will be loaded
1012 // as a word, not a long.
1013 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1014 __ ldrw(dest->as_register(), as_Address(from_addr));
1015 } else {
1016 __ ldr(dest->as_register(), as_Address(from_addr));
1017 }
1018 break;
1019 case T_INT:
1020 __ ldrw(dest->as_register(), as_Address(from_addr));
1021 break;
1022
1023 case T_LONG: {
1024 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1025 break;
1026 }
1027
1028 case T_BYTE:
1029 __ ldrsb(dest->as_register(), as_Address(from_addr));
1030 break;
1031 case T_BOOLEAN: {
1032 __ ldrb(dest->as_register(), as_Address(from_addr));
1033 break;
1034 }
1035
1036 case T_CHAR:
1037 __ ldrh(dest->as_register(), as_Address(from_addr));
1038 break;
1039 case T_SHORT:
1040 __ ldrsh(dest->as_register(), as_Address(from_addr));
1041 break;
1042
1043 default:
1044 ShouldNotReachHere();
1045 }
1046
1047 if (type == T_ARRAY || type == T_OBJECT) {
1048 if (UseCompressedOops && !wide) {
1049 __ decode_heap_oop(dest->as_register());
1050 }
1051 __ verify_oop(dest->as_register());
1052 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1053 if (UseCompressedClassPointers) {
1054 __ decode_klass_not_null(dest->as_register());
1055 }
1056 }
1057 }
1058
1059
1060 int LIR_Assembler::array_element_size(BasicType type) const {
1061 int elem_size = type2aelembytes(type);
1062 return exact_log2(elem_size);
1063 }
1064
1065 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1066 Register Rdividend = op->in_opr1()->as_register();
1067 Register Rdivisor = op->in_opr2()->as_register();
1068 Register Rscratch = op->in_opr3()->as_register();
1069 Register Rresult = op->result_opr()->as_register();
1070 int divisor = -1;
1071
1072 /*
1073 TODO: For some reason, using the Rscratch that gets passed in is
1074 not possible because the register allocator does not see the tmp reg
1075 as used, and assignes it the same register as Rdividend. We use rscratch1
1076 instead.
1077
1078 assert(Rdividend != Rscratch, "");
1079 assert(Rdivisor != Rscratch, "");
1080 */
1081
1082 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1083 // convert division by a power of two into some shifts and logical operations
1084 }
1085
1086 if (op->code() == lir_irem) {
1087 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1088 } else if (op->code() == lir_idiv) {
1089 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1090 } else
1091 ShouldNotReachHere();
1092 }
1093
1094 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1095 #ifdef ASSERT
1096 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1097 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1098 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1099 #endif
1100
1101 if (op->cond() == lir_cond_always) {
1102 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1103 __ b(*(op->label()));
1104 } else {
1105 Assembler::Condition acond;
1106 if (op->code() == lir_cond_float_branch) {
1107 bool is_unordered = (op->ublock() == op->block());
1108 // Assembler::EQ does not permit unordered branches, so we add
1109 // another branch here. Likewise, Assembler::NE does not permit
1110 // ordered branches.
1111 if (is_unordered && op->cond() == lir_cond_equal
1112 || !is_unordered && op->cond() == lir_cond_notEqual)
1113 __ br(Assembler::VS, *(op->ublock()->label()));
1114 switch(op->cond()) {
1115 case lir_cond_equal: acond = Assembler::EQ; break;
1116 case lir_cond_notEqual: acond = Assembler::NE; break;
1117 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1118 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1119 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1120 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1121 default: ShouldNotReachHere();
1122 }
1123 } else {
1124 switch (op->cond()) {
1125 case lir_cond_equal: acond = Assembler::EQ; break;
1126 case lir_cond_notEqual: acond = Assembler::NE; break;
1127 case lir_cond_less: acond = Assembler::LT; break;
1128 case lir_cond_lessEqual: acond = Assembler::LE; break;
1129 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1130 case lir_cond_greater: acond = Assembler::GT; break;
1131 case lir_cond_belowEqual: acond = Assembler::LS; break;
1132 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1133 default: ShouldNotReachHere();
1134 }
1135 }
1136 __ br(acond,*(op->label()));
1137 }
1138 }
1139
1140
1141
1142 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1143 LIR_Opr src = op->in_opr();
1144 LIR_Opr dest = op->result_opr();
1145
1146 switch (op->bytecode()) {
1147 case Bytecodes::_i2f:
1148 {
1149 __ scvtfws(dest->as_float_reg(), src->as_register());
1150 break;
1151 }
1152 case Bytecodes::_i2d:
1153 {
1154 __ scvtfwd(dest->as_double_reg(), src->as_register());
1155 break;
1156 }
1157 case Bytecodes::_l2d:
1158 {
1159 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1160 break;
1161 }
1162 case Bytecodes::_l2f:
1163 {
1164 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1165 break;
1166 }
1167 case Bytecodes::_f2d:
1168 {
1169 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1170 break;
1171 }
1172 case Bytecodes::_d2f:
1173 {
1174 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1175 break;
1176 }
1177 case Bytecodes::_i2c:
1178 {
1179 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1180 break;
1181 }
1182 case Bytecodes::_i2l:
1183 {
1184 __ sxtw(dest->as_register_lo(), src->as_register());
1185 break;
1186 }
1187 case Bytecodes::_i2s:
1188 {
1189 __ sxth(dest->as_register(), src->as_register());
1190 break;
1191 }
1192 case Bytecodes::_i2b:
1193 {
1194 __ sxtb(dest->as_register(), src->as_register());
1195 break;
1196 }
1197 case Bytecodes::_l2i:
1198 {
1199 _masm->block_comment("FIXME: This could be a no-op");
1200 __ uxtw(dest->as_register(), src->as_register_lo());
1201 break;
1202 }
1203 case Bytecodes::_d2l:
1204 {
1205 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1206 break;
1207 }
1208 case Bytecodes::_f2i:
1209 {
1210 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1211 break;
1212 }
1213 case Bytecodes::_f2l:
1214 {
1215 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1216 break;
1217 }
1218 case Bytecodes::_d2i:
1219 {
1220 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1221 break;
1222 }
1223 default: ShouldNotReachHere();
1224 }
1225 }
1226
1227 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1228 if (op->init_check()) {
1229 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1230 InstanceKlass::init_state_offset()));
1231 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1232 add_debug_info_for_null_check_here(op->stub()->info());
1233 __ br(Assembler::NE, *op->stub()->entry());
1234 }
1235 __ allocate_object(op->obj()->as_register(),
1236 op->tmp1()->as_register(),
1237 op->tmp2()->as_register(),
1238 op->header_size(),
1239 op->object_size(),
1240 op->klass()->as_register(),
1241 *op->stub()->entry());
1242 __ bind(*op->stub()->continuation());
1243 }
1244
1245 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1246 Register len = op->len()->as_register();
1247 __ uxtw(len, len);
1248
1249 if (UseSlowPath ||
1250 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1251 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1252 __ b(*op->stub()->entry());
1253 } else {
1254 Register tmp1 = op->tmp1()->as_register();
1255 Register tmp2 = op->tmp2()->as_register();
1256 Register tmp3 = op->tmp3()->as_register();
1257 if (len == tmp1) {
1258 tmp1 = tmp3;
1259 } else if (len == tmp2) {
1260 tmp2 = tmp3;
1261 } else if (len == tmp3) {
1262 // everything is ok
1263 } else {
1264 __ mov(tmp3, len);
1265 }
1266 __ allocate_array(op->obj()->as_register(),
1267 len,
1268 tmp1,
1269 tmp2,
1270 arrayOopDesc::header_size(op->type()),
1271 array_element_size(op->type()),
1272 op->klass()->as_register(),
1273 *op->stub()->entry());
1274 }
1275 __ bind(*op->stub()->continuation());
1276 }
1277
1278 void LIR_Assembler::type_profile_helper(Register mdo,
1279 ciMethodData *md, ciProfileData *data,
1280 Register recv, Label* update_done) {
1281 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1282 Label next_test;
1283 // See if the receiver is receiver[n].
1284 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1285 __ ldr(rscratch1, Address(rscratch2));
1286 __ cmp(recv, rscratch1);
1287 __ br(Assembler::NE, next_test);
1288 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1289 __ addptr(data_addr, DataLayout::counter_increment);
1290 __ b(*update_done);
1291 __ bind(next_test);
1292 }
1293
1294 // Didn't find receiver; find next empty slot and fill it in
1295 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1296 Label next_test;
1297 __ lea(rscratch2,
1298 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1299 Address recv_addr(rscratch2);
1300 __ ldr(rscratch1, recv_addr);
1301 __ cbnz(rscratch1, next_test);
1302 __ str(recv, recv_addr);
1303 __ mov(rscratch1, DataLayout::counter_increment);
1304 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1305 __ str(rscratch1, Address(rscratch2));
1306 __ b(*update_done);
1307 __ bind(next_test);
1308 }
1309 }
1310
1311 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1312 // we always need a stub for the failure case.
1313 CodeStub* stub = op->stub();
1314 Register obj = op->object()->as_register();
1315 Register k_RInfo = op->tmp1()->as_register();
1316 Register klass_RInfo = op->tmp2()->as_register();
1317 Register dst = op->result_opr()->as_register();
1318 ciKlass* k = op->klass();
1319 Register Rtmp1 = noreg;
1320
1321 // check if it needs to be profiled
1322 ciMethodData* md;
1323 ciProfileData* data;
1324
1325 if (op->should_profile()) {
1326 ciMethod* method = op->profiled_method();
1327 assert(method != NULL, "Should have method");
1328 int bci = op->profiled_bci();
1329 md = method->method_data_or_null();
1330 assert(md != NULL, "Sanity");
1331 data = md->bci_to_data(bci);
1332 assert(data != NULL, "need data for type check");
1333 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1334 }
1335 Label profile_cast_success, profile_cast_failure;
1336 Label *success_target = op->should_profile() ? &profile_cast_success : success;
1337 Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1338
1339 if (obj == k_RInfo) {
1340 k_RInfo = dst;
1341 } else if (obj == klass_RInfo) {
1342 klass_RInfo = dst;
1343 }
1344 if (k->is_loaded() && !UseCompressedClassPointers) {
1345 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1346 } else {
1347 Rtmp1 = op->tmp3()->as_register();
1348 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1349 }
1350
1351 assert_different_registers(obj, k_RInfo, klass_RInfo);
1352
1353 if (op->should_profile()) {
1354 Label not_null;
1355 __ cbnz(obj, not_null);
1356 // Object is null; update MDO and exit
1357 Register mdo = klass_RInfo;
1358 __ mov_metadata(mdo, md->constant_encoding());
1359 Address data_addr
1360 = __ form_address(rscratch2, mdo,
1361 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1362 LogBytesPerWord);
1363 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1364 __ ldr(rscratch1, data_addr);
1365 __ orr(rscratch1, rscratch1, header_bits);
1366 __ str(rscratch1, data_addr);
1367 __ b(*obj_is_null);
1368 __ bind(not_null);
1369 } else {
1370 __ cbz(obj, *obj_is_null);
1371 }
1372
1373 if (!k->is_loaded()) {
1374 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1375 } else {
1376 __ mov_metadata(k_RInfo, k->constant_encoding());
1377 }
1378 __ verify_oop(obj);
1379
1380 if (op->fast_check()) {
1381 // get object class
1382 // not a safepoint as obj null check happens earlier
1383 __ load_klass(rscratch1, obj);
1384 __ cmp( rscratch1, k_RInfo);
1385
1386 __ br(Assembler::NE, *failure_target);
1387 // successful cast, fall through to profile or jump
1388 } else {
1389 // get object class
1390 // not a safepoint as obj null check happens earlier
1391 __ load_klass(klass_RInfo, obj);
1392 if (k->is_loaded()) {
1393 // See if we get an immediate positive hit
1394 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1395 __ cmp(k_RInfo, rscratch1);
1396 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1397 __ br(Assembler::NE, *failure_target);
1398 // successful cast, fall through to profile or jump
1399 } else {
1400 // See if we get an immediate positive hit
1401 __ br(Assembler::EQ, *success_target);
1402 // check for self
1403 __ cmp(klass_RInfo, k_RInfo);
1404 __ br(Assembler::EQ, *success_target);
1405
1406 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1407 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1408 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1409 // result is a boolean
1410 __ cbzw(klass_RInfo, *failure_target);
1411 // successful cast, fall through to profile or jump
1412 }
1413 } else {
1414 // perform the fast part of the checking logic
1415 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1416 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1417 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1418 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1419 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1420 // result is a boolean
1421 __ cbz(k_RInfo, *failure_target);
1422 // successful cast, fall through to profile or jump
1423 }
1424 }
1425 if (op->should_profile()) {
1426 Register mdo = klass_RInfo, recv = k_RInfo;
1427 __ bind(profile_cast_success);
1428 __ mov_metadata(mdo, md->constant_encoding());
1429 __ load_klass(recv, obj);
1430 Label update_done;
1431 type_profile_helper(mdo, md, data, recv, success);
1432 __ b(*success);
1433
1434 __ bind(profile_cast_failure);
1435 __ mov_metadata(mdo, md->constant_encoding());
1436 Address counter_addr
1437 = __ form_address(rscratch2, mdo,
1438 md->byte_offset_of_slot(data, CounterData::count_offset()),
1439 LogBytesPerWord);
1440 __ ldr(rscratch1, counter_addr);
1441 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1442 __ str(rscratch1, counter_addr);
1443 __ b(*failure);
1444 }
1445 __ b(*success);
1446 }
1447
1448
1449 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1450 LIR_Code code = op->code();
1451 if (code == lir_store_check) {
1452 Register value = op->object()->as_register();
1453 Register array = op->array()->as_register();
1454 Register k_RInfo = op->tmp1()->as_register();
1455 Register klass_RInfo = op->tmp2()->as_register();
1456 Register Rtmp1 = op->tmp3()->as_register();
1457
1458 CodeStub* stub = op->stub();
1459
1460 // check if it needs to be profiled
1461 ciMethodData* md;
1462 ciProfileData* data;
1463
1464 if (op->should_profile()) {
1465 ciMethod* method = op->profiled_method();
1466 assert(method != NULL, "Should have method");
1467 int bci = op->profiled_bci();
1468 md = method->method_data_or_null();
1469 assert(md != NULL, "Sanity");
1470 data = md->bci_to_data(bci);
1471 assert(data != NULL, "need data for type check");
1472 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1473 }
1474 Label profile_cast_success, profile_cast_failure, done;
1475 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1476 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1477
1478 if (op->should_profile()) {
1479 Label not_null;
1480 __ cbnz(value, not_null);
1481 // Object is null; update MDO and exit
1482 Register mdo = klass_RInfo;
1483 __ mov_metadata(mdo, md->constant_encoding());
1484 Address data_addr
1485 = __ form_address(rscratch2, mdo,
1486 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1487 LogBytesPerInt);
1488 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1489 __ ldrw(rscratch1, data_addr);
1490 __ orrw(rscratch1, rscratch1, header_bits);
1491 __ strw(rscratch1, data_addr);
1492 __ b(done);
1493 __ bind(not_null);
1494 } else {
1495 __ cbz(value, done);
1496 }
1497
1498 add_debug_info_for_null_check_here(op->info_for_exception());
1499 __ load_klass(k_RInfo, array);
1500 __ load_klass(klass_RInfo, value);
1501
1502 // get instance klass (it's already uncompressed)
1503 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1504 // perform the fast part of the checking logic
1505 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1506 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1507 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1508 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1509 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1510 // result is a boolean
1511 __ cbzw(k_RInfo, *failure_target);
1512 // fall through to the success case
1513
1514 if (op->should_profile()) {
1515 Register mdo = klass_RInfo, recv = k_RInfo;
1516 __ bind(profile_cast_success);
1517 __ mov_metadata(mdo, md->constant_encoding());
1518 __ load_klass(recv, value);
1519 Label update_done;
1520 type_profile_helper(mdo, md, data, recv, &done);
1521 __ b(done);
1522
1523 __ bind(profile_cast_failure);
1524 __ mov_metadata(mdo, md->constant_encoding());
1525 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1526 __ lea(rscratch2, counter_addr);
1527 __ ldr(rscratch1, Address(rscratch2));
1528 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1529 __ str(rscratch1, Address(rscratch2));
1530 __ b(*stub->entry());
1531 }
1532
1533 __ bind(done);
1534 } else if (code == lir_checkcast) {
1535 Register obj = op->object()->as_register();
1536 Register dst = op->result_opr()->as_register();
1537 Label success;
1538 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1539 __ bind(success);
1540 if (dst != obj) {
1541 __ mov(dst, obj);
1542 }
1543 } else if (code == lir_instanceof) {
1544 Register obj = op->object()->as_register();
1545 Register dst = op->result_opr()->as_register();
1546 Label success, failure, done;
1547 emit_typecheck_helper(op, &success, &failure, &failure);
1548 __ bind(failure);
1549 __ mov(dst, zr);
1550 __ b(done);
1551 __ bind(success);
1552 __ mov(dst, 1);
1553 __ bind(done);
1554 } else {
1555 ShouldNotReachHere();
1556 }
1557 }
1558
1559 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1560 Label retry_load, nope;
1561 // flush and load exclusive from the memory location
1562 // and fail if it is not what we expect
1563 __ bind(retry_load);
1564 __ ldaxrw(rscratch1, addr);
1565 __ cmpw(rscratch1, cmpval);
1566 __ cset(rscratch1, Assembler::NE);
1567 __ br(Assembler::NE, nope);
1568 // if we store+flush with no intervening write rscratch1 wil be zero
1569 __ stlxrw(rscratch1, newval, addr);
1570 // retry so we only ever return after a load fails to compare
1571 // ensures we don't return a stale value after a failed write.
1572 __ cbnzw(rscratch1, retry_load);
1573 __ bind(nope);
1574 __ membar(__ AnyAny);
1575 }
1576
1577 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1578 Label retry_load, nope;
1579 // flush and load exclusive from the memory location
1580 // and fail if it is not what we expect
1581 __ bind(retry_load);
1582 __ ldaxr(rscratch1, addr);
1583 __ cmp(rscratch1, cmpval);
1584 __ cset(rscratch1, Assembler::NE);
1585 __ br(Assembler::NE, nope);
1586 // if we store+flush with no intervening write rscratch1 wil be zero
1587 __ stlxr(rscratch1, newval, addr);
1588 // retry so we only ever return after a load fails to compare
1589 // ensures we don't return a stale value after a failed write.
1590 __ cbnz(rscratch1, retry_load);
1591 __ bind(nope);
1592 __ membar(__ AnyAny);
1593 }
1594
1595
1596 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1597 assert(VM_Version::supports_cx8(), "wrong machine");
1598 Register addr = as_reg(op->addr());
1599 Register newval = as_reg(op->new_value());
1600 Register cmpval = as_reg(op->cmp_value());
1601 Label succeed, fail, around;
1602
1603 if (op->code() == lir_cas_obj) {
1604 if (UseCompressedOops) {
1605 Register t1 = op->tmp1()->as_register();
1606 assert(op->tmp1()->is_valid(), "must be");
1607 __ encode_heap_oop(t1, cmpval);
1608 cmpval = t1;
1609 __ encode_heap_oop(rscratch2, newval);
1610 newval = rscratch2;
1611 casw(addr, newval, cmpval);
1612 } else {
1613 casl(addr, newval, cmpval);
1614 }
1615 } else if (op->code() == lir_cas_int) {
1616 casw(addr, newval, cmpval);
1617 } else {
1618 casl(addr, newval, cmpval);
1619 }
1620 }
1621
1622
1623 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1624
1625 Assembler::Condition acond, ncond;
1626 switch (condition) {
1627 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1628 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1629 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1630 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1631 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1632 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1633 case lir_cond_belowEqual: Unimplemented(); break;
1634 case lir_cond_aboveEqual: Unimplemented(); break;
1635 default: ShouldNotReachHere();
1636 }
1637
1638 assert(result->is_single_cpu() || result->is_double_cpu(),
1639 "expect single register for result");
1640 if (opr1->is_constant() && opr2->is_constant()
1641 && opr1->type() == T_INT && opr2->type() == T_INT) {
1642 jint val1 = opr1->as_jint();
1643 jint val2 = opr2->as_jint();
1644 if (val1 == 0 && val2 == 1) {
1645 __ cset(result->as_register(), ncond);
1646 return;
1647 } else if (val1 == 1 && val2 == 0) {
1648 __ cset(result->as_register(), acond);
1649 return;
1650 }
1651 }
1652
1653 if (opr1->is_constant() && opr2->is_constant()
1654 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1655 jlong val1 = opr1->as_jlong();
1656 jlong val2 = opr2->as_jlong();
1657 if (val1 == 0 && val2 == 1) {
1658 __ cset(result->as_register_lo(), ncond);
1659 return;
1660 } else if (val1 == 1 && val2 == 0) {
1661 __ cset(result->as_register_lo(), acond);
1662 return;
1663 }
1664 }
1665
1666 if (opr1->is_stack()) {
1667 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1668 opr1 = FrameMap::rscratch1_opr;
1669 } else if (opr1->is_constant()) {
1670 LIR_Opr tmp
1671 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1672 const2reg(opr1, tmp, lir_patch_none, NULL);
1673 opr1 = tmp;
1674 }
1675
1676 if (opr2->is_stack()) {
1677 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1678 opr2 = FrameMap::rscratch2_opr;
1679 } else if (opr2->is_constant()) {
1680 LIR_Opr tmp
1681 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1682 const2reg(opr2, tmp, lir_patch_none, NULL);
1683 opr2 = tmp;
1684 }
1685
1686 if (result->type() == T_LONG)
1687 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1688 else
1689 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1690 }
1691
1692 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1693 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1694
1695 if (left->is_single_cpu()) {
1696 Register lreg = left->as_register();
1697 Register dreg = as_reg(dest);
1698
1699 if (right->is_single_cpu()) {
1700 // cpu register - cpu register
1701
1702 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1703 "should be");
1704 Register rreg = right->as_register();
1705 switch (code) {
1706 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1707 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1708 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1709 default: ShouldNotReachHere();
1710 }
1711
1712 } else if (right->is_double_cpu()) {
1713 Register rreg = right->as_register_lo();
1714 // single_cpu + double_cpu: can happen with obj+long
1715 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1716 switch (code) {
1717 case lir_add: __ add(dreg, lreg, rreg); break;
1718 case lir_sub: __ sub(dreg, lreg, rreg); break;
1719 default: ShouldNotReachHere();
1720 }
1721 } else if (right->is_constant()) {
1722 // cpu register - constant
1723 jlong c;
1724
1725 // FIXME. This is fugly: we really need to factor all this logic.
1726 switch(right->type()) {
1727 case T_LONG:
1728 c = right->as_constant_ptr()->as_jlong();
1729 break;
1730 case T_INT:
1731 case T_ADDRESS:
1732 c = right->as_constant_ptr()->as_jint();
1733 break;
1734 default:
1735 ShouldNotReachHere();
1736 break;
1737 }
1738
1739 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1740 if (c == 0 && dreg == lreg) {
1741 COMMENT("effective nop elided");
1742 return;
1743 }
1744 switch(left->type()) {
1745 case T_INT:
1746 switch (code) {
1747 case lir_add: __ addw(dreg, lreg, c); break;
1748 case lir_sub: __ subw(dreg, lreg, c); break;
1749 default: ShouldNotReachHere();
1750 }
1751 break;
1752 case T_OBJECT:
1753 case T_ADDRESS:
1754 switch (code) {
1755 case lir_add: __ add(dreg, lreg, c); break;
1756 case lir_sub: __ sub(dreg, lreg, c); break;
1757 default: ShouldNotReachHere();
1758 }
1759 break;
1760 ShouldNotReachHere();
1761 }
1762 } else {
1763 ShouldNotReachHere();
1764 }
1765
1766 } else if (left->is_double_cpu()) {
1767 Register lreg_lo = left->as_register_lo();
1768
1769 if (right->is_double_cpu()) {
1770 // cpu register - cpu register
1771 Register rreg_lo = right->as_register_lo();
1772 switch (code) {
1773 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1774 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1775 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1776 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1777 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1778 default:
1779 ShouldNotReachHere();
1780 }
1781
1782 } else if (right->is_constant()) {
1783 jlong c = right->as_constant_ptr()->as_jlong_bits();
1784 Register dreg = as_reg(dest);
1785 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1786 if (c == 0 && dreg == lreg_lo) {
1787 COMMENT("effective nop elided");
1788 return;
1789 }
1790 switch (code) {
1791 case lir_add: __ add(dreg, lreg_lo, c); break;
1792 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1793 default:
1794 ShouldNotReachHere();
1795 }
1796 } else {
1797 ShouldNotReachHere();
1798 }
1799 } else if (left->is_single_fpu()) {
1800 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1801 switch (code) {
1802 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1803 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1804 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1805 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1806 default:
1807 ShouldNotReachHere();
1808 }
1809 } else if (left->is_double_fpu()) {
1810 if (right->is_double_fpu()) {
1811 // cpu register - cpu register
1812 switch (code) {
1813 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1814 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1815 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1816 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1817 default:
1818 ShouldNotReachHere();
1819 }
1820 } else {
1821 if (right->is_constant()) {
1822 ShouldNotReachHere();
1823 }
1824 ShouldNotReachHere();
1825 }
1826 } else if (left->is_single_stack() || left->is_address()) {
1827 assert(left == dest, "left and dest must be equal");
1828 ShouldNotReachHere();
1829 } else {
1830 ShouldNotReachHere();
1831 }
1832 }
1833
1834 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1835
1836
1837 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1838 switch(code) {
1839 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1840 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1841 default : ShouldNotReachHere();
1842 }
1843 }
1844
1845 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1846
1847 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1848 Register Rleft = left->is_single_cpu() ? left->as_register() :
1849 left->as_register_lo();
1850 if (dst->is_single_cpu()) {
1851 Register Rdst = dst->as_register();
1852 if (right->is_constant()) {
1853 switch (code) {
1854 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1855 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1856 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1857 default: ShouldNotReachHere(); break;
1858 }
1859 } else {
1860 Register Rright = right->is_single_cpu() ? right->as_register() :
1861 right->as_register_lo();
1862 switch (code) {
1863 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1864 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1865 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1866 default: ShouldNotReachHere(); break;
1867 }
1868 }
1869 } else {
1870 Register Rdst = dst->as_register_lo();
1871 if (right->is_constant()) {
1872 switch (code) {
1873 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1874 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1875 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1876 default: ShouldNotReachHere(); break;
1877 }
1878 } else {
1879 Register Rright = right->is_single_cpu() ? right->as_register() :
1880 right->as_register_lo();
1881 switch (code) {
1882 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1883 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1884 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1885 default: ShouldNotReachHere(); break;
1886 }
1887 }
1888 }
1889 }
1890
1891
1892
1893 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1894
1895
1896 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1897 if (opr1->is_constant() && opr2->is_single_cpu()) {
1898 // tableswitch
1899 Register reg = as_reg(opr2);
1900 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1901 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1902 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1903 Register reg1 = as_reg(opr1);
1904 if (opr2->is_single_cpu()) {
1905 // cpu register - cpu register
1906 Register reg2 = opr2->as_register();
1907 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1908 __ cmp(reg1, reg2);
1909 } else {
1910 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1911 __ cmpw(reg1, reg2);
1912 }
1913 return;
1914 }
1915 if (opr2->is_double_cpu()) {
1916 // cpu register - cpu register
1917 Register reg2 = opr2->as_register_lo();
1918 __ cmp(reg1, reg2);
1919 return;
1920 }
1921
1922 if (opr2->is_constant()) {
1923 jlong imm;
1924 switch(opr2->type()) {
1925 case T_LONG:
1926 imm = opr2->as_constant_ptr()->as_jlong();
1927 break;
1928 case T_INT:
1929 case T_ADDRESS:
1930 imm = opr2->as_constant_ptr()->as_jint();
1931 break;
1932 case T_OBJECT:
1933 case T_ARRAY:
1934 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1935 break;
1936 default:
1937 ShouldNotReachHere();
1938 break;
1939 }
1940
1941 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1942 if (type2aelembytes(opr1->type()) <= 4)
1943 __ cmpw(reg1, imm);
1944 else
1945 __ cmp(reg1, imm);
1946 return;
1947 } else {
1948 __ mov(rscratch1, imm);
1949 if (type2aelembytes(opr1->type()) <= 4)
1950 __ cmpw(reg1, rscratch1);
1951 else
1952 __ cmp(reg1, rscratch1);
1953 return;
1954 }
1955 } else
1956 ShouldNotReachHere();
1957 } else if (opr1->is_single_fpu()) {
1958 FloatRegister reg1 = opr1->as_float_reg();
1959 assert(opr2->is_single_fpu(), "expect single float register");
1960 FloatRegister reg2 = opr2->as_float_reg();
1961 __ fcmps(reg1, reg2);
1962 } else if (opr1->is_double_fpu()) {
1963 FloatRegister reg1 = opr1->as_double_reg();
1964 assert(opr2->is_double_fpu(), "expect double float register");
1965 FloatRegister reg2 = opr2->as_double_reg();
1966 __ fcmpd(reg1, reg2);
1967 } else {
1968 ShouldNotReachHere();
1969 }
1970 }
1971
1972 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1973 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1974 bool is_unordered_less = (code == lir_ucmp_fd2i);
1975 if (left->is_single_fpu()) {
1976 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1977 } else if (left->is_double_fpu()) {
1978 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1979 } else {
1980 ShouldNotReachHere();
1981 }
1982 } else if (code == lir_cmp_l2i) {
1983 Label done;
1984 __ cmp(left->as_register_lo(), right->as_register_lo());
1985 __ mov(dst->as_register(), (u_int64_t)-1L);
1986 __ br(Assembler::LT, done);
1987 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1988 __ bind(done);
1989 } else {
1990 ShouldNotReachHere();
1991 }
1992 }
1993
1994
1995 void LIR_Assembler::align_call(LIR_Code code) { }
1996
1997
1998 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1999 __ trampoline_call(Address(op->addr(), rtype));
2000 add_call_info(code_offset(), op->info());
2001 }
2002
2003
2004 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2005 __ ic_call(op->addr());
2006 add_call_info(code_offset(), op->info());
2007 }
2008
2009
2010 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2011 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2012 ShouldNotReachHere();
2013 }
2014
2015
2016 void LIR_Assembler::emit_static_call_stub() {
2017 address call_pc = __ pc();
2018 address stub = __ start_a_stub(call_stub_size);
2019 if (stub == NULL) {
2020 bailout("static call stub overflow");
2021 return;
2022 }
2023
2024 int start = __ offset();
2025
2026 __ relocate(static_stub_Relocation::spec(call_pc));
2027 __ mov_metadata(rmethod, (Metadata*)NULL);
2028 __ movptr(rscratch1, 0);
2029 __ br(rscratch1);
2030
2031 assert(__ offset() - start <= call_stub_size, "stub too big");
2032 __ end_a_stub();
2033 }
2034
2035
2036 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2037 assert(exceptionOop->as_register() == r0, "must match");
2038 assert(exceptionPC->as_register() == r3, "must match");
2039
2040 // exception object is not added to oop map by LinearScan
2041 // (LinearScan assumes that no oops are in fixed registers)
2042 info->add_register_oop(exceptionOop);
2043 Runtime1::StubID unwind_id;
2044
2045 // get current pc information
2046 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2047 int pc_for_athrow_offset = __ offset();
2048 InternalAddress pc_for_athrow(__ pc());
2049 __ adr(exceptionPC->as_register(), pc_for_athrow);
2050 add_call_info(pc_for_athrow_offset, info); // for exception handler
2051
2052 __ verify_not_null_oop(r0);
2053 // search an exception handler (r0: exception oop, r3: throwing pc)
2054 if (compilation()->has_fpu_code()) {
2055 unwind_id = Runtime1::handle_exception_id;
2056 } else {
2057 unwind_id = Runtime1::handle_exception_nofpu_id;
2058 }
2059 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2060
2061 // FIXME: enough room for two byte trap ????
2062 __ nop();
2063 }
2064
2065
2066 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2067 assert(exceptionOop->as_register() == r0, "must match");
2068
2069 __ b(_unwind_handler_entry);
2070 }
2071
2072
2073 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2074 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2075 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2076
2077 switch (left->type()) {
2078 case T_INT: {
2079 switch (code) {
2080 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2081 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2082 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2083 default:
2084 ShouldNotReachHere();
2085 break;
2086 }
2087 break;
2088 case T_LONG:
2089 case T_ADDRESS:
2090 case T_OBJECT:
2091 switch (code) {
2092 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2093 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2094 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2095 default:
2096 ShouldNotReachHere();
2097 break;
2098 }
2099 break;
2100 default:
2101 ShouldNotReachHere();
2102 break;
2103 }
2104 }
2105 }
2106
2107
2108 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2109 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2110 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2111
2112 switch (left->type()) {
2113 case T_INT: {
2114 switch (code) {
2115 case lir_shl: __ lslw (dreg, lreg, count); break;
2116 case lir_shr: __ asrw (dreg, lreg, count); break;
2117 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2118 default:
2119 ShouldNotReachHere();
2120 break;
2121 }
2122 break;
2123 case T_LONG:
2124 case T_ADDRESS:
2125 case T_OBJECT:
2126 switch (code) {
2127 case lir_shl: __ lsl (dreg, lreg, count); break;
2128 case lir_shr: __ asr (dreg, lreg, count); break;
2129 case lir_ushr: __ lsr (dreg, lreg, count); break;
2130 default:
2131 ShouldNotReachHere();
2132 break;
2133 }
2134 break;
2135 default:
2136 ShouldNotReachHere();
2137 break;
2138 }
2139 }
2140 }
2141
2142
2143 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2144 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2145 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2146 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2147 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2148 }
2149
2150
2151 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2152 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2153 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2154 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2155 __ mov (rscratch1, c);
2156 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2157 }
2158
2159
2160 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2161 ShouldNotReachHere();
2162 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2163 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2164 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2165 __ lea(rscratch1, __ constant_oop_address(o));
2166 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2167 }
2168
2169
2170 // This code replaces a call to arraycopy; no exception may
2171 // be thrown in this code, they must be thrown in the System.arraycopy
2172 // activation frame; we could save some checks if this would not be the case
2173 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2174 ciArrayKlass* default_type = op->expected_type();
2175 Register src = op->src()->as_register();
2176 Register dst = op->dst()->as_register();
2177 Register src_pos = op->src_pos()->as_register();
2178 Register dst_pos = op->dst_pos()->as_register();
2179 Register length = op->length()->as_register();
2180 Register tmp = op->tmp()->as_register();
2181
2182 CodeStub* stub = op->stub();
2183 int flags = op->flags();
2184 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2185 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2186
2187 // if we don't know anything, just go through the generic arraycopy
2188 if (default_type == NULL // || basic_type == T_OBJECT
2189 ) {
2190 Label done;
2191 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2192
2193 // Save the arguments in case the generic arraycopy fails and we
2194 // have to fall back to the JNI stub
2195 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2196 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2197 __ str(src, Address(sp, 4*BytesPerWord));
2198
2199 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2200 address copyfunc_addr = StubRoutines::generic_arraycopy();
2201
2202 // The arguments are in java calling convention so we shift them
2203 // to C convention
2204 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2205 __ mov(c_rarg0, j_rarg0);
2206 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2207 __ mov(c_rarg1, j_rarg1);
2208 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2209 __ mov(c_rarg2, j_rarg2);
2210 assert_different_registers(c_rarg3, j_rarg4);
2211 __ mov(c_rarg3, j_rarg3);
2212 __ mov(c_rarg4, j_rarg4);
2213 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2214 __ mov(rscratch1, RuntimeAddress(C_entry));
2215 __ blrt(rscratch1, 5, 0, 1);
2216 } else {
2217 #ifndef PRODUCT
2218 if (PrintC1Statistics) {
2219 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2220 }
2221 #endif
2222 __ far_call(RuntimeAddress(copyfunc_addr));
2223 }
2224
2225 __ cbz(r0, *stub->continuation());
2226
2227 // Reload values from the stack so they are where the stub
2228 // expects them.
2229 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2230 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2231 __ ldr(src, Address(sp, 4*BytesPerWord));
2232
2233 if (copyfunc_addr != NULL) {
2234 // r0 is -1^K where K == partial copied count
2235 __ eonw(rscratch1, r0, 0);
2236 // adjust length down and src/end pos up by partial copied count
2237 __ subw(length, length, rscratch1);
2238 __ addw(src_pos, src_pos, rscratch1);
2239 __ addw(dst_pos, dst_pos, rscratch1);
2240 }
2241 __ b(*stub->entry());
2242
2243 __ bind(*stub->continuation());
2244 return;
2245 }
2246
2247 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2248
2249 int elem_size = type2aelembytes(basic_type);
2250 int shift_amount;
2251 int scale = exact_log2(elem_size);
2252
2253 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2254 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2255 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2256 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2257
2258 // test for NULL
2259 if (flags & LIR_OpArrayCopy::src_null_check) {
2260 __ cbz(src, *stub->entry());
2261 }
2262 if (flags & LIR_OpArrayCopy::dst_null_check) {
2263 __ cbz(dst, *stub->entry());
2264 }
2265
2266 // check if negative
2267 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2268 __ cmpw(src_pos, 0);
2269 __ br(Assembler::LT, *stub->entry());
2270 }
2271 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2272 __ cmpw(dst_pos, 0);
2273 __ br(Assembler::LT, *stub->entry());
2274 }
2275
2276 if (flags & LIR_OpArrayCopy::length_positive_check) {
2277 __ cmpw(length, 0);
2278 __ br(Assembler::LT, *stub->entry());
2279 }
2280
2281 if (flags & LIR_OpArrayCopy::src_range_check) {
2282 __ addw(tmp, src_pos, length);
2283 __ ldrw(rscratch1, src_length_addr);
2284 __ cmpw(tmp, rscratch1);
2285 __ br(Assembler::HI, *stub->entry());
2286 }
2287 if (flags & LIR_OpArrayCopy::dst_range_check) {
2288 __ addw(tmp, dst_pos, length);
2289 __ ldrw(rscratch1, dst_length_addr);
2290 __ cmpw(tmp, rscratch1);
2291 __ br(Assembler::HI, *stub->entry());
2292 }
2293
2294 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2295 // length_positive_check if the source of our length operand is an
2296 // arraylength. However, that arraylength might be zero, and the
2297 // stub that we're about to call contains an assertion that count !=
2298 // 0 . So we make this check purely in order not to trigger an
2299 // assertion failure.
2300 __ cbzw(length, *stub->continuation());
2301
2302 if (flags & LIR_OpArrayCopy::type_check) {
2303 // We don't know the array types are compatible
2304 if (basic_type != T_OBJECT) {
2305 // Simple test for basic type arrays
2306 if (UseCompressedClassPointers) {
2307 __ ldrw(tmp, src_klass_addr);
2308 __ ldrw(rscratch1, dst_klass_addr);
2309 __ cmpw(tmp, rscratch1);
2310 } else {
2311 __ ldr(tmp, src_klass_addr);
2312 __ ldr(rscratch1, dst_klass_addr);
2313 __ cmp(tmp, rscratch1);
2314 }
2315 __ br(Assembler::NE, *stub->entry());
2316 } else {
2317 // For object arrays, if src is a sub class of dst then we can
2318 // safely do the copy.
2319 Label cont, slow;
2320
2321 #define PUSH(r1, r2) \
2322 stp(r1, r2, __ pre(sp, -2 * wordSize));
2323
2324 #define POP(r1, r2) \
2325 ldp(r1, r2, __ post(sp, 2 * wordSize));
2326
2327 __ PUSH(src, dst);
2328
2329 __ load_klass(src, src);
2330 __ load_klass(dst, dst);
2331
2332 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2333
2334 __ PUSH(src, dst);
2335 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2336 __ POP(src, dst);
2337
2338 __ cbnz(src, cont);
2339
2340 __ bind(slow);
2341 __ POP(src, dst);
2342
2343 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2344 if (copyfunc_addr != NULL) { // use stub if available
2345 // src is not a sub class of dst so we have to do a
2346 // per-element check.
2347
2348 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2349 if ((flags & mask) != mask) {
2350 // Check that at least both of them object arrays.
2351 assert(flags & mask, "one of the two should be known to be an object array");
2352
2353 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2354 __ load_klass(tmp, src);
2355 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2356 __ load_klass(tmp, dst);
2357 }
2358 int lh_offset = in_bytes(Klass::layout_helper_offset());
2359 Address klass_lh_addr(tmp, lh_offset);
2360 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2361 __ ldrw(rscratch1, klass_lh_addr);
2362 __ mov(rscratch2, objArray_lh);
2363 __ eorw(rscratch1, rscratch1, rscratch2);
2364 __ cbnzw(rscratch1, *stub->entry());
2365 }
2366
2367 // Spill because stubs can use any register they like and it's
2368 // easier to restore just those that we care about.
2369 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2370 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2371 __ str(src, Address(sp, 4*BytesPerWord));
2372
2373 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2374 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2375 assert_different_registers(c_rarg0, dst, dst_pos, length);
2376 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2377 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2378 assert_different_registers(c_rarg1, dst, length);
2379 __ uxtw(c_rarg2, length);
2380 assert_different_registers(c_rarg2, dst);
2381
2382 __ load_klass(c_rarg4, dst);
2383 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2384 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2385 __ far_call(RuntimeAddress(copyfunc_addr));
2386
2387 #ifndef PRODUCT
2388 if (PrintC1Statistics) {
2389 Label failed;
2390 __ cbnz(r0, failed);
2391 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2392 __ bind(failed);
2393 }
2394 #endif
2395
2396 __ cbz(r0, *stub->continuation());
2397
2398 #ifndef PRODUCT
2399 if (PrintC1Statistics) {
2400 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2401 }
2402 #endif
2403 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2404
2405 // Restore previously spilled arguments
2406 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2407 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2408 __ ldr(src, Address(sp, 4*BytesPerWord));
2409
2410 // return value is -1^K where K is partial copied count
2411 __ eonw(rscratch1, r0, zr);
2412 // adjust length down and src/end pos up by partial copied count
2413 __ subw(length, length, rscratch1);
2414 __ addw(src_pos, src_pos, rscratch1);
2415 __ addw(dst_pos, dst_pos, rscratch1);
2416 }
2417
2418 __ b(*stub->entry());
2419
2420 __ bind(cont);
2421 __ POP(src, dst);
2422 }
2423 }
2424
2425 #ifdef ASSERT
2426 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2427 // Sanity check the known type with the incoming class. For the
2428 // primitive case the types must match exactly with src.klass and
2429 // dst.klass each exactly matching the default type. For the
2430 // object array case, if no type check is needed then either the
2431 // dst type is exactly the expected type and the src type is a
2432 // subtype which we can't check or src is the same array as dst
2433 // but not necessarily exactly of type default_type.
2434 Label known_ok, halt;
2435 __ mov_metadata(tmp, default_type->constant_encoding());
2436 if (UseCompressedClassPointers) {
2437 __ encode_klass_not_null(tmp);
2438 }
2439
2440 if (basic_type != T_OBJECT) {
2441
2442 if (UseCompressedClassPointers) {
2443 __ ldrw(rscratch1, dst_klass_addr);
2444 __ cmpw(tmp, rscratch1);
2445 } else {
2446 __ ldr(rscratch1, dst_klass_addr);
2447 __ cmp(tmp, rscratch1);
2448 }
2449 __ br(Assembler::NE, halt);
2450 if (UseCompressedClassPointers) {
2451 __ ldrw(rscratch1, src_klass_addr);
2452 __ cmpw(tmp, rscratch1);
2453 } else {
2454 __ ldr(rscratch1, src_klass_addr);
2455 __ cmp(tmp, rscratch1);
2456 }
2457 __ br(Assembler::EQ, known_ok);
2458 } else {
2459 if (UseCompressedClassPointers) {
2460 __ ldrw(rscratch1, dst_klass_addr);
2461 __ cmpw(tmp, rscratch1);
2462 } else {
2463 __ ldr(rscratch1, dst_klass_addr);
2464 __ cmp(tmp, rscratch1);
2465 }
2466 __ br(Assembler::EQ, known_ok);
2467 __ cmp(src, dst);
2468 __ br(Assembler::EQ, known_ok);
2469 }
2470 __ bind(halt);
2471 __ stop("incorrect type information in arraycopy");
2472 __ bind(known_ok);
2473 }
2474 #endif
2475
2476 #ifndef PRODUCT
2477 if (PrintC1Statistics) {
2478 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2479 }
2480 #endif
2481
2482 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2483 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2484 assert_different_registers(c_rarg0, dst, dst_pos, length);
2485 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2486 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2487 assert_different_registers(c_rarg1, dst, length);
2488 __ uxtw(c_rarg2, length);
2489 assert_different_registers(c_rarg2, dst);
2490
2491 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2492 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2493 const char *name;
2494 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2495
2496 CodeBlob *cb = CodeCache::find_blob(entry);
2497 if (cb) {
2498 __ far_call(RuntimeAddress(entry));
2499 } else {
2500 __ call_VM_leaf(entry, 3);
2501 }
2502
2503 __ bind(*stub->continuation());
2504 }
2505
2506
2507
2508
2509 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2510 Register obj = op->obj_opr()->as_register(); // may not be an oop
2511 Register hdr = op->hdr_opr()->as_register();
2512 Register lock = op->lock_opr()->as_register();
2513 if (!UseFastLocking) {
2514 __ b(*op->stub()->entry());
2515 } else if (op->code() == lir_lock) {
2516 Register scratch = noreg;
2517 if (UseBiasedLocking) {
2518 scratch = op->scratch_opr()->as_register();
2519 }
2520 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2521 // add debug info for NullPointerException only if one is possible
2522 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2523 if (op->info() != NULL) {
2524 add_debug_info_for_null_check(null_check_offset, op->info());
2525 }
2526 // done
2527 } else if (op->code() == lir_unlock) {
2528 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2529 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2530 } else {
2531 Unimplemented();
2532 }
2533 __ bind(*op->stub()->continuation());
2534 }
2535
2536
2537 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2538 ciMethod* method = op->profiled_method();
2539 int bci = op->profiled_bci();
2540 ciMethod* callee = op->profiled_callee();
2541
2542 // Update counter for all call types
2543 ciMethodData* md = method->method_data_or_null();
2544 assert(md != NULL, "Sanity");
2545 ciProfileData* data = md->bci_to_data(bci);
2546 assert(data->is_CounterData(), "need CounterData for calls");
2547 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2548 Register mdo = op->mdo()->as_register();
2549 __ mov_metadata(mdo, md->constant_encoding());
2550 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2551 Bytecodes::Code bc = method->java_code_at_bci(bci);
2552 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2553 // Perform additional virtual call profiling for invokevirtual and
2554 // invokeinterface bytecodes
2555 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2556 !callee_is_static && // required for optimized MH invokes
2557 C1ProfileVirtualCalls) {
2558 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2559 Register recv = op->recv()->as_register();
2560 assert_different_registers(mdo, recv);
2561 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2562 ciKlass* known_klass = op->known_holder();
2563 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2564 // We know the type that will be seen at this call site; we can
2565 // statically update the MethodData* rather than needing to do
2566 // dynamic tests on the receiver type
2567
2568 // NOTE: we should probably put a lock around this search to
2569 // avoid collisions by concurrent compilations
2570 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2571 uint i;
2572 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2573 ciKlass* receiver = vc_data->receiver(i);
2574 if (known_klass->equals(receiver)) {
2575 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2576 __ addptr(data_addr, DataLayout::counter_increment);
2577 return;
2578 }
2579 }
2580
2581 // Receiver type not found in profile data; select an empty slot
2582
2583 // Note that this is less efficient than it should be because it
2584 // always does a write to the receiver part of the
2585 // VirtualCallData rather than just the first time
2586 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2587 ciKlass* receiver = vc_data->receiver(i);
2588 if (receiver == NULL) {
2589 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2590 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2591 __ lea(rscratch2, recv_addr);
2592 __ str(rscratch1, Address(rscratch2));
2593 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2594 __ addptr(data_addr, DataLayout::counter_increment);
2595 return;
2596 }
2597 }
2598 } else {
2599 __ load_klass(recv, recv);
2600 Label update_done;
2601 type_profile_helper(mdo, md, data, recv, &update_done);
2602 // Receiver did not match any saved receiver and there is no empty row for it.
2603 // Increment total counter to indicate polymorphic case.
2604 __ addptr(counter_addr, DataLayout::counter_increment);
2605
2606 __ bind(update_done);
2607 }
2608 } else {
2609 // Static call
2610 __ addptr(counter_addr, DataLayout::counter_increment);
2611 }
2612 }
2613
2614
2615 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2616 Unimplemented();
2617 }
2618
2619
2620 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2621 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2622 }
2623
2624 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2625 assert(op->crc()->is_single_cpu(), "crc must be register");
2626 assert(op->val()->is_single_cpu(), "byte value must be register");
2627 assert(op->result_opr()->is_single_cpu(), "result must be register");
2628 Register crc = op->crc()->as_register();
2629 Register val = op->val()->as_register();
2630 Register res = op->result_opr()->as_register();
2631
2632 assert_different_registers(val, crc, res);
2633 unsigned long offset;
2634 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2635 if (offset) __ add(res, res, offset);
2636
2637 __ ornw(crc, zr, crc); // ~crc
2638 __ update_byte_crc32(crc, val, res);
2639 __ ornw(res, zr, crc); // ~crc
2640 }
2641
2642 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2643 COMMENT("emit_profile_type {");
2644 Register obj = op->obj()->as_register();
2645 Register tmp = op->tmp()->as_pointer_register();
2646 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2647 ciKlass* exact_klass = op->exact_klass();
2648 intptr_t current_klass = op->current_klass();
2649 bool not_null = op->not_null();
2650 bool no_conflict = op->no_conflict();
2651
2652 Label update, next, none;
2653
2654 bool do_null = !not_null;
2655 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2656 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2657
2658 assert(do_null || do_update, "why are we here?");
2659 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2660 assert(mdo_addr.base() != rscratch1, "wrong register");
2661
2662 __ verify_oop(obj);
2663
2664 if (tmp != obj) {
2665 __ mov(tmp, obj);
2666 }
2667 if (do_null) {
2668 __ cbnz(tmp, update);
2669 if (!TypeEntries::was_null_seen(current_klass)) {
2670 __ ldr(rscratch2, mdo_addr);
2671 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2672 __ str(rscratch2, mdo_addr);
2673 }
2674 if (do_update) {
2675 #ifndef ASSERT
2676 __ b(next);
2677 }
2678 #else
2679 __ b(next);
2680 }
2681 } else {
2682 __ cbnz(tmp, update);
2683 __ stop("unexpected null obj");
2684 #endif
2685 }
2686
2687 __ bind(update);
2688
2689 if (do_update) {
2690 #ifdef ASSERT
2691 if (exact_klass != NULL) {
2692 Label ok;
2693 __ load_klass(tmp, tmp);
2694 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2695 __ eor(rscratch1, tmp, rscratch1);
2696 __ cbz(rscratch1, ok);
2697 __ stop("exact klass and actual klass differ");
2698 __ bind(ok);
2699 }
2700 #endif
2701 if (!no_conflict) {
2702 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2703 if (exact_klass != NULL) {
2704 __ mov_metadata(tmp, exact_klass->constant_encoding());
2705 } else {
2706 __ load_klass(tmp, tmp);
2707 }
2708
2709 __ ldr(rscratch2, mdo_addr);
2710 __ eor(tmp, tmp, rscratch2);
2711 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2712 // klass seen before, nothing to do. The unknown bit may have been
2713 // set already but no need to check.
2714 __ cbz(rscratch1, next);
2715
2716 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2717 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2718
2719 if (TypeEntries::is_type_none(current_klass)) {
2720 __ cbz(rscratch2, none);
2721 __ cmp(rscratch2, TypeEntries::null_seen);
2722 __ br(Assembler::EQ, none);
2723 // There is a chance that the checks above (re-reading profiling
2724 // data from memory) fail if another thread has just set the
2725 // profiling to this obj's klass
2726 __ dmb(Assembler::ISHLD);
2727 __ ldr(rscratch2, mdo_addr);
2728 __ eor(tmp, tmp, rscratch2);
2729 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2730 __ cbz(rscratch1, next);
2731 }
2732 } else {
2733 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2734 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2735
2736 __ ldr(tmp, mdo_addr);
2737 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2738 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2739 }
2740
2741 // different than before. Cannot keep accurate profile.
2742 __ ldr(rscratch2, mdo_addr);
2743 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2744 __ str(rscratch2, mdo_addr);
2745
2746 if (TypeEntries::is_type_none(current_klass)) {
2747 __ b(next);
2748
2749 __ bind(none);
2750 // first time here. Set profile type.
2751 __ str(tmp, mdo_addr);
2752 }
2753 } else {
2754 // There's a single possible klass at this profile point
2755 assert(exact_klass != NULL, "should be");
2756 if (TypeEntries::is_type_none(current_klass)) {
2757 __ mov_metadata(tmp, exact_klass->constant_encoding());
2758 __ ldr(rscratch2, mdo_addr);
2759 __ eor(tmp, tmp, rscratch2);
2760 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2761 __ cbz(rscratch1, next);
2762 #ifdef ASSERT
2763 {
2764 Label ok;
2765 __ ldr(rscratch1, mdo_addr);
2766 __ cbz(rscratch1, ok);
2767 __ cmp(rscratch1, TypeEntries::null_seen);
2768 __ br(Assembler::EQ, ok);
2769 // may have been set by another thread
2770 __ dmb(Assembler::ISHLD);
2771 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2772 __ ldr(rscratch2, mdo_addr);
2773 __ eor(rscratch2, rscratch1, rscratch2);
2774 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2775 __ cbz(rscratch2, ok);
2776
2777 __ stop("unexpected profiling mismatch");
2778 __ bind(ok);
2779 }
2780 #endif
2781 // first time here. Set profile type.
2782 __ ldr(tmp, mdo_addr);
2783 } else {
2784 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2785 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2786
2787 __ ldr(tmp, mdo_addr);
2788 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2789 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2790
2791 __ orr(tmp, tmp, TypeEntries::type_unknown);
2792 __ str(tmp, mdo_addr);
2793 // FIXME: Write barrier needed here?
2794 }
2795 }
2796
2797 __ bind(next);
2798 }
2799 COMMENT("} emit_profile_type");
2800 }
2801
2802
2803 void LIR_Assembler::align_backward_branch_target() {
2804 }
2805
2806
2807 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2808 if (left->is_single_cpu()) {
2809 assert(dest->is_single_cpu(), "expect single result reg");
2810 __ negw(dest->as_register(), left->as_register());
2811 } else if (left->is_double_cpu()) {
2812 assert(dest->is_double_cpu(), "expect double result reg");
2813 __ neg(dest->as_register_lo(), left->as_register_lo());
2814 } else if (left->is_single_fpu()) {
2815 assert(dest->is_single_fpu(), "expect single float result reg");
2816 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2817 } else {
2818 assert(left->is_double_fpu(), "expect double float operand reg");
2819 assert(dest->is_double_fpu(), "expect double float result reg");
2820 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2821 }
2822 }
2823
2824
2825 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2826 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2827 }
2828
2829
2830 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2831 assert(!tmp->is_valid(), "don't need temporary");
2832
2833 CodeBlob *cb = CodeCache::find_blob(dest);
2834 if (cb) {
2835 __ far_call(RuntimeAddress(dest));
2836 } else {
2837 __ mov(rscratch1, RuntimeAddress(dest));
2838 int len = args->length();
2839 int type = 0;
2840 if (! result->is_illegal()) {
2841 switch (result->type()) {
2842 case T_VOID:
2843 type = 0;
2844 break;
2845 case T_INT:
2846 case T_LONG:
2847 case T_OBJECT:
2848 type = 1;
2849 break;
2850 case T_FLOAT:
2851 type = 2;
2852 break;
2853 case T_DOUBLE:
2854 type = 3;
2855 break;
2856 default:
2857 ShouldNotReachHere();
2858 break;
2859 }
2860 }
2861 int num_gpargs = 0;
2862 int num_fpargs = 0;
2863 for (int i = 0; i < args->length(); i++) {
2864 LIR_Opr arg = args->at(i);
2865 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2866 num_fpargs++;
2867 } else {
2868 num_gpargs++;
2869 }
2870 }
2871 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2872 }
2873
2874 if (info != NULL) {
2875 add_call_info_here(info);
2876 }
2877 __ maybe_isb();
2878 }
2879
2880 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2881 if (dest->is_address() || src->is_address()) {
2882 move_op(src, dest, type, lir_patch_none, info,
2883 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2884 } else {
2885 ShouldNotReachHere();
2886 }
2887 }
2888
2889 #ifdef ASSERT
2890 // emit run-time assertion
2891 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2892 assert(op->code() == lir_assert, "must be");
2893
2894 if (op->in_opr1()->is_valid()) {
2895 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2896 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2897 } else {
2898 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2899 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2900 }
2901
2902 Label ok;
2903 if (op->condition() != lir_cond_always) {
2904 Assembler::Condition acond = Assembler::AL;
2905 switch (op->condition()) {
2906 case lir_cond_equal: acond = Assembler::EQ; break;
2907 case lir_cond_notEqual: acond = Assembler::NE; break;
2908 case lir_cond_less: acond = Assembler::LT; break;
2909 case lir_cond_lessEqual: acond = Assembler::LE; break;
2910 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2911 case lir_cond_greater: acond = Assembler::GT; break;
2912 case lir_cond_belowEqual: acond = Assembler::LS; break;
2913 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2914 default: ShouldNotReachHere();
2915 }
2916 __ br(acond, ok);
2917 }
2918 if (op->halt()) {
2919 const char* str = __ code_string(op->msg());
2920 __ stop(str);
2921 } else {
2922 breakpoint();
2923 }
2924 __ bind(ok);
2925 }
2926 #endif
2927
2928 #ifndef PRODUCT
2929 #define COMMENT(x) do { __ block_comment(x); } while (0)
2930 #else
2931 #define COMMENT(x)
2932 #endif
2933
2934 void LIR_Assembler::membar() {
2935 COMMENT("membar");
2936 __ membar(MacroAssembler::AnyAny);
2937 }
2938
2939 void LIR_Assembler::membar_acquire() {
2940 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2941 }
2942
2943 void LIR_Assembler::membar_release() {
2944 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2945 }
2946
2947 void LIR_Assembler::membar_loadload() {
2948 __ membar(Assembler::LoadLoad);
2949 }
2950
2951 void LIR_Assembler::membar_storestore() {
2952 __ membar(MacroAssembler::StoreStore);
2953 }
2954
2955 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2956
2957 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2958
2959 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2960 __ mov(result_reg->as_register(), rthread);
2961 }
2962
2963
2964 void LIR_Assembler::peephole(LIR_List *lir) {
2965 #if 0
2966 if (tableswitch_count >= max_tableswitches)
2967 return;
2968
2969 /*
2970 This finite-state automaton recognizes sequences of compare-and-
2971 branch instructions. We will turn them into a tableswitch. You
2972 could argue that C1 really shouldn't be doing this sort of
2973 optimization, but without it the code is really horrible.
2974 */
2975
2976 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2977 int first_key, last_key = -2147483648;
2978 int next_key = 0;
2979 int start_insn = -1;
2980 int last_insn = -1;
2981 Register reg = noreg;
2982 LIR_Opr reg_opr;
2983 state = start_s;
2984
2985 LIR_OpList* inst = lir->instructions_list();
2986 for (int i = 0; i < inst->length(); i++) {
2987 LIR_Op* op = inst->at(i);
2988 switch (state) {
2989 case start_s:
2990 first_key = -1;
2991 start_insn = i;
2992 switch (op->code()) {
2993 case lir_cmp:
2994 LIR_Opr opr1 = op->as_Op2()->in_opr1();
2995 LIR_Opr opr2 = op->as_Op2()->in_opr2();
2996 if (opr1->is_cpu_register() && opr1->is_single_cpu()
2997 && opr2->is_constant()
2998 && opr2->type() == T_INT) {
2999 reg_opr = opr1;
3000 reg = opr1->as_register();
3001 first_key = opr2->as_constant_ptr()->as_jint();
3002 next_key = first_key + 1;
3003 state = cmp_s;
3004 goto next_state;
3005 }
3006 break;
3007 }
3008 break;
3009 case cmp_s:
3010 switch (op->code()) {
3011 case lir_branch:
3012 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3013 state = beq_s;
3014 last_insn = i;
3015 goto next_state;
3016 }
3017 }
3018 state = start_s;
3019 break;
3020 case beq_s:
3021 switch (op->code()) {
3022 case lir_cmp: {
3023 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3024 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3025 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3026 && opr1->as_register() == reg
3027 && opr2->is_constant()
3028 && opr2->type() == T_INT
3029 && opr2->as_constant_ptr()->as_jint() == next_key) {
3030 last_key = next_key;
3031 next_key++;
3032 state = cmp_s;
3033 goto next_state;
3034 }
3035 }
3036 }
3037 last_key = next_key;
3038 state = start_s;
3039 break;
3040 default:
3041 assert(false, "impossible state");
3042 }
3043 if (state == start_s) {
3044 if (first_key < last_key - 5L && reg != noreg) {
3045 {
3046 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3047 // reg->encoding(),
3048 // start_insn, first_key, last_key);
3049 // for (int i = 0; i < inst->length(); i++) {
3050 // inst->at(i)->print();
3051 // tty->print("\n");
3052 // }
3053 // tty->print("\n");
3054 }
3055
3056 struct tableswitch *sw = &switches[tableswitch_count];
3057 sw->_insn_index = start_insn, sw->_first_key = first_key,
3058 sw->_last_key = last_key, sw->_reg = reg;
3059 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3060 {
3061 // Insert the new table of branches
3062 int offset = last_insn;
3063 for (int n = first_key; n < last_key; n++) {
3064 inst->insert_before
3065 (last_insn + 1,
3066 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3067 inst->at(offset)->as_OpBranch()->label()));
3068 offset -= 2, i++;
3069 }
3070 }
3071 // Delete all the old compare-and-branch instructions
3072 for (int n = first_key; n < last_key; n++) {
3073 inst->remove_at(start_insn);
3074 inst->remove_at(start_insn);
3075 }
3076 // Insert the tableswitch instruction
3077 inst->insert_before(start_insn,
3078 new LIR_Op2(lir_cmp, lir_cond_always,
3079 LIR_OprFact::intConst(tableswitch_count),
3080 reg_opr));
3081 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3082 tableswitch_count++;
3083 }
3084 reg = noreg;
3085 last_key = -2147483648;
3086 }
3087 next_state:
3088 ;
3089 }
3090 #endif
3091 }
3092
3093 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3094 Address addr = as_Address(src->as_address_ptr(), noreg);
3095 BasicType type = src->type();
3096 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3097
3098 void (MacroAssembler::* lda)(Register Rd, Register Ra);
3099 void (MacroAssembler::* add)(Register Rd, Register Rn, RegisterOrConstant increment);
3100 void (MacroAssembler::* stl)(Register Rs, Register Rt, Register Rn);
3101
3102 switch(type) {
3103 case T_INT:
3104 lda = &MacroAssembler::ldaxrw;
3105 add = &MacroAssembler::addw;
3106 stl = &MacroAssembler::stlxrw;
3107 break;
3108 case T_LONG:
3109 lda = &MacroAssembler::ldaxr;
3110 add = &MacroAssembler::add;
3111 stl = &MacroAssembler::stlxr;
3112 break;
3113 case T_OBJECT:
3114 case T_ARRAY:
3115 if (UseCompressedOops) {
3116 lda = &MacroAssembler::ldaxrw;
3117 add = &MacroAssembler::addw;
3118 stl = &MacroAssembler::stlxrw;
3119 } else {
3120 lda = &MacroAssembler::ldaxr;
3121 add = &MacroAssembler::add;
3122 stl = &MacroAssembler::stlxr;
3123 }
3124 break;
3125 default:
3126 ShouldNotReachHere();
3127 }
3128
3129 switch (code) {
3130 case lir_xadd:
3131 {
3132 RegisterOrConstant inc;
3133 Register tmp = as_reg(tmp_op);
3134 Register dst = as_reg(dest);
3135 if (data->is_constant()) {
3136 inc = RegisterOrConstant(as_long(data));
3137 assert_different_registers(dst, addr.base(), tmp,
3138 rscratch1, rscratch2);
3139 } else {
3140 inc = RegisterOrConstant(as_reg(data));
3141 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3142 rscratch1, rscratch2);
3143 }
3144 Label again;
3145 __ lea(tmp, addr);
3146 __ bind(again);
3147 (_masm->*lda)(dst, tmp);
3148 (_masm->*add)(rscratch1, dst, inc);
3149 (_masm->*stl)(rscratch2, rscratch1, tmp);
3150 __ cbnzw(rscratch2, again);
3151 break;
3152 }
3153 case lir_xchg:
3154 {
3155 Register tmp = tmp_op->as_register();
3156 Register obj = as_reg(data);
3157 Register dst = as_reg(dest);
3158 if (is_oop && UseCompressedOops) {
3159 __ encode_heap_oop(obj);
3160 }
3161 assert_different_registers(obj, addr.base(), tmp, rscratch2, dst);
3162 Label again;
3163 __ lea(tmp, addr);
3164 __ bind(again);
3165 (_masm->*lda)(dst, tmp);
3166 (_masm->*stl)(rscratch2, obj, tmp);
3167 __ cbnzw(rscratch2, again);
3168 if (is_oop && UseCompressedOops) {
3169 __ decode_heap_oop(dst);
3170 }
3171 }
3172 break;
3173 default:
3174 ShouldNotReachHere();
3175 }
3176 __ membar(__ AnyAny);
3177 }
3178
3179 #undef __