1 /*
2 * Copyright (c) 2014, Red Hat Inc.
3 * Copyright (c) 2000, 2012, Oracle and/or its affiliates.
4 * All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "gc_interface/collectedHeap.hpp"
38 #include "memory/barrierSet.hpp"
39 #include "memory/cardTableModRefBS.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "vmreg_aarch64.inline.hpp"
44
45
46
47 #ifndef PRODUCT
48 #define COMMENT(x) do { __ block_comment(x); } while (0)
49 #else
50 #define COMMENT(x)
51 #endif
52
53 NEEDS_CLEANUP // remove this definitions ?
54 const Register IC_Klass = rscratch2; // where the IC klass is cached
55 const Register SYNC_header = r0; // synchronization header
56 const Register SHIFT_count = r0; // where count for shift operations must be
57
58 #define __ _masm->
59
60
61 static void select_different_registers(Register preserve,
62 Register extra,
63 Register &tmp1,
64 Register &tmp2) {
65 if (tmp1 == preserve) {
66 assert_different_registers(tmp1, tmp2, extra);
67 tmp1 = extra;
68 } else if (tmp2 == preserve) {
69 assert_different_registers(tmp1, tmp2, extra);
70 tmp2 = extra;
71 }
72 assert_different_registers(preserve, tmp1, tmp2);
73 }
74
75
76
77 static void select_different_registers(Register preserve,
78 Register extra,
79 Register &tmp1,
80 Register &tmp2,
81 Register &tmp3) {
82 if (tmp1 == preserve) {
83 assert_different_registers(tmp1, tmp2, tmp3, extra);
84 tmp1 = extra;
85 } else if (tmp2 == preserve) {
86 assert_different_registers(tmp1, tmp2, tmp3, extra);
87 tmp2 = extra;
88 } else if (tmp3 == preserve) {
89 assert_different_registers(tmp1, tmp2, tmp3, extra);
90 tmp3 = extra;
91 }
92 assert_different_registers(preserve, tmp1, tmp2, tmp3);
93 }
94
95
96 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
97
98
99 LIR_Opr LIR_Assembler::receiverOpr() {
100 return FrameMap::receiver_opr;
101 }
102
103 LIR_Opr LIR_Assembler::osrBufferPointer() {
104 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
105 }
106
107 //--------------fpu register translations-----------------------
108
109
110 address LIR_Assembler::float_constant(float f) {
111 address const_addr = __ float_constant(f);
112 if (const_addr == NULL) {
113 bailout("const section overflow");
114 return __ code()->consts()->start();
115 } else {
116 return const_addr;
117 }
118 }
119
120
121 address LIR_Assembler::double_constant(double d) {
122 address const_addr = __ double_constant(d);
123 if (const_addr == NULL) {
124 bailout("const section overflow");
125 return __ code()->consts()->start();
126 } else {
127 return const_addr;
128 }
129 }
130
131 address LIR_Assembler::int_constant(jlong n) {
132 address const_addr = __ long_constant(n);
133 if (const_addr == NULL) {
134 bailout("const section overflow");
135 return __ code()->consts()->start();
136 } else {
137 return const_addr;
138 }
139 }
140
141 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
142
143 void LIR_Assembler::reset_FPU() { Unimplemented(); }
144
145 void LIR_Assembler::fpop() { Unimplemented(); }
146
147 void LIR_Assembler::fxch(int i) { Unimplemented(); }
148
149 void LIR_Assembler::fld(int i) { Unimplemented(); }
150
151 void LIR_Assembler::ffree(int i) { Unimplemented(); }
152
153 void LIR_Assembler::breakpoint() { Unimplemented(); }
154
155 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
156
157 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
158
159 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
160 //-------------------------------------------
161
162 static Register as_reg(LIR_Opr op) {
163 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
164 }
165
166 static jlong as_long(LIR_Opr data) {
167 jlong result;
168 switch (data->type()) {
169 case T_INT:
170 result = (data->as_jint());
171 break;
172 case T_LONG:
173 result = (data->as_jlong());
174 break;
175 default:
176 ShouldNotReachHere();
177 }
178 return result;
179 }
180
181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
182 Register base = addr->base()->as_pointer_register();
183 LIR_Opr opr = addr->index();
184 if (opr->is_cpu_register()) {
185 Register index;
186 if (opr->is_single_cpu())
187 index = opr->as_register();
188 else
189 index = opr->as_register_lo();
190 assert(addr->disp() == 0, "must be");
191 switch(opr->type()) {
192 case T_INT:
193 return Address(base, index, Address::sxtw(addr->scale()));
194 case T_LONG:
195 return Address(base, index, Address::lsl(addr->scale()));
196 default:
197 ShouldNotReachHere();
198 }
199 } else {
200 intptr_t addr_offset = intptr_t(addr->disp());
201 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
202 return Address(base, addr_offset, Address::lsl(addr->scale()));
203 else {
204 __ mov(tmp, addr_offset);
205 return Address(base, tmp, Address::lsl(addr->scale()));
206 }
207 }
208 }
209
210 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
211 ShouldNotReachHere();
212 }
213
214 Address LIR_Assembler::as_Address(LIR_Address* addr) {
215 return as_Address(addr, rscratch1);
216 }
217
218 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
219 return as_Address(addr, rscratch1); // Ouch
220 // FIXME: This needs to be much more clever. See x86.
221 }
222
223
224 void LIR_Assembler::osr_entry() {
225 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
226 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
227 ValueStack* entry_state = osr_entry->state();
228 int number_of_locks = entry_state->locks_size();
229
230 // we jump here if osr happens with the interpreter
231 // state set up to continue at the beginning of the
232 // loop that triggered osr - in particular, we have
233 // the following registers setup:
234 //
235 // r2: osr buffer
236 //
237
238 // build frame
239 ciMethod* m = compilation()->method();
240 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
241
242 // OSR buffer is
243 //
244 // locals[nlocals-1..0]
245 // monitors[0..number_of_locks]
246 //
247 // locals is a direct copy of the interpreter frame so in the osr buffer
248 // so first slot in the local array is the last local from the interpreter
249 // and last slot is local[0] (receiver) from the interpreter
250 //
251 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
252 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
253 // in the interpreter frame (the method lock if a sync method)
254
255 // Initialize monitors in the compiled activation.
256 // r2: pointer to osr buffer
257 //
258 // All other registers are dead at this point and the locals will be
259 // copied into place by code emitted in the IR.
260
261 Register OSR_buf = osrBufferPointer()->as_pointer_register();
262 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
263 int monitor_offset = BytesPerWord * method()->max_locals() +
264 (2 * BytesPerWord) * (number_of_locks - 1);
265 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
266 // the OSR buffer using 2 word entries: first the lock and then
267 // the oop.
268 for (int i = 0; i < number_of_locks; i++) {
269 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
270 #ifdef ASSERT
271 // verify the interpreter's monitor has a non-null object
272 {
273 Label L;
274 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
275 __ cbnz(rscratch1, L);
276 __ stop("locked object is NULL");
277 __ bind(L);
278 }
279 #endif
280 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
281 __ str(r19, frame_map()->address_for_monitor_lock(i));
282 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
283 __ str(r19, frame_map()->address_for_monitor_object(i));
284 }
285 }
286 }
287
288
289 // inline cache check; done before the frame is built.
290 int LIR_Assembler::check_icache() {
291 Register receiver = FrameMap::receiver_opr->as_register();
292 Register ic_klass = IC_Klass;
293 int start_offset = __ offset();
294 __ inline_cache_check(receiver, ic_klass);
295
296 // if icache check fails, then jump to runtime routine
297 // Note: RECEIVER must still contain the receiver!
298 Label dont;
299 __ br(Assembler::EQ, dont);
300 __ b(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
301
302 // We align the verified entry point unless the method body
303 // (including its inline cache check) will fit in a single 64-byte
304 // icache line.
305 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
306 // force alignment after the cache check.
307 __ align(CodeEntryAlignment);
308 }
309
310 __ bind(dont);
311 return start_offset;
312 }
313
314
315 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
316 if (o == NULL) {
317 __ mov(reg, zr);
318 } else {
319 __ movoop(reg, o, /*immediate*/true);
320 }
321 }
322
323 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
324 address target = NULL;
325 relocInfo::relocType reloc_type = relocInfo::none;
326
327 switch (patching_id(info)) {
328 case PatchingStub::access_field_id:
329 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
330 reloc_type = relocInfo::section_word_type;
331 break;
332 case PatchingStub::load_klass_id:
333 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
334 reloc_type = relocInfo::metadata_type;
335 break;
336 case PatchingStub::load_mirror_id:
337 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
338 reloc_type = relocInfo::oop_type;
339 break;
340 case PatchingStub::load_appendix_id:
341 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
342 reloc_type = relocInfo::oop_type;
343 break;
344 default: ShouldNotReachHere();
345 }
346
347 __ bl(RuntimeAddress(target));
348 add_call_info_here(info);
349 }
350
351 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
352 deoptimize_trap(info);
353 }
354
355
356 // This specifies the rsp decrement needed to build the frame
357 int LIR_Assembler::initial_frame_size_in_bytes() const {
358 // if rounding, must let FrameMap know!
359
360 // The frame_map records size in slots (32bit word)
361
362 // subtract two words to account for return address and link
363 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
364 }
365
366
367 int LIR_Assembler::emit_exception_handler() {
368 // if the last instruction is a call (typically to do a throw which
369 // is coming at the end after block reordering) the return address
370 // must still point into the code area in order to avoid assertion
371 // failures when searching for the corresponding bci => add a nop
372 // (was bug 5/14/1999 - gri)
373 __ nop();
374
375 // generate code for exception handler
376 address handler_base = __ start_a_stub(exception_handler_size);
377 if (handler_base == NULL) {
378 // not enough space left for the handler
379 bailout("exception handler overflow");
380 return -1;
381 }
382
383 int offset = code_offset();
384
385 // the exception oop and pc are in r0, and r3
386 // no other registers need to be preserved, so invalidate them
387 __ invalidate_registers(false, true, true, false, true, true);
388
389 // check that there is really an exception
390 __ verify_not_null_oop(r0);
391
392 // search an exception handler (r0: exception oop, r3: throwing pc)
393 __ bl(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
394 __ should_not_reach_here();
395 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
396 __ end_a_stub();
397
398 return offset;
399 }
400
401
402 // Emit the code to remove the frame from the stack in the exception
403 // unwind path.
404 int LIR_Assembler::emit_unwind_handler() {
405 #ifndef PRODUCT
406 if (CommentedAssembly) {
407 _masm->block_comment("Unwind handler");
408 }
409 #endif
410
411 int offset = code_offset();
412
413 // Fetch the exception from TLS and clear out exception related thread state
414 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
415 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
417
418 __ bind(_unwind_handler_entry);
419 __ verify_not_null_oop(r0);
420 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
421 __ mov(r19, r0); // Preserve the exception
422 }
423
424 // Preform needed unlocking
425 MonitorExitStub* stub = NULL;
426 if (method()->is_synchronized()) {
427 monitor_address(0, FrameMap::r0_opr);
428 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
429 __ unlock_object(r5, r4, r0, *stub->entry());
430 __ bind(*stub->continuation());
431 }
432
433 if (compilation()->env()->dtrace_method_probes()) {
434 __ call_Unimplemented();
435 #if 0
436 __ movptr(Address(rsp, 0), rax);
437 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
438 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
439 #endif
440 }
441
442 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
443 __ mov(r0, r19); // Restore the exception
444 }
445
446 // remove the activation and dispatch to the unwind handler
447 __ block_comment("remove_frame and dispatch to the unwind handler");
448 __ remove_frame(initial_frame_size_in_bytes());
449 __ b(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
450
451 // Emit the slow path assembly
452 if (stub != NULL) {
453 stub->emit_code(this);
454 }
455
456 return offset;
457 }
458
459
460 int LIR_Assembler::emit_deopt_handler() {
461 // if the last instruction is a call (typically to do a throw which
462 // is coming at the end after block reordering) the return address
463 // must still point into the code area in order to avoid assertion
464 // failures when searching for the corresponding bci => add a nop
465 // (was bug 5/14/1999 - gri)
466 __ nop();
467
468 // generate code for exception handler
469 address handler_base = __ start_a_stub(deopt_handler_size);
470 if (handler_base == NULL) {
471 // not enough space left for the handler
472 bailout("deopt handler overflow");
473 return -1;
474 }
475
476 int offset = code_offset();
477
478 __ adr(lr, pc());
479 __ b(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
480 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
481 __ end_a_stub();
482
483 return offset;
484 }
485
486
487 // This is the fast version of java.lang.String.compare; it has not
488 // OSR-entry and therefore, we generate a slow version for OSR's
489 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
490 __ mov(r2, (address)__FUNCTION__);
491 __ call_Unimplemented();
492 }
493
494
495 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
496 _masm->code_section()->relocate(adr, relocInfo::poll_type);
497 int pc_offset = code_offset();
498 flush_debug_info(pc_offset);
499 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
500 if (info->exception_handlers() != NULL) {
501 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
502 }
503 }
504
505 // Rather than take a segfault when the polling page is protected,
506 // explicitly check for a safepoint in progress and if there is one,
507 // fake a call to the handler as if a segfault had been caught.
508 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
509 __ mov(rscratch1, SafepointSynchronize::address_of_state());
510 __ ldrb(rscratch1, Address(rscratch1));
511 Label nope, poll;
512 __ cbz(rscratch1, nope);
513 __ block_comment("safepoint");
514 __ enter();
515 __ push(0x3, sp); // r0 & r1
516 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
517 __ adr(r0, poll);
518 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
519 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
520 __ blrt(rscratch1, 1, 0, 1);
521 __ maybe_isb();
522 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
523 __ mov(rscratch1, r0);
524 __ pop(0x3, sp); // r0 & r1
525 __ leave();
526 __ br(rscratch1);
527 address polling_page(os::get_polling_page());
528 assert(os::is_poll_address(polling_page), "should be");
529 unsigned long off;
530 __ adrp(rscratch1, Address(polling_page, rtype), off);
531 __ bind(poll);
532 if (info)
533 add_debug_info_for_branch(info); // This isn't just debug info:
534 // it's the oop map
535 else
536 __ code_section()->relocate(pc(), rtype);
537 __ ldrw(zr, Address(rscratch1, off));
538 __ bind(nope);
539 }
540
541 void LIR_Assembler::return_op(LIR_Opr result) {
542 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
543 // Pop the stack before the safepoint code
544 __ remove_frame(initial_frame_size_in_bytes());
545 if (UseCompilerSafepoints) {
546 address polling_page(os::get_polling_page());
547 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
548 } else {
549 poll_for_safepoint(relocInfo::poll_return_type);
550 }
551 __ ret(lr);
552 }
553
554 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
555 address polling_page(os::get_polling_page());
556 if (UseCompilerSafepoints) {
557 guarantee(info != NULL, "Shouldn't be NULL");
558 assert(os::is_poll_address(polling_page), "should be");
559 unsigned long off;
560 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
561 assert(off == 0, "must be");
562 add_debug_info_for_branch(info); // This isn't just debug info:
563 // it's the oop map
564 __ read_polling_page(rscratch1, relocInfo::poll_type);
565 } else {
566 poll_for_safepoint(relocInfo::poll_type, info);
567 }
568
569 return __ offset();
570 }
571
572
573 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
574 if (from_reg == r31_sp)
575 from_reg = sp;
576 if (to_reg == r31_sp)
577 to_reg = sp;
578 __ mov(to_reg, from_reg);
579 }
580
581 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
582
583
584 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
585 assert(src->is_constant(), "should not call otherwise");
586 assert(dest->is_register(), "should not call otherwise");
587 LIR_Const* c = src->as_constant_ptr();
588
589 switch (c->type()) {
590 case T_INT: {
591 assert(patch_code == lir_patch_none, "no patching handled here");
592 __ movw(dest->as_register(), c->as_jint());
593 break;
594 }
595
596 case T_ADDRESS: {
597 assert(patch_code == lir_patch_none, "no patching handled here");
598 __ mov(dest->as_register(), c->as_jint());
599 break;
600 }
601
602 case T_LONG: {
603 assert(patch_code == lir_patch_none, "no patching handled here");
604 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
605 break;
606 }
607
608 case T_OBJECT: {
609 if (patch_code == lir_patch_none) {
610 jobject2reg(c->as_jobject(), dest->as_register());
611 } else {
612 jobject2reg_with_patching(dest->as_register(), info);
613 }
614 break;
615 }
616
617 case T_METADATA: {
618 if (patch_code != lir_patch_none) {
619 klass2reg_with_patching(dest->as_register(), info);
620 } else {
621 __ mov_metadata(dest->as_register(), c->as_metadata());
622 }
623 break;
624 }
625
626 case T_FLOAT: {
627 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
628 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
629 } else {
630 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
631 __ ldrs(dest->as_float_reg(), Address(rscratch1));
632 }
633 break;
634 }
635
636 case T_DOUBLE: {
637 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
638 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
639 } else {
640 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
641 __ ldrd(dest->as_double_reg(), Address(rscratch1));
642 }
643 break;
644 }
645
646 default:
647 ShouldNotReachHere();
648 }
649 }
650
651 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
652 LIR_Const* c = src->as_constant_ptr();
653 switch (c->type()) {
654 case T_OBJECT:
655 {
656 if (! c->as_jobject())
657 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
658 else {
659 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
660 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
661 }
662 }
663 break;
664 case T_ADDRESS:
665 {
666 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
667 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
668 }
669 case T_INT:
670 case T_FLOAT:
671 {
672 Register reg = zr;
673 if (c->as_jint_bits() == 0)
674 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
675 else {
676 __ movw(rscratch1, c->as_jint_bits());
677 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
678 }
679 }
680 break;
681 case T_LONG:
682 case T_DOUBLE:
683 {
684 Register reg = zr;
685 if (c->as_jlong_bits() == 0)
686 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
687 lo_word_offset_in_bytes));
688 else {
689 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
690 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
691 lo_word_offset_in_bytes));
692 }
693 }
694 break;
695 default:
696 ShouldNotReachHere();
697 }
698 }
699
700 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
701 assert(src->is_constant(), "should not call otherwise");
702 LIR_Const* c = src->as_constant_ptr();
703 LIR_Address* to_addr = dest->as_address_ptr();
704
705 void (Assembler::* insn)(Register Rt, const Address &adr);
706
707 switch (type) {
708 case T_ADDRESS:
709 assert(c->as_jint() == 0, "should be");
710 insn = &Assembler::str;
711 break;
712 case T_LONG:
713 assert(c->as_jlong() == 0, "should be");
714 insn = &Assembler::str;
715 break;
716 case T_INT:
717 assert(c->as_jint() == 0, "should be");
718 insn = &Assembler::strw;
719 break;
720 case T_OBJECT:
721 case T_ARRAY:
722 assert(c->as_jobject() == 0, "should be");
723 if (UseCompressedOops && !wide) {
724 insn = &Assembler::strw;
725 } else {
726 insn = &Assembler::str;
727 }
728 break;
729 case T_CHAR:
730 case T_SHORT:
731 assert(c->as_jint() == 0, "should be");
732 insn = &Assembler::strh;
733 break;
734 case T_BOOLEAN:
735 case T_BYTE:
736 assert(c->as_jint() == 0, "should be");
737 insn = &Assembler::strb;
738 break;
739 default:
740 ShouldNotReachHere();
741 }
742
743 if (info) add_debug_info_for_null_check_here(info);
744 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
745 }
746
747 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
748 assert(src->is_register(), "should not call otherwise");
749 assert(dest->is_register(), "should not call otherwise");
750
751 // move between cpu-registers
752 if (dest->is_single_cpu()) {
753 if (src->type() == T_LONG) {
754 // Can do LONG -> OBJECT
755 move_regs(src->as_register_lo(), dest->as_register());
756 return;
757 }
758 assert(src->is_single_cpu(), "must match");
759 if (src->type() == T_OBJECT) {
760 __ verify_oop(src->as_register());
761 }
762 move_regs(src->as_register(), dest->as_register());
763
764 } else if (dest->is_double_cpu()) {
765 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
766 // Surprising to me but we can see move of a long to t_object
767 __ verify_oop(src->as_register());
768 move_regs(src->as_register(), dest->as_register_lo());
769 return;
770 }
771 assert(src->is_double_cpu(), "must match");
772 Register f_lo = src->as_register_lo();
773 Register f_hi = src->as_register_hi();
774 Register t_lo = dest->as_register_lo();
775 Register t_hi = dest->as_register_hi();
776 assert(f_hi == f_lo, "must be same");
777 assert(t_hi == t_lo, "must be same");
778 move_regs(f_lo, t_lo);
779
780 } else if (dest->is_single_fpu()) {
781 __ fmovs(dest->as_float_reg(), src->as_float_reg());
782
783 } else if (dest->is_double_fpu()) {
784 __ fmovd(dest->as_double_reg(), src->as_double_reg());
785
786 } else {
787 ShouldNotReachHere();
788 }
789 }
790
791 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
792 if (src->is_single_cpu()) {
793 if (type == T_ARRAY || type == T_OBJECT) {
794 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
795 __ verify_oop(src->as_register());
796 } else if (type == T_METADATA || type == T_DOUBLE) {
797 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
798 } else {
799 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
800 }
801
802 } else if (src->is_double_cpu()) {
803 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
804 __ str(src->as_register_lo(), dest_addr_LO);
805
806 } else if (src->is_single_fpu()) {
807 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
808 __ strs(src->as_float_reg(), dest_addr);
809
810 } else if (src->is_double_fpu()) {
811 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
812 __ strd(src->as_double_reg(), dest_addr);
813
814 } else {
815 ShouldNotReachHere();
816 }
817
818 }
819
820
821 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
822 LIR_Address* to_addr = dest->as_address_ptr();
823 PatchingStub* patch = NULL;
824 Register compressed_src = rscratch1;
825
826 if (patch_code != lir_patch_none) {
827 deoptimize_trap(info);
828 return;
829 }
830
831 if (type == T_ARRAY || type == T_OBJECT) {
832 __ verify_oop(src->as_register());
833
834 if (UseCompressedOops && !wide) {
835 __ encode_heap_oop(compressed_src, src->as_register());
836 } else {
837 compressed_src = src->as_register();
838 }
839 }
840
841 int null_check_here = code_offset();
842 switch (type) {
843 case T_FLOAT: {
844 __ strs(src->as_float_reg(), as_Address(to_addr));
845 break;
846 }
847
848 case T_DOUBLE: {
849 __ strd(src->as_double_reg(), as_Address(to_addr));
850 break;
851 }
852
853 case T_ARRAY: // fall through
854 case T_OBJECT: // fall through
855 if (UseCompressedOops && !wide) {
856 __ strw(compressed_src, as_Address(to_addr, rscratch2));
857 } else {
858 __ str(compressed_src, as_Address(to_addr));
859 }
860 break;
861 case T_METADATA:
862 // We get here to store a method pointer to the stack to pass to
863 // a dtrace runtime call. This can't work on 64 bit with
864 // compressed klass ptrs: T_METADATA can be a compressed klass
865 // ptr or a 64 bit method pointer.
866 LP64_ONLY(ShouldNotReachHere());
867 __ str(src->as_register(), as_Address(to_addr));
868 break;
869 case T_ADDRESS:
870 __ str(src->as_register(), as_Address(to_addr));
871 break;
872 case T_INT:
873 __ strw(src->as_register(), as_Address(to_addr));
874 break;
875
876 case T_LONG: {
877 __ str(src->as_register_lo(), as_Address_lo(to_addr));
878 break;
879 }
880
881 case T_BYTE: // fall through
882 case T_BOOLEAN: {
883 __ strb(src->as_register(), as_Address(to_addr));
884 break;
885 }
886
887 case T_CHAR: // fall through
888 case T_SHORT:
889 __ strh(src->as_register(), as_Address(to_addr));
890 break;
891
892 default:
893 ShouldNotReachHere();
894 }
895 if (info != NULL) {
896 add_debug_info_for_null_check(null_check_here, info);
897 }
898 }
899
900
901 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
902 assert(src->is_stack(), "should not call otherwise");
903 assert(dest->is_register(), "should not call otherwise");
904
905 if (dest->is_single_cpu()) {
906 if (type == T_ARRAY || type == T_OBJECT) {
907 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
908 __ verify_oop(dest->as_register());
909 } else if (type == T_METADATA) {
910 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
911 } else {
912 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
913 }
914
915 } else if (dest->is_double_cpu()) {
916 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
917 __ ldr(dest->as_register_lo(), src_addr_LO);
918
919 } else if (dest->is_single_fpu()) {
920 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
921 __ ldrs(dest->as_float_reg(), src_addr);
922
923 } else if (dest->is_double_fpu()) {
924 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
925 __ ldrd(dest->as_double_reg(), src_addr);
926
927 } else {
928 ShouldNotReachHere();
929 }
930 }
931
932
933 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
934 address target = NULL;
935 relocInfo::relocType reloc_type = relocInfo::none;
936
937 switch (patching_id(info)) {
938 case PatchingStub::access_field_id:
939 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
940 reloc_type = relocInfo::section_word_type;
941 break;
942 case PatchingStub::load_klass_id:
943 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
944 reloc_type = relocInfo::metadata_type;
945 break;
946 case PatchingStub::load_mirror_id:
947 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
948 reloc_type = relocInfo::oop_type;
949 break;
950 case PatchingStub::load_appendix_id:
951 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
952 reloc_type = relocInfo::oop_type;
953 break;
954 default: ShouldNotReachHere();
955 }
956
957 __ bl(RuntimeAddress(target));
958 add_call_info_here(info);
959 }
960
961 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
962
963 LIR_Opr temp;
964 if (type == T_LONG || type == T_DOUBLE)
965 temp = FrameMap::rscratch1_long_opr;
966 else
967 temp = FrameMap::rscratch1_opr;
968
969 stack2reg(src, temp, src->type());
970 reg2stack(temp, dest, dest->type(), false);
971 }
972
973
974 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
975 LIR_Address* addr = src->as_address_ptr();
976 LIR_Address* from_addr = src->as_address_ptr();
977
978 if (addr->base()->type() == T_OBJECT) {
979 __ verify_oop(addr->base()->as_pointer_register());
980 }
981
982 if (patch_code != lir_patch_none) {
983 deoptimize_trap(info);
984 return;
985 }
986
987 if (info != NULL) {
988 add_debug_info_for_null_check_here(info);
989 }
990 int null_check_here = code_offset();
991 switch (type) {
992 case T_FLOAT: {
993 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
994 break;
995 }
996
997 case T_DOUBLE: {
998 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
999 break;
1000 }
1001
1002 case T_ARRAY: // fall through
1003 case T_OBJECT: // fall through
1004 if (UseCompressedOops && !wide) {
1005 __ ldrw(dest->as_register(), as_Address(from_addr));
1006 } else {
1007 __ ldr(dest->as_register(), as_Address(from_addr));
1008 }
1009 break;
1010 case T_METADATA:
1011 // We get here to store a method pointer to the stack to pass to
1012 // a dtrace runtime call. This can't work on 64 bit with
1013 // compressed klass ptrs: T_METADATA can be a compressed klass
1014 // ptr or a 64 bit method pointer.
1015 LP64_ONLY(ShouldNotReachHere());
1016 __ ldr(dest->as_register(), as_Address(from_addr));
1017 break;
1018 case T_ADDRESS:
1019 // FIXME: OMG this is a horrible kludge. Any offset from an
1020 // address that matches klass_offset_in_bytes() will be loaded
1021 // as a word, not a long.
1022 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1023 __ ldrw(dest->as_register(), as_Address(from_addr));
1024 } else {
1025 __ ldr(dest->as_register(), as_Address(from_addr));
1026 }
1027 break;
1028 case T_INT:
1029 __ ldrw(dest->as_register(), as_Address(from_addr));
1030 break;
1031
1032 case T_LONG: {
1033 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1034 break;
1035 }
1036
1037 case T_BYTE:
1038 __ ldrsb(dest->as_register(), as_Address(from_addr));
1039 break;
1040 case T_BOOLEAN: {
1041 __ ldrb(dest->as_register(), as_Address(from_addr));
1042 break;
1043 }
1044
1045 case T_CHAR:
1046 __ ldrh(dest->as_register(), as_Address(from_addr));
1047 break;
1048 case T_SHORT:
1049 __ ldrsh(dest->as_register(), as_Address(from_addr));
1050 break;
1051
1052 default:
1053 ShouldNotReachHere();
1054 }
1055
1056 if (type == T_ARRAY || type == T_OBJECT) {
1057 #ifdef _LP64
1058 if (UseCompressedOops && !wide) {
1059 __ decode_heap_oop(dest->as_register());
1060 }
1061 #endif
1062 __ verify_oop(dest->as_register());
1063 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1064 #ifdef _LP64
1065 if (UseCompressedClassPointers) {
1066 __ decode_klass_not_null(dest->as_register());
1067 }
1068 #endif
1069 }
1070 }
1071
1072
1073 void LIR_Assembler::prefetchr(LIR_Opr src) { Unimplemented(); }
1074
1075
1076 void LIR_Assembler::prefetchw(LIR_Opr src) { Unimplemented(); }
1077
1078
1079 int LIR_Assembler::array_element_size(BasicType type) const {
1080 int elem_size = type2aelembytes(type);
1081 return exact_log2(elem_size);
1082 }
1083
1084 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1085 Register Rdividend = op->in_opr1()->as_register();
1086 Register Rdivisor = op->in_opr2()->as_register();
1087 Register Rscratch = op->in_opr3()->as_register();
1088 Register Rresult = op->result_opr()->as_register();
1089 int divisor = -1;
1090
1091 /*
1092 TODO: For some reason, using the Rscratch that gets passed in is
1093 not possible because the register allocator does not see the tmp reg
1094 as used, and assignes it the same register as Rdividend. We use rscratch1
1095 instead.
1096
1097 assert(Rdividend != Rscratch, "");
1098 assert(Rdivisor != Rscratch, "");
1099 */
1100
1101 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1102 // convert division by a power of two into some shifts and logical operations
1103 }
1104
1105 if (op->code() == lir_irem) {
1106 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1107 } else if (op->code() == lir_idiv) {
1108 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1109 } else
1110 ShouldNotReachHere();
1111 }
1112
1113 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1114 #ifdef ASSERT
1115 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1116 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1117 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1118 #endif
1119
1120 if (op->cond() == lir_cond_always) {
1121 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1122 __ b(*(op->label()));
1123 } else {
1124 Assembler::Condition acond;
1125 if (op->code() == lir_cond_float_branch) {
1126 bool is_unordered = (op->ublock() == op->block());
1127 // Assembler::EQ does not permit unordered branches, so we add
1128 // another branch here. Likewise, Assembler::NE does not permit
1129 // ordered branches.
1130 if (is_unordered && op->cond() == lir_cond_equal
1131 || !is_unordered && op->cond() == lir_cond_notEqual)
1132 __ br(Assembler::VS, *(op->ublock()->label()));
1133 switch(op->cond()) {
1134 case lir_cond_equal: acond = Assembler::EQ; break;
1135 case lir_cond_notEqual: acond = Assembler::NE; break;
1136 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1137 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1138 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1139 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1140 default: ShouldNotReachHere();
1141 }
1142 } else {
1143 switch (op->cond()) {
1144 case lir_cond_equal: acond = Assembler::EQ; break;
1145 case lir_cond_notEqual: acond = Assembler::NE; break;
1146 case lir_cond_less: acond = Assembler::LT; break;
1147 case lir_cond_lessEqual: acond = Assembler::LE; break;
1148 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1149 case lir_cond_greater: acond = Assembler::GT; break;
1150 case lir_cond_belowEqual: acond = Assembler::LS; break;
1151 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1152 default: ShouldNotReachHere();
1153 }
1154 }
1155 __ br(acond,*(op->label()));
1156 }
1157 }
1158
1159
1160
1161 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1162 LIR_Opr src = op->in_opr();
1163 LIR_Opr dest = op->result_opr();
1164
1165 switch (op->bytecode()) {
1166 case Bytecodes::_i2f:
1167 {
1168 __ scvtfws(dest->as_float_reg(), src->as_register());
1169 break;
1170 }
1171 case Bytecodes::_i2d:
1172 {
1173 __ scvtfwd(dest->as_double_reg(), src->as_register());
1174 break;
1175 }
1176 case Bytecodes::_l2d:
1177 {
1178 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1179 break;
1180 }
1181 case Bytecodes::_l2f:
1182 {
1183 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1184 break;
1185 }
1186 case Bytecodes::_f2d:
1187 {
1188 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1189 break;
1190 }
1191 case Bytecodes::_d2f:
1192 {
1193 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1194 break;
1195 }
1196 case Bytecodes::_i2c:
1197 {
1198 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1199 break;
1200 }
1201 case Bytecodes::_i2l:
1202 {
1203 __ sxtw(dest->as_register_lo(), src->as_register());
1204 break;
1205 }
1206 case Bytecodes::_i2s:
1207 {
1208 __ sxth(dest->as_register(), src->as_register());
1209 break;
1210 }
1211 case Bytecodes::_i2b:
1212 {
1213 __ sxtb(dest->as_register(), src->as_register());
1214 break;
1215 }
1216 case Bytecodes::_l2i:
1217 {
1218 _masm->block_comment("FIXME: This could be a no-op");
1219 __ uxtw(dest->as_register(), src->as_register_lo());
1220 break;
1221 }
1222 case Bytecodes::_d2l:
1223 {
1224 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1225 break;
1226 }
1227 case Bytecodes::_f2i:
1228 {
1229 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1230 break;
1231 }
1232 case Bytecodes::_f2l:
1233 {
1234 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1235 break;
1236 }
1237 case Bytecodes::_d2i:
1238 {
1239 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1240 break;
1241 }
1242 default: ShouldNotReachHere();
1243 }
1244 }
1245
1246 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1247 if (op->init_check()) {
1248 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1249 InstanceKlass::init_state_offset()));
1250 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1251 add_debug_info_for_null_check_here(op->stub()->info());
1252 __ br(Assembler::NE, *op->stub()->entry());
1253 }
1254 __ allocate_object(op->obj()->as_register(),
1255 op->tmp1()->as_register(),
1256 op->tmp2()->as_register(),
1257 op->header_size(),
1258 op->object_size(),
1259 op->klass()->as_register(),
1260 *op->stub()->entry());
1261 __ bind(*op->stub()->continuation());
1262 }
1263
1264 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1265 Register len = op->len()->as_register();
1266 __ uxtw(len, len);
1267
1268 if (UseSlowPath ||
1269 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1270 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1271 __ b(*op->stub()->entry());
1272 } else {
1273 Register tmp1 = op->tmp1()->as_register();
1274 Register tmp2 = op->tmp2()->as_register();
1275 Register tmp3 = op->tmp3()->as_register();
1276 if (len == tmp1) {
1277 tmp1 = tmp3;
1278 } else if (len == tmp2) {
1279 tmp2 = tmp3;
1280 } else if (len == tmp3) {
1281 // everything is ok
1282 } else {
1283 __ mov(tmp3, len);
1284 }
1285 __ allocate_array(op->obj()->as_register(),
1286 len,
1287 tmp1,
1288 tmp2,
1289 arrayOopDesc::header_size(op->type()),
1290 array_element_size(op->type()),
1291 op->klass()->as_register(),
1292 *op->stub()->entry());
1293 }
1294 __ bind(*op->stub()->continuation());
1295 }
1296
1297 void LIR_Assembler::type_profile_helper(Register mdo,
1298 ciMethodData *md, ciProfileData *data,
1299 Register recv, Label* update_done) {
1300 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1301 Label next_test;
1302 // See if the receiver is receiver[n].
1303 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1304 __ ldr(rscratch1, Address(rscratch2));
1305 __ cmp(recv, rscratch1);
1306 __ br(Assembler::NE, next_test);
1307 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1308 __ addptr(data_addr, DataLayout::counter_increment);
1309 __ b(*update_done);
1310 __ bind(next_test);
1311 }
1312
1313 // Didn't find receiver; find next empty slot and fill it in
1314 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1315 Label next_test;
1316 __ lea(rscratch2,
1317 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1318 Address recv_addr(rscratch2);
1319 __ ldr(rscratch1, recv_addr);
1320 __ cbnz(rscratch1, next_test);
1321 __ str(recv, recv_addr);
1322 __ mov(rscratch1, DataLayout::counter_increment);
1323 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1324 __ str(rscratch1, Address(rscratch2));
1325 __ b(*update_done);
1326 __ bind(next_test);
1327 }
1328 }
1329
1330 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1331 // we always need a stub for the failure case.
1332 CodeStub* stub = op->stub();
1333 Register obj = op->object()->as_register();
1334 Register k_RInfo = op->tmp1()->as_register();
1335 Register klass_RInfo = op->tmp2()->as_register();
1336 Register dst = op->result_opr()->as_register();
1337 ciKlass* k = op->klass();
1338 Register Rtmp1 = noreg;
1339
1340 // check if it needs to be profiled
1341 ciMethodData* md;
1342 ciProfileData* data;
1343
1344 if (op->should_profile()) {
1345 ciMethod* method = op->profiled_method();
1346 assert(method != NULL, "Should have method");
1347 int bci = op->profiled_bci();
1348 md = method->method_data_or_null();
1349 assert(md != NULL, "Sanity");
1350 data = md->bci_to_data(bci);
1351 assert(data != NULL, "need data for type check");
1352 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1353 }
1354 Label profile_cast_success, profile_cast_failure;
1355 Label *success_target = op->should_profile() ? &profile_cast_success : success;
1356 Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1357
1358 if (obj == k_RInfo) {
1359 k_RInfo = dst;
1360 } else if (obj == klass_RInfo) {
1361 klass_RInfo = dst;
1362 }
1363 if (k->is_loaded() && !UseCompressedClassPointers) {
1364 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1365 } else {
1366 Rtmp1 = op->tmp3()->as_register();
1367 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1368 }
1369
1370 assert_different_registers(obj, k_RInfo, klass_RInfo);
1371
1372 if (op->should_profile()) {
1373 Label not_null;
1374 __ cbnz(obj, not_null);
1375 // Object is null; update MDO and exit
1376 Register mdo = klass_RInfo;
1377 __ mov_metadata(mdo, md->constant_encoding());
1378 Address data_addr
1379 = __ form_address(rscratch2, mdo,
1380 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1381 LogBytesPerWord);
1382 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1383 __ ldr(rscratch1, data_addr);
1384 __ orr(rscratch1, rscratch1, header_bits);
1385 __ str(rscratch1, data_addr);
1386 __ b(*obj_is_null);
1387 __ bind(not_null);
1388 } else {
1389 __ cbz(obj, *obj_is_null);
1390 }
1391
1392 if (!k->is_loaded()) {
1393 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1394 } else {
1395 #ifdef _LP64
1396 __ mov_metadata(k_RInfo, k->constant_encoding());
1397 #endif // _LP64
1398 }
1399 __ verify_oop(obj);
1400
1401 if (op->fast_check()) {
1402 // get object class
1403 // not a safepoint as obj null check happens earlier
1404 __ load_klass(rscratch1, obj);
1405 __ cmp( rscratch1, k_RInfo);
1406
1407 __ br(Assembler::NE, *failure_target);
1408 // successful cast, fall through to profile or jump
1409 } else {
1410 // get object class
1411 // not a safepoint as obj null check happens earlier
1412 __ load_klass(klass_RInfo, obj);
1413 if (k->is_loaded()) {
1414 // See if we get an immediate positive hit
1415 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1416 __ cmp(k_RInfo, rscratch1);
1417 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1418 __ br(Assembler::NE, *failure_target);
1419 // successful cast, fall through to profile or jump
1420 } else {
1421 // See if we get an immediate positive hit
1422 __ br(Assembler::EQ, *success_target);
1423 // check for self
1424 __ cmp(klass_RInfo, k_RInfo);
1425 __ br(Assembler::EQ, *success_target);
1426
1427 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1428 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1429 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1430 // result is a boolean
1431 __ cbzw(klass_RInfo, *failure_target);
1432 // successful cast, fall through to profile or jump
1433 }
1434 } else {
1435 // perform the fast part of the checking logic
1436 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1437 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1438 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1439 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1440 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1441 // result is a boolean
1442 __ cbz(k_RInfo, *failure_target);
1443 // successful cast, fall through to profile or jump
1444 }
1445 }
1446 if (op->should_profile()) {
1447 Register mdo = klass_RInfo, recv = k_RInfo;
1448 __ bind(profile_cast_success);
1449 __ mov_metadata(mdo, md->constant_encoding());
1450 __ load_klass(recv, obj);
1451 Label update_done;
1452 type_profile_helper(mdo, md, data, recv, success);
1453 __ b(*success);
1454
1455 __ bind(profile_cast_failure);
1456 __ mov_metadata(mdo, md->constant_encoding());
1457 Address counter_addr
1458 = __ form_address(rscratch2, mdo,
1459 md->byte_offset_of_slot(data, CounterData::count_offset()),
1460 LogBytesPerWord);
1461 __ ldr(rscratch1, counter_addr);
1462 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1463 __ str(rscratch1, counter_addr);
1464 __ b(*failure);
1465 }
1466 __ b(*success);
1467 }
1468
1469
1470 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1471 LIR_Code code = op->code();
1472 if (code == lir_store_check) {
1473 Register value = op->object()->as_register();
1474 Register array = op->array()->as_register();
1475 Register k_RInfo = op->tmp1()->as_register();
1476 Register klass_RInfo = op->tmp2()->as_register();
1477 Register Rtmp1 = op->tmp3()->as_register();
1478
1479 CodeStub* stub = op->stub();
1480
1481 // check if it needs to be profiled
1482 ciMethodData* md;
1483 ciProfileData* data;
1484
1485 if (op->should_profile()) {
1486 ciMethod* method = op->profiled_method();
1487 assert(method != NULL, "Should have method");
1488 int bci = op->profiled_bci();
1489 md = method->method_data_or_null();
1490 assert(md != NULL, "Sanity");
1491 data = md->bci_to_data(bci);
1492 assert(data != NULL, "need data for type check");
1493 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1494 }
1495 Label profile_cast_success, profile_cast_failure, done;
1496 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1497 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1498
1499 if (op->should_profile()) {
1500 Label not_null;
1501 __ cbnz(value, not_null);
1502 // Object is null; update MDO and exit
1503 Register mdo = klass_RInfo;
1504 __ mov_metadata(mdo, md->constant_encoding());
1505 Address data_addr
1506 = __ form_address(rscratch2, mdo,
1507 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1508 LogBytesPerInt);
1509 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1510 __ ldrw(rscratch1, data_addr);
1511 __ orrw(rscratch1, rscratch1, header_bits);
1512 __ strw(rscratch1, data_addr);
1513 __ b(done);
1514 __ bind(not_null);
1515 } else {
1516 __ cbz(value, done);
1517 }
1518
1519 add_debug_info_for_null_check_here(op->info_for_exception());
1520 __ load_klass(k_RInfo, array);
1521 __ load_klass(klass_RInfo, value);
1522
1523 // get instance klass (it's already uncompressed)
1524 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1525 // perform the fast part of the checking logic
1526 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1527 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1528 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1529 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1530 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1531 // result is a boolean
1532 __ cbzw(k_RInfo, *failure_target);
1533 // fall through to the success case
1534
1535 if (op->should_profile()) {
1536 Register mdo = klass_RInfo, recv = k_RInfo;
1537 __ bind(profile_cast_success);
1538 __ mov_metadata(mdo, md->constant_encoding());
1539 __ load_klass(recv, value);
1540 Label update_done;
1541 type_profile_helper(mdo, md, data, recv, &done);
1542 __ b(done);
1543
1544 __ bind(profile_cast_failure);
1545 __ mov_metadata(mdo, md->constant_encoding());
1546 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1547 __ lea(rscratch2, counter_addr);
1548 __ ldr(rscratch1, Address(rscratch2));
1549 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1550 __ str(rscratch1, Address(rscratch2));
1551 __ b(*stub->entry());
1552 }
1553
1554 __ bind(done);
1555 } else if (code == lir_checkcast) {
1556 Register obj = op->object()->as_register();
1557 Register dst = op->result_opr()->as_register();
1558 Label success;
1559 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1560 __ bind(success);
1561 if (dst != obj) {
1562 __ mov(dst, obj);
1563 }
1564 } else if (code == lir_instanceof) {
1565 Register obj = op->object()->as_register();
1566 Register dst = op->result_opr()->as_register();
1567 Label success, failure, done;
1568 emit_typecheck_helper(op, &success, &failure, &failure);
1569 __ bind(failure);
1570 __ mov(dst, zr);
1571 __ b(done);
1572 __ bind(success);
1573 __ mov(dst, 1);
1574 __ bind(done);
1575 } else {
1576 ShouldNotReachHere();
1577 }
1578 }
1579
1580 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1581 Label retry_load, nope;
1582 // flush and load exclusive from the memory location
1583 // and fail if it is not what we expect
1584 __ bind(retry_load);
1585 __ ldaxrw(rscratch1, addr);
1586 __ cmpw(rscratch1, cmpval);
1587 __ cset(rscratch1, Assembler::NE);
1588 __ br(Assembler::NE, nope);
1589 // if we store+flush with no intervening write rscratch1 wil be zero
1590 __ stlxrw(rscratch1, newval, addr);
1591 // retry so we only ever return after a load fails to compare
1592 // ensures we don't return a stale value after a failed write.
1593 __ cbnzw(rscratch1, retry_load);
1594 __ bind(nope);
1595 __ membar(__ AnyAny);
1596 }
1597
1598 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1599 Label retry_load, nope;
1600 // flush and load exclusive from the memory location
1601 // and fail if it is not what we expect
1602 __ bind(retry_load);
1603 __ ldaxr(rscratch1, addr);
1604 __ cmp(rscratch1, cmpval);
1605 __ cset(rscratch1, Assembler::NE);
1606 __ br(Assembler::NE, nope);
1607 // if we store+flush with no intervening write rscratch1 wil be zero
1608 __ stlxr(rscratch1, newval, addr);
1609 // retry so we only ever return after a load fails to compare
1610 // ensures we don't return a stale value after a failed write.
1611 __ cbnz(rscratch1, retry_load);
1612 __ bind(nope);
1613 __ membar(__ AnyAny);
1614 }
1615
1616
1617 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1618 assert(VM_Version::supports_cx8(), "wrong machine");
1619 Register addr = as_reg(op->addr());
1620 Register newval = as_reg(op->new_value());
1621 Register cmpval = as_reg(op->cmp_value());
1622 Label succeed, fail, around;
1623
1624 if (op->code() == lir_cas_obj) {
1625 if (UseCompressedOops) {
1626 Register t1 = op->tmp1()->as_register();
1627 assert(op->tmp1()->is_valid(), "must be");
1628 __ encode_heap_oop(t1, cmpval);
1629 cmpval = t1;
1630 __ encode_heap_oop(rscratch2, newval);
1631 newval = rscratch2;
1632 casw(addr, newval, cmpval);
1633 } else {
1634 casl(addr, newval, cmpval);
1635 }
1636 } else if (op->code() == lir_cas_int) {
1637 casw(addr, newval, cmpval);
1638 } else {
1639 casl(addr, newval, cmpval);
1640 }
1641 }
1642
1643
1644 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1645
1646 Assembler::Condition acond, ncond;
1647 switch (condition) {
1648 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1649 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1650 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1651 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1652 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1653 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1654 case lir_cond_belowEqual: Unimplemented(); break;
1655 case lir_cond_aboveEqual: Unimplemented(); break;
1656 default: ShouldNotReachHere();
1657 }
1658
1659 assert(result->is_single_cpu() || result->is_double_cpu(),
1660 "expect single register for result");
1661 if (opr1->is_constant() && opr2->is_constant()
1662 && opr1->type() == T_INT && opr2->type() == T_INT) {
1663 jint val1 = opr1->as_jint();
1664 jint val2 = opr2->as_jint();
1665 if (val1 == 0 && val2 == 1) {
1666 __ cset(result->as_register(), ncond);
1667 return;
1668 } else if (val1 == 1 && val2 == 0) {
1669 __ cset(result->as_register(), acond);
1670 return;
1671 }
1672 }
1673
1674 if (opr1->is_constant() && opr2->is_constant()
1675 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1676 jlong val1 = opr1->as_jlong();
1677 jlong val2 = opr2->as_jlong();
1678 if (val1 == 0 && val2 == 1) {
1679 __ cset(result->as_register_lo(), ncond);
1680 return;
1681 } else if (val1 == 1 && val2 == 0) {
1682 __ cset(result->as_register_lo(), acond);
1683 return;
1684 }
1685 }
1686
1687 if (opr1->is_stack()) {
1688 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1689 opr1 = FrameMap::rscratch1_opr;
1690 } else if (opr1->is_constant()) {
1691 LIR_Opr tmp
1692 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1693 const2reg(opr1, tmp, lir_patch_none, NULL);
1694 opr1 = tmp;
1695 }
1696
1697 if (opr2->is_stack()) {
1698 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1699 opr2 = FrameMap::rscratch2_opr;
1700 } else if (opr2->is_constant()) {
1701 LIR_Opr tmp
1702 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1703 const2reg(opr2, tmp, lir_patch_none, NULL);
1704 opr2 = tmp;
1705 }
1706
1707 if (result->type() == T_LONG)
1708 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1709 else
1710 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1711 }
1712
1713 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1714 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1715
1716 if (left->is_single_cpu()) {
1717 Register lreg = left->as_register();
1718 Register dreg = as_reg(dest);
1719
1720 if (right->is_single_cpu()) {
1721 // cpu register - cpu register
1722
1723 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1724 "should be");
1725 Register rreg = right->as_register();
1726 switch (code) {
1727 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1728 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1729 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1730 default: ShouldNotReachHere();
1731 }
1732
1733 } else if (right->is_double_cpu()) {
1734 Register rreg = right->as_register_lo();
1735 // single_cpu + double_cpu: can happen with obj+long
1736 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1737 switch (code) {
1738 case lir_add: __ add(dreg, lreg, rreg); break;
1739 case lir_sub: __ sub(dreg, lreg, rreg); break;
1740 default: ShouldNotReachHere();
1741 }
1742 } else if (right->is_constant()) {
1743 // cpu register - constant
1744 jlong c;
1745
1746 // FIXME. This is fugly: we really need to factor all this logic.
1747 switch(right->type()) {
1748 case T_LONG:
1749 c = right->as_constant_ptr()->as_jlong();
1750 break;
1751 case T_INT:
1752 case T_ADDRESS:
1753 c = right->as_constant_ptr()->as_jint();
1754 break;
1755 default:
1756 ShouldNotReachHere();
1757 break;
1758 }
1759
1760 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1761 if (c == 0 && dreg == lreg) {
1762 COMMENT("effective nop elided");
1763 return;
1764 }
1765 switch(left->type()) {
1766 case T_INT:
1767 switch (code) {
1768 case lir_add: __ addw(dreg, lreg, c); break;
1769 case lir_sub: __ subw(dreg, lreg, c); break;
1770 default: ShouldNotReachHere();
1771 }
1772 break;
1773 case T_OBJECT:
1774 case T_ADDRESS:
1775 switch (code) {
1776 case lir_add: __ add(dreg, lreg, c); break;
1777 case lir_sub: __ sub(dreg, lreg, c); break;
1778 default: ShouldNotReachHere();
1779 }
1780 break;
1781 ShouldNotReachHere();
1782 }
1783 } else {
1784 ShouldNotReachHere();
1785 }
1786
1787 } else if (left->is_double_cpu()) {
1788 Register lreg_lo = left->as_register_lo();
1789
1790 if (right->is_double_cpu()) {
1791 // cpu register - cpu register
1792 Register rreg_lo = right->as_register_lo();
1793 switch (code) {
1794 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1795 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1796 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1797 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1798 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1799 default:
1800 ShouldNotReachHere();
1801 }
1802
1803 } else if (right->is_constant()) {
1804 jlong c = right->as_constant_ptr()->as_jlong_bits();
1805 Register dreg = as_reg(dest);
1806 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1807 if (c == 0 && dreg == lreg_lo) {
1808 COMMENT("effective nop elided");
1809 return;
1810 }
1811 switch (code) {
1812 case lir_add: __ add(dreg, lreg_lo, c); break;
1813 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1814 default:
1815 ShouldNotReachHere();
1816 }
1817 } else {
1818 ShouldNotReachHere();
1819 }
1820 } else if (left->is_single_fpu()) {
1821 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1822 switch (code) {
1823 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1824 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1825 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1826 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1827 default:
1828 ShouldNotReachHere();
1829 }
1830 } else if (left->is_double_fpu()) {
1831 if (right->is_double_fpu()) {
1832 // cpu register - cpu register
1833 switch (code) {
1834 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1835 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1836 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1837 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1838 default:
1839 ShouldNotReachHere();
1840 }
1841 } else {
1842 if (right->is_constant()) {
1843 ShouldNotReachHere();
1844 }
1845 ShouldNotReachHere();
1846 }
1847 } else if (left->is_single_stack() || left->is_address()) {
1848 assert(left == dest, "left and dest must be equal");
1849 ShouldNotReachHere();
1850 } else {
1851 ShouldNotReachHere();
1852 }
1853 }
1854
1855 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1856
1857
1858 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1859 switch(code) {
1860 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1861 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1862 default : ShouldNotReachHere();
1863 }
1864 }
1865
1866 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1867
1868 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1869 Register Rleft = left->is_single_cpu() ? left->as_register() :
1870 left->as_register_lo();
1871 if (dst->is_single_cpu()) {
1872 Register Rdst = dst->as_register();
1873 if (right->is_constant()) {
1874 switch (code) {
1875 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1876 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1877 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1878 default: ShouldNotReachHere(); break;
1879 }
1880 } else {
1881 Register Rright = right->is_single_cpu() ? right->as_register() :
1882 right->as_register_lo();
1883 switch (code) {
1884 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1885 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1886 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1887 default: ShouldNotReachHere(); break;
1888 }
1889 }
1890 } else {
1891 Register Rdst = dst->as_register_lo();
1892 if (right->is_constant()) {
1893 switch (code) {
1894 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1895 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1896 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1897 default: ShouldNotReachHere(); break;
1898 }
1899 } else {
1900 Register Rright = right->is_single_cpu() ? right->as_register() :
1901 right->as_register_lo();
1902 switch (code) {
1903 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1904 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1905 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1906 default: ShouldNotReachHere(); break;
1907 }
1908 }
1909 }
1910 }
1911
1912
1913
1914 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1915
1916
1917 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1918 if (opr1->is_constant() && opr2->is_single_cpu()) {
1919 // tableswitch
1920 Register reg = as_reg(opr2);
1921 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1922 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1923 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1924 Register reg1 = as_reg(opr1);
1925 if (opr2->is_single_cpu()) {
1926 // cpu register - cpu register
1927 Register reg2 = opr2->as_register();
1928 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1929 __ cmp(reg1, reg2);
1930 } else {
1931 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1932 __ cmpw(reg1, reg2);
1933 }
1934 return;
1935 }
1936 if (opr2->is_double_cpu()) {
1937 // cpu register - cpu register
1938 Register reg2 = opr2->as_register_lo();
1939 __ cmp(reg1, reg2);
1940 return;
1941 }
1942
1943 if (opr2->is_constant()) {
1944 jlong imm;
1945 switch(opr2->type()) {
1946 case T_LONG:
1947 imm = opr2->as_constant_ptr()->as_jlong();
1948 break;
1949 case T_INT:
1950 case T_ADDRESS:
1951 imm = opr2->as_constant_ptr()->as_jint();
1952 break;
1953 case T_OBJECT:
1954 case T_ARRAY:
1955 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1956 break;
1957 default:
1958 ShouldNotReachHere();
1959 break;
1960 }
1961
1962 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1963 if (type2aelembytes(opr1->type()) <= 4)
1964 __ cmpw(reg1, imm);
1965 else
1966 __ cmp(reg1, imm);
1967 return;
1968 } else {
1969 __ mov(rscratch1, imm);
1970 if (type2aelembytes(opr1->type()) <= 4)
1971 __ cmpw(reg1, rscratch1);
1972 else
1973 __ cmp(reg1, rscratch1);
1974 return;
1975 }
1976 } else
1977 ShouldNotReachHere();
1978 } else if (opr1->is_single_fpu()) {
1979 FloatRegister reg1 = opr1->as_float_reg();
1980 assert(opr2->is_single_fpu(), "expect single float register");
1981 FloatRegister reg2 = opr2->as_float_reg();
1982 __ fcmps(reg1, reg2);
1983 } else if (opr1->is_double_fpu()) {
1984 FloatRegister reg1 = opr1->as_double_reg();
1985 assert(opr2->is_double_fpu(), "expect double float register");
1986 FloatRegister reg2 = opr2->as_double_reg();
1987 __ fcmpd(reg1, reg2);
1988 } else {
1989 ShouldNotReachHere();
1990 }
1991 }
1992
1993 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1994 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1995 bool is_unordered_less = (code == lir_ucmp_fd2i);
1996 if (left->is_single_fpu()) {
1997 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1998 } else if (left->is_double_fpu()) {
1999 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2000 } else {
2001 ShouldNotReachHere();
2002 }
2003 } else if (code == lir_cmp_l2i) {
2004 Label done;
2005 __ cmp(left->as_register_lo(), right->as_register_lo());
2006 __ mov(dst->as_register(), (u_int64_t)-1L);
2007 __ br(Assembler::LT, done);
2008 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2009 __ bind(done);
2010 } else {
2011 ShouldNotReachHere();
2012 }
2013 }
2014
2015
2016 void LIR_Assembler::align_call(LIR_Code code) { }
2017
2018
2019 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2020 __ bl(Address(op->addr(), rtype));
2021 add_call_info(code_offset(), op->info());
2022 }
2023
2024
2025 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2026 __ ic_call(op->addr());
2027 add_call_info(code_offset(), op->info());
2028 }
2029
2030
2031 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2032 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2033 ShouldNotReachHere();
2034 }
2035
2036
2037 void LIR_Assembler::emit_static_call_stub() {
2038 address call_pc = __ pc();
2039 address stub = __ start_a_stub(call_stub_size);
2040 if (stub == NULL) {
2041 bailout("static call stub overflow");
2042 return;
2043 }
2044
2045 int start = __ offset();
2046
2047 __ relocate(static_stub_Relocation::spec(call_pc));
2048 __ mov_metadata(rmethod, (Metadata*)NULL);
2049 __ b(__ pc());
2050
2051 assert(__ offset() - start <= call_stub_size, "stub too big");
2052 __ end_a_stub();
2053 }
2054
2055
2056 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2057 assert(exceptionOop->as_register() == r0, "must match");
2058 assert(exceptionPC->as_register() == r3, "must match");
2059
2060 // exception object is not added to oop map by LinearScan
2061 // (LinearScan assumes that no oops are in fixed registers)
2062 info->add_register_oop(exceptionOop);
2063 Runtime1::StubID unwind_id;
2064
2065 // get current pc information
2066 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2067 int pc_for_athrow_offset = __ offset();
2068 InternalAddress pc_for_athrow(__ pc());
2069 __ adr(exceptionPC->as_register(), pc_for_athrow);
2070 add_call_info(pc_for_athrow_offset, info); // for exception handler
2071
2072 __ verify_not_null_oop(r0);
2073 // search an exception handler (r0: exception oop, r3: throwing pc)
2074 if (compilation()->has_fpu_code()) {
2075 unwind_id = Runtime1::handle_exception_id;
2076 } else {
2077 unwind_id = Runtime1::handle_exception_nofpu_id;
2078 }
2079 __ bl(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2080
2081 // FIXME: enough room for two byte trap ????
2082 __ nop();
2083 }
2084
2085
2086 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2087 assert(exceptionOop->as_register() == r0, "must match");
2088
2089 __ b(_unwind_handler_entry);
2090 }
2091
2092
2093 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2094 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2095 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2096
2097 switch (left->type()) {
2098 case T_INT: {
2099 switch (code) {
2100 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2101 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2102 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2103 default:
2104 ShouldNotReachHere();
2105 break;
2106 }
2107 break;
2108 case T_LONG:
2109 case T_ADDRESS:
2110 case T_OBJECT:
2111 switch (code) {
2112 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2113 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2114 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2115 default:
2116 ShouldNotReachHere();
2117 break;
2118 }
2119 break;
2120 default:
2121 ShouldNotReachHere();
2122 break;
2123 }
2124 }
2125 }
2126
2127
2128 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2129 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2130 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2131
2132 switch (left->type()) {
2133 case T_INT: {
2134 switch (code) {
2135 case lir_shl: __ lslw (dreg, lreg, count); break;
2136 case lir_shr: __ asrw (dreg, lreg, count); break;
2137 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2138 default:
2139 ShouldNotReachHere();
2140 break;
2141 }
2142 break;
2143 case T_LONG:
2144 case T_ADDRESS:
2145 case T_OBJECT:
2146 switch (code) {
2147 case lir_shl: __ lsl (dreg, lreg, count); break;
2148 case lir_shr: __ asr (dreg, lreg, count); break;
2149 case lir_ushr: __ lsr (dreg, lreg, count); break;
2150 default:
2151 ShouldNotReachHere();
2152 break;
2153 }
2154 break;
2155 default:
2156 ShouldNotReachHere();
2157 break;
2158 }
2159 }
2160 }
2161
2162
2163 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2164 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2165 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2166 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2167 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2168 }
2169
2170
2171 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2172 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2173 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2174 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2175 __ mov (rscratch1, c);
2176 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2177 }
2178
2179
2180 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2181 ShouldNotReachHere();
2182 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2183 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2184 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2185 __ lea(rscratch1, __ constant_oop_address(o));
2186 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2187 }
2188
2189
2190 // This code replaces a call to arraycopy; no exception may
2191 // be thrown in this code, they must be thrown in the System.arraycopy
2192 // activation frame; we could save some checks if this would not be the case
2193 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2194 ciArrayKlass* default_type = op->expected_type();
2195 Register src = op->src()->as_register();
2196 Register dst = op->dst()->as_register();
2197 Register src_pos = op->src_pos()->as_register();
2198 Register dst_pos = op->dst_pos()->as_register();
2199 Register length = op->length()->as_register();
2200 Register tmp = op->tmp()->as_register();
2201
2202 CodeStub* stub = op->stub();
2203 int flags = op->flags();
2204 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2205 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2206
2207 // if we don't know anything, just go through the generic arraycopy
2208 if (default_type == NULL // || basic_type == T_OBJECT
2209 ) {
2210 Label done;
2211 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2212
2213 // Save the arguments in case the generic arraycopy fails and we
2214 // have to fall back to the JNI stub
2215 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2216 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2217 __ str(src, Address(sp, 4*BytesPerWord));
2218
2219 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2220 address copyfunc_addr = StubRoutines::generic_arraycopy();
2221
2222 // The arguments are in java calling convention so we shift them
2223 // to C convention
2224 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2225 __ mov(c_rarg0, j_rarg0);
2226 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2227 __ mov(c_rarg1, j_rarg1);
2228 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2229 __ mov(c_rarg2, j_rarg2);
2230 assert_different_registers(c_rarg3, j_rarg4);
2231 __ mov(c_rarg3, j_rarg3);
2232 __ mov(c_rarg4, j_rarg4);
2233 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2234 __ mov(rscratch1, RuntimeAddress(C_entry));
2235 __ blrt(rscratch1, 5, 0, 1);
2236 } else {
2237 #ifndef PRODUCT
2238 if (PrintC1Statistics) {
2239 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2240 }
2241 #endif
2242 __ bl(RuntimeAddress(copyfunc_addr));
2243 }
2244
2245 __ cbz(r0, *stub->continuation());
2246
2247 // Reload values from the stack so they are where the stub
2248 // expects them.
2249 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2250 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2251 __ ldr(src, Address(sp, 4*BytesPerWord));
2252
2253 if (copyfunc_addr != NULL) {
2254 // r0 is -1^K where K == partial copied count
2255 __ eonw(rscratch1, r0, 0);
2256 // adjust length down and src/end pos up by partial copied count
2257 __ subw(length, length, rscratch1);
2258 __ addw(src_pos, src_pos, rscratch1);
2259 __ addw(dst_pos, dst_pos, rscratch1);
2260 }
2261 __ b(*stub->entry());
2262
2263 __ bind(*stub->continuation());
2264 return;
2265 }
2266
2267 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2268
2269 int elem_size = type2aelembytes(basic_type);
2270 int shift_amount;
2271 int scale = exact_log2(elem_size);
2272
2273 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2274 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2275 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2276 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2277
2278 // test for NULL
2279 if (flags & LIR_OpArrayCopy::src_null_check) {
2280 __ cbz(src, *stub->entry());
2281 }
2282 if (flags & LIR_OpArrayCopy::dst_null_check) {
2283 __ cbz(dst, *stub->entry());
2284 }
2285
2286 // check if negative
2287 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2288 __ cmpw(src_pos, 0);
2289 __ br(Assembler::LT, *stub->entry());
2290 }
2291 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2292 __ cmpw(dst_pos, 0);
2293 __ br(Assembler::LT, *stub->entry());
2294 }
2295
2296 if (flags & LIR_OpArrayCopy::length_positive_check) {
2297 __ cmpw(length, 0);
2298 __ br(Assembler::LT, *stub->entry());
2299 }
2300
2301 if (flags & LIR_OpArrayCopy::src_range_check) {
2302 __ addw(tmp, src_pos, length);
2303 __ ldrw(rscratch1, src_length_addr);
2304 __ cmpw(tmp, rscratch1);
2305 __ br(Assembler::HI, *stub->entry());
2306 }
2307 if (flags & LIR_OpArrayCopy::dst_range_check) {
2308 __ addw(tmp, dst_pos, length);
2309 __ ldrw(rscratch1, dst_length_addr);
2310 __ cmpw(tmp, rscratch1);
2311 __ br(Assembler::HI, *stub->entry());
2312 }
2313
2314 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2315 // length_positive_check if the source of our length operand is an
2316 // arraylength. However, that arraylength might be zero, and the
2317 // stub that we're about to call contains an assertion that count !=
2318 // 0 . So we make this check purely in order not to trigger an
2319 // assertion failure.
2320 __ cbzw(length, *stub->continuation());
2321
2322 if (flags & LIR_OpArrayCopy::type_check) {
2323 // We don't know the array types are compatible
2324 if (basic_type != T_OBJECT) {
2325 // Simple test for basic type arrays
2326 if (UseCompressedClassPointers) {
2327 __ ldrw(tmp, src_klass_addr);
2328 __ ldrw(rscratch1, dst_klass_addr);
2329 __ cmpw(tmp, rscratch1);
2330 } else {
2331 __ ldr(tmp, src_klass_addr);
2332 __ ldr(rscratch1, dst_klass_addr);
2333 __ cmp(tmp, rscratch1);
2334 }
2335 __ br(Assembler::NE, *stub->entry());
2336 } else {
2337 // For object arrays, if src is a sub class of dst then we can
2338 // safely do the copy.
2339 Label cont, slow;
2340
2341 #define PUSH(r1, r2) \
2342 stp(r1, r2, __ pre(sp, -2 * wordSize));
2343
2344 #define POP(r1, r2) \
2345 ldp(r1, r2, __ post(sp, 2 * wordSize));
2346
2347 __ PUSH(src, dst);
2348
2349 __ load_klass(src, src);
2350 __ load_klass(dst, dst);
2351
2352 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2353
2354 __ PUSH(src, dst);
2355 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2356 __ POP(src, dst);
2357
2358 __ cbnz(src, cont);
2359
2360 __ bind(slow);
2361 __ POP(src, dst);
2362
2363 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2364 if (copyfunc_addr != NULL) { // use stub if available
2365 // src is not a sub class of dst so we have to do a
2366 // per-element check.
2367
2368 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2369 if ((flags & mask) != mask) {
2370 // Check that at least both of them object arrays.
2371 assert(flags & mask, "one of the two should be known to be an object array");
2372
2373 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2374 __ load_klass(tmp, src);
2375 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2376 __ load_klass(tmp, dst);
2377 }
2378 int lh_offset = in_bytes(Klass::layout_helper_offset());
2379 Address klass_lh_addr(tmp, lh_offset);
2380 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2381 __ ldrw(rscratch1, klass_lh_addr);
2382 __ mov(rscratch2, objArray_lh);
2383 __ eorw(rscratch1, rscratch1, rscratch2);
2384 __ cbnzw(rscratch1, *stub->entry());
2385 }
2386
2387 // Spill because stubs can use any register they like and it's
2388 // easier to restore just those that we care about.
2389 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2390 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2391 __ str(src, Address(sp, 4*BytesPerWord));
2392
2393 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2394 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2395 assert_different_registers(c_rarg0, dst, dst_pos, length);
2396 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2397 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2398 assert_different_registers(c_rarg1, dst, length);
2399 __ uxtw(c_rarg2, length);
2400 assert_different_registers(c_rarg2, dst);
2401
2402 __ load_klass(c_rarg4, dst);
2403 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2404 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2405 __ call(RuntimeAddress(copyfunc_addr));
2406
2407 #ifndef PRODUCT
2408 if (PrintC1Statistics) {
2409 Label failed;
2410 __ cbnz(r0, failed);
2411 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2412 __ bind(failed);
2413 }
2414 #endif
2415
2416 __ cbz(r0, *stub->continuation());
2417
2418 #ifndef PRODUCT
2419 if (PrintC1Statistics) {
2420 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2421 }
2422 #endif
2423 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2424
2425 // Restore previously spilled arguments
2426 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2427 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2428 __ ldr(src, Address(sp, 4*BytesPerWord));
2429
2430 // return value is -1^K where K is partial copied count
2431 __ eonw(rscratch1, r0, zr);
2432 // adjust length down and src/end pos up by partial copied count
2433 __ subw(length, length, rscratch1);
2434 __ addw(src_pos, src_pos, rscratch1);
2435 __ addw(dst_pos, dst_pos, rscratch1);
2436 }
2437
2438 __ b(*stub->entry());
2439
2440 __ bind(cont);
2441 __ POP(src, dst);
2442 }
2443 }
2444
2445 #ifdef ASSERT
2446 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2447 // Sanity check the known type with the incoming class. For the
2448 // primitive case the types must match exactly with src.klass and
2449 // dst.klass each exactly matching the default type. For the
2450 // object array case, if no type check is needed then either the
2451 // dst type is exactly the expected type and the src type is a
2452 // subtype which we can't check or src is the same array as dst
2453 // but not necessarily exactly of type default_type.
2454 Label known_ok, halt;
2455 __ mov_metadata(tmp, default_type->constant_encoding());
2456 #ifdef _LP64
2457 if (UseCompressedClassPointers) {
2458 __ encode_klass_not_null(tmp);
2459 }
2460 #endif
2461
2462 if (basic_type != T_OBJECT) {
2463
2464 if (UseCompressedClassPointers) {
2465 __ ldrw(rscratch1, dst_klass_addr);
2466 __ cmpw(tmp, rscratch1);
2467 } else {
2468 __ ldr(rscratch1, dst_klass_addr);
2469 __ cmp(tmp, rscratch1);
2470 }
2471 __ br(Assembler::NE, halt);
2472 if (UseCompressedClassPointers) {
2473 __ ldrw(rscratch1, src_klass_addr);
2474 __ cmpw(tmp, rscratch1);
2475 } else {
2476 __ ldr(rscratch1, src_klass_addr);
2477 __ cmp(tmp, rscratch1);
2478 }
2479 __ br(Assembler::EQ, known_ok);
2480 } else {
2481 if (UseCompressedClassPointers) {
2482 __ ldrw(rscratch1, dst_klass_addr);
2483 __ cmpw(tmp, rscratch1);
2484 } else {
2485 __ ldr(rscratch1, dst_klass_addr);
2486 __ cmp(tmp, rscratch1);
2487 }
2488 __ br(Assembler::EQ, known_ok);
2489 __ cmp(src, dst);
2490 __ br(Assembler::EQ, known_ok);
2491 }
2492 __ bind(halt);
2493 __ stop("incorrect type information in arraycopy");
2494 __ bind(known_ok);
2495 }
2496 #endif
2497
2498 #ifndef PRODUCT
2499 if (PrintC1Statistics) {
2500 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2501 }
2502 #endif
2503
2504 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2505 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2506 assert_different_registers(c_rarg0, dst, dst_pos, length);
2507 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2508 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2509 assert_different_registers(c_rarg1, dst, length);
2510 __ uxtw(c_rarg2, length);
2511 assert_different_registers(c_rarg2, dst);
2512
2513 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2514 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2515 const char *name;
2516 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2517
2518 CodeBlob *cb = CodeCache::find_blob(entry);
2519 if (cb) {
2520 __ bl(RuntimeAddress(entry));
2521 } else {
2522 __ call_VM_leaf(entry, 3);
2523 }
2524
2525 __ bind(*stub->continuation());
2526 }
2527
2528
2529
2530
2531 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2532 Register obj = op->obj_opr()->as_register(); // may not be an oop
2533 Register hdr = op->hdr_opr()->as_register();
2534 Register lock = op->lock_opr()->as_register();
2535 if (!UseFastLocking) {
2536 __ b(*op->stub()->entry());
2537 } else if (op->code() == lir_lock) {
2538 Register scratch = noreg;
2539 if (UseBiasedLocking) {
2540 scratch = op->scratch_opr()->as_register();
2541 }
2542 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2543 // add debug info for NullPointerException only if one is possible
2544 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2545 if (op->info() != NULL) {
2546 add_debug_info_for_null_check(null_check_offset, op->info());
2547 }
2548 // done
2549 } else if (op->code() == lir_unlock) {
2550 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2551 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2552 } else {
2553 Unimplemented();
2554 }
2555 __ bind(*op->stub()->continuation());
2556 }
2557
2558
2559 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2560 ciMethod* method = op->profiled_method();
2561 int bci = op->profiled_bci();
2562 ciMethod* callee = op->profiled_callee();
2563
2564 // Update counter for all call types
2565 ciMethodData* md = method->method_data_or_null();
2566 assert(md != NULL, "Sanity");
2567 ciProfileData* data = md->bci_to_data(bci);
2568 assert(data->is_CounterData(), "need CounterData for calls");
2569 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2570 Register mdo = op->mdo()->as_register();
2571 __ mov_metadata(mdo, md->constant_encoding());
2572 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2573 Bytecodes::Code bc = method->java_code_at_bci(bci);
2574 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2575 // Perform additional virtual call profiling for invokevirtual and
2576 // invokeinterface bytecodes
2577 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2578 !callee_is_static && // required for optimized MH invokes
2579 C1ProfileVirtualCalls) {
2580 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2581 Register recv = op->recv()->as_register();
2582 assert_different_registers(mdo, recv);
2583 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2584 ciKlass* known_klass = op->known_holder();
2585 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2586 // We know the type that will be seen at this call site; we can
2587 // statically update the MethodData* rather than needing to do
2588 // dynamic tests on the receiver type
2589
2590 // NOTE: we should probably put a lock around this search to
2591 // avoid collisions by concurrent compilations
2592 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2593 uint i;
2594 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2595 ciKlass* receiver = vc_data->receiver(i);
2596 if (known_klass->equals(receiver)) {
2597 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2598 __ addptr(data_addr, DataLayout::counter_increment);
2599 return;
2600 }
2601 }
2602
2603 // Receiver type not found in profile data; select an empty slot
2604
2605 // Note that this is less efficient than it should be because it
2606 // always does a write to the receiver part of the
2607 // VirtualCallData rather than just the first time
2608 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2609 ciKlass* receiver = vc_data->receiver(i);
2610 if (receiver == NULL) {
2611 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2612 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2613 __ lea(rscratch2, recv_addr);
2614 __ str(rscratch1, Address(rscratch2));
2615 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2616 __ addptr(data_addr, DataLayout::counter_increment);
2617 return;
2618 }
2619 }
2620 } else {
2621 __ load_klass(recv, recv);
2622 Label update_done;
2623 type_profile_helper(mdo, md, data, recv, &update_done);
2624 // Receiver did not match any saved receiver and there is no empty row for it.
2625 // Increment total counter to indicate polymorphic case.
2626 __ addptr(counter_addr, DataLayout::counter_increment);
2627
2628 __ bind(update_done);
2629 }
2630 } else {
2631 // Static call
2632 __ addptr(counter_addr, DataLayout::counter_increment);
2633 }
2634 }
2635
2636
2637 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2638 Unimplemented();
2639 }
2640
2641
2642 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2643 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2644 }
2645
2646 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2647 assert(op->crc()->is_single_cpu(), "crc must be register");
2648 assert(op->val()->is_single_cpu(), "byte value must be register");
2649 assert(op->result_opr()->is_single_cpu(), "result must be register");
2650 Register crc = op->crc()->as_register();
2651 Register val = op->val()->as_register();
2652 Register res = op->result_opr()->as_register();
2653
2654 assert_different_registers(val, crc, res);
2655 unsigned long offset;
2656 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2657 if (offset) __ add(res, res, offset);
2658
2659 __ ornw(crc, zr, crc); // ~crc
2660 __ update_byte_crc32(crc, val, res);
2661 __ ornw(res, zr, crc); // ~crc
2662 }
2663
2664 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2665 COMMENT("emit_profile_type {");
2666 Register obj = op->obj()->as_register();
2667 Register tmp = op->tmp()->as_pointer_register();
2668 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2669 ciKlass* exact_klass = op->exact_klass();
2670 intptr_t current_klass = op->current_klass();
2671 bool not_null = op->not_null();
2672 bool no_conflict = op->no_conflict();
2673
2674 Label update, next, none;
2675
2676 bool do_null = !not_null;
2677 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2678 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2679
2680 assert(do_null || do_update, "why are we here?");
2681 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2682 assert(mdo_addr.base() != rscratch1, "wrong register");
2683
2684 __ verify_oop(obj);
2685
2686 if (tmp != obj) {
2687 __ mov(tmp, obj);
2688 }
2689 if (do_null) {
2690 __ cbnz(tmp, update);
2691 if (!TypeEntries::was_null_seen(current_klass)) {
2692 __ ldr(rscratch2, mdo_addr);
2693 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2694 __ str(rscratch2, mdo_addr);
2695 }
2696 if (do_update) {
2697 #ifndef ASSERT
2698 __ b(next);
2699 }
2700 #else
2701 __ b(next);
2702 }
2703 } else {
2704 __ cbnz(tmp, update);
2705 __ stop("unexpected null obj");
2706 #endif
2707 }
2708
2709 __ bind(update);
2710
2711 if (do_update) {
2712 #ifdef ASSERT
2713 if (exact_klass != NULL) {
2714 Label ok;
2715 __ load_klass(tmp, tmp);
2716 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2717 __ eor(rscratch1, tmp, rscratch1);
2718 __ cbz(rscratch1, ok);
2719 __ stop("exact klass and actual klass differ");
2720 __ bind(ok);
2721 }
2722 #endif
2723 if (!no_conflict) {
2724 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2725 if (exact_klass != NULL) {
2726 __ mov_metadata(tmp, exact_klass->constant_encoding());
2727 } else {
2728 __ load_klass(tmp, tmp);
2729 }
2730
2731 __ ldr(rscratch2, mdo_addr);
2732 __ eor(tmp, tmp, rscratch2);
2733 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2734 // klass seen before, nothing to do. The unknown bit may have been
2735 // set already but no need to check.
2736 __ cbz(rscratch1, next);
2737
2738 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2739 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2740
2741 if (TypeEntries::is_type_none(current_klass)) {
2742 __ cbz(rscratch2, none);
2743 __ cmp(rscratch2, TypeEntries::null_seen);
2744 __ br(Assembler::EQ, none);
2745 // There is a chance that the checks above (re-reading profiling
2746 // data from memory) fail if another thread has just set the
2747 // profiling to this obj's klass
2748 __ dmb(Assembler::ISHLD);
2749 __ ldr(rscratch2, mdo_addr);
2750 __ eor(tmp, tmp, rscratch2);
2751 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2752 __ cbz(rscratch1, next);
2753 }
2754 } else {
2755 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2756 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2757
2758 __ ldr(tmp, mdo_addr);
2759 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2760 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2761 }
2762
2763 // different than before. Cannot keep accurate profile.
2764 __ ldr(rscratch2, mdo_addr);
2765 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2766 __ str(rscratch2, mdo_addr);
2767
2768 if (TypeEntries::is_type_none(current_klass)) {
2769 __ b(next);
2770
2771 __ bind(none);
2772 // first time here. Set profile type.
2773 __ str(tmp, mdo_addr);
2774 }
2775 } else {
2776 // There's a single possible klass at this profile point
2777 assert(exact_klass != NULL, "should be");
2778 if (TypeEntries::is_type_none(current_klass)) {
2779 __ mov_metadata(tmp, exact_klass->constant_encoding());
2780 __ ldr(rscratch2, mdo_addr);
2781 __ eor(tmp, tmp, rscratch2);
2782 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2783 __ cbz(rscratch1, next);
2784 #ifdef ASSERT
2785 {
2786 Label ok;
2787 __ ldr(rscratch1, mdo_addr);
2788 __ cbz(rscratch1, ok);
2789 __ cmp(rscratch1, TypeEntries::null_seen);
2790 __ br(Assembler::EQ, ok);
2791 // may have been set by another thread
2792 __ dmb(Assembler::ISHLD);
2793 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2794 __ ldr(rscratch2, mdo_addr);
2795 __ eor(rscratch2, rscratch1, rscratch2);
2796 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2797 __ cbz(rscratch2, ok);
2798
2799 __ stop("unexpected profiling mismatch");
2800 __ bind(ok);
2801 __ pop(tmp);
2802 }
2803 #endif
2804 // first time here. Set profile type.
2805 __ ldr(tmp, mdo_addr);
2806 } else {
2807 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2808 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2809
2810 __ ldr(tmp, mdo_addr);
2811 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2812 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2813
2814 __ orr(tmp, tmp, TypeEntries::type_unknown);
2815 __ str(tmp, mdo_addr);
2816 // FIXME: Write barrier needed here?
2817 }
2818 }
2819
2820 __ bind(next);
2821 }
2822 COMMENT("} emit_profile_type");
2823 }
2824
2825
2826 void LIR_Assembler::align_backward_branch_target() {
2827 }
2828
2829
2830 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2831 if (left->is_single_cpu()) {
2832 assert(dest->is_single_cpu(), "expect single result reg");
2833 __ negw(dest->as_register(), left->as_register());
2834 } else if (left->is_double_cpu()) {
2835 assert(dest->is_double_cpu(), "expect double result reg");
2836 __ neg(dest->as_register_lo(), left->as_register_lo());
2837 } else if (left->is_single_fpu()) {
2838 assert(dest->is_single_fpu(), "expect single float result reg");
2839 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2840 } else {
2841 assert(left->is_double_fpu(), "expect double float operand reg");
2842 assert(dest->is_double_fpu(), "expect double float result reg");
2843 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2844 }
2845 }
2846
2847
2848 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2849 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2850 }
2851
2852
2853 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2854 assert(!tmp->is_valid(), "don't need temporary");
2855
2856 CodeBlob *cb = CodeCache::find_blob(dest);
2857 if (cb) {
2858 __ bl(RuntimeAddress(dest));
2859 } else {
2860 __ mov(rscratch1, RuntimeAddress(dest));
2861 int len = args->length();
2862 int type = 0;
2863 if (! result->is_illegal()) {
2864 switch (result->type()) {
2865 case T_VOID:
2866 type = 0;
2867 break;
2868 case T_INT:
2869 case T_LONG:
2870 case T_OBJECT:
2871 type = 1;
2872 break;
2873 case T_FLOAT:
2874 type = 2;
2875 break;
2876 case T_DOUBLE:
2877 type = 3;
2878 break;
2879 default:
2880 ShouldNotReachHere();
2881 break;
2882 }
2883 }
2884 int num_gpargs = 0;
2885 int num_fpargs = 0;
2886 for (int i = 0; i < args->length(); i++) {
2887 LIR_Opr arg = args->at(i);
2888 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2889 num_fpargs++;
2890 } else {
2891 num_gpargs++;
2892 }
2893 }
2894 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2895 }
2896
2897 if (info != NULL) {
2898 add_call_info_here(info);
2899 }
2900 __ maybe_isb();
2901 }
2902
2903 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2904 if (dest->is_address() || src->is_address()) {
2905 move_op(src, dest, type, lir_patch_none, info,
2906 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2907 } else {
2908 ShouldNotReachHere();
2909 }
2910 }
2911
2912 #ifdef ASSERT
2913 // emit run-time assertion
2914 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2915 assert(op->code() == lir_assert, "must be");
2916
2917 if (op->in_opr1()->is_valid()) {
2918 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2919 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2920 } else {
2921 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2922 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2923 }
2924
2925 Label ok;
2926 if (op->condition() != lir_cond_always) {
2927 Assembler::Condition acond = Assembler::AL;
2928 switch (op->condition()) {
2929 case lir_cond_equal: acond = Assembler::EQ; break;
2930 case lir_cond_notEqual: acond = Assembler::NE; break;
2931 case lir_cond_less: acond = Assembler::LT; break;
2932 case lir_cond_lessEqual: acond = Assembler::LE; break;
2933 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2934 case lir_cond_greater: acond = Assembler::GT; break;
2935 case lir_cond_belowEqual: acond = Assembler::LS; break;
2936 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2937 default: ShouldNotReachHere();
2938 }
2939 __ br(acond, ok);
2940 }
2941 if (op->halt()) {
2942 const char* str = __ code_string(op->msg());
2943 __ stop(str);
2944 } else {
2945 breakpoint();
2946 }
2947 __ bind(ok);
2948 }
2949 #endif
2950
2951 #ifndef PRODUCT
2952 #define COMMENT(x) do { __ block_comment(x); } while (0)
2953 #else
2954 #define COMMENT(x)
2955 #endif
2956
2957 void LIR_Assembler::membar() {
2958 COMMENT("membar");
2959 __ membar(MacroAssembler::AnyAny);
2960 }
2961
2962 void LIR_Assembler::membar_acquire() {
2963 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2964 }
2965
2966 void LIR_Assembler::membar_release() {
2967 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2968 }
2969
2970 void LIR_Assembler::membar_loadload() {
2971 __ membar(Assembler::LoadLoad);
2972 }
2973
2974 void LIR_Assembler::membar_storestore() {
2975 __ membar(MacroAssembler::StoreStore);
2976 }
2977
2978 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2979
2980 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2981
2982 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2983 __ mov(result_reg->as_register(), rthread);
2984 }
2985
2986
2987 void LIR_Assembler::peephole(LIR_List *lir) {
2988 #if 0
2989 if (tableswitch_count >= max_tableswitches)
2990 return;
2991
2992 /*
2993 This finite-state automaton recognizes sequences of compare-and-
2994 branch instructions. We will turn them into a tableswitch. You
2995 could argue that C1 really shouldn't be doing this sort of
2996 optimization, but without it the code is really horrible.
2997 */
2998
2999 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3000 int first_key, last_key = -2147483648;
3001 int next_key = 0;
3002 int start_insn = -1;
3003 int last_insn = -1;
3004 Register reg = noreg;
3005 LIR_Opr reg_opr;
3006 state = start_s;
3007
3008 LIR_OpList* inst = lir->instructions_list();
3009 for (int i = 0; i < inst->length(); i++) {
3010 LIR_Op* op = inst->at(i);
3011 switch (state) {
3012 case start_s:
3013 first_key = -1;
3014 start_insn = i;
3015 switch (op->code()) {
3016 case lir_cmp:
3017 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3018 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3019 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3020 && opr2->is_constant()
3021 && opr2->type() == T_INT) {
3022 reg_opr = opr1;
3023 reg = opr1->as_register();
3024 first_key = opr2->as_constant_ptr()->as_jint();
3025 next_key = first_key + 1;
3026 state = cmp_s;
3027 goto next_state;
3028 }
3029 break;
3030 }
3031 break;
3032 case cmp_s:
3033 switch (op->code()) {
3034 case lir_branch:
3035 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3036 state = beq_s;
3037 last_insn = i;
3038 goto next_state;
3039 }
3040 }
3041 state = start_s;
3042 break;
3043 case beq_s:
3044 switch (op->code()) {
3045 case lir_cmp: {
3046 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3047 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3048 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3049 && opr1->as_register() == reg
3050 && opr2->is_constant()
3051 && opr2->type() == T_INT
3052 && opr2->as_constant_ptr()->as_jint() == next_key) {
3053 last_key = next_key;
3054 next_key++;
3055 state = cmp_s;
3056 goto next_state;
3057 }
3058 }
3059 }
3060 last_key = next_key;
3061 state = start_s;
3062 break;
3063 default:
3064 assert(false, "impossible state");
3065 }
3066 if (state == start_s) {
3067 if (first_key < last_key - 5L && reg != noreg) {
3068 {
3069 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3070 // reg->encoding(),
3071 // start_insn, first_key, last_key);
3072 // for (int i = 0; i < inst->length(); i++) {
3073 // inst->at(i)->print();
3074 // tty->print("\n");
3075 // }
3076 // tty->print("\n");
3077 }
3078
3079 struct tableswitch *sw = &switches[tableswitch_count];
3080 sw->_insn_index = start_insn, sw->_first_key = first_key,
3081 sw->_last_key = last_key, sw->_reg = reg;
3082 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3083 {
3084 // Insert the new table of branches
3085 int offset = last_insn;
3086 for (int n = first_key; n < last_key; n++) {
3087 inst->insert_before
3088 (last_insn + 1,
3089 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3090 inst->at(offset)->as_OpBranch()->label()));
3091 offset -= 2, i++;
3092 }
3093 }
3094 // Delete all the old compare-and-branch instructions
3095 for (int n = first_key; n < last_key; n++) {
3096 inst->remove_at(start_insn);
3097 inst->remove_at(start_insn);
3098 }
3099 // Insert the tableswitch instruction
3100 inst->insert_before(start_insn,
3101 new LIR_Op2(lir_cmp, lir_cond_always,
3102 LIR_OprFact::intConst(tableswitch_count),
3103 reg_opr));
3104 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3105 tableswitch_count++;
3106 }
3107 reg = noreg;
3108 last_key = -2147483648;
3109 }
3110 next_state:
3111 ;
3112 }
3113 #endif
3114 }
3115
3116 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3117 Address addr = as_Address(src->as_address_ptr(), noreg);
3118 BasicType type = src->type();
3119 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3120
3121 void (MacroAssembler::* lda)(Register Rd, Register Ra);
3122 void (MacroAssembler::* add)(Register Rd, Register Rn, RegisterOrConstant increment);
3123 void (MacroAssembler::* stl)(Register Rs, Register Rt, Register Rn);
3124
3125 switch(type) {
3126 case T_INT:
3127 lda = &MacroAssembler::ldaxrw;
3128 add = &MacroAssembler::addw;
3129 stl = &MacroAssembler::stlxrw;
3130 break;
3131 case T_LONG:
3132 lda = &MacroAssembler::ldaxr;
3133 add = &MacroAssembler::add;
3134 stl = &MacroAssembler::stlxr;
3135 break;
3136 case T_OBJECT:
3137 case T_ARRAY:
3138 if (UseCompressedOops) {
3139 lda = &MacroAssembler::ldaxrw;
3140 add = &MacroAssembler::addw;
3141 stl = &MacroAssembler::stlxrw;
3142 } else {
3143 lda = &MacroAssembler::ldaxr;
3144 add = &MacroAssembler::add;
3145 stl = &MacroAssembler::stlxr;
3146 }
3147 break;
3148 default:
3149 ShouldNotReachHere();
3150 }
3151
3152 switch (code) {
3153 case lir_xadd:
3154 {
3155 RegisterOrConstant inc;
3156 Register tmp = as_reg(tmp_op);
3157 Register dst = as_reg(dest);
3158 if (data->is_constant()) {
3159 inc = RegisterOrConstant(as_long(data));
3160 assert_different_registers(dst, addr.base(), tmp,
3161 rscratch1, rscratch2);
3162 } else {
3163 inc = RegisterOrConstant(as_reg(data));
3164 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3165 rscratch1, rscratch2);
3166 }
3167 Label again;
3168 __ lea(tmp, addr);
3169 __ bind(again);
3170 (_masm->*lda)(dst, tmp);
3171 (_masm->*add)(rscratch1, dst, inc);
3172 (_masm->*stl)(rscratch2, rscratch1, tmp);
3173 __ cbnzw(rscratch2, again);
3174 break;
3175 }
3176 case lir_xchg:
3177 {
3178 Register tmp = tmp_op->as_register();
3179 Register obj = as_reg(data);
3180 Register dst = as_reg(dest);
3181 if (is_oop && UseCompressedOops) {
3182 __ encode_heap_oop(obj);
3183 }
3184 assert_different_registers(obj, addr.base(), tmp, rscratch2, dst);
3185 Label again;
3186 __ lea(tmp, addr);
3187 __ bind(again);
3188 (_masm->*lda)(dst, tmp);
3189 (_masm->*stl)(rscratch2, obj, tmp);
3190 __ cbnzw(rscratch2, again);
3191 if (is_oop && UseCompressedOops) {
3192 __ decode_heap_oop(dst);
3193 }
3194 }
3195 break;
3196 default:
3197 ShouldNotReachHere();
3198 }
3199 __ membar(__ AnyAny);
3200 }
3201
3202 #undef __