1 /*
2 * Copyright (c) 2013, Red Hat Inc.
3 * Copyright (c) 2000, 2012, Oracle and/or its affiliates.
4 * All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "gc_interface/collectedHeap.hpp"
38 #include "memory/barrierSet.hpp"
39 #include "memory/cardTableModRefBS.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "vmreg_aarch64.inline.hpp"
44
45
46
47 #ifndef PRODUCT
48 #define COMMENT(x) do { __ block_comment(x); } while (0)
49 #else
50 #define COMMENT(x)
51 #endif
52
53 NEEDS_CLEANUP // remove this definitions ?
54 const Register IC_Klass = rscratch2; // where the IC klass is cached
55 const Register SYNC_header = r0; // synchronization header
56 const Register SHIFT_count = r0; // where count for shift operations must be
57
58 #define __ _masm->
59
60
61 static void select_different_registers(Register preserve,
62 Register extra,
63 Register &tmp1,
64 Register &tmp2) {
65 if (tmp1 == preserve) {
66 assert_different_registers(tmp1, tmp2, extra);
67 tmp1 = extra;
68 } else if (tmp2 == preserve) {
69 assert_different_registers(tmp1, tmp2, extra);
70 tmp2 = extra;
71 }
72 assert_different_registers(preserve, tmp1, tmp2);
73 }
74
75
76
77 static void select_different_registers(Register preserve,
78 Register extra,
79 Register &tmp1,
80 Register &tmp2,
81 Register &tmp3) {
82 if (tmp1 == preserve) {
83 assert_different_registers(tmp1, tmp2, tmp3, extra);
84 tmp1 = extra;
85 } else if (tmp2 == preserve) {
86 assert_different_registers(tmp1, tmp2, tmp3, extra);
87 tmp2 = extra;
88 } else if (tmp3 == preserve) {
89 assert_different_registers(tmp1, tmp2, tmp3, extra);
90 tmp3 = extra;
91 }
92 assert_different_registers(preserve, tmp1, tmp2, tmp3);
93 }
94
95
96 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
97
98
99 LIR_Opr LIR_Assembler::receiverOpr() {
100 return FrameMap::receiver_opr;
101 }
102
103 LIR_Opr LIR_Assembler::osrBufferPointer() {
104 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
105 }
106
107 //--------------fpu register translations-----------------------
108
109
110 address LIR_Assembler::float_constant(float f) {
111 address const_addr = __ float_constant(f);
112 if (const_addr == NULL) {
113 bailout("const section overflow");
114 return __ code()->consts()->start();
115 } else {
116 return const_addr;
117 }
118 }
119
120
121 address LIR_Assembler::double_constant(double d) {
122 address const_addr = __ double_constant(d);
123 if (const_addr == NULL) {
124 bailout("const section overflow");
125 return __ code()->consts()->start();
126 } else {
127 return const_addr;
128 }
129 }
130
131 address LIR_Assembler::int_constant(jlong n) {
132 address const_addr = __ long_constant(n);
133 if (const_addr == NULL) {
134 bailout("const section overflow");
135 return __ code()->consts()->start();
136 } else {
137 return const_addr;
138 }
139 }
140
141 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
142
143 void LIR_Assembler::reset_FPU() { Unimplemented(); }
144
145 void LIR_Assembler::fpop() { Unimplemented(); }
146
147 void LIR_Assembler::fxch(int i) { Unimplemented(); }
148
149 void LIR_Assembler::fld(int i) { Unimplemented(); }
150
151 void LIR_Assembler::ffree(int i) { Unimplemented(); }
152
153 void LIR_Assembler::breakpoint() { Unimplemented(); }
154
155 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
156
157 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
158
159 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
160 //-------------------------------------------
161
162 static Register as_reg(LIR_Opr op) {
163 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
164 }
165
166 static jlong as_long(LIR_Opr data) {
167 jlong result;
168 switch (data->type()) {
169 case T_INT:
170 result = (data->as_jint());
171 break;
172 case T_LONG:
173 result = (data->as_jlong());
174 break;
175 default:
176 ShouldNotReachHere();
177 }
178 return result;
179 }
180
181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
182 Register base = addr->base()->as_pointer_register();
183 LIR_Opr opr = addr->index();
184 if (opr->is_cpu_register()) {
185 Register index;
186 if (opr->is_single_cpu())
187 index = opr->as_register();
188 else
189 index = opr->as_register_lo();
190 assert(addr->disp() == 0, "must be");
191 switch(opr->type()) {
192 case T_INT:
193 return Address(base, index, Address::sxtw(addr->scale()));
194 case T_LONG:
195 return Address(base, index, Address::lsl(addr->scale()));
196 default:
197 ShouldNotReachHere();
198 }
199 } else {
200 intptr_t addr_offset = intptr_t(addr->disp());
201 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
202 return Address(base, addr_offset, Address::lsl(addr->scale()));
203 else {
204 __ mov(tmp, addr_offset);
205 return Address(base, tmp, Address::lsl(addr->scale()));
206 }
207 }
208 }
209
210 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
211 ShouldNotReachHere();
212 return Address();
213 }
214
215 Address LIR_Assembler::as_Address(LIR_Address* addr) {
216 return as_Address(addr, rscratch1);
217 }
218
219 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
220 return as_Address(addr, rscratch1); // Ouch
221 // FIXME: This needs to be much more clever. See x86.
222 }
223
224
225 void LIR_Assembler::osr_entry() {
226 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
227 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
228 ValueStack* entry_state = osr_entry->state();
229 int number_of_locks = entry_state->locks_size();
230
231 // we jump here if osr happens with the interpreter
232 // state set up to continue at the beginning of the
233 // loop that triggered osr - in particular, we have
234 // the following registers setup:
235 //
236 // r2: osr buffer
237 //
238
239 // build frame
240 ciMethod* m = compilation()->method();
241 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
242
243 // OSR buffer is
244 //
245 // locals[nlocals-1..0]
246 // monitors[0..number_of_locks]
247 //
248 // locals is a direct copy of the interpreter frame so in the osr buffer
249 // so first slot in the local array is the last local from the interpreter
250 // and last slot is local[0] (receiver) from the interpreter
251 //
252 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
253 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
254 // in the interpreter frame (the method lock if a sync method)
255
256 // Initialize monitors in the compiled activation.
257 // r2: pointer to osr buffer
258 //
259 // All other registers are dead at this point and the locals will be
260 // copied into place by code emitted in the IR.
261
262 Register OSR_buf = osrBufferPointer()->as_pointer_register();
263 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
264 int monitor_offset = BytesPerWord * method()->max_locals() +
265 (2 * BytesPerWord) * (number_of_locks - 1);
266 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
267 // the OSR buffer using 2 word entries: first the lock and then
268 // the oop.
269 for (int i = 0; i < number_of_locks; i++) {
270 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
271 #ifdef ASSERT
272 // verify the interpreter's monitor has a non-null object
273 {
274 Label L;
275 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
276 __ cbnz(rscratch1, L);
277 __ stop("locked object is NULL");
278 __ bind(L);
279 }
280 #endif
281 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
282 __ str(r19, frame_map()->address_for_monitor_lock(i));
283 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
284 __ str(r19, frame_map()->address_for_monitor_object(i));
285 }
286 }
287 }
288
289
290 // inline cache check; done before the frame is built.
291 int LIR_Assembler::check_icache() {
292 Register receiver = FrameMap::receiver_opr->as_register();
293 Register ic_klass = IC_Klass;
294 int start_offset = __ offset();
295 __ inline_cache_check(receiver, ic_klass);
296
297 // if icache check fails, then jump to runtime routine
298 // Note: RECEIVER must still contain the receiver!
299 Label dont;
300 __ br(Assembler::EQ, dont);
301 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
302
303 // We align the verified entry point unless the method body
304 // (including its inline cache check) will fit in a single 64-byte
305 // icache line.
306 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
307 // force alignment after the cache check.
308 __ align(CodeEntryAlignment);
309 }
310
311 __ bind(dont);
312 return start_offset;
313 }
314
315
316 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
317 if (o == NULL) {
318 __ mov(reg, zr);
319 } else {
320 __ movoop(reg, o, /*immediate*/true);
321 }
322 }
323
324 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
325 address target = NULL;
326 relocInfo::relocType reloc_type = relocInfo::none;
327
328 switch (patching_id(info)) {
329 case PatchingStub::access_field_id:
330 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
331 reloc_type = relocInfo::section_word_type;
332 break;
333 case PatchingStub::load_klass_id:
334 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
335 reloc_type = relocInfo::metadata_type;
336 break;
337 case PatchingStub::load_mirror_id:
338 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
339 reloc_type = relocInfo::oop_type;
340 break;
341 case PatchingStub::load_appendix_id:
342 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
343 reloc_type = relocInfo::oop_type;
344 break;
345 default: ShouldNotReachHere();
346 }
347
348 __ far_call(RuntimeAddress(target));
349 add_call_info_here(info);
350 }
351
352 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
353 deoptimize_trap(info);
354 }
355
356
357 // This specifies the rsp decrement needed to build the frame
358 int LIR_Assembler::initial_frame_size_in_bytes() const {
359 // if rounding, must let FrameMap know!
360
361 // The frame_map records size in slots (32bit word)
362
363 // subtract two words to account for return address and link
364 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
365 }
366
367
368 int LIR_Assembler::emit_exception_handler() {
369 // if the last instruction is a call (typically to do a throw which
370 // is coming at the end after block reordering) the return address
371 // must still point into the code area in order to avoid assertion
372 // failures when searching for the corresponding bci => add a nop
373 // (was bug 5/14/1999 - gri)
374 __ nop();
375
376 // generate code for exception handler
377 address handler_base = __ start_a_stub(exception_handler_size);
378 if (handler_base == NULL) {
379 // not enough space left for the handler
380 bailout("exception handler overflow");
381 return -1;
382 }
383
384 int offset = code_offset();
385
386 // the exception oop and pc are in r0, and r3
387 // no other registers need to be preserved, so invalidate them
388 __ invalidate_registers(false, true, true, false, true, true);
389
390 // check that there is really an exception
391 __ verify_not_null_oop(r0);
392
393 // search an exception handler (r0: exception oop, r3: throwing pc)
394 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
395 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
396 __ end_a_stub();
397
398 return offset;
399 }
400
401
402 // Emit the code to remove the frame from the stack in the exception
403 // unwind path.
404 int LIR_Assembler::emit_unwind_handler() {
405 #ifndef PRODUCT
406 if (CommentedAssembly) {
407 _masm->block_comment("Unwind handler");
408 }
409 #endif
410
411 int offset = code_offset();
412
413 // Fetch the exception from TLS and clear out exception related thread state
414 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
415 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
417
418 __ bind(_unwind_handler_entry);
419 __ verify_not_null_oop(r0);
420 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
421 __ mov(r19, r0); // Preserve the exception
422 }
423
424 // Preform needed unlocking
425 MonitorExitStub* stub = NULL;
426 if (method()->is_synchronized()) {
427 monitor_address(0, FrameMap::r0_opr);
428 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
429 __ unlock_object(r5, r4, r0, *stub->entry());
430 __ bind(*stub->continuation());
431 }
432
433 if (compilation()->env()->dtrace_method_probes()) {
434 __ call_Unimplemented();
435 #if 0
436 __ movptr(Address(rsp, 0), rax);
437 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
438 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
439 #endif
440 }
441
442 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
443 __ mov(r0, r19); // Restore the exception
444 }
445
446 // remove the activation and dispatch to the unwind handler
447 __ block_comment("remove_frame and dispatch to the unwind handler");
448 __ remove_frame(initial_frame_size_in_bytes());
449 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
450
451 // Emit the slow path assembly
452 if (stub != NULL) {
453 stub->emit_code(this);
454 }
455
456 return offset;
457 }
458
459
460 int LIR_Assembler::emit_deopt_handler() {
461 // if the last instruction is a call (typically to do a throw which
462 // is coming at the end after block reordering) the return address
463 // must still point into the code area in order to avoid assertion
464 // failures when searching for the corresponding bci => add a nop
465 // (was bug 5/14/1999 - gri)
466 __ nop();
467
468 // generate code for exception handler
469 address handler_base = __ start_a_stub(deopt_handler_size);
470 if (handler_base == NULL) {
471 // not enough space left for the handler
472 bailout("deopt handler overflow");
473 return -1;
474 }
475
476 int offset = code_offset();
477
478 __ adr(lr, pc());
479 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
480 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
481 __ end_a_stub();
482
483 return offset;
484 }
485
486
487 // This is the fast version of java.lang.String.compare; it has not
488 // OSR-entry and therefore, we generate a slow version for OSR's
489 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
490 __ mov(r2, (address)__FUNCTION__);
491 __ call_Unimplemented();
492 }
493
494
495 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
496 _masm->code_section()->relocate(adr, relocInfo::poll_type);
497 int pc_offset = code_offset();
498 flush_debug_info(pc_offset);
499 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
500 if (info->exception_handlers() != NULL) {
501 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
502 }
503 }
504
505 // Rather than take a segfault when the polling page is protected,
506 // explicitly check for a safepoint in progress and if there is one,
507 // fake a call to the handler as if a segfault had been caught.
508 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
509 __ mov(rscratch1, SafepointSynchronize::address_of_state());
510 __ ldrb(rscratch1, Address(rscratch1));
511 Label nope, poll;
512 __ cbz(rscratch1, nope);
513 __ block_comment("safepoint");
514 __ enter();
515 __ push(0x3, sp); // r0 & r1
516 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
517 __ adr(r0, poll);
518 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
519 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
520 __ blrt(rscratch1, 1, 0, 1);
521 __ maybe_isb();
522 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
523 __ mov(rscratch1, r0);
524 __ pop(0x3, sp); // r0 & r1
525 __ leave();
526 __ br(rscratch1);
527 address polling_page(os::get_polling_page());
528 assert(os::is_poll_address(polling_page), "should be");
529 unsigned long off;
530 __ adrp(rscratch1, Address(polling_page, rtype), off);
531 __ bind(poll);
532 if (info)
533 add_debug_info_for_branch(info); // This isn't just debug info:
534 // it's the oop map
535 else
536 __ code_section()->relocate(pc(), rtype);
537 __ ldrw(zr, Address(rscratch1, off));
538 __ bind(nope);
539 }
540
541 void LIR_Assembler::return_op(LIR_Opr result) {
542 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
543 // Pop the stack before the safepoint code
544 __ remove_frame(initial_frame_size_in_bytes());
545 if (UseCompilerSafepoints) {
546 address polling_page(os::get_polling_page());
547 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
548 } else {
549 poll_for_safepoint(relocInfo::poll_return_type);
550 }
551 __ ret(lr);
552 }
553
554 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
555 address polling_page(os::get_polling_page());
556 if (UseCompilerSafepoints) {
557 guarantee(info != NULL, "Shouldn't be NULL");
558 assert(os::is_poll_address(polling_page), "should be");
559 unsigned long off;
560 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
561 assert(off == 0, "must be");
562 add_debug_info_for_branch(info); // This isn't just debug info:
563 // it's the oop map
564 __ read_polling_page(rscratch1, relocInfo::poll_type);
565 } else {
566 poll_for_safepoint(relocInfo::poll_type, info);
567 }
568
569 return __ offset();
570 }
571
572
573 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
574 if (from_reg == r31_sp)
575 from_reg = sp;
576 if (to_reg == r31_sp)
577 to_reg = sp;
578 __ mov(to_reg, from_reg);
579 }
580
581 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
582
583
584 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
585 assert(src->is_constant(), "should not call otherwise");
586 assert(dest->is_register(), "should not call otherwise");
587 LIR_Const* c = src->as_constant_ptr();
588
589 switch (c->type()) {
590 case T_INT: {
591 assert(patch_code == lir_patch_none, "no patching handled here");
592 __ movw(dest->as_register(), c->as_jint());
593 break;
594 }
595
596 case T_ADDRESS: {
597 assert(patch_code == lir_patch_none, "no patching handled here");
598 __ mov(dest->as_register(), c->as_jint());
599 break;
600 }
601
602 case T_LONG: {
603 assert(patch_code == lir_patch_none, "no patching handled here");
604 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
605 break;
606 }
607
608 case T_OBJECT: {
609 if (patch_code == lir_patch_none) {
610 jobject2reg(c->as_jobject(), dest->as_register());
611 } else {
612 jobject2reg_with_patching(dest->as_register(), info);
613 }
614 break;
615 }
616
617 case T_METADATA: {
618 if (patch_code != lir_patch_none) {
619 klass2reg_with_patching(dest->as_register(), info);
620 } else {
621 __ mov_metadata(dest->as_register(), c->as_metadata());
622 }
623 break;
624 }
625
626 case T_FLOAT: {
627 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
628 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
629 } else {
630 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
631 __ ldrs(dest->as_float_reg(), Address(rscratch1));
632 }
633 break;
634 }
635
636 case T_DOUBLE: {
637 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
638 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
639 } else {
640 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
641 __ ldrd(dest->as_double_reg(), Address(rscratch1));
642 }
643 break;
644 }
645
646 default:
647 ShouldNotReachHere();
648 }
649 }
650
651 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
652 LIR_Const* c = src->as_constant_ptr();
653 switch (c->type()) {
654 case T_OBJECT:
655 {
656 if (! c->as_jobject())
657 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
658 else {
659 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
660 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
661 }
662 }
663 break;
664 case T_ADDRESS:
665 {
666 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
667 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
668 }
669 case T_INT:
670 case T_FLOAT:
671 {
672 Register reg = zr;
673 if (c->as_jint_bits() == 0)
674 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
675 else {
676 __ movw(rscratch1, c->as_jint_bits());
677 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
678 }
679 }
680 break;
681 case T_LONG:
682 case T_DOUBLE:
683 {
684 Register reg = zr;
685 if (c->as_jlong_bits() == 0)
686 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
687 lo_word_offset_in_bytes));
688 else {
689 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
690 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
691 lo_word_offset_in_bytes));
692 }
693 }
694 break;
695 default:
696 ShouldNotReachHere();
697 }
698 }
699
700 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
701 assert(src->is_constant(), "should not call otherwise");
702 LIR_Const* c = src->as_constant_ptr();
703 LIR_Address* to_addr = dest->as_address_ptr();
704
705 void (Assembler::* insn)(Register Rt, const Address &adr);
706
707 switch (type) {
708 case T_ADDRESS:
709 assert(c->as_jint() == 0, "should be");
710 insn = &Assembler::str;
711 break;
712 case T_LONG:
713 assert(c->as_jlong() == 0, "should be");
714 insn = &Assembler::str;
715 break;
716 case T_INT:
717 assert(c->as_jint() == 0, "should be");
718 insn = &Assembler::strw;
719 break;
720 case T_OBJECT:
721 case T_ARRAY:
722 assert(c->as_jobject() == 0, "should be");
723 if (UseCompressedOops && !wide) {
724 insn = &Assembler::strw;
725 } else {
726 insn = &Assembler::str;
727 }
728 break;
729 case T_CHAR:
730 case T_SHORT:
731 assert(c->as_jint() == 0, "should be");
732 insn = &Assembler::strh;
733 break;
734 case T_BOOLEAN:
735 case T_BYTE:
736 assert(c->as_jint() == 0, "should be");
737 insn = &Assembler::strb;
738 break;
739 default:
740 ShouldNotReachHere();
741 }
742
743 if (info) add_debug_info_for_null_check_here(info);
744 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
745 }
746
747 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
748 assert(src->is_register(), "should not call otherwise");
749 assert(dest->is_register(), "should not call otherwise");
750
751 // move between cpu-registers
752 if (dest->is_single_cpu()) {
753 if (src->type() == T_LONG) {
754 // Can do LONG -> OBJECT
755 move_regs(src->as_register_lo(), dest->as_register());
756 return;
757 }
758 assert(src->is_single_cpu(), "must match");
759 if (src->type() == T_OBJECT) {
760 __ verify_oop(src->as_register());
761 }
762 move_regs(src->as_register(), dest->as_register());
763
764 } else if (dest->is_double_cpu()) {
765 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
766 // Surprising to me but we can see move of a long to t_object
767 __ verify_oop(src->as_register());
768 move_regs(src->as_register(), dest->as_register_lo());
769 return;
770 }
771 assert(src->is_double_cpu(), "must match");
772 Register f_lo = src->as_register_lo();
773 Register f_hi = src->as_register_hi();
774 Register t_lo = dest->as_register_lo();
775 Register t_hi = dest->as_register_hi();
776 assert(f_hi == f_lo, "must be same");
777 assert(t_hi == t_lo, "must be same");
778 move_regs(f_lo, t_lo);
779
780 } else if (dest->is_single_fpu()) {
781 __ fmovs(dest->as_float_reg(), src->as_float_reg());
782
783 } else if (dest->is_double_fpu()) {
784 __ fmovd(dest->as_double_reg(), src->as_double_reg());
785
786 } else {
787 ShouldNotReachHere();
788 }
789 }
790
791 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
792 if (src->is_single_cpu()) {
793 if (type == T_ARRAY || type == T_OBJECT) {
794 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
795 __ verify_oop(src->as_register());
796 } else if (type == T_METADATA || type == T_DOUBLE) {
797 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
798 } else {
799 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
800 }
801
802 } else if (src->is_double_cpu()) {
803 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
804 __ str(src->as_register_lo(), dest_addr_LO);
805
806 } else if (src->is_single_fpu()) {
807 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
808 __ strs(src->as_float_reg(), dest_addr);
809
810 } else if (src->is_double_fpu()) {
811 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
812 __ strd(src->as_double_reg(), dest_addr);
813
814 } else {
815 ShouldNotReachHere();
816 }
817
818 }
819
820
821 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
822 LIR_Address* to_addr = dest->as_address_ptr();
823 PatchingStub* patch = NULL;
824 Register compressed_src = rscratch1;
825
826 if (patch_code != lir_patch_none) {
827 deoptimize_trap(info);
828 return;
829 }
830
831 if (type == T_ARRAY || type == T_OBJECT) {
832 __ verify_oop(src->as_register());
833
834 if (UseCompressedOops && !wide) {
835 __ encode_heap_oop(compressed_src, src->as_register());
836 } else {
837 compressed_src = src->as_register();
838 }
839 }
840
841 int null_check_here = code_offset();
842 switch (type) {
843 case T_FLOAT: {
844 __ strs(src->as_float_reg(), as_Address(to_addr));
845 break;
846 }
847
848 case T_DOUBLE: {
849 __ strd(src->as_double_reg(), as_Address(to_addr));
850 break;
851 }
852
853 case T_ARRAY: // fall through
854 case T_OBJECT: // fall through
855 if (UseCompressedOops && !wide) {
856 __ strw(compressed_src, as_Address(to_addr, rscratch2));
857 } else {
858 __ str(compressed_src, as_Address(to_addr));
859 }
860 break;
861 case T_METADATA:
862 // We get here to store a method pointer to the stack to pass to
863 // a dtrace runtime call. This can't work on 64 bit with
864 // compressed klass ptrs: T_METADATA can be a compressed klass
865 // ptr or a 64 bit method pointer.
866 LP64_ONLY(ShouldNotReachHere());
867 __ str(src->as_register(), as_Address(to_addr));
868 break;
869 case T_ADDRESS:
870 __ str(src->as_register(), as_Address(to_addr));
871 break;
872 case T_INT:
873 __ strw(src->as_register(), as_Address(to_addr));
874 break;
875
876 case T_LONG: {
877 __ str(src->as_register_lo(), as_Address_lo(to_addr));
878 break;
879 }
880
881 case T_BYTE: // fall through
882 case T_BOOLEAN: {
883 __ strb(src->as_register(), as_Address(to_addr));
884 break;
885 }
886
887 case T_CHAR: // fall through
888 case T_SHORT:
889 __ strh(src->as_register(), as_Address(to_addr));
890 break;
891
892 default:
893 ShouldNotReachHere();
894 }
895 if (info != NULL) {
896 add_debug_info_for_null_check(null_check_here, info);
897 }
898 }
899
900
901 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
902 assert(src->is_stack(), "should not call otherwise");
903 assert(dest->is_register(), "should not call otherwise");
904
905 if (dest->is_single_cpu()) {
906 if (type == T_ARRAY || type == T_OBJECT) {
907 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
908 __ verify_oop(dest->as_register());
909 } else if (type == T_METADATA) {
910 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
911 } else {
912 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
913 }
914
915 } else if (dest->is_double_cpu()) {
916 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
917 __ ldr(dest->as_register_lo(), src_addr_LO);
918
919 } else if (dest->is_single_fpu()) {
920 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
921 __ ldrs(dest->as_float_reg(), src_addr);
922
923 } else if (dest->is_double_fpu()) {
924 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
925 __ ldrd(dest->as_double_reg(), src_addr);
926
927 } else {
928 ShouldNotReachHere();
929 }
930 }
931
932
933 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
934 address target = NULL;
935 relocInfo::relocType reloc_type = relocInfo::none;
936
937 switch (patching_id(info)) {
938 case PatchingStub::access_field_id:
939 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
940 reloc_type = relocInfo::section_word_type;
941 break;
942 case PatchingStub::load_klass_id:
943 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
944 reloc_type = relocInfo::metadata_type;
945 break;
946 case PatchingStub::load_mirror_id:
947 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
948 reloc_type = relocInfo::oop_type;
949 break;
950 case PatchingStub::load_appendix_id:
951 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
952 reloc_type = relocInfo::oop_type;
953 break;
954 default: ShouldNotReachHere();
955 }
956
957 __ far_call(RuntimeAddress(target));
958 add_call_info_here(info);
959 }
960
961 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
962
963 LIR_Opr temp;
964 if (type == T_LONG || type == T_DOUBLE)
965 temp = FrameMap::rscratch1_long_opr;
966 else
967 temp = FrameMap::rscratch1_opr;
968
969 stack2reg(src, temp, src->type());
970 reg2stack(temp, dest, dest->type(), false);
971 }
972
973 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
974 LIR_Address* addr = src->as_address_ptr();
975 LIR_Address* from_addr = src->as_address_ptr();
976
977 if (addr->base()->type() == T_OBJECT) {
978 __ verify_oop(addr->base()->as_pointer_register());
979 }
980
981 if (patch_code != lir_patch_none) {
982 deoptimize_trap(info);
983 return;
984 }
985
986 if (info != NULL) {
987 add_debug_info_for_null_check_here(info);
988 }
989 int null_check_here = code_offset();
990 switch (type) {
991 case T_FLOAT: {
992 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
993 break;
994 }
995
996 case T_DOUBLE: {
997 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
998 break;
999 }
1000
1001 case T_ARRAY: // fall through
1002 case T_OBJECT: // fall through
1003 if (UseCompressedOops && !wide) {
1004 __ ldrw(dest->as_register(), as_Address(from_addr));
1005 } else {
1006 __ ldr(dest->as_register(), as_Address(from_addr));
1007 }
1008 break;
1009 case T_METADATA:
1010 // We get here to store a method pointer to the stack to pass to
1011 // a dtrace runtime call. This can't work on 64 bit with
1012 // compressed klass ptrs: T_METADATA can be a compressed klass
1013 // ptr or a 64 bit method pointer.
1014 LP64_ONLY(ShouldNotReachHere());
1015 __ ldr(dest->as_register(), as_Address(from_addr));
1016 break;
1017 case T_ADDRESS:
1018 // FIXME: OMG this is a horrible kludge. Any offset from an
1019 // address that matches klass_offset_in_bytes() will be loaded
1020 // as a word, not a long.
1021 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1022 __ ldrw(dest->as_register(), as_Address(from_addr));
1023 } else {
1024 __ ldr(dest->as_register(), as_Address(from_addr));
1025 }
1026 break;
1027 case T_INT:
1028 __ ldrw(dest->as_register(), as_Address(from_addr));
1029 break;
1030
1031 case T_LONG: {
1032 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1033 break;
1034 }
1035
1036 case T_BYTE:
1037 __ ldrsb(dest->as_register(), as_Address(from_addr));
1038 break;
1039 case T_BOOLEAN: {
1040 __ ldrb(dest->as_register(), as_Address(from_addr));
1041 break;
1042 }
1043
1044 case T_CHAR:
1045 __ ldrh(dest->as_register(), as_Address(from_addr));
1046 break;
1047 case T_SHORT:
1048 __ ldrsh(dest->as_register(), as_Address(from_addr));
1049 break;
1050
1051 default:
1052 ShouldNotReachHere();
1053 }
1054
1055 if (type == T_ARRAY || type == T_OBJECT) {
1056 #ifdef _LP64
1057 if (UseCompressedOops && !wide) {
1058 __ decode_heap_oop(dest->as_register());
1059 }
1060 #endif
1061 __ verify_oop(dest->as_register());
1062 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1063 #ifdef _LP64
1064 if (UseCompressedClassPointers) {
1065 __ decode_klass_not_null(dest->as_register());
1066 }
1067 #endif
1068 }
1069 }
1070
1071 void LIR_Assembler::prefetchr(LIR_Opr src) { Unimplemented(); }
1072
1073
1074 void LIR_Assembler::prefetchw(LIR_Opr src) { Unimplemented(); }
1075
1076
1077 int LIR_Assembler::array_element_size(BasicType type) const {
1078 int elem_size = type2aelembytes(type);
1079 return exact_log2(elem_size);
1080 }
1081
1082 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1083 Register Rdividend = op->in_opr1()->as_register();
1084 Register Rdivisor = op->in_opr2()->as_register();
1085 Register Rscratch = op->in_opr3()->as_register();
1086 Register Rresult = op->result_opr()->as_register();
1087 int divisor = -1;
1088
1089 /*
1090 TODO: For some reason, using the Rscratch that gets passed in is
1091 not possible because the register allocator does not see the tmp reg
1092 as used, and assignes it the same register as Rdividend. We use rscratch1
1093 instead.
1094
1095 assert(Rdividend != Rscratch, "");
1096 assert(Rdivisor != Rscratch, "");
1097 */
1098
1099 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1100 // convert division by a power of two into some shifts and logical operations
1101 }
1102
1103 if (op->code() == lir_irem) {
1104 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1105 } else if (op->code() == lir_idiv) {
1106 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1107 } else
1108 ShouldNotReachHere();
1109 }
1110
1111 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1112 #ifdef ASSERT
1113 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1114 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1115 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1116 #endif
1117
1118 if (op->cond() == lir_cond_always) {
1119 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1120 __ b(*(op->label()));
1121 } else {
1122 Assembler::Condition acond;
1123 if (op->code() == lir_cond_float_branch) {
1124 bool is_unordered = (op->ublock() == op->block());
1125 // Assembler::EQ does not permit unordered branches, so we add
1126 // another branch here. Likewise, Assembler::NE does not permit
1127 // ordered branches.
1128 if (is_unordered && op->cond() == lir_cond_equal
1129 || !is_unordered && op->cond() == lir_cond_notEqual)
1130 __ br(Assembler::VS, *(op->ublock()->label()));
1131 switch(op->cond()) {
1132 case lir_cond_equal: acond = Assembler::EQ; break;
1133 case lir_cond_notEqual: acond = Assembler::NE; break;
1134 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1135 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1136 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1137 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1138 default: ShouldNotReachHere();
1139 }
1140 } else {
1141 switch (op->cond()) {
1142 case lir_cond_equal: acond = Assembler::EQ; break;
1143 case lir_cond_notEqual: acond = Assembler::NE; break;
1144 case lir_cond_less: acond = Assembler::LT; break;
1145 case lir_cond_lessEqual: acond = Assembler::LE; break;
1146 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1147 case lir_cond_greater: acond = Assembler::GT; break;
1148 case lir_cond_belowEqual: acond = Assembler::LS; break;
1149 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1150 default: ShouldNotReachHere();
1151 }
1152 }
1153 __ br(acond,*(op->label()));
1154 }
1155 }
1156
1157 #if INCLUDE_ALL_GCS
1158 void LIR_Assembler::emit_opShenandoahWriteBarrier(LIR_OpShenandoahWriteBarrier* op) {
1159
1160 Register obj = op->in_opr()->as_register();
1161 Register res = op->result_opr()->as_register();
1162
1163 Label done;
1164
1165 __ block_comment("Shenandoah write barrier {");
1166
1167 if (res != obj) {
1168 __ mov(res, obj);
1169 }
1170 // Check for null.
1171 if (op->need_null_check()) {
1172 __ cbz(res, done);
1173 }
1174
1175 __ shenandoah_write_barrier(res);
1176
1177 __ bind(done);
1178
1179 __ block_comment("} Shenandoah write barrier");
1180
1181 }
1182 #endif
1183
1184 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1185 LIR_Opr src = op->in_opr();
1186 LIR_Opr dest = op->result_opr();
1187
1188 switch (op->bytecode()) {
1189 case Bytecodes::_i2f:
1190 {
1191 __ scvtfws(dest->as_float_reg(), src->as_register());
1192 break;
1193 }
1194 case Bytecodes::_i2d:
1195 {
1196 __ scvtfwd(dest->as_double_reg(), src->as_register());
1197 break;
1198 }
1199 case Bytecodes::_l2d:
1200 {
1201 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1202 break;
1203 }
1204 case Bytecodes::_l2f:
1205 {
1206 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1207 break;
1208 }
1209 case Bytecodes::_f2d:
1210 {
1211 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1212 break;
1213 }
1214 case Bytecodes::_d2f:
1215 {
1216 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1217 break;
1218 }
1219 case Bytecodes::_i2c:
1220 {
1221 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1222 break;
1223 }
1224 case Bytecodes::_i2l:
1225 {
1226 __ sxtw(dest->as_register_lo(), src->as_register());
1227 break;
1228 }
1229 case Bytecodes::_i2s:
1230 {
1231 __ sxth(dest->as_register(), src->as_register());
1232 break;
1233 }
1234 case Bytecodes::_i2b:
1235 {
1236 __ sxtb(dest->as_register(), src->as_register());
1237 break;
1238 }
1239 case Bytecodes::_l2i:
1240 {
1241 _masm->block_comment("FIXME: This could be a no-op");
1242 __ uxtw(dest->as_register(), src->as_register_lo());
1243 break;
1244 }
1245 case Bytecodes::_d2l:
1246 {
1247 Register tmp = op->tmp1()->as_register();
1248 __ clear_fpsr();
1249 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1250 __ get_fpsr(tmp);
1251 __ tst(tmp, 1); // FPSCR.IOC
1252 __ br(Assembler::NE, *(op->stub()->entry()));
1253 __ bind(*op->stub()->continuation());
1254 break;
1255 }
1256 case Bytecodes::_f2i:
1257 {
1258 Register tmp = op->tmp1()->as_register();
1259 __ clear_fpsr();
1260 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1261 __ get_fpsr(tmp);
1262 __ tst(tmp, 1); // FPSCR.IOC
1263 __ br(Assembler::NE, *(op->stub()->entry()));
1264 __ bind(*op->stub()->continuation());
1265 break;
1266 }
1267 case Bytecodes::_f2l:
1268 {
1269 Register tmp = op->tmp1()->as_register();
1270 __ clear_fpsr();
1271 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1272 __ get_fpsr(tmp);
1273 __ tst(tmp, 1); // FPSCR.IOC
1274 __ br(Assembler::NE, *(op->stub()->entry()));
1275 __ bind(*op->stub()->continuation());
1276 break;
1277 }
1278 case Bytecodes::_d2i:
1279 {
1280 Register tmp = op->tmp1()->as_register();
1281 __ clear_fpsr();
1282 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1283 __ get_fpsr(tmp);
1284 __ tst(tmp, 1); // FPSCR.IOC
1285 __ br(Assembler::NE, *(op->stub()->entry()));
1286 __ bind(*op->stub()->continuation());
1287 break;
1288 }
1289 default: ShouldNotReachHere();
1290 }
1291 }
1292
1293 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1294 if (op->init_check()) {
1295 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1296 InstanceKlass::init_state_offset()));
1297 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1298 add_debug_info_for_null_check_here(op->stub()->info());
1299 __ br(Assembler::NE, *op->stub()->entry());
1300 }
1301 __ allocate_object(op->obj()->as_register(),
1302 op->tmp1()->as_register(),
1303 op->tmp2()->as_register(),
1304 op->header_size(),
1305 op->object_size(),
1306 op->klass()->as_register(),
1307 *op->stub()->entry());
1308 __ bind(*op->stub()->continuation());
1309 }
1310
1311 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1312 Register len = op->len()->as_register();
1313 __ uxtw(len, len);
1314
1315 if (UseSlowPath ||
1316 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1317 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1318 __ b(*op->stub()->entry());
1319 } else {
1320 Register tmp1 = op->tmp1()->as_register();
1321 Register tmp2 = op->tmp2()->as_register();
1322 Register tmp3 = op->tmp3()->as_register();
1323 if (len == tmp1) {
1324 tmp1 = tmp3;
1325 } else if (len == tmp2) {
1326 tmp2 = tmp3;
1327 } else if (len == tmp3) {
1328 // everything is ok
1329 } else {
1330 __ mov(tmp3, len);
1331 }
1332 __ allocate_array(op->obj()->as_register(),
1333 len,
1334 tmp1,
1335 tmp2,
1336 arrayOopDesc::header_size(op->type()),
1337 array_element_size(op->type()),
1338 op->klass()->as_register(),
1339 *op->stub()->entry());
1340 }
1341 __ bind(*op->stub()->continuation());
1342 }
1343
1344 void LIR_Assembler::type_profile_helper(Register mdo,
1345 ciMethodData *md, ciProfileData *data,
1346 Register recv, Label* update_done) {
1347 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1348 Label next_test;
1349 // See if the receiver is receiver[n].
1350 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1351 __ ldr(rscratch1, Address(rscratch2));
1352 __ cmp(recv, rscratch1);
1353 __ br(Assembler::NE, next_test);
1354 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1355 __ addptr(data_addr, DataLayout::counter_increment);
1356 __ b(*update_done);
1357 __ bind(next_test);
1358 }
1359
1360 // Didn't find receiver; find next empty slot and fill it in
1361 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1362 Label next_test;
1363 __ lea(rscratch2,
1364 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1365 Address recv_addr(rscratch2);
1366 __ ldr(rscratch1, recv_addr);
1367 __ cbnz(rscratch1, next_test);
1368 __ str(recv, recv_addr);
1369 __ mov(rscratch1, DataLayout::counter_increment);
1370 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1371 __ str(rscratch1, Address(rscratch2));
1372 __ b(*update_done);
1373 __ bind(next_test);
1374 }
1375 }
1376
1377 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1378 // we always need a stub for the failure case.
1379 CodeStub* stub = op->stub();
1380 Register obj = op->object()->as_register();
1381 Register k_RInfo = op->tmp1()->as_register();
1382 Register klass_RInfo = op->tmp2()->as_register();
1383 Register dst = op->result_opr()->as_register();
1384 ciKlass* k = op->klass();
1385 Register Rtmp1 = noreg;
1386
1387 // check if it needs to be profiled
1388 ciMethodData* md;
1389 ciProfileData* data;
1390
1391 if (op->should_profile()) {
1392 ciMethod* method = op->profiled_method();
1393 assert(method != NULL, "Should have method");
1394 int bci = op->profiled_bci();
1395 md = method->method_data_or_null();
1396 assert(md != NULL, "Sanity");
1397 data = md->bci_to_data(bci);
1398 assert(data != NULL, "need data for type check");
1399 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1400 }
1401 Label profile_cast_success, profile_cast_failure;
1402 Label *success_target = op->should_profile() ? &profile_cast_success : success;
1403 Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1404
1405 if (obj == k_RInfo) {
1406 k_RInfo = dst;
1407 } else if (obj == klass_RInfo) {
1408 klass_RInfo = dst;
1409 }
1410 if (k->is_loaded() && !UseCompressedClassPointers) {
1411 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1412 } else {
1413 Rtmp1 = op->tmp3()->as_register();
1414 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1415 }
1416
1417 assert_different_registers(obj, k_RInfo, klass_RInfo);
1418
1419 if (op->should_profile()) {
1420 Label not_null;
1421 __ cbnz(obj, not_null);
1422 // Object is null; update MDO and exit
1423 Register mdo = klass_RInfo;
1424 __ mov_metadata(mdo, md->constant_encoding());
1425 Address data_addr
1426 = __ form_address(rscratch2, mdo,
1427 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1428 LogBytesPerWord);
1429 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1430 __ ldr(rscratch1, data_addr);
1431 __ orr(rscratch1, rscratch1, header_bits);
1432 __ str(rscratch1, data_addr);
1433 __ b(*obj_is_null);
1434 __ bind(not_null);
1435 } else {
1436 __ cbz(obj, *obj_is_null);
1437 }
1438
1439 if (!k->is_loaded()) {
1440 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1441 } else {
1442 #ifdef _LP64
1443 __ mov_metadata(k_RInfo, k->constant_encoding());
1444 #endif // _LP64
1445 }
1446 __ verify_oop(obj);
1447
1448 if (op->fast_check()) {
1449 // get object class
1450 // not a safepoint as obj null check happens earlier
1451 __ load_klass(rscratch1, obj);
1452 __ cmp( rscratch1, k_RInfo);
1453
1454 __ br(Assembler::NE, *failure_target);
1455 // successful cast, fall through to profile or jump
1456 } else {
1457 // get object class
1458 // not a safepoint as obj null check happens earlier
1459 __ load_klass(klass_RInfo, obj);
1460 if (k->is_loaded()) {
1461 // See if we get an immediate positive hit
1462 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1463 __ cmp(k_RInfo, rscratch1);
1464 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1465 __ br(Assembler::NE, *failure_target);
1466 // successful cast, fall through to profile or jump
1467 } else {
1468 // See if we get an immediate positive hit
1469 __ br(Assembler::EQ, *success_target);
1470 // check for self
1471 __ cmp(klass_RInfo, k_RInfo);
1472 __ br(Assembler::EQ, *success_target);
1473
1474 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1475 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1476 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1477 // result is a boolean
1478 __ cbzw(klass_RInfo, *failure_target);
1479 // successful cast, fall through to profile or jump
1480 }
1481 } else {
1482 // perform the fast part of the checking logic
1483 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1484 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1485 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1486 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1487 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1488 // result is a boolean
1489 __ cbz(k_RInfo, *failure_target);
1490 // successful cast, fall through to profile or jump
1491 }
1492 }
1493 if (op->should_profile()) {
1494 Register mdo = klass_RInfo, recv = k_RInfo;
1495 __ bind(profile_cast_success);
1496 __ mov_metadata(mdo, md->constant_encoding());
1497 __ load_klass(recv, obj);
1498 Label update_done;
1499 type_profile_helper(mdo, md, data, recv, success);
1500 __ b(*success);
1501
1502 __ bind(profile_cast_failure);
1503 __ mov_metadata(mdo, md->constant_encoding());
1504 Address counter_addr
1505 = __ form_address(rscratch2, mdo,
1506 md->byte_offset_of_slot(data, CounterData::count_offset()),
1507 LogBytesPerWord);
1508 __ ldr(rscratch1, counter_addr);
1509 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1510 __ str(rscratch1, counter_addr);
1511 __ b(*failure);
1512 }
1513 __ b(*success);
1514 }
1515
1516
1517 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1518 LIR_Code code = op->code();
1519 if (code == lir_store_check) {
1520 Register value = op->object()->as_register();
1521 Register array = op->array()->as_register();
1522 Register k_RInfo = op->tmp1()->as_register();
1523 Register klass_RInfo = op->tmp2()->as_register();
1524 Register Rtmp1 = op->tmp3()->as_register();
1525
1526 CodeStub* stub = op->stub();
1527
1528 // check if it needs to be profiled
1529 ciMethodData* md;
1530 ciProfileData* data;
1531
1532 if (op->should_profile()) {
1533 ciMethod* method = op->profiled_method();
1534 assert(method != NULL, "Should have method");
1535 int bci = op->profiled_bci();
1536 md = method->method_data_or_null();
1537 assert(md != NULL, "Sanity");
1538 data = md->bci_to_data(bci);
1539 assert(data != NULL, "need data for type check");
1540 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1541 }
1542 Label profile_cast_success, profile_cast_failure, done;
1543 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1544 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1545
1546 if (op->should_profile()) {
1547 Label not_null;
1548 __ cbnz(value, not_null);
1549 // Object is null; update MDO and exit
1550 Register mdo = klass_RInfo;
1551 __ mov_metadata(mdo, md->constant_encoding());
1552 Address data_addr
1553 = __ form_address(rscratch2, mdo,
1554 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1555 LogBytesPerInt);
1556 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1557 __ ldrw(rscratch1, data_addr);
1558 __ orrw(rscratch1, rscratch1, header_bits);
1559 __ strw(rscratch1, data_addr);
1560 __ b(done);
1561 __ bind(not_null);
1562 } else {
1563 __ cbz(value, done);
1564 }
1565
1566 add_debug_info_for_null_check_here(op->info_for_exception());
1567 __ load_klass(k_RInfo, array);
1568 __ load_klass(klass_RInfo, value);
1569
1570 // get instance klass (it's already uncompressed)
1571 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1572 // perform the fast part of the checking logic
1573 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1574 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1575 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1576 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1577 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1578 // result is a boolean
1579 __ cbzw(k_RInfo, *failure_target);
1580 // fall through to the success case
1581
1582 if (op->should_profile()) {
1583 Register mdo = klass_RInfo, recv = k_RInfo;
1584 __ bind(profile_cast_success);
1585 __ mov_metadata(mdo, md->constant_encoding());
1586 __ load_klass(recv, value);
1587 Label update_done;
1588 type_profile_helper(mdo, md, data, recv, &done);
1589 __ b(done);
1590
1591 __ bind(profile_cast_failure);
1592 __ mov_metadata(mdo, md->constant_encoding());
1593 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1594 __ lea(rscratch2, counter_addr);
1595 __ ldr(rscratch1, Address(rscratch2));
1596 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1597 __ str(rscratch1, Address(rscratch2));
1598 __ b(*stub->entry());
1599 }
1600
1601 __ bind(done);
1602 } else if (code == lir_checkcast) {
1603 Register obj = op->object()->as_register();
1604 Register dst = op->result_opr()->as_register();
1605 Label success;
1606 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1607 __ bind(success);
1608 if (dst != obj) {
1609 __ mov(dst, obj);
1610 }
1611 } else if (code == lir_instanceof) {
1612 Register obj = op->object()->as_register();
1613 Register dst = op->result_opr()->as_register();
1614 Label success, failure, done;
1615 emit_typecheck_helper(op, &success, &failure, &failure);
1616 __ bind(failure);
1617 __ mov(dst, zr);
1618 __ b(done);
1619 __ bind(success);
1620 __ mov(dst, 1);
1621 __ bind(done);
1622 } else {
1623 ShouldNotReachHere();
1624 }
1625 }
1626
1627 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1628 if (UseLSE) {
1629 __ mov(rscratch1, cmpval);
1630 __ casal(Assembler::word, rscratch1, newval, addr);
1631 __ cmpw(rscratch1, cmpval);
1632 __ cset(rscratch1, Assembler::NE);
1633 } else {
1634 Label retry_load, nope;
1635 // flush and load exclusive from the memory location
1636 // and fail if it is not what we expect
1637 if ((VM_Version::cpu_cpuFeatures() & VM_Version::CPU_STXR_PREFETCH))
1638 __ prfm(Address(addr), PSTL1STRM);
1639 __ bind(retry_load);
1640 __ ldaxrw(rscratch1, addr);
1641 __ cmpw(rscratch1, cmpval);
1642 __ cset(rscratch1, Assembler::NE);
1643 __ br(Assembler::NE, nope);
1644 // if we store+flush with no intervening write rscratch1 wil be zero
1645 __ stlxrw(rscratch1, newval, addr);
1646 // retry so we only ever return after a load fails to compare
1647 // ensures we don't return a stale value after a failed write.
1648 __ cbnzw(rscratch1, retry_load);
1649 __ bind(nope);
1650 }
1651 __ membar(__ AnyAny);
1652 }
1653
1654 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1655 if (UseLSE) {
1656 __ mov(rscratch1, cmpval);
1657 __ casal(Assembler::xword, rscratch1, newval, addr);
1658 __ cmp(rscratch1, cmpval);
1659 __ cset(rscratch1, Assembler::NE);
1660 } else {
1661 Label retry_load, nope;
1662 // flush and load exclusive from the memory location
1663 // and fail if it is not what we expect
1664 if ((VM_Version::cpu_cpuFeatures() & VM_Version::CPU_STXR_PREFETCH))
1665 __ prfm(Address(addr), PSTL1STRM);
1666 __ bind(retry_load);
1667 __ ldaxr(rscratch1, addr);
1668 __ cmp(rscratch1, cmpval);
1669 __ cset(rscratch1, Assembler::NE);
1670 __ br(Assembler::NE, nope);
1671 // if we store+flush with no intervening write rscratch1 wil be zero
1672 __ stlxr(rscratch1, newval, addr);
1673 // retry so we only ever return after a load fails to compare
1674 // ensures we don't return a stale value after a failed write.
1675 __ cbnz(rscratch1, retry_load);
1676 __ bind(nope);
1677 }
1678 __ membar(__ AnyAny);
1679 }
1680
1681 // Return 1 in rscratch1 if the CAS fails.
1682 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1683 assert(VM_Version::supports_cx8(), "wrong machine");
1684 Register addr = as_reg(op->addr());
1685 Register newval = as_reg(op->new_value());
1686 Register cmpval = as_reg(op->cmp_value());
1687 Register res = op->result_opr()->as_register();
1688
1689 if (op->code() == lir_cas_obj) {
1690 assert(op->tmp1()->is_valid(), "must be");
1691 Register t1 = op->tmp1()->as_register();
1692 if (UseCompressedOops) {
1693 #if INCLUDE_ALL_GCS
1694 if (UseShenandoahGC && ShenandoahCASBarrier) {
1695 __ encode_heap_oop(t1, cmpval);
1696 cmpval = t1;
1697 assert(op->tmp2()->is_valid(), "must be");
1698 Register t2 = op->tmp2()->as_register();
1699 __ encode_heap_oop(t2, newval);
1700 newval = t2;
1701 __ cmpxchg_oop_shenandoah(addr, cmpval, newval, Assembler::word, /*acquire*/ false, /*release*/ true, /*weak*/ false);
1702 __ csetw(res, Assembler::EQ);
1703 } else
1704 #endif
1705 {
1706 __ encode_heap_oop(t1, cmpval);
1707 cmpval = t1;
1708 __ encode_heap_oop(rscratch2, newval);
1709 newval = rscratch2;
1710 casw(addr, newval, cmpval);
1711 __ eorw (res, r8, 1);
1712 }
1713 } else {
1714 #if INCLUDE_ALL_GCS
1715 if (UseShenandoahGC && ShenandoahCASBarrier) {
1716 __ cmpxchg_oop_shenandoah(addr, cmpval, newval, Assembler::xword, /*acquire*/ false, /*release*/ true, /*weak*/ false);
1717 __ csetw(res, Assembler::EQ);
1718 } else
1719 #endif
1720 {
1721 casl(addr, newval, cmpval);
1722 __ eorw (res, r8, 1);
1723 }
1724 }
1725 } else if (op->code() == lir_cas_int) {
1726 casw(addr, newval, cmpval);
1727 __ eorw (res, r8, 1);
1728 } else {
1729 casl(addr, newval, cmpval);
1730 __ eorw (res, r8, 1);
1731 }
1732 }
1733
1734
1735 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1736
1737 Assembler::Condition acond, ncond;
1738 switch (condition) {
1739 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1740 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1741 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1742 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1743 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1744 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1745 case lir_cond_belowEqual: Unimplemented(); break;
1746 case lir_cond_aboveEqual: Unimplemented(); break;
1747 default: ShouldNotReachHere();
1748 }
1749
1750 assert(result->is_single_cpu() || result->is_double_cpu(),
1751 "expect single register for result");
1752 if (opr1->is_constant() && opr2->is_constant()
1753 && opr1->type() == T_INT && opr2->type() == T_INT) {
1754 jint val1 = opr1->as_jint();
1755 jint val2 = opr2->as_jint();
1756 if (val1 == 0 && val2 == 1) {
1757 __ cset(result->as_register(), ncond);
1758 return;
1759 } else if (val1 == 1 && val2 == 0) {
1760 __ cset(result->as_register(), acond);
1761 return;
1762 }
1763 }
1764
1765 if (opr1->is_constant() && opr2->is_constant()
1766 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1767 jlong val1 = opr1->as_jlong();
1768 jlong val2 = opr2->as_jlong();
1769 if (val1 == 0 && val2 == 1) {
1770 __ cset(result->as_register_lo(), ncond);
1771 return;
1772 } else if (val1 == 1 && val2 == 0) {
1773 __ cset(result->as_register_lo(), acond);
1774 return;
1775 }
1776 }
1777
1778 if (opr1->is_stack()) {
1779 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1780 opr1 = FrameMap::rscratch1_opr;
1781 } else if (opr1->is_constant()) {
1782 LIR_Opr tmp
1783 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1784 const2reg(opr1, tmp, lir_patch_none, NULL);
1785 opr1 = tmp;
1786 }
1787
1788 if (opr2->is_stack()) {
1789 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1790 opr2 = FrameMap::rscratch2_opr;
1791 } else if (opr2->is_constant()) {
1792 LIR_Opr tmp
1793 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1794 const2reg(opr2, tmp, lir_patch_none, NULL);
1795 opr2 = tmp;
1796 }
1797
1798 if (result->type() == T_LONG)
1799 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1800 else
1801 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1802 }
1803
1804 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1805 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1806
1807 if (left->is_single_cpu()) {
1808 Register lreg = left->as_register();
1809 Register dreg = as_reg(dest);
1810
1811 if (right->is_single_cpu()) {
1812 // cpu register - cpu register
1813
1814 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1815 "should be");
1816 Register rreg = right->as_register();
1817 switch (code) {
1818 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1819 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1820 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1821 default: ShouldNotReachHere();
1822 }
1823
1824 } else if (right->is_double_cpu()) {
1825 Register rreg = right->as_register_lo();
1826 // single_cpu + double_cpu: can happen with obj+long
1827 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1828 switch (code) {
1829 case lir_add: __ add(dreg, lreg, rreg); break;
1830 case lir_sub: __ sub(dreg, lreg, rreg); break;
1831 default: ShouldNotReachHere();
1832 }
1833 } else if (right->is_constant()) {
1834 // cpu register - constant
1835 jlong c;
1836
1837 // FIXME. This is fugly: we really need to factor all this logic.
1838 switch(right->type()) {
1839 case T_LONG:
1840 c = right->as_constant_ptr()->as_jlong();
1841 break;
1842 case T_INT:
1843 case T_ADDRESS:
1844 c = right->as_constant_ptr()->as_jint();
1845 break;
1846 default:
1847 ShouldNotReachHere();
1848 break;
1849 }
1850
1851 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1852 if (c == 0 && dreg == lreg) {
1853 COMMENT("effective nop elided");
1854 return;
1855 }
1856 switch(left->type()) {
1857 case T_INT:
1858 switch (code) {
1859 case lir_add: __ addw(dreg, lreg, c); break;
1860 case lir_sub: __ subw(dreg, lreg, c); break;
1861 default: ShouldNotReachHere();
1862 }
1863 break;
1864 case T_OBJECT:
1865 case T_ADDRESS:
1866 switch (code) {
1867 case lir_add: __ add(dreg, lreg, c); break;
1868 case lir_sub: __ sub(dreg, lreg, c); break;
1869 default: ShouldNotReachHere();
1870 }
1871 break;
1872 ShouldNotReachHere();
1873 }
1874 } else {
1875 ShouldNotReachHere();
1876 }
1877
1878 } else if (left->is_double_cpu()) {
1879 Register lreg_lo = left->as_register_lo();
1880
1881 if (right->is_double_cpu()) {
1882 // cpu register - cpu register
1883 Register rreg_lo = right->as_register_lo();
1884 switch (code) {
1885 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1886 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1887 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1888 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1889 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1890 default:
1891 ShouldNotReachHere();
1892 }
1893
1894 } else if (right->is_constant()) {
1895 jlong c = right->as_constant_ptr()->as_jlong_bits();
1896 Register dreg = as_reg(dest);
1897 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1898 if (c == 0 && dreg == lreg_lo) {
1899 COMMENT("effective nop elided");
1900 return;
1901 }
1902 switch (code) {
1903 case lir_add: __ add(dreg, lreg_lo, c); break;
1904 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1905 default:
1906 ShouldNotReachHere();
1907 }
1908 } else {
1909 ShouldNotReachHere();
1910 }
1911 } else if (left->is_single_fpu()) {
1912 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1913 switch (code) {
1914 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1915 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1916 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1917 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1918 default:
1919 ShouldNotReachHere();
1920 }
1921 } else if (left->is_double_fpu()) {
1922 if (right->is_double_fpu()) {
1923 // cpu register - cpu register
1924 switch (code) {
1925 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1926 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1927 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1928 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1929 default:
1930 ShouldNotReachHere();
1931 }
1932 } else {
1933 if (right->is_constant()) {
1934 ShouldNotReachHere();
1935 }
1936 ShouldNotReachHere();
1937 }
1938 } else if (left->is_single_stack() || left->is_address()) {
1939 assert(left == dest, "left and dest must be equal");
1940 ShouldNotReachHere();
1941 } else {
1942 ShouldNotReachHere();
1943 }
1944 }
1945
1946 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1947
1948
1949 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1950 switch(code) {
1951 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1952 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1953 default : ShouldNotReachHere();
1954 }
1955 }
1956
1957 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1958
1959 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1960 Register Rleft = left->is_single_cpu() ? left->as_register() :
1961 left->as_register_lo();
1962 if (dst->is_single_cpu()) {
1963 Register Rdst = dst->as_register();
1964 if (right->is_constant()) {
1965 switch (code) {
1966 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1967 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1968 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1969 default: ShouldNotReachHere(); break;
1970 }
1971 } else {
1972 Register Rright = right->is_single_cpu() ? right->as_register() :
1973 right->as_register_lo();
1974 switch (code) {
1975 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1976 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1977 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1978 default: ShouldNotReachHere(); break;
1979 }
1980 }
1981 } else {
1982 Register Rdst = dst->as_register_lo();
1983 if (right->is_constant()) {
1984 switch (code) {
1985 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1986 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1987 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1988 default: ShouldNotReachHere(); break;
1989 }
1990 } else {
1991 Register Rright = right->is_single_cpu() ? right->as_register() :
1992 right->as_register_lo();
1993 switch (code) {
1994 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1995 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1996 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1997 default: ShouldNotReachHere(); break;
1998 }
1999 }
2000 }
2001 }
2002
2003
2004
2005 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
2006
2007
2008 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2009 if (opr1->is_constant() && opr2->is_single_cpu()) {
2010 // tableswitch
2011 Register reg = as_reg(opr2);
2012 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
2013 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
2014 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
2015 Register reg1 = as_reg(opr1);
2016 if (opr2->is_single_cpu()) {
2017 // cpu register - cpu register
2018 Register reg2 = opr2->as_register();
2019 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
2020 __ cmpoops(reg1, reg2);
2021 } else {
2022 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
2023 __ cmpw(reg1, reg2);
2024 }
2025 return;
2026 }
2027 if (opr2->is_double_cpu()) {
2028 guarantee(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "need acmp barrier?");
2029 guarantee(opr1->type() != T_OBJECT && opr1->type() != T_ARRAY, "need acmp barrier?");
2030 // cpu register - cpu register
2031 Register reg2 = opr2->as_register_lo();
2032 __ cmp(reg1, reg2);
2033 return;
2034 }
2035
2036 if (opr2->is_constant()) {
2037 bool is_32bit = false; // width of register operand
2038 jlong imm;
2039
2040 switch(opr2->type()) {
2041 case T_INT:
2042 imm = opr2->as_constant_ptr()->as_jint();
2043 is_32bit = true;
2044 break;
2045 case T_LONG:
2046 imm = opr2->as_constant_ptr()->as_jlong();
2047 break;
2048 case T_ADDRESS:
2049 imm = opr2->as_constant_ptr()->as_jint();
2050 break;
2051 case T_OBJECT:
2052 case T_ARRAY:
2053 imm = jlong(opr2->as_constant_ptr()->as_jobject());
2054 break;
2055 default:
2056 ShouldNotReachHere();
2057 break;
2058 }
2059
2060 if (opr2->type() == T_OBJECT || opr2->type() == T_ARRAY) {
2061 jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
2062 __ cmpoops(reg1, rscratch1);
2063 return;
2064 }
2065
2066 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
2067 if (is_32bit)
2068 __ cmpw(reg1, imm);
2069 else
2070 __ cmp(reg1, imm);
2071 return;
2072 } else {
2073 __ mov(rscratch1, imm);
2074 if (is_32bit)
2075 __ cmpw(reg1, rscratch1);
2076 else
2077 __ cmp(reg1, rscratch1);
2078 return;
2079 }
2080 } else
2081 ShouldNotReachHere();
2082 } else if (opr1->is_single_fpu()) {
2083 FloatRegister reg1 = opr1->as_float_reg();
2084 assert(opr2->is_single_fpu(), "expect single float register");
2085 FloatRegister reg2 = opr2->as_float_reg();
2086 __ fcmps(reg1, reg2);
2087 } else if (opr1->is_double_fpu()) {
2088 FloatRegister reg1 = opr1->as_double_reg();
2089 assert(opr2->is_double_fpu(), "expect double float register");
2090 FloatRegister reg2 = opr2->as_double_reg();
2091 __ fcmpd(reg1, reg2);
2092 } else {
2093 ShouldNotReachHere();
2094 }
2095 }
2096
2097 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2098 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2099 bool is_unordered_less = (code == lir_ucmp_fd2i);
2100 if (left->is_single_fpu()) {
2101 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2102 } else if (left->is_double_fpu()) {
2103 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2104 } else {
2105 ShouldNotReachHere();
2106 }
2107 } else if (code == lir_cmp_l2i) {
2108 Label done;
2109 __ cmp(left->as_register_lo(), right->as_register_lo());
2110 __ mov(dst->as_register(), (u_int64_t)-1L);
2111 __ br(Assembler::LT, done);
2112 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2113 __ bind(done);
2114 } else {
2115 ShouldNotReachHere();
2116 }
2117 }
2118
2119
2120 void LIR_Assembler::align_call(LIR_Code code) { }
2121
2122
2123 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2124 address call = __ trampoline_call(Address(op->addr(), rtype));
2125 if (call == NULL) {
2126 bailout("trampoline stub overflow");
2127 return;
2128 }
2129 add_call_info(code_offset(), op->info());
2130 }
2131
2132
2133 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2134 address call = __ ic_call(op->addr());
2135 if (call == NULL) {
2136 bailout("trampoline stub overflow");
2137 return;
2138 }
2139 add_call_info(code_offset(), op->info());
2140 }
2141
2142
2143 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2144 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2145 ShouldNotReachHere();
2146 }
2147
2148
2149 void LIR_Assembler::emit_static_call_stub() {
2150 address call_pc = __ pc();
2151 address stub = __ start_a_stub(call_stub_size);
2152 if (stub == NULL) {
2153 bailout("static call stub overflow");
2154 return;
2155 }
2156
2157 int start = __ offset();
2158
2159 __ relocate(static_stub_Relocation::spec(call_pc));
2160 __ mov_metadata(rmethod, (Metadata*)NULL);
2161 __ movptr(rscratch1, 0);
2162 __ br(rscratch1);
2163
2164 assert(__ offset() - start <= call_stub_size, "stub too big");
2165 __ end_a_stub();
2166 }
2167
2168
2169 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2170 assert(exceptionOop->as_register() == r0, "must match");
2171 assert(exceptionPC->as_register() == r3, "must match");
2172
2173 // exception object is not added to oop map by LinearScan
2174 // (LinearScan assumes that no oops are in fixed registers)
2175 info->add_register_oop(exceptionOop);
2176 Runtime1::StubID unwind_id;
2177
2178 // get current pc information
2179 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2180 int pc_for_athrow_offset = __ offset();
2181 InternalAddress pc_for_athrow(__ pc());
2182 __ adr(exceptionPC->as_register(), pc_for_athrow);
2183 add_call_info(pc_for_athrow_offset, info); // for exception handler
2184
2185 __ verify_not_null_oop(r0);
2186 // search an exception handler (r0: exception oop, r3: throwing pc)
2187 if (compilation()->has_fpu_code()) {
2188 unwind_id = Runtime1::handle_exception_id;
2189 } else {
2190 unwind_id = Runtime1::handle_exception_nofpu_id;
2191 }
2192 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2193
2194 // FIXME: enough room for two byte trap ????
2195 __ nop();
2196 }
2197
2198
2199 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2200 assert(exceptionOop->as_register() == r0, "must match");
2201
2202 __ b(_unwind_handler_entry);
2203 }
2204
2205
2206 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2207 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2208 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2209
2210 switch (left->type()) {
2211 case T_INT: {
2212 switch (code) {
2213 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2214 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2215 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2216 default:
2217 ShouldNotReachHere();
2218 break;
2219 }
2220 break;
2221 case T_LONG:
2222 case T_ADDRESS:
2223 case T_OBJECT:
2224 switch (code) {
2225 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2226 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2227 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2228 default:
2229 ShouldNotReachHere();
2230 break;
2231 }
2232 break;
2233 default:
2234 ShouldNotReachHere();
2235 break;
2236 }
2237 }
2238 }
2239
2240
2241 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2242 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2243 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2244
2245 switch (left->type()) {
2246 case T_INT: {
2247 switch (code) {
2248 case lir_shl: __ lslw (dreg, lreg, count); break;
2249 case lir_shr: __ asrw (dreg, lreg, count); break;
2250 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2251 default:
2252 ShouldNotReachHere();
2253 break;
2254 }
2255 break;
2256 case T_LONG:
2257 case T_ADDRESS:
2258 case T_OBJECT:
2259 switch (code) {
2260 case lir_shl: __ lsl (dreg, lreg, count); break;
2261 case lir_shr: __ asr (dreg, lreg, count); break;
2262 case lir_ushr: __ lsr (dreg, lreg, count); break;
2263 default:
2264 ShouldNotReachHere();
2265 break;
2266 }
2267 break;
2268 default:
2269 ShouldNotReachHere();
2270 break;
2271 }
2272 }
2273 }
2274
2275
2276 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2277 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2278 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2279 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2280 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2281 }
2282
2283
2284 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2285 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2286 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2287 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2288 __ mov (rscratch1, c);
2289 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2290 }
2291
2292
2293 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2294 ShouldNotReachHere();
2295 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2296 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2297 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2298 __ lea(rscratch1, __ constant_oop_address(o));
2299 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2300 }
2301
2302
2303 // This code replaces a call to arraycopy; no exception may
2304 // be thrown in this code, they must be thrown in the System.arraycopy
2305 // activation frame; we could save some checks if this would not be the case
2306 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2307 ciArrayKlass* default_type = op->expected_type();
2308 Register src = op->src()->as_register();
2309 Register dst = op->dst()->as_register();
2310 Register src_pos = op->src_pos()->as_register();
2311 Register dst_pos = op->dst_pos()->as_register();
2312 Register length = op->length()->as_register();
2313 Register tmp = op->tmp()->as_register();
2314
2315 CodeStub* stub = op->stub();
2316 int flags = op->flags();
2317 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2318 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2319
2320 // if we don't know anything, just go through the generic arraycopy
2321 if (default_type == NULL // || basic_type == T_OBJECT
2322 ) {
2323 Label done;
2324 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2325
2326 // Save the arguments in case the generic arraycopy fails and we
2327 // have to fall back to the JNI stub
2328 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2329 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2330 __ str(src, Address(sp, 4*BytesPerWord));
2331
2332 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2333 address copyfunc_addr = StubRoutines::generic_arraycopy();
2334
2335 // The arguments are in java calling convention so we shift them
2336 // to C convention
2337 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2338 __ mov(c_rarg0, j_rarg0);
2339 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2340 __ mov(c_rarg1, j_rarg1);
2341 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2342 __ mov(c_rarg2, j_rarg2);
2343 assert_different_registers(c_rarg3, j_rarg4);
2344 __ mov(c_rarg3, j_rarg3);
2345 __ mov(c_rarg4, j_rarg4);
2346 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2347 __ mov(rscratch1, RuntimeAddress(C_entry));
2348 __ blrt(rscratch1, 5, 0, 1);
2349 } else {
2350 #ifndef PRODUCT
2351 if (PrintC1Statistics) {
2352 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2353 }
2354 #endif
2355 __ far_call(RuntimeAddress(copyfunc_addr));
2356 }
2357
2358 __ cbz(r0, *stub->continuation());
2359
2360 // Reload values from the stack so they are where the stub
2361 // expects them.
2362 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2363 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2364 __ ldr(src, Address(sp, 4*BytesPerWord));
2365
2366 if (copyfunc_addr != NULL) {
2367 // r0 is -1^K where K == partial copied count
2368 __ eonw(rscratch1, r0, 0);
2369 // adjust length down and src/end pos up by partial copied count
2370 __ subw(length, length, rscratch1);
2371 __ addw(src_pos, src_pos, rscratch1);
2372 __ addw(dst_pos, dst_pos, rscratch1);
2373 }
2374 __ b(*stub->entry());
2375
2376 __ bind(*stub->continuation());
2377 return;
2378 }
2379
2380 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2381
2382 int elem_size = type2aelembytes(basic_type);
2383 int shift_amount;
2384 int scale = exact_log2(elem_size);
2385
2386 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2387 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2388 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2389 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2390
2391 // test for NULL
2392 if (flags & LIR_OpArrayCopy::src_null_check) {
2393 __ cbz(src, *stub->entry());
2394 }
2395 if (flags & LIR_OpArrayCopy::dst_null_check) {
2396 __ cbz(dst, *stub->entry());
2397 }
2398
2399 // If the compiler was not able to prove that exact type of the source or the destination
2400 // of the arraycopy is an array type, check at runtime if the source or the destination is
2401 // an instance type.
2402 if (flags & LIR_OpArrayCopy::type_check) {
2403 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2404 __ load_klass(tmp, dst);
2405 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2406 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2407 __ br(Assembler::GE, *stub->entry());
2408 }
2409
2410 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2411 __ load_klass(tmp, src);
2412 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2413 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2414 __ br(Assembler::GE, *stub->entry());
2415 }
2416 }
2417
2418 // check if negative
2419 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2420 __ cmpw(src_pos, 0);
2421 __ br(Assembler::LT, *stub->entry());
2422 }
2423 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2424 __ cmpw(dst_pos, 0);
2425 __ br(Assembler::LT, *stub->entry());
2426 }
2427
2428 if (flags & LIR_OpArrayCopy::length_positive_check) {
2429 __ cmpw(length, 0);
2430 __ br(Assembler::LT, *stub->entry());
2431 }
2432
2433 if (flags & LIR_OpArrayCopy::src_range_check) {
2434 __ addw(tmp, src_pos, length);
2435 __ ldrw(rscratch1, src_length_addr);
2436 __ cmpw(tmp, rscratch1);
2437 __ br(Assembler::HI, *stub->entry());
2438 }
2439 if (flags & LIR_OpArrayCopy::dst_range_check) {
2440 __ addw(tmp, dst_pos, length);
2441 __ ldrw(rscratch1, dst_length_addr);
2442 __ cmpw(tmp, rscratch1);
2443 __ br(Assembler::HI, *stub->entry());
2444 }
2445
2446 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2447 // length_positive_check if the source of our length operand is an
2448 // arraylength. However, that arraylength might be zero, and the
2449 // stub that we're about to call contains an assertion that count !=
2450 // 0 . So we make this check purely in order not to trigger an
2451 // assertion failure.
2452 __ cbzw(length, *stub->continuation());
2453
2454 if (flags & LIR_OpArrayCopy::type_check) {
2455 // We don't know the array types are compatible
2456 if (basic_type != T_OBJECT) {
2457 // Simple test for basic type arrays
2458 if (UseCompressedClassPointers) {
2459 __ ldrw(tmp, src_klass_addr);
2460 __ ldrw(rscratch1, dst_klass_addr);
2461 __ cmpw(tmp, rscratch1);
2462 } else {
2463 __ ldr(tmp, src_klass_addr);
2464 __ ldr(rscratch1, dst_klass_addr);
2465 __ cmp(tmp, rscratch1);
2466 }
2467 __ br(Assembler::NE, *stub->entry());
2468 } else {
2469 // For object arrays, if src is a sub class of dst then we can
2470 // safely do the copy.
2471 Label cont, slow;
2472
2473 #define PUSH(r1, r2) \
2474 stp(r1, r2, __ pre(sp, -2 * wordSize));
2475
2476 #define POP(r1, r2) \
2477 ldp(r1, r2, __ post(sp, 2 * wordSize));
2478
2479 __ PUSH(src, dst);
2480
2481 __ load_klass(src, src);
2482 __ load_klass(dst, dst);
2483
2484 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2485
2486 __ PUSH(src, dst);
2487 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2488 __ POP(src, dst);
2489
2490 __ cbnz(src, cont);
2491
2492 __ bind(slow);
2493 __ POP(src, dst);
2494
2495 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2496 if (copyfunc_addr != NULL) { // use stub if available
2497 // src is not a sub class of dst so we have to do a
2498 // per-element check.
2499
2500 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2501 if ((flags & mask) != mask) {
2502 // Check that at least both of them object arrays.
2503 assert(flags & mask, "one of the two should be known to be an object array");
2504
2505 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2506 __ load_klass(tmp, src);
2507 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2508 __ load_klass(tmp, dst);
2509 }
2510 int lh_offset = in_bytes(Klass::layout_helper_offset());
2511 Address klass_lh_addr(tmp, lh_offset);
2512 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2513 __ ldrw(rscratch1, klass_lh_addr);
2514 __ mov(rscratch2, objArray_lh);
2515 __ eorw(rscratch1, rscratch1, rscratch2);
2516 __ cbnzw(rscratch1, *stub->entry());
2517 }
2518
2519 // Spill because stubs can use any register they like and it's
2520 // easier to restore just those that we care about.
2521 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2522 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2523 __ str(src, Address(sp, 4*BytesPerWord));
2524
2525 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2526 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2527 assert_different_registers(c_rarg0, dst, dst_pos, length);
2528 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2529 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2530 assert_different_registers(c_rarg1, dst, length);
2531 __ uxtw(c_rarg2, length);
2532 assert_different_registers(c_rarg2, dst);
2533
2534 __ load_klass(c_rarg4, dst);
2535 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2536 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2537 __ far_call(RuntimeAddress(copyfunc_addr));
2538
2539 #ifndef PRODUCT
2540 if (PrintC1Statistics) {
2541 Label failed;
2542 __ cbnz(r0, failed);
2543 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2544 __ bind(failed);
2545 }
2546 #endif
2547
2548 __ cbz(r0, *stub->continuation());
2549
2550 #ifndef PRODUCT
2551 if (PrintC1Statistics) {
2552 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2553 }
2554 #endif
2555 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2556
2557 // Restore previously spilled arguments
2558 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2559 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2560 __ ldr(src, Address(sp, 4*BytesPerWord));
2561
2562 // return value is -1^K where K is partial copied count
2563 __ eonw(rscratch1, r0, zr);
2564 // adjust length down and src/end pos up by partial copied count
2565 __ subw(length, length, rscratch1);
2566 __ addw(src_pos, src_pos, rscratch1);
2567 __ addw(dst_pos, dst_pos, rscratch1);
2568 }
2569
2570 __ b(*stub->entry());
2571
2572 __ bind(cont);
2573 __ POP(src, dst);
2574 }
2575 }
2576
2577 #ifdef ASSERT
2578 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2579 // Sanity check the known type with the incoming class. For the
2580 // primitive case the types must match exactly with src.klass and
2581 // dst.klass each exactly matching the default type. For the
2582 // object array case, if no type check is needed then either the
2583 // dst type is exactly the expected type and the src type is a
2584 // subtype which we can't check or src is the same array as dst
2585 // but not necessarily exactly of type default_type.
2586 Label known_ok, halt;
2587 __ mov_metadata(tmp, default_type->constant_encoding());
2588 #ifdef _LP64
2589 if (UseCompressedClassPointers) {
2590 __ encode_klass_not_null(tmp);
2591 }
2592 #endif
2593
2594 if (basic_type != T_OBJECT) {
2595
2596 if (UseCompressedClassPointers) {
2597 __ ldrw(rscratch1, dst_klass_addr);
2598 __ cmpw(tmp, rscratch1);
2599 } else {
2600 __ ldr(rscratch1, dst_klass_addr);
2601 __ cmp(tmp, rscratch1);
2602 }
2603 __ br(Assembler::NE, halt);
2604 if (UseCompressedClassPointers) {
2605 __ ldrw(rscratch1, src_klass_addr);
2606 __ cmpw(tmp, rscratch1);
2607 } else {
2608 __ ldr(rscratch1, src_klass_addr);
2609 __ cmp(tmp, rscratch1);
2610 }
2611 __ br(Assembler::EQ, known_ok);
2612 } else {
2613 if (UseCompressedClassPointers) {
2614 __ ldrw(rscratch1, dst_klass_addr);
2615 __ cmpw(tmp, rscratch1);
2616 } else {
2617 __ ldr(rscratch1, dst_klass_addr);
2618 __ cmp(tmp, rscratch1);
2619 }
2620 __ br(Assembler::EQ, known_ok);
2621 __ cmp(src, dst);
2622 __ br(Assembler::EQ, known_ok);
2623 }
2624 __ bind(halt);
2625 __ stop("incorrect type information in arraycopy");
2626 __ bind(known_ok);
2627 }
2628 #endif
2629
2630 #ifndef PRODUCT
2631 if (PrintC1Statistics) {
2632 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2633 }
2634 #endif
2635
2636 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2637 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2638 assert_different_registers(c_rarg0, dst, dst_pos, length);
2639 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2640 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2641 assert_different_registers(c_rarg1, dst, length);
2642 __ uxtw(c_rarg2, length);
2643 assert_different_registers(c_rarg2, dst);
2644
2645 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2646 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2647 const char *name;
2648 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2649
2650 CodeBlob *cb = CodeCache::find_blob(entry);
2651 if (cb) {
2652 __ far_call(RuntimeAddress(entry));
2653 } else {
2654 __ call_VM_leaf(entry, 3);
2655 }
2656
2657 __ bind(*stub->continuation());
2658 }
2659
2660
2661
2662
2663 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2664 Register obj = op->obj_opr()->as_register(); // may not be an oop
2665 Register hdr = op->hdr_opr()->as_register();
2666 Register lock = op->lock_opr()->as_register();
2667 if (!UseFastLocking) {
2668 __ b(*op->stub()->entry());
2669 } else if (op->code() == lir_lock) {
2670 Register scratch = noreg;
2671 if (UseBiasedLocking) {
2672 scratch = op->scratch_opr()->as_register();
2673 }
2674 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2675 // add debug info for NullPointerException only if one is possible
2676 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2677 if (op->info() != NULL) {
2678 add_debug_info_for_null_check(null_check_offset, op->info());
2679 }
2680 // done
2681 } else if (op->code() == lir_unlock) {
2682 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2683 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2684 } else {
2685 Unimplemented();
2686 }
2687 __ bind(*op->stub()->continuation());
2688 }
2689
2690
2691 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2692 ciMethod* method = op->profiled_method();
2693 int bci = op->profiled_bci();
2694 ciMethod* callee = op->profiled_callee();
2695
2696 // Update counter for all call types
2697 ciMethodData* md = method->method_data_or_null();
2698 assert(md != NULL, "Sanity");
2699 ciProfileData* data = md->bci_to_data(bci);
2700 assert(data->is_CounterData(), "need CounterData for calls");
2701 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2702 Register mdo = op->mdo()->as_register();
2703 __ mov_metadata(mdo, md->constant_encoding());
2704 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2705 Bytecodes::Code bc = method->java_code_at_bci(bci);
2706 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2707 // Perform additional virtual call profiling for invokevirtual and
2708 // invokeinterface bytecodes
2709 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2710 !callee_is_static && // required for optimized MH invokes
2711 C1ProfileVirtualCalls) {
2712 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2713 Register recv = op->recv()->as_register();
2714 assert_different_registers(mdo, recv);
2715 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2716 ciKlass* known_klass = op->known_holder();
2717 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2718 // We know the type that will be seen at this call site; we can
2719 // statically update the MethodData* rather than needing to do
2720 // dynamic tests on the receiver type
2721
2722 // NOTE: we should probably put a lock around this search to
2723 // avoid collisions by concurrent compilations
2724 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2725 uint i;
2726 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2727 ciKlass* receiver = vc_data->receiver(i);
2728 if (known_klass->equals(receiver)) {
2729 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2730 __ addptr(data_addr, DataLayout::counter_increment);
2731 return;
2732 }
2733 }
2734
2735 // Receiver type not found in profile data; select an empty slot
2736
2737 // Note that this is less efficient than it should be because it
2738 // always does a write to the receiver part of the
2739 // VirtualCallData rather than just the first time
2740 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2741 ciKlass* receiver = vc_data->receiver(i);
2742 if (receiver == NULL) {
2743 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2744 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2745 __ lea(rscratch2, recv_addr);
2746 __ str(rscratch1, Address(rscratch2));
2747 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2748 __ addptr(data_addr, DataLayout::counter_increment);
2749 return;
2750 }
2751 }
2752 } else {
2753 __ load_klass(recv, recv);
2754 Label update_done;
2755 type_profile_helper(mdo, md, data, recv, &update_done);
2756 // Receiver did not match any saved receiver and there is no empty row for it.
2757 // Increment total counter to indicate polymorphic case.
2758 __ addptr(counter_addr, DataLayout::counter_increment);
2759
2760 __ bind(update_done);
2761 }
2762 } else {
2763 // Static call
2764 __ addptr(counter_addr, DataLayout::counter_increment);
2765 }
2766 }
2767
2768
2769 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2770 Unimplemented();
2771 }
2772
2773
2774 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2775 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2776 }
2777
2778 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2779 assert(op->crc()->is_single_cpu(), "crc must be register");
2780 assert(op->val()->is_single_cpu(), "byte value must be register");
2781 assert(op->result_opr()->is_single_cpu(), "result must be register");
2782 Register crc = op->crc()->as_register();
2783 Register val = op->val()->as_register();
2784 Register res = op->result_opr()->as_register();
2785
2786 assert_different_registers(val, crc, res);
2787 unsigned long offset;
2788 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2789 if (offset) __ add(res, res, offset);
2790
2791 __ ornw(crc, zr, crc); // ~crc
2792 __ update_byte_crc32(crc, val, res);
2793 __ ornw(res, zr, crc); // ~crc
2794 }
2795
2796 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2797 COMMENT("emit_profile_type {");
2798 Register obj = op->obj()->as_register();
2799 Register tmp = op->tmp()->as_pointer_register();
2800 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2801 ciKlass* exact_klass = op->exact_klass();
2802 intptr_t current_klass = op->current_klass();
2803 bool not_null = op->not_null();
2804 bool no_conflict = op->no_conflict();
2805
2806 Label update, next, none;
2807
2808 bool do_null = !not_null;
2809 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2810 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2811
2812 assert(do_null || do_update, "why are we here?");
2813 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2814 assert(mdo_addr.base() != rscratch1, "wrong register");
2815
2816 __ verify_oop(obj);
2817
2818 if (tmp != obj) {
2819 __ mov(tmp, obj);
2820 }
2821 if (do_null) {
2822 __ cbnz(tmp, update);
2823 if (!TypeEntries::was_null_seen(current_klass)) {
2824 __ ldr(rscratch2, mdo_addr);
2825 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2826 __ str(rscratch2, mdo_addr);
2827 }
2828 if (do_update) {
2829 #ifndef ASSERT
2830 __ b(next);
2831 }
2832 #else
2833 __ b(next);
2834 }
2835 } else {
2836 __ cbnz(tmp, update);
2837 __ stop("unexpected null obj");
2838 #endif
2839 }
2840
2841 __ bind(update);
2842
2843 if (do_update) {
2844 #ifdef ASSERT
2845 if (exact_klass != NULL) {
2846 Label ok;
2847 __ load_klass(tmp, tmp);
2848 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2849 __ eor(rscratch1, tmp, rscratch1);
2850 __ cbz(rscratch1, ok);
2851 __ stop("exact klass and actual klass differ");
2852 __ bind(ok);
2853 }
2854 #endif
2855 if (!no_conflict) {
2856 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2857 if (exact_klass != NULL) {
2858 __ mov_metadata(tmp, exact_klass->constant_encoding());
2859 } else {
2860 __ load_klass(tmp, tmp);
2861 }
2862
2863 __ ldr(rscratch2, mdo_addr);
2864 __ eor(tmp, tmp, rscratch2);
2865 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2866 // klass seen before, nothing to do. The unknown bit may have been
2867 // set already but no need to check.
2868 __ cbz(rscratch1, next);
2869
2870 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2871 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2872
2873 if (TypeEntries::is_type_none(current_klass)) {
2874 __ cbz(rscratch2, none);
2875 __ cmp(rscratch2, TypeEntries::null_seen);
2876 __ br(Assembler::EQ, none);
2877 // There is a chance that the checks above (re-reading profiling
2878 // data from memory) fail if another thread has just set the
2879 // profiling to this obj's klass
2880 __ dmb(Assembler::ISHLD);
2881 __ ldr(rscratch2, mdo_addr);
2882 __ eor(tmp, tmp, rscratch2);
2883 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2884 __ cbz(rscratch1, next);
2885 }
2886 } else {
2887 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2888 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2889
2890 __ ldr(tmp, mdo_addr);
2891 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2892 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2893 }
2894
2895 // different than before. Cannot keep accurate profile.
2896 __ ldr(rscratch2, mdo_addr);
2897 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2898 __ str(rscratch2, mdo_addr);
2899
2900 if (TypeEntries::is_type_none(current_klass)) {
2901 __ b(next);
2902
2903 __ bind(none);
2904 // first time here. Set profile type.
2905 __ str(tmp, mdo_addr);
2906 }
2907 } else {
2908 // There's a single possible klass at this profile point
2909 assert(exact_klass != NULL, "should be");
2910 if (TypeEntries::is_type_none(current_klass)) {
2911 __ mov_metadata(tmp, exact_klass->constant_encoding());
2912 __ ldr(rscratch2, mdo_addr);
2913 __ eor(tmp, tmp, rscratch2);
2914 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2915 __ cbz(rscratch1, next);
2916 #ifdef ASSERT
2917 {
2918 Label ok;
2919 __ ldr(rscratch1, mdo_addr);
2920 __ cbz(rscratch1, ok);
2921 __ cmp(rscratch1, TypeEntries::null_seen);
2922 __ br(Assembler::EQ, ok);
2923 // may have been set by another thread
2924 __ dmb(Assembler::ISHLD);
2925 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2926 __ ldr(rscratch2, mdo_addr);
2927 __ eor(rscratch2, rscratch1, rscratch2);
2928 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2929 __ cbz(rscratch2, ok);
2930
2931 __ stop("unexpected profiling mismatch");
2932 __ bind(ok);
2933 }
2934 #endif
2935 // first time here. Set profile type.
2936 __ ldr(tmp, mdo_addr);
2937 } else {
2938 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2939 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2940
2941 __ ldr(tmp, mdo_addr);
2942 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2943 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2944
2945 __ orr(tmp, tmp, TypeEntries::type_unknown);
2946 __ str(tmp, mdo_addr);
2947 // FIXME: Write barrier needed here?
2948 }
2949 }
2950
2951 __ bind(next);
2952 }
2953 COMMENT("} emit_profile_type");
2954 }
2955
2956
2957 void LIR_Assembler::align_backward_branch_target() {
2958 }
2959
2960
2961 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2962 if (left->is_single_cpu()) {
2963 assert(dest->is_single_cpu(), "expect single result reg");
2964 __ negw(dest->as_register(), left->as_register());
2965 } else if (left->is_double_cpu()) {
2966 assert(dest->is_double_cpu(), "expect double result reg");
2967 __ neg(dest->as_register_lo(), left->as_register_lo());
2968 } else if (left->is_single_fpu()) {
2969 assert(dest->is_single_fpu(), "expect single float result reg");
2970 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2971 } else {
2972 assert(left->is_double_fpu(), "expect double float operand reg");
2973 assert(dest->is_double_fpu(), "expect double float result reg");
2974 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2975 }
2976 }
2977
2978
2979 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2980 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2981 }
2982
2983
2984 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2985 assert(!tmp->is_valid(), "don't need temporary");
2986
2987 CodeBlob *cb = CodeCache::find_blob(dest);
2988 if (cb) {
2989 __ far_call(RuntimeAddress(dest));
2990 } else {
2991 __ mov(rscratch1, RuntimeAddress(dest));
2992 int len = args->length();
2993 int type = 0;
2994 if (! result->is_illegal()) {
2995 switch (result->type()) {
2996 case T_VOID:
2997 type = 0;
2998 break;
2999 case T_INT:
3000 case T_LONG:
3001 case T_OBJECT:
3002 type = 1;
3003 break;
3004 case T_FLOAT:
3005 type = 2;
3006 break;
3007 case T_DOUBLE:
3008 type = 3;
3009 break;
3010 default:
3011 ShouldNotReachHere();
3012 break;
3013 }
3014 }
3015 int num_gpargs = 0;
3016 int num_fpargs = 0;
3017 for (int i = 0; i < args->length(); i++) {
3018 LIR_Opr arg = args->at(i);
3019 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
3020 num_fpargs++;
3021 } else {
3022 num_gpargs++;
3023 }
3024 }
3025 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
3026 }
3027
3028 if (info != NULL) {
3029 add_call_info_here(info);
3030 }
3031 __ maybe_isb();
3032 }
3033
3034 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3035 if (dest->is_address() || src->is_address()) {
3036 move_op(src, dest, type, lir_patch_none, info,
3037 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
3038 } else {
3039 ShouldNotReachHere();
3040 }
3041 }
3042
3043 #ifdef ASSERT
3044 // emit run-time assertion
3045 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3046 assert(op->code() == lir_assert, "must be");
3047
3048 if (op->in_opr1()->is_valid()) {
3049 assert(op->in_opr2()->is_valid(), "both operands must be valid");
3050 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3051 } else {
3052 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3053 assert(op->condition() == lir_cond_always, "no other conditions allowed");
3054 }
3055
3056 Label ok;
3057 if (op->condition() != lir_cond_always) {
3058 Assembler::Condition acond = Assembler::AL;
3059 switch (op->condition()) {
3060 case lir_cond_equal: acond = Assembler::EQ; break;
3061 case lir_cond_notEqual: acond = Assembler::NE; break;
3062 case lir_cond_less: acond = Assembler::LT; break;
3063 case lir_cond_lessEqual: acond = Assembler::LE; break;
3064 case lir_cond_greaterEqual: acond = Assembler::GE; break;
3065 case lir_cond_greater: acond = Assembler::GT; break;
3066 case lir_cond_belowEqual: acond = Assembler::LS; break;
3067 case lir_cond_aboveEqual: acond = Assembler::HS; break;
3068 default: ShouldNotReachHere();
3069 }
3070 __ br(acond, ok);
3071 }
3072 if (op->halt()) {
3073 const char* str = __ code_string(op->msg());
3074 __ stop(str);
3075 } else {
3076 breakpoint();
3077 }
3078 __ bind(ok);
3079 }
3080 #endif
3081
3082 #ifndef PRODUCT
3083 #define COMMENT(x) do { __ block_comment(x); } while (0)
3084 #else
3085 #define COMMENT(x)
3086 #endif
3087
3088 void LIR_Assembler::membar() {
3089 COMMENT("membar");
3090 __ membar(MacroAssembler::AnyAny);
3091 }
3092
3093 void LIR_Assembler::membar_acquire() {
3094 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3095 }
3096
3097 void LIR_Assembler::membar_release() {
3098 __ membar(Assembler::LoadStore|Assembler::StoreStore);
3099 }
3100
3101 void LIR_Assembler::membar_loadload() {
3102 __ membar(Assembler::LoadLoad);
3103 }
3104
3105 void LIR_Assembler::membar_storestore() {
3106 __ membar(MacroAssembler::StoreStore);
3107 }
3108
3109 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3110
3111 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3112
3113 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3114 __ mov(result_reg->as_register(), rthread);
3115 }
3116
3117
3118 void LIR_Assembler::peephole(LIR_List *lir) {
3119 #if 0
3120 if (tableswitch_count >= max_tableswitches)
3121 return;
3122
3123 /*
3124 This finite-state automaton recognizes sequences of compare-and-
3125 branch instructions. We will turn them into a tableswitch. You
3126 could argue that C1 really shouldn't be doing this sort of
3127 optimization, but without it the code is really horrible.
3128 */
3129
3130 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3131 int first_key, last_key = -2147483648;
3132 int next_key = 0;
3133 int start_insn = -1;
3134 int last_insn = -1;
3135 Register reg = noreg;
3136 LIR_Opr reg_opr;
3137 state = start_s;
3138
3139 LIR_OpList* inst = lir->instructions_list();
3140 for (int i = 0; i < inst->length(); i++) {
3141 LIR_Op* op = inst->at(i);
3142 switch (state) {
3143 case start_s:
3144 first_key = -1;
3145 start_insn = i;
3146 switch (op->code()) {
3147 case lir_cmp:
3148 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3149 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3150 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3151 && opr2->is_constant()
3152 && opr2->type() == T_INT) {
3153 reg_opr = opr1;
3154 reg = opr1->as_register();
3155 first_key = opr2->as_constant_ptr()->as_jint();
3156 next_key = first_key + 1;
3157 state = cmp_s;
3158 goto next_state;
3159 }
3160 break;
3161 }
3162 break;
3163 case cmp_s:
3164 switch (op->code()) {
3165 case lir_branch:
3166 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3167 state = beq_s;
3168 last_insn = i;
3169 goto next_state;
3170 }
3171 }
3172 state = start_s;
3173 break;
3174 case beq_s:
3175 switch (op->code()) {
3176 case lir_cmp: {
3177 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3178 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3179 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3180 && opr1->as_register() == reg
3181 && opr2->is_constant()
3182 && opr2->type() == T_INT
3183 && opr2->as_constant_ptr()->as_jint() == next_key) {
3184 last_key = next_key;
3185 next_key++;
3186 state = cmp_s;
3187 goto next_state;
3188 }
3189 }
3190 }
3191 last_key = next_key;
3192 state = start_s;
3193 break;
3194 default:
3195 assert(false, "impossible state");
3196 }
3197 if (state == start_s) {
3198 if (first_key < last_key - 5L && reg != noreg) {
3199 {
3200 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3201 // reg->encoding(),
3202 // start_insn, first_key, last_key);
3203 // for (int i = 0; i < inst->length(); i++) {
3204 // inst->at(i)->print();
3205 // tty->print("\n");
3206 // }
3207 // tty->print("\n");
3208 }
3209
3210 struct tableswitch *sw = &switches[tableswitch_count];
3211 sw->_insn_index = start_insn, sw->_first_key = first_key,
3212 sw->_last_key = last_key, sw->_reg = reg;
3213 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3214 {
3215 // Insert the new table of branches
3216 int offset = last_insn;
3217 for (int n = first_key; n < last_key; n++) {
3218 inst->insert_before
3219 (last_insn + 1,
3220 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3221 inst->at(offset)->as_OpBranch()->label()));
3222 offset -= 2, i++;
3223 }
3224 }
3225 // Delete all the old compare-and-branch instructions
3226 for (int n = first_key; n < last_key; n++) {
3227 inst->remove_at(start_insn);
3228 inst->remove_at(start_insn);
3229 }
3230 // Insert the tableswitch instruction
3231 inst->insert_before(start_insn,
3232 new LIR_Op2(lir_cmp, lir_cond_always,
3233 LIR_OprFact::intConst(tableswitch_count),
3234 reg_opr));
3235 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3236 tableswitch_count++;
3237 }
3238 reg = noreg;
3239 last_key = -2147483648;
3240 }
3241 next_state:
3242 ;
3243 }
3244 #endif
3245 }
3246
3247 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3248 Address addr = as_Address(src->as_address_ptr(), noreg);
3249 BasicType type = src->type();
3250 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3251 Assembler::operand_size sz = Assembler::xword;
3252
3253 void (MacroAssembler::* lda)(Register Rd, Register Ra);
3254 void (MacroAssembler::* add)(Register Rd, Register Rn, RegisterOrConstant increment);
3255 void (MacroAssembler::* stl)(Register Rs, Register Rt, Register Rn);
3256
3257 switch(type) {
3258 case T_INT:
3259 lda = &MacroAssembler::ldaxrw;
3260 add = &MacroAssembler::addw;
3261 stl = &MacroAssembler::stlxrw;
3262 sz = Assembler::word;
3263 break;
3264 case T_LONG:
3265 lda = &MacroAssembler::ldaxr;
3266 add = &MacroAssembler::add;
3267 stl = &MacroAssembler::stlxr;
3268 break;
3269 case T_OBJECT:
3270 case T_ARRAY:
3271 if (UseCompressedOops) {
3272 lda = &MacroAssembler::ldaxrw;
3273 add = &MacroAssembler::addw;
3274 stl = &MacroAssembler::stlxrw;
3275 sz = Assembler::word;
3276 } else {
3277 lda = &MacroAssembler::ldaxr;
3278 add = &MacroAssembler::add;
3279 stl = &MacroAssembler::stlxr;
3280 }
3281 break;
3282 default:
3283 ShouldNotReachHere();
3284 }
3285
3286 switch (code) {
3287 case lir_xadd:
3288 {
3289 RegisterOrConstant inc;
3290 Register tmp = as_reg(tmp_op);
3291 Register dst = as_reg(dest);
3292 if (data->is_constant()) {
3293 inc = RegisterOrConstant(as_long(data));
3294 assert_different_registers(dst, addr.base(), tmp,
3295 rscratch1, rscratch2);
3296 } else {
3297 inc = RegisterOrConstant(as_reg(data));
3298 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3299 rscratch1, rscratch2);
3300 }
3301 __ lea(tmp, addr);
3302 if (UseLSE) {
3303 if (inc.is_register()) {
3304 __ ldaddal(sz, inc.as_register(), dst, tmp);
3305 } else {
3306 __ mov(rscratch2, inc.as_constant());
3307 __ ldaddal(sz, rscratch2, dst, tmp);
3308 }
3309 } else {
3310 Label again;
3311 if ((VM_Version::cpu_cpuFeatures() & VM_Version::CPU_STXR_PREFETCH))
3312 __ prfm(Address(tmp), PSTL1STRM);
3313 __ bind(again);
3314 (_masm->*lda)(dst, tmp);
3315 (_masm->*add)(rscratch1, dst, inc);
3316 (_masm->*stl)(rscratch2, rscratch1, tmp);
3317 __ cbnzw(rscratch2, again);
3318 }
3319 break;
3320 }
3321 case lir_xchg:
3322 {
3323 Register tmp = tmp_op->as_register();
3324 Register obj = as_reg(data);
3325 Register dst = as_reg(dest);
3326 if (is_oop && UseCompressedOops) {
3327 __ encode_heap_oop(rscratch1, obj);
3328 obj = rscratch1;
3329 }
3330 assert_different_registers(obj, addr.base(), tmp, rscratch2, dst);
3331 __ lea(tmp, addr);
3332 if (UseLSE) {
3333 __ swp(sz, obj, dst, tmp);
3334 } else {
3335 Label again;
3336 if ((VM_Version::cpu_cpuFeatures() & VM_Version::CPU_STXR_PREFETCH))
3337 __ prfm(Address(tmp), PSTL1STRM);
3338 __ bind(again);
3339 (_masm->*lda)(dst, tmp);
3340 (_masm->*stl)(rscratch2, obj, tmp);
3341 __ cbnzw(rscratch2, again);
3342 }
3343 if (is_oop && UseCompressedOops) {
3344 __ decode_heap_oop(dst);
3345 }
3346 }
3347 break;
3348 default:
3349 ShouldNotReachHere();
3350 }
3351 __ membar(__ AnyAny);
3352 }
3353
3354 #undef __