1 /*
2 * Copyright (c) 2013, Red Hat Inc.
3 * Copyright (c) 2000, 2019, Oracle and/or its affiliates.
4 * All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "gc_interface/collectedHeap.hpp"
38 #include "memory/barrierSet.hpp"
39 #include "memory/cardTableModRefBS.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "vmreg_aarch64.inline.hpp"
44
45 #if INCLUDE_ALL_GCS
46 #include "shenandoahBarrierSetAssembler_aarch64.hpp"
47 #endif
48
49 #ifndef PRODUCT
50 #define COMMENT(x) do { __ block_comment(x); } while (0)
51 #else
52 #define COMMENT(x)
53 #endif
54
55 NEEDS_CLEANUP // remove this definitions ?
56 const Register IC_Klass = rscratch2; // where the IC klass is cached
57 const Register SYNC_header = r0; // synchronization header
58 const Register SHIFT_count = r0; // where count for shift operations must be
59
60 #define __ _masm->
61
62
63 static void select_different_registers(Register preserve,
64 Register extra,
65 Register &tmp1,
66 Register &tmp2) {
67 if (tmp1 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp1 = extra;
70 } else if (tmp2 == preserve) {
71 assert_different_registers(tmp1, tmp2, extra);
72 tmp2 = extra;
73 }
74 assert_different_registers(preserve, tmp1, tmp2);
75 }
76
77
78
79 static void select_different_registers(Register preserve,
80 Register extra,
81 Register &tmp1,
82 Register &tmp2,
83 Register &tmp3) {
84 if (tmp1 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp1 = extra;
87 } else if (tmp2 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp2 = extra;
90 } else if (tmp3 == preserve) {
91 assert_different_registers(tmp1, tmp2, tmp3, extra);
92 tmp3 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2, tmp3);
95 }
96
97
98 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
99
100
101 LIR_Opr LIR_Assembler::receiverOpr() {
102 return FrameMap::receiver_opr;
103 }
104
105 LIR_Opr LIR_Assembler::osrBufferPointer() {
106 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
107 }
108
109 //--------------fpu register translations-----------------------
110
111
112 address LIR_Assembler::float_constant(float f) {
113 address const_addr = __ float_constant(f);
114 if (const_addr == NULL) {
115 bailout("const section overflow");
116 return __ code()->consts()->start();
117 } else {
118 return const_addr;
119 }
120 }
121
122
123 address LIR_Assembler::double_constant(double d) {
124 address const_addr = __ double_constant(d);
125 if (const_addr == NULL) {
126 bailout("const section overflow");
127 return __ code()->consts()->start();
128 } else {
129 return const_addr;
130 }
131 }
132
133 address LIR_Assembler::int_constant(jlong n) {
134 address const_addr = __ long_constant(n);
135 if (const_addr == NULL) {
136 bailout("const section overflow");
137 return __ code()->consts()->start();
138 } else {
139 return const_addr;
140 }
141 }
142
143 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
144
145 void LIR_Assembler::reset_FPU() { Unimplemented(); }
146
147 void LIR_Assembler::fpop() { Unimplemented(); }
148
149 void LIR_Assembler::fxch(int i) { Unimplemented(); }
150
151 void LIR_Assembler::fld(int i) { Unimplemented(); }
152
153 void LIR_Assembler::ffree(int i) { Unimplemented(); }
154
155 void LIR_Assembler::breakpoint() { Unimplemented(); }
156
157 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
158
159 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
160
161 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
162 //-------------------------------------------
163
164 static Register as_reg(LIR_Opr op) {
165 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
166 }
167
168 static jlong as_long(LIR_Opr data) {
169 jlong result;
170 switch (data->type()) {
171 case T_INT:
172 result = (data->as_jint());
173 break;
174 case T_LONG:
175 result = (data->as_jlong());
176 break;
177 default:
178 ShouldNotReachHere();
179 }
180 return result;
181 }
182
183 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
184 Register base = addr->base()->as_pointer_register();
185 LIR_Opr opr = addr->index();
186 if (opr->is_cpu_register()) {
187 Register index;
188 if (opr->is_single_cpu())
189 index = opr->as_register();
190 else
191 index = opr->as_register_lo();
192 assert(addr->disp() == 0, "must be");
193 switch(opr->type()) {
194 case T_INT:
195 return Address(base, index, Address::sxtw(addr->scale()));
196 case T_LONG:
197 return Address(base, index, Address::lsl(addr->scale()));
198 default:
199 ShouldNotReachHere();
200 }
201 } else {
202 intptr_t addr_offset = intptr_t(addr->disp());
203 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
204 return Address(base, addr_offset, Address::lsl(addr->scale()));
205 else {
206 __ mov(tmp, addr_offset);
207 return Address(base, tmp, Address::lsl(addr->scale()));
208 }
209 }
210 }
211
212 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
213 ShouldNotReachHere();
214 return Address();
215 }
216
217 Address LIR_Assembler::as_Address(LIR_Address* addr) {
218 return as_Address(addr, rscratch1);
219 }
220
221 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
222 return as_Address(addr, rscratch1); // Ouch
223 // FIXME: This needs to be much more clever. See x86.
224 }
225
226
227 void LIR_Assembler::osr_entry() {
228 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
229 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
230 ValueStack* entry_state = osr_entry->state();
231 int number_of_locks = entry_state->locks_size();
232
233 // we jump here if osr happens with the interpreter
234 // state set up to continue at the beginning of the
235 // loop that triggered osr - in particular, we have
236 // the following registers setup:
237 //
238 // r2: osr buffer
239 //
240
241 // build frame
242 ciMethod* m = compilation()->method();
243 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
244
245 // OSR buffer is
246 //
247 // locals[nlocals-1..0]
248 // monitors[0..number_of_locks]
249 //
250 // locals is a direct copy of the interpreter frame so in the osr buffer
251 // so first slot in the local array is the last local from the interpreter
252 // and last slot is local[0] (receiver) from the interpreter
253 //
254 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
255 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
256 // in the interpreter frame (the method lock if a sync method)
257
258 // Initialize monitors in the compiled activation.
259 // r2: pointer to osr buffer
260 //
261 // All other registers are dead at this point and the locals will be
262 // copied into place by code emitted in the IR.
263
264 Register OSR_buf = osrBufferPointer()->as_pointer_register();
265 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
266 int monitor_offset = BytesPerWord * method()->max_locals() +
267 (2 * BytesPerWord) * (number_of_locks - 1);
268 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
269 // the OSR buffer using 2 word entries: first the lock and then
270 // the oop.
271 for (int i = 0; i < number_of_locks; i++) {
272 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
273 #ifdef ASSERT
274 // verify the interpreter's monitor has a non-null object
275 {
276 Label L;
277 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
278 __ cbnz(rscratch1, L);
279 __ stop("locked object is NULL");
280 __ bind(L);
281 }
282 #endif
283 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
284 __ str(r19, frame_map()->address_for_monitor_lock(i));
285 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
286 __ str(r19, frame_map()->address_for_monitor_object(i));
287 }
288 }
289 }
290
291
292 // inline cache check; done before the frame is built.
293 int LIR_Assembler::check_icache() {
294 Register receiver = FrameMap::receiver_opr->as_register();
295 Register ic_klass = IC_Klass;
296 int start_offset = __ offset();
297 __ inline_cache_check(receiver, ic_klass);
298
299 // if icache check fails, then jump to runtime routine
300 // Note: RECEIVER must still contain the receiver!
301 Label dont;
302 __ br(Assembler::EQ, dont);
303 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
304
305 // We align the verified entry point unless the method body
306 // (including its inline cache check) will fit in a single 64-byte
307 // icache line.
308 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
309 // force alignment after the cache check.
310 __ align(CodeEntryAlignment);
311 }
312
313 __ bind(dont);
314 return start_offset;
315 }
316
317
318 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
319 if (o == NULL) {
320 __ mov(reg, zr);
321 } else {
322 __ movoop(reg, o, /*immediate*/true);
323 }
324 }
325
326 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
327 address target = NULL;
328 relocInfo::relocType reloc_type = relocInfo::none;
329
330 switch (patching_id(info)) {
331 case PatchingStub::access_field_id:
332 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
333 reloc_type = relocInfo::section_word_type;
334 break;
335 case PatchingStub::load_klass_id:
336 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
337 reloc_type = relocInfo::metadata_type;
338 break;
339 case PatchingStub::load_mirror_id:
340 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
341 reloc_type = relocInfo::oop_type;
342 break;
343 case PatchingStub::load_appendix_id:
344 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
345 reloc_type = relocInfo::oop_type;
346 break;
347 default: ShouldNotReachHere();
348 }
349
350 __ far_call(RuntimeAddress(target));
351 add_call_info_here(info);
352 }
353
354 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
355 deoptimize_trap(info);
356 }
357
358
359 // This specifies the rsp decrement needed to build the frame
360 int LIR_Assembler::initial_frame_size_in_bytes() const {
361 // if rounding, must let FrameMap know!
362
363 // The frame_map records size in slots (32bit word)
364
365 // subtract two words to account for return address and link
366 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
367 }
368
369
370 int LIR_Assembler::emit_exception_handler() {
371 // if the last instruction is a call (typically to do a throw which
372 // is coming at the end after block reordering) the return address
373 // must still point into the code area in order to avoid assertion
374 // failures when searching for the corresponding bci => add a nop
375 // (was bug 5/14/1999 - gri)
376 __ nop();
377
378 // generate code for exception handler
379 address handler_base = __ start_a_stub(exception_handler_size);
380 if (handler_base == NULL) {
381 // not enough space left for the handler
382 bailout("exception handler overflow");
383 return -1;
384 }
385
386 int offset = code_offset();
387
388 // the exception oop and pc are in r0, and r3
389 // no other registers need to be preserved, so invalidate them
390 __ invalidate_registers(false, true, true, false, true, true);
391
392 // check that there is really an exception
393 __ verify_not_null_oop(r0);
394
395 // search an exception handler (r0: exception oop, r3: throwing pc)
396 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
397 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
398 __ end_a_stub();
399
400 return offset;
401 }
402
403
404 // Emit the code to remove the frame from the stack in the exception
405 // unwind path.
406 int LIR_Assembler::emit_unwind_handler() {
407 #ifndef PRODUCT
408 if (CommentedAssembly) {
409 _masm->block_comment("Unwind handler");
410 }
411 #endif
412
413 int offset = code_offset();
414
415 // Fetch the exception from TLS and clear out exception related thread state
416 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
417 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
418 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
419
420 __ bind(_unwind_handler_entry);
421 __ verify_not_null_oop(r0);
422 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
423 __ mov(r19, r0); // Preserve the exception
424 }
425
426 // Preform needed unlocking
427 MonitorExitStub* stub = NULL;
428 if (method()->is_synchronized()) {
429 monitor_address(0, FrameMap::r0_opr);
430 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
431 __ unlock_object(r5, r4, r0, *stub->entry());
432 __ bind(*stub->continuation());
433 }
434
435 if (compilation()->env()->dtrace_method_probes()) {
436 __ call_Unimplemented();
437 #if 0
438 __ movptr(Address(rsp, 0), rax);
439 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
440 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
441 #endif
442 }
443
444 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
445 __ mov(r0, r19); // Restore the exception
446 }
447
448 // remove the activation and dispatch to the unwind handler
449 __ block_comment("remove_frame and dispatch to the unwind handler");
450 __ remove_frame(initial_frame_size_in_bytes());
451 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
452
453 // Emit the slow path assembly
454 if (stub != NULL) {
455 stub->emit_code(this);
456 }
457
458 return offset;
459 }
460
461
462 int LIR_Assembler::emit_deopt_handler() {
463 // if the last instruction is a call (typically to do a throw which
464 // is coming at the end after block reordering) the return address
465 // must still point into the code area in order to avoid assertion
466 // failures when searching for the corresponding bci => add a nop
467 // (was bug 5/14/1999 - gri)
468 __ nop();
469
470 // generate code for exception handler
471 address handler_base = __ start_a_stub(deopt_handler_size);
472 if (handler_base == NULL) {
473 // not enough space left for the handler
474 bailout("deopt handler overflow");
475 return -1;
476 }
477
478 int offset = code_offset();
479
480 __ adr(lr, pc());
481 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
482 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
483 __ end_a_stub();
484
485 return offset;
486 }
487
488
489 // This is the fast version of java.lang.String.compare; it has not
490 // OSR-entry and therefore, we generate a slow version for OSR's
491 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
492 __ mov(r2, (address)__FUNCTION__);
493 __ call_Unimplemented();
494 }
495
496
497 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
498 _masm->code_section()->relocate(adr, relocInfo::poll_type);
499 int pc_offset = code_offset();
500 flush_debug_info(pc_offset);
501 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
502 if (info->exception_handlers() != NULL) {
503 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
504 }
505 }
506
507 // Rather than take a segfault when the polling page is protected,
508 // explicitly check for a safepoint in progress and if there is one,
509 // fake a call to the handler as if a segfault had been caught.
510 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
511 __ mov(rscratch1, SafepointSynchronize::address_of_state());
512 __ ldrb(rscratch1, Address(rscratch1));
513 Label nope, poll;
514 __ cbz(rscratch1, nope);
515 __ block_comment("safepoint");
516 __ enter();
517 __ push(0x3, sp); // r0 & r1
518 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
519 __ adr(r0, poll);
520 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
521 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
522 __ blr(rscratch1);
523 __ maybe_isb();
524 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
525 __ mov(rscratch1, r0);
526 __ pop(0x3, sp); // r0 & r1
527 __ leave();
528 __ br(rscratch1);
529 address polling_page(os::get_polling_page());
530 assert(os::is_poll_address(polling_page), "should be");
531 unsigned long off;
532 __ adrp(rscratch1, Address(polling_page, rtype), off);
533 __ bind(poll);
534 if (info)
535 add_debug_info_for_branch(info); // This isn't just debug info:
536 // it's the oop map
537 else
538 __ code_section()->relocate(pc(), rtype);
539 __ ldrw(zr, Address(rscratch1, off));
540 __ bind(nope);
541 }
542
543 void LIR_Assembler::return_op(LIR_Opr result) {
544 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
545 // Pop the stack before the safepoint code
546 __ remove_frame(initial_frame_size_in_bytes());
547 if (UseCompilerSafepoints) {
548 address polling_page(os::get_polling_page());
549 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
550 } else {
551 poll_for_safepoint(relocInfo::poll_return_type);
552 }
553 __ ret(lr);
554 }
555
556 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
557 address polling_page(os::get_polling_page());
558 if (UseCompilerSafepoints) {
559 guarantee(info != NULL, "Shouldn't be NULL");
560 assert(os::is_poll_address(polling_page), "should be");
561 unsigned long off;
562 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
563 assert(off == 0, "must be");
564 add_debug_info_for_branch(info); // This isn't just debug info:
565 // it's the oop map
566 __ read_polling_page(rscratch1, relocInfo::poll_type);
567 } else {
568 poll_for_safepoint(relocInfo::poll_type, info);
569 }
570
571 return __ offset();
572 }
573
574
575 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
576 if (from_reg == r31_sp)
577 from_reg = sp;
578 if (to_reg == r31_sp)
579 to_reg = sp;
580 __ mov(to_reg, from_reg);
581 }
582
583 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
584
585
586 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
587 assert(src->is_constant(), "should not call otherwise");
588 assert(dest->is_register(), "should not call otherwise");
589 LIR_Const* c = src->as_constant_ptr();
590
591 switch (c->type()) {
592 case T_INT: {
593 assert(patch_code == lir_patch_none, "no patching handled here");
594 __ movw(dest->as_register(), c->as_jint());
595 break;
596 }
597
598 case T_ADDRESS: {
599 assert(patch_code == lir_patch_none, "no patching handled here");
600 __ mov(dest->as_register(), c->as_jint());
601 break;
602 }
603
604 case T_LONG: {
605 assert(patch_code == lir_patch_none, "no patching handled here");
606 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
607 break;
608 }
609
610 case T_OBJECT: {
611 if (patch_code == lir_patch_none) {
612 jobject2reg(c->as_jobject(), dest->as_register());
613 } else {
614 jobject2reg_with_patching(dest->as_register(), info);
615 }
616 break;
617 }
618
619 case T_METADATA: {
620 if (patch_code != lir_patch_none) {
621 klass2reg_with_patching(dest->as_register(), info);
622 } else {
623 __ mov_metadata(dest->as_register(), c->as_metadata());
624 }
625 break;
626 }
627
628 case T_FLOAT: {
629 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
630 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
631 } else {
632 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
633 __ ldrs(dest->as_float_reg(), Address(rscratch1));
634 }
635 break;
636 }
637
638 case T_DOUBLE: {
639 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
640 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
641 } else {
642 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
643 __ ldrd(dest->as_double_reg(), Address(rscratch1));
644 }
645 break;
646 }
647
648 default:
649 ShouldNotReachHere();
650 }
651 }
652
653 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
654 LIR_Const* c = src->as_constant_ptr();
655 switch (c->type()) {
656 case T_OBJECT:
657 {
658 if (! c->as_jobject())
659 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
660 else {
661 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
662 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
663 }
664 }
665 break;
666 case T_ADDRESS:
667 {
668 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
669 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
670 }
671 case T_INT:
672 case T_FLOAT:
673 {
674 Register reg = zr;
675 if (c->as_jint_bits() == 0)
676 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
677 else {
678 __ movw(rscratch1, c->as_jint_bits());
679 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
680 }
681 }
682 break;
683 case T_LONG:
684 case T_DOUBLE:
685 {
686 Register reg = zr;
687 if (c->as_jlong_bits() == 0)
688 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
689 lo_word_offset_in_bytes));
690 else {
691 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
692 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
693 lo_word_offset_in_bytes));
694 }
695 }
696 break;
697 default:
698 ShouldNotReachHere();
699 }
700 }
701
702 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
703 assert(src->is_constant(), "should not call otherwise");
704 LIR_Const* c = src->as_constant_ptr();
705 LIR_Address* to_addr = dest->as_address_ptr();
706
707 void (Assembler::* insn)(Register Rt, const Address &adr);
708
709 switch (type) {
710 case T_ADDRESS:
711 assert(c->as_jint() == 0, "should be");
712 insn = &Assembler::str;
713 break;
714 case T_LONG:
715 assert(c->as_jlong() == 0, "should be");
716 insn = &Assembler::str;
717 break;
718 case T_INT:
719 assert(c->as_jint() == 0, "should be");
720 insn = &Assembler::strw;
721 break;
722 case T_OBJECT:
723 case T_ARRAY:
724 assert(c->as_jobject() == 0, "should be");
725 if (UseCompressedOops && !wide) {
726 insn = &Assembler::strw;
727 } else {
728 insn = &Assembler::str;
729 }
730 break;
731 case T_CHAR:
732 case T_SHORT:
733 assert(c->as_jint() == 0, "should be");
734 insn = &Assembler::strh;
735 break;
736 case T_BOOLEAN:
737 case T_BYTE:
738 assert(c->as_jint() == 0, "should be");
739 insn = &Assembler::strb;
740 break;
741 default:
742 ShouldNotReachHere();
743 }
744
745 if (info) add_debug_info_for_null_check_here(info);
746 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
747 }
748
749 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
750 assert(src->is_register(), "should not call otherwise");
751 assert(dest->is_register(), "should not call otherwise");
752
753 // move between cpu-registers
754 if (dest->is_single_cpu()) {
755 if (src->type() == T_LONG) {
756 // Can do LONG -> OBJECT
757 move_regs(src->as_register_lo(), dest->as_register());
758 return;
759 }
760 assert(src->is_single_cpu(), "must match");
761 if (src->type() == T_OBJECT) {
762 __ verify_oop(src->as_register());
763 }
764 move_regs(src->as_register(), dest->as_register());
765
766 } else if (dest->is_double_cpu()) {
767 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
768 // Surprising to me but we can see move of a long to t_object
769 __ verify_oop(src->as_register());
770 move_regs(src->as_register(), dest->as_register_lo());
771 return;
772 }
773 assert(src->is_double_cpu(), "must match");
774 Register f_lo = src->as_register_lo();
775 Register f_hi = src->as_register_hi();
776 Register t_lo = dest->as_register_lo();
777 Register t_hi = dest->as_register_hi();
778 assert(f_hi == f_lo, "must be same");
779 assert(t_hi == t_lo, "must be same");
780 move_regs(f_lo, t_lo);
781
782 } else if (dest->is_single_fpu()) {
783 __ fmovs(dest->as_float_reg(), src->as_float_reg());
784
785 } else if (dest->is_double_fpu()) {
786 __ fmovd(dest->as_double_reg(), src->as_double_reg());
787
788 } else {
789 ShouldNotReachHere();
790 }
791 }
792
793 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
794 if (src->is_single_cpu()) {
795 if (type == T_ARRAY || type == T_OBJECT) {
796 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
797 __ verify_oop(src->as_register());
798 } else if (type == T_METADATA || type == T_DOUBLE) {
799 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
800 } else {
801 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
802 }
803
804 } else if (src->is_double_cpu()) {
805 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
806 __ str(src->as_register_lo(), dest_addr_LO);
807
808 } else if (src->is_single_fpu()) {
809 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
810 __ strs(src->as_float_reg(), dest_addr);
811
812 } else if (src->is_double_fpu()) {
813 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
814 __ strd(src->as_double_reg(), dest_addr);
815
816 } else {
817 ShouldNotReachHere();
818 }
819
820 }
821
822
823 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
824 LIR_Address* to_addr = dest->as_address_ptr();
825 PatchingStub* patch = NULL;
826 Register compressed_src = rscratch1;
827
828 if (patch_code != lir_patch_none) {
829 deoptimize_trap(info);
830 return;
831 }
832
833 if (type == T_ARRAY || type == T_OBJECT) {
834 __ verify_oop(src->as_register());
835
836 if (UseCompressedOops && !wide) {
837 __ encode_heap_oop(compressed_src, src->as_register());
838 } else {
839 compressed_src = src->as_register();
840 }
841 }
842
843 int null_check_here = code_offset();
844 switch (type) {
845 case T_FLOAT: {
846 __ strs(src->as_float_reg(), as_Address(to_addr));
847 break;
848 }
849
850 case T_DOUBLE: {
851 __ strd(src->as_double_reg(), as_Address(to_addr));
852 break;
853 }
854
855 case T_ARRAY: // fall through
856 case T_OBJECT: // fall through
857 if (UseCompressedOops && !wide) {
858 __ strw(compressed_src, as_Address(to_addr, rscratch2));
859 } else {
860 __ str(compressed_src, as_Address(to_addr));
861 }
862 break;
863 case T_METADATA:
864 // We get here to store a method pointer to the stack to pass to
865 // a dtrace runtime call. This can't work on 64 bit with
866 // compressed klass ptrs: T_METADATA can be a compressed klass
867 // ptr or a 64 bit method pointer.
868 LP64_ONLY(ShouldNotReachHere());
869 __ str(src->as_register(), as_Address(to_addr));
870 break;
871 case T_ADDRESS:
872 __ str(src->as_register(), as_Address(to_addr));
873 break;
874 case T_INT:
875 __ strw(src->as_register(), as_Address(to_addr));
876 break;
877
878 case T_LONG: {
879 __ str(src->as_register_lo(), as_Address_lo(to_addr));
880 break;
881 }
882
883 case T_BYTE: // fall through
884 case T_BOOLEAN: {
885 __ strb(src->as_register(), as_Address(to_addr));
886 break;
887 }
888
889 case T_CHAR: // fall through
890 case T_SHORT:
891 __ strh(src->as_register(), as_Address(to_addr));
892 break;
893
894 default:
895 ShouldNotReachHere();
896 }
897 if (info != NULL) {
898 add_debug_info_for_null_check(null_check_here, info);
899 }
900 }
901
902
903 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
904 assert(src->is_stack(), "should not call otherwise");
905 assert(dest->is_register(), "should not call otherwise");
906
907 if (dest->is_single_cpu()) {
908 if (type == T_ARRAY || type == T_OBJECT) {
909 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
910 __ verify_oop(dest->as_register());
911 } else if (type == T_METADATA) {
912 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
913 } else {
914 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
915 }
916
917 } else if (dest->is_double_cpu()) {
918 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
919 __ ldr(dest->as_register_lo(), src_addr_LO);
920
921 } else if (dest->is_single_fpu()) {
922 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
923 __ ldrs(dest->as_float_reg(), src_addr);
924
925 } else if (dest->is_double_fpu()) {
926 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
927 __ ldrd(dest->as_double_reg(), src_addr);
928
929 } else {
930 ShouldNotReachHere();
931 }
932 }
933
934
935 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
936 address target = NULL;
937 relocInfo::relocType reloc_type = relocInfo::none;
938
939 switch (patching_id(info)) {
940 case PatchingStub::access_field_id:
941 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
942 reloc_type = relocInfo::section_word_type;
943 break;
944 case PatchingStub::load_klass_id:
945 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
946 reloc_type = relocInfo::metadata_type;
947 break;
948 case PatchingStub::load_mirror_id:
949 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
950 reloc_type = relocInfo::oop_type;
951 break;
952 case PatchingStub::load_appendix_id:
953 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
954 reloc_type = relocInfo::oop_type;
955 break;
956 default: ShouldNotReachHere();
957 }
958
959 __ far_call(RuntimeAddress(target));
960 add_call_info_here(info);
961 }
962
963 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
964
965 LIR_Opr temp;
966 if (type == T_LONG || type == T_DOUBLE)
967 temp = FrameMap::rscratch1_long_opr;
968 else
969 temp = FrameMap::rscratch1_opr;
970
971 stack2reg(src, temp, src->type());
972 reg2stack(temp, dest, dest->type(), false);
973 }
974
975
976 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
977 LIR_Address* addr = src->as_address_ptr();
978 LIR_Address* from_addr = src->as_address_ptr();
979
980 if (addr->base()->type() == T_OBJECT) {
981 __ verify_oop(addr->base()->as_pointer_register());
982 }
983
984 if (patch_code != lir_patch_none) {
985 deoptimize_trap(info);
986 return;
987 }
988
989 if (info != NULL) {
990 add_debug_info_for_null_check_here(info);
991 }
992 int null_check_here = code_offset();
993 switch (type) {
994 case T_FLOAT: {
995 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
996 break;
997 }
998
999 case T_DOUBLE: {
1000 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
1001 break;
1002 }
1003
1004 case T_ARRAY: // fall through
1005 case T_OBJECT: // fall through
1006 if (UseCompressedOops && !wide) {
1007 __ ldrw(dest->as_register(), as_Address(from_addr));
1008 } else {
1009 __ ldr(dest->as_register(), as_Address(from_addr));
1010 }
1011 break;
1012 case T_METADATA:
1013 // We get here to store a method pointer to the stack to pass to
1014 // a dtrace runtime call. This can't work on 64 bit with
1015 // compressed klass ptrs: T_METADATA can be a compressed klass
1016 // ptr or a 64 bit method pointer.
1017 LP64_ONLY(ShouldNotReachHere());
1018 __ ldr(dest->as_register(), as_Address(from_addr));
1019 break;
1020 case T_ADDRESS:
1021 // FIXME: OMG this is a horrible kludge. Any offset from an
1022 // address that matches klass_offset_in_bytes() will be loaded
1023 // as a word, not a long.
1024 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1025 __ ldrw(dest->as_register(), as_Address(from_addr));
1026 } else {
1027 __ ldr(dest->as_register(), as_Address(from_addr));
1028 }
1029 break;
1030 case T_INT:
1031 __ ldrw(dest->as_register(), as_Address(from_addr));
1032 break;
1033
1034 case T_LONG: {
1035 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1036 break;
1037 }
1038
1039 case T_BYTE:
1040 __ ldrsb(dest->as_register(), as_Address(from_addr));
1041 break;
1042 case T_BOOLEAN: {
1043 __ ldrb(dest->as_register(), as_Address(from_addr));
1044 break;
1045 }
1046
1047 case T_CHAR:
1048 __ ldrh(dest->as_register(), as_Address(from_addr));
1049 break;
1050 case T_SHORT:
1051 __ ldrsh(dest->as_register(), as_Address(from_addr));
1052 break;
1053
1054 default:
1055 ShouldNotReachHere();
1056 }
1057
1058 if (type == T_ARRAY || type == T_OBJECT) {
1059 #ifdef _LP64
1060 if (UseCompressedOops && !wide) {
1061 __ decode_heap_oop(dest->as_register());
1062 }
1063 #endif
1064 __ verify_oop(dest->as_register());
1065 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1066 #ifdef _LP64
1067 if (UseCompressedClassPointers) {
1068 __ decode_klass_not_null(dest->as_register());
1069 }
1070 #endif
1071 }
1072 }
1073
1074
1075 void LIR_Assembler::prefetchr(LIR_Opr src) { Unimplemented(); }
1076
1077
1078 void LIR_Assembler::prefetchw(LIR_Opr src) { Unimplemented(); }
1079
1080
1081 int LIR_Assembler::array_element_size(BasicType type) const {
1082 int elem_size = type2aelembytes(type);
1083 return exact_log2(elem_size);
1084 }
1085
1086 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1087 Register Rdividend = op->in_opr1()->as_register();
1088 Register Rdivisor = op->in_opr2()->as_register();
1089 Register Rscratch = op->in_opr3()->as_register();
1090 Register Rresult = op->result_opr()->as_register();
1091 int divisor = -1;
1092
1093 /*
1094 TODO: For some reason, using the Rscratch that gets passed in is
1095 not possible because the register allocator does not see the tmp reg
1096 as used, and assignes it the same register as Rdividend. We use rscratch1
1097 instead.
1098
1099 assert(Rdividend != Rscratch, "");
1100 assert(Rdivisor != Rscratch, "");
1101 */
1102
1103 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1104 // convert division by a power of two into some shifts and logical operations
1105 }
1106
1107 if (op->code() == lir_irem) {
1108 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1109 } else if (op->code() == lir_idiv) {
1110 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1111 } else
1112 ShouldNotReachHere();
1113 }
1114
1115 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1116 #ifdef ASSERT
1117 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1118 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1119 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1120 #endif
1121
1122 if (op->cond() == lir_cond_always) {
1123 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1124 __ b(*(op->label()));
1125 } else {
1126 Assembler::Condition acond;
1127 if (op->code() == lir_cond_float_branch) {
1128 bool is_unordered = (op->ublock() == op->block());
1129 // Assembler::EQ does not permit unordered branches, so we add
1130 // another branch here. Likewise, Assembler::NE does not permit
1131 // ordered branches.
1132 if ((is_unordered && op->cond() == lir_cond_equal)
1133 || (!is_unordered && op->cond() == lir_cond_notEqual))
1134 __ br(Assembler::VS, *(op->ublock()->label()));
1135 switch(op->cond()) {
1136 case lir_cond_equal: acond = Assembler::EQ; break;
1137 case lir_cond_notEqual: acond = Assembler::NE; break;
1138 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1139 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1140 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1141 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1142 default: ShouldNotReachHere();
1143 }
1144 } else {
1145 switch (op->cond()) {
1146 case lir_cond_equal: acond = Assembler::EQ; break;
1147 case lir_cond_notEqual: acond = Assembler::NE; break;
1148 case lir_cond_less: acond = Assembler::LT; break;
1149 case lir_cond_lessEqual: acond = Assembler::LE; break;
1150 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1151 case lir_cond_greater: acond = Assembler::GT; break;
1152 case lir_cond_belowEqual: acond = Assembler::LS; break;
1153 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1154 default: ShouldNotReachHere();
1155 }
1156 }
1157 __ br(acond,*(op->label()));
1158 }
1159 }
1160
1161
1162
1163 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1164 LIR_Opr src = op->in_opr();
1165 LIR_Opr dest = op->result_opr();
1166
1167 switch (op->bytecode()) {
1168 case Bytecodes::_i2f:
1169 {
1170 __ scvtfws(dest->as_float_reg(), src->as_register());
1171 break;
1172 }
1173 case Bytecodes::_i2d:
1174 {
1175 __ scvtfwd(dest->as_double_reg(), src->as_register());
1176 break;
1177 }
1178 case Bytecodes::_l2d:
1179 {
1180 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1181 break;
1182 }
1183 case Bytecodes::_l2f:
1184 {
1185 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1186 break;
1187 }
1188 case Bytecodes::_f2d:
1189 {
1190 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1191 break;
1192 }
1193 case Bytecodes::_d2f:
1194 {
1195 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1196 break;
1197 }
1198 case Bytecodes::_i2c:
1199 {
1200 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1201 break;
1202 }
1203 case Bytecodes::_i2l:
1204 {
1205 __ sxtw(dest->as_register_lo(), src->as_register());
1206 break;
1207 }
1208 case Bytecodes::_i2s:
1209 {
1210 __ sxth(dest->as_register(), src->as_register());
1211 break;
1212 }
1213 case Bytecodes::_i2b:
1214 {
1215 __ sxtb(dest->as_register(), src->as_register());
1216 break;
1217 }
1218 case Bytecodes::_l2i:
1219 {
1220 _masm->block_comment("FIXME: This could be a no-op");
1221 __ uxtw(dest->as_register(), src->as_register_lo());
1222 break;
1223 }
1224 case Bytecodes::_d2l:
1225 {
1226 Register tmp = op->tmp1()->as_register();
1227 __ clear_fpsr();
1228 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1229 __ get_fpsr(tmp);
1230 __ tst(tmp, 1); // FPSCR.IOC
1231 __ br(Assembler::NE, *(op->stub()->entry()));
1232 __ bind(*op->stub()->continuation());
1233 break;
1234 }
1235 case Bytecodes::_f2i:
1236 {
1237 Register tmp = op->tmp1()->as_register();
1238 __ clear_fpsr();
1239 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1240 __ get_fpsr(tmp);
1241 __ tst(tmp, 1); // FPSCR.IOC
1242 __ br(Assembler::NE, *(op->stub()->entry()));
1243 __ bind(*op->stub()->continuation());
1244 break;
1245 }
1246 case Bytecodes::_f2l:
1247 {
1248 Register tmp = op->tmp1()->as_register();
1249 __ clear_fpsr();
1250 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1251 __ get_fpsr(tmp);
1252 __ tst(tmp, 1); // FPSCR.IOC
1253 __ br(Assembler::NE, *(op->stub()->entry()));
1254 __ bind(*op->stub()->continuation());
1255 break;
1256 }
1257 case Bytecodes::_d2i:
1258 {
1259 Register tmp = op->tmp1()->as_register();
1260 __ clear_fpsr();
1261 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1262 __ get_fpsr(tmp);
1263 __ tst(tmp, 1); // FPSCR.IOC
1264 __ br(Assembler::NE, *(op->stub()->entry()));
1265 __ bind(*op->stub()->continuation());
1266 break;
1267 }
1268 default: ShouldNotReachHere();
1269 }
1270 }
1271
1272 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1273 if (op->init_check()) {
1274 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1275 InstanceKlass::init_state_offset()));
1276 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1277 add_debug_info_for_null_check_here(op->stub()->info());
1278 __ br(Assembler::NE, *op->stub()->entry());
1279 }
1280 __ allocate_object(op->obj()->as_register(),
1281 op->tmp1()->as_register(),
1282 op->tmp2()->as_register(),
1283 op->header_size(),
1284 op->object_size(),
1285 op->klass()->as_register(),
1286 *op->stub()->entry());
1287 __ bind(*op->stub()->continuation());
1288 }
1289
1290 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1291 Register len = op->len()->as_register();
1292 __ uxtw(len, len);
1293
1294 if (UseSlowPath ||
1295 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1296 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1297 __ b(*op->stub()->entry());
1298 } else {
1299 Register tmp1 = op->tmp1()->as_register();
1300 Register tmp2 = op->tmp2()->as_register();
1301 Register tmp3 = op->tmp3()->as_register();
1302 if (len == tmp1) {
1303 tmp1 = tmp3;
1304 } else if (len == tmp2) {
1305 tmp2 = tmp3;
1306 } else if (len == tmp3) {
1307 // everything is ok
1308 } else {
1309 __ mov(tmp3, len);
1310 }
1311 __ allocate_array(op->obj()->as_register(),
1312 len,
1313 tmp1,
1314 tmp2,
1315 arrayOopDesc::header_size(op->type()),
1316 array_element_size(op->type()),
1317 op->klass()->as_register(),
1318 *op->stub()->entry());
1319 }
1320 __ bind(*op->stub()->continuation());
1321 }
1322
1323 void LIR_Assembler::type_profile_helper(Register mdo,
1324 ciMethodData *md, ciProfileData *data,
1325 Register recv, Label* update_done) {
1326 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1327 Label next_test;
1328 // See if the receiver is receiver[n].
1329 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1330 __ ldr(rscratch1, Address(rscratch2));
1331 __ cmp(recv, rscratch1);
1332 __ br(Assembler::NE, next_test);
1333 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1334 __ addptr(data_addr, DataLayout::counter_increment);
1335 __ b(*update_done);
1336 __ bind(next_test);
1337 }
1338
1339 // Didn't find receiver; find next empty slot and fill it in
1340 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1341 Label next_test;
1342 __ lea(rscratch2,
1343 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1344 Address recv_addr(rscratch2);
1345 __ ldr(rscratch1, recv_addr);
1346 __ cbnz(rscratch1, next_test);
1347 __ str(recv, recv_addr);
1348 __ mov(rscratch1, DataLayout::counter_increment);
1349 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1350 __ str(rscratch1, Address(rscratch2));
1351 __ b(*update_done);
1352 __ bind(next_test);
1353 }
1354 }
1355
1356 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1357 // we always need a stub for the failure case.
1358 CodeStub* stub = op->stub();
1359 Register obj = op->object()->as_register();
1360 Register k_RInfo = op->tmp1()->as_register();
1361 Register klass_RInfo = op->tmp2()->as_register();
1362 Register dst = op->result_opr()->as_register();
1363 ciKlass* k = op->klass();
1364 Register Rtmp1 = noreg;
1365
1366 // check if it needs to be profiled
1367 ciMethodData* md;
1368 ciProfileData* data;
1369
1370 if (op->should_profile()) {
1371 ciMethod* method = op->profiled_method();
1372 assert(method != NULL, "Should have method");
1373 int bci = op->profiled_bci();
1374 md = method->method_data_or_null();
1375 assert(md != NULL, "Sanity");
1376 data = md->bci_to_data(bci);
1377 assert(data != NULL, "need data for type check");
1378 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1379 }
1380 Label profile_cast_success, profile_cast_failure;
1381 Label *success_target = op->should_profile() ? &profile_cast_success : success;
1382 Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1383
1384 if (obj == k_RInfo) {
1385 k_RInfo = dst;
1386 } else if (obj == klass_RInfo) {
1387 klass_RInfo = dst;
1388 }
1389 if (k->is_loaded() && !UseCompressedClassPointers) {
1390 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1391 } else {
1392 Rtmp1 = op->tmp3()->as_register();
1393 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1394 }
1395
1396 assert_different_registers(obj, k_RInfo, klass_RInfo);
1397
1398 if (op->should_profile()) {
1399 Label not_null;
1400 __ cbnz(obj, not_null);
1401 // Object is null; update MDO and exit
1402 Register mdo = klass_RInfo;
1403 __ mov_metadata(mdo, md->constant_encoding());
1404 Address data_addr
1405 = __ form_address(rscratch2, mdo,
1406 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1407 LogBytesPerWord);
1408 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1409 __ ldr(rscratch1, data_addr);
1410 __ orr(rscratch1, rscratch1, header_bits);
1411 __ str(rscratch1, data_addr);
1412 __ b(*obj_is_null);
1413 __ bind(not_null);
1414 } else {
1415 __ cbz(obj, *obj_is_null);
1416 }
1417
1418 if (!k->is_loaded()) {
1419 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1420 } else {
1421 #ifdef _LP64
1422 __ mov_metadata(k_RInfo, k->constant_encoding());
1423 #endif // _LP64
1424 }
1425 __ verify_oop(obj);
1426
1427 if (op->fast_check()) {
1428 // get object class
1429 // not a safepoint as obj null check happens earlier
1430 __ load_klass(rscratch1, obj);
1431 __ cmp( rscratch1, k_RInfo);
1432
1433 __ br(Assembler::NE, *failure_target);
1434 // successful cast, fall through to profile or jump
1435 } else {
1436 // get object class
1437 // not a safepoint as obj null check happens earlier
1438 __ load_klass(klass_RInfo, obj);
1439 if (k->is_loaded()) {
1440 // See if we get an immediate positive hit
1441 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1442 __ cmp(k_RInfo, rscratch1);
1443 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1444 __ br(Assembler::NE, *failure_target);
1445 // successful cast, fall through to profile or jump
1446 } else {
1447 // See if we get an immediate positive hit
1448 __ br(Assembler::EQ, *success_target);
1449 // check for self
1450 __ cmp(klass_RInfo, k_RInfo);
1451 __ br(Assembler::EQ, *success_target);
1452
1453 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1454 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1455 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1456 // result is a boolean
1457 __ cbzw(klass_RInfo, *failure_target);
1458 // successful cast, fall through to profile or jump
1459 }
1460 } else {
1461 // perform the fast part of the checking logic
1462 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1463 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1464 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1465 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1466 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1467 // result is a boolean
1468 __ cbz(k_RInfo, *failure_target);
1469 // successful cast, fall through to profile or jump
1470 }
1471 }
1472 if (op->should_profile()) {
1473 Register mdo = klass_RInfo, recv = k_RInfo;
1474 __ bind(profile_cast_success);
1475 __ mov_metadata(mdo, md->constant_encoding());
1476 __ load_klass(recv, obj);
1477 Label update_done;
1478 type_profile_helper(mdo, md, data, recv, success);
1479 __ b(*success);
1480
1481 __ bind(profile_cast_failure);
1482 __ mov_metadata(mdo, md->constant_encoding());
1483 Address counter_addr
1484 = __ form_address(rscratch2, mdo,
1485 md->byte_offset_of_slot(data, CounterData::count_offset()),
1486 LogBytesPerWord);
1487 __ ldr(rscratch1, counter_addr);
1488 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1489 __ str(rscratch1, counter_addr);
1490 __ b(*failure);
1491 }
1492 __ b(*success);
1493 }
1494
1495
1496 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1497 LIR_Code code = op->code();
1498 if (code == lir_store_check) {
1499 Register value = op->object()->as_register();
1500 Register array = op->array()->as_register();
1501 Register k_RInfo = op->tmp1()->as_register();
1502 Register klass_RInfo = op->tmp2()->as_register();
1503 Register Rtmp1 = op->tmp3()->as_register();
1504
1505 CodeStub* stub = op->stub();
1506
1507 // check if it needs to be profiled
1508 ciMethodData* md;
1509 ciProfileData* data;
1510
1511 if (op->should_profile()) {
1512 ciMethod* method = op->profiled_method();
1513 assert(method != NULL, "Should have method");
1514 int bci = op->profiled_bci();
1515 md = method->method_data_or_null();
1516 assert(md != NULL, "Sanity");
1517 data = md->bci_to_data(bci);
1518 assert(data != NULL, "need data for type check");
1519 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1520 }
1521 Label profile_cast_success, profile_cast_failure, done;
1522 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1523 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1524
1525 if (op->should_profile()) {
1526 Label not_null;
1527 __ cbnz(value, not_null);
1528 // Object is null; update MDO and exit
1529 Register mdo = klass_RInfo;
1530 __ mov_metadata(mdo, md->constant_encoding());
1531 Address data_addr
1532 = __ form_address(rscratch2, mdo,
1533 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1534 LogBytesPerInt);
1535 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1536 __ ldrw(rscratch1, data_addr);
1537 __ orrw(rscratch1, rscratch1, header_bits);
1538 __ strw(rscratch1, data_addr);
1539 __ b(done);
1540 __ bind(not_null);
1541 } else {
1542 __ cbz(value, done);
1543 }
1544
1545 add_debug_info_for_null_check_here(op->info_for_exception());
1546 __ load_klass(k_RInfo, array);
1547 __ load_klass(klass_RInfo, value);
1548
1549 // get instance klass (it's already uncompressed)
1550 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1551 // perform the fast part of the checking logic
1552 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1553 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1554 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1555 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1556 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1557 // result is a boolean
1558 __ cbzw(k_RInfo, *failure_target);
1559 // fall through to the success case
1560
1561 if (op->should_profile()) {
1562 Register mdo = klass_RInfo, recv = k_RInfo;
1563 __ bind(profile_cast_success);
1564 __ mov_metadata(mdo, md->constant_encoding());
1565 __ load_klass(recv, value);
1566 Label update_done;
1567 type_profile_helper(mdo, md, data, recv, &done);
1568 __ b(done);
1569
1570 __ bind(profile_cast_failure);
1571 __ mov_metadata(mdo, md->constant_encoding());
1572 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1573 __ lea(rscratch2, counter_addr);
1574 __ ldr(rscratch1, Address(rscratch2));
1575 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1576 __ str(rscratch1, Address(rscratch2));
1577 __ b(*stub->entry());
1578 }
1579
1580 __ bind(done);
1581 } else if (code == lir_checkcast) {
1582 Register obj = op->object()->as_register();
1583 Register dst = op->result_opr()->as_register();
1584 Label success;
1585 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1586 __ bind(success);
1587 if (dst != obj) {
1588 __ mov(dst, obj);
1589 }
1590 } else if (code == lir_instanceof) {
1591 Register obj = op->object()->as_register();
1592 Register dst = op->result_opr()->as_register();
1593 Label success, failure, done;
1594 emit_typecheck_helper(op, &success, &failure, &failure);
1595 __ bind(failure);
1596 __ mov(dst, zr);
1597 __ b(done);
1598 __ bind(success);
1599 __ mov(dst, 1);
1600 __ bind(done);
1601 } else {
1602 ShouldNotReachHere();
1603 }
1604 }
1605
1606 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1607 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, rscratch1);
1608 __ cset(rscratch1, Assembler::NE);
1609 __ membar(__ AnyAny);
1610 }
1611
1612 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1613 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, rscratch1);
1614 __ cset(rscratch1, Assembler::NE);
1615 __ membar(__ AnyAny);
1616 }
1617
1618
1619 // Return 1 in rscratch1 if the CAS fails.
1620 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1621 assert(VM_Version::supports_cx8(), "wrong machine");
1622 Register addr = as_reg(op->addr());
1623 Register newval = as_reg(op->new_value());
1624 Register cmpval = as_reg(op->cmp_value());
1625 Label succeed, fail, around;
1626 Register res = op->result_opr()->as_register();
1627
1628 if (op->code() == lir_cas_obj) {
1629 assert(op->tmp1()->is_valid(), "must be");
1630 Register t1 = op->tmp1()->as_register();
1631 if (UseCompressedOops) {
1632 #if INCLUDE_ALL_GCS
1633 if (UseShenandoahGC && ShenandoahCASBarrier) {
1634 __ encode_heap_oop(t1, cmpval);
1635 cmpval = t1;
1636 assert(op->tmp2()->is_valid(), "must be");
1637 Register t2 = op->tmp2()->as_register();
1638 __ encode_heap_oop(t2, newval);
1639 newval = t2;
1640 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(_masm, addr, cmpval, newval, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, res);
1641 } else
1642 #endif
1643 {
1644 __ encode_heap_oop(t1, cmpval);
1645 cmpval = t1;
1646 __ encode_heap_oop(rscratch2, newval);
1647 newval = rscratch2;
1648 casw(addr, newval, cmpval);
1649 __ eorw (res, r8, 1);
1650 }
1651 } else {
1652 #if INCLUDE_ALL_GCS
1653 if (UseShenandoahGC && ShenandoahCASBarrier) {
1654 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(_masm, addr, cmpval, newval, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, res);
1655 } else
1656 #endif
1657 {
1658 casl(addr, newval, cmpval);
1659 __ eorw (res, r8, 1);
1660 }
1661 }
1662 } else if (op->code() == lir_cas_int) {
1663 casw(addr, newval, cmpval);
1664 __ eorw (res, r8, 1);
1665 } else {
1666 casl(addr, newval, cmpval);
1667 __ eorw (res, r8, 1);
1668 }
1669 }
1670
1671
1672 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1673
1674 Assembler::Condition acond, ncond;
1675 switch (condition) {
1676 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1677 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1678 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1679 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1680 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1681 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1682 case lir_cond_belowEqual: Unimplemented(); break;
1683 case lir_cond_aboveEqual: Unimplemented(); break;
1684 default: ShouldNotReachHere();
1685 }
1686
1687 assert(result->is_single_cpu() || result->is_double_cpu(),
1688 "expect single register for result");
1689 if (opr1->is_constant() && opr2->is_constant()
1690 && opr1->type() == T_INT && opr2->type() == T_INT) {
1691 jint val1 = opr1->as_jint();
1692 jint val2 = opr2->as_jint();
1693 if (val1 == 0 && val2 == 1) {
1694 __ cset(result->as_register(), ncond);
1695 return;
1696 } else if (val1 == 1 && val2 == 0) {
1697 __ cset(result->as_register(), acond);
1698 return;
1699 }
1700 }
1701
1702 if (opr1->is_constant() && opr2->is_constant()
1703 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1704 jlong val1 = opr1->as_jlong();
1705 jlong val2 = opr2->as_jlong();
1706 if (val1 == 0 && val2 == 1) {
1707 __ cset(result->as_register_lo(), ncond);
1708 return;
1709 } else if (val1 == 1 && val2 == 0) {
1710 __ cset(result->as_register_lo(), acond);
1711 return;
1712 }
1713 }
1714
1715 if (opr1->is_stack()) {
1716 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1717 opr1 = FrameMap::rscratch1_opr;
1718 } else if (opr1->is_constant()) {
1719 LIR_Opr tmp
1720 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1721 const2reg(opr1, tmp, lir_patch_none, NULL);
1722 opr1 = tmp;
1723 }
1724
1725 if (opr2->is_stack()) {
1726 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1727 opr2 = FrameMap::rscratch2_opr;
1728 } else if (opr2->is_constant()) {
1729 LIR_Opr tmp
1730 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1731 const2reg(opr2, tmp, lir_patch_none, NULL);
1732 opr2 = tmp;
1733 }
1734
1735 if (result->type() == T_LONG)
1736 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1737 else
1738 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1739 }
1740
1741 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1742 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1743
1744 if (left->is_single_cpu()) {
1745 Register lreg = left->as_register();
1746 Register dreg = as_reg(dest);
1747
1748 if (right->is_single_cpu()) {
1749 // cpu register - cpu register
1750
1751 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1752 "should be");
1753 Register rreg = right->as_register();
1754 switch (code) {
1755 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1756 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1757 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1758 default: ShouldNotReachHere();
1759 }
1760
1761 } else if (right->is_double_cpu()) {
1762 Register rreg = right->as_register_lo();
1763 // single_cpu + double_cpu: can happen with obj+long
1764 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1765 switch (code) {
1766 case lir_add: __ add(dreg, lreg, rreg); break;
1767 case lir_sub: __ sub(dreg, lreg, rreg); break;
1768 default: ShouldNotReachHere();
1769 }
1770 } else if (right->is_constant()) {
1771 // cpu register - constant
1772 jlong c;
1773
1774 // FIXME. This is fugly: we really need to factor all this logic.
1775 switch(right->type()) {
1776 case T_LONG:
1777 c = right->as_constant_ptr()->as_jlong();
1778 break;
1779 case T_INT:
1780 case T_ADDRESS:
1781 c = right->as_constant_ptr()->as_jint();
1782 break;
1783 default:
1784 ShouldNotReachHere();
1785 break;
1786 }
1787
1788 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1789 if (c == 0 && dreg == lreg) {
1790 COMMENT("effective nop elided");
1791 return;
1792 }
1793 switch(left->type()) {
1794 case T_INT:
1795 switch (code) {
1796 case lir_add: __ addw(dreg, lreg, c); break;
1797 case lir_sub: __ subw(dreg, lreg, c); break;
1798 default: ShouldNotReachHere();
1799 }
1800 break;
1801 case T_OBJECT:
1802 case T_ADDRESS:
1803 switch (code) {
1804 case lir_add: __ add(dreg, lreg, c); break;
1805 case lir_sub: __ sub(dreg, lreg, c); break;
1806 default: ShouldNotReachHere();
1807 }
1808 break;
1809 ShouldNotReachHere();
1810 }
1811 } else {
1812 ShouldNotReachHere();
1813 }
1814
1815 } else if (left->is_double_cpu()) {
1816 Register lreg_lo = left->as_register_lo();
1817
1818 if (right->is_double_cpu()) {
1819 // cpu register - cpu register
1820 Register rreg_lo = right->as_register_lo();
1821 switch (code) {
1822 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1823 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1824 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1825 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1826 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1827 default:
1828 ShouldNotReachHere();
1829 }
1830
1831 } else if (right->is_constant()) {
1832 jlong c = right->as_constant_ptr()->as_jlong_bits();
1833 Register dreg = as_reg(dest);
1834 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1835 if (c == 0 && dreg == lreg_lo) {
1836 COMMENT("effective nop elided");
1837 return;
1838 }
1839 switch (code) {
1840 case lir_add: __ add(dreg, lreg_lo, c); break;
1841 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1842 default:
1843 ShouldNotReachHere();
1844 }
1845 } else {
1846 ShouldNotReachHere();
1847 }
1848 } else if (left->is_single_fpu()) {
1849 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1850 switch (code) {
1851 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1852 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1853 case lir_mul_strictfp: // fall through
1854 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1855 case lir_div_strictfp: // fall through
1856 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1857 default:
1858 ShouldNotReachHere();
1859 }
1860 } else if (left->is_double_fpu()) {
1861 if (right->is_double_fpu()) {
1862 // fpu register - fpu register
1863 switch (code) {
1864 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1865 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1866 case lir_mul_strictfp: // fall through
1867 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1868 case lir_div_strictfp: // fall through
1869 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1870 default:
1871 ShouldNotReachHere();
1872 }
1873 } else {
1874 if (right->is_constant()) {
1875 ShouldNotReachHere();
1876 }
1877 ShouldNotReachHere();
1878 }
1879 } else if (left->is_single_stack() || left->is_address()) {
1880 assert(left == dest, "left and dest must be equal");
1881 ShouldNotReachHere();
1882 } else {
1883 ShouldNotReachHere();
1884 }
1885 }
1886
1887 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1888
1889
1890 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1891 switch(code) {
1892 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1893 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1894 default : ShouldNotReachHere();
1895 }
1896 }
1897
1898 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1899
1900 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1901 Register Rleft = left->is_single_cpu() ? left->as_register() :
1902 left->as_register_lo();
1903 if (dst->is_single_cpu()) {
1904 Register Rdst = dst->as_register();
1905 if (right->is_constant()) {
1906 switch (code) {
1907 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1908 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1909 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1910 default: ShouldNotReachHere(); break;
1911 }
1912 } else {
1913 Register Rright = right->is_single_cpu() ? right->as_register() :
1914 right->as_register_lo();
1915 switch (code) {
1916 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1917 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1918 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1919 default: ShouldNotReachHere(); break;
1920 }
1921 }
1922 } else {
1923 Register Rdst = dst->as_register_lo();
1924 if (right->is_constant()) {
1925 switch (code) {
1926 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1927 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1928 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1929 default: ShouldNotReachHere(); break;
1930 }
1931 } else {
1932 Register Rright = right->is_single_cpu() ? right->as_register() :
1933 right->as_register_lo();
1934 switch (code) {
1935 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1936 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1937 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1938 default: ShouldNotReachHere(); break;
1939 }
1940 }
1941 }
1942 }
1943
1944
1945
1946 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1947
1948
1949 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1950 if (opr1->is_constant() && opr2->is_single_cpu()) {
1951 // tableswitch
1952 Register reg = as_reg(opr2);
1953 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1954 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1955 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1956 Register reg1 = as_reg(opr1);
1957 if (opr2->is_single_cpu()) {
1958 // cpu register - cpu register
1959 Register reg2 = opr2->as_register();
1960 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1961 __ cmp(reg1, reg2);
1962 } else {
1963 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1964 __ cmpw(reg1, reg2);
1965 }
1966 return;
1967 }
1968 if (opr2->is_double_cpu()) {
1969 // cpu register - cpu register
1970 Register reg2 = opr2->as_register_lo();
1971 __ cmp(reg1, reg2);
1972 return;
1973 }
1974
1975 if (opr2->is_constant()) {
1976 bool is_32bit = false; // width of register operand
1977 jlong imm;
1978
1979 switch(opr2->type()) {
1980 case T_INT:
1981 imm = opr2->as_constant_ptr()->as_jint();
1982 is_32bit = true;
1983 break;
1984 case T_LONG:
1985 imm = opr2->as_constant_ptr()->as_jlong();
1986 break;
1987 case T_ADDRESS:
1988 imm = opr2->as_constant_ptr()->as_jint();
1989 break;
1990 case T_OBJECT:
1991 case T_ARRAY:
1992 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1993 break;
1994 default:
1995 ShouldNotReachHere();
1996 break;
1997 }
1998
1999 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
2000 if (is_32bit)
2001 __ cmpw(reg1, imm);
2002 else
2003 __ cmp(reg1, imm);
2004 return;
2005 } else {
2006 __ mov(rscratch1, imm);
2007 if (is_32bit)
2008 __ cmpw(reg1, rscratch1);
2009 else
2010 __ cmp(reg1, rscratch1);
2011 return;
2012 }
2013 } else
2014 ShouldNotReachHere();
2015 } else if (opr1->is_single_fpu()) {
2016 FloatRegister reg1 = opr1->as_float_reg();
2017 assert(opr2->is_single_fpu(), "expect single float register");
2018 FloatRegister reg2 = opr2->as_float_reg();
2019 __ fcmps(reg1, reg2);
2020 } else if (opr1->is_double_fpu()) {
2021 FloatRegister reg1 = opr1->as_double_reg();
2022 assert(opr2->is_double_fpu(), "expect double float register");
2023 FloatRegister reg2 = opr2->as_double_reg();
2024 __ fcmpd(reg1, reg2);
2025 } else {
2026 ShouldNotReachHere();
2027 }
2028 }
2029
2030 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2031 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2032 bool is_unordered_less = (code == lir_ucmp_fd2i);
2033 if (left->is_single_fpu()) {
2034 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2035 } else if (left->is_double_fpu()) {
2036 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2037 } else {
2038 ShouldNotReachHere();
2039 }
2040 } else if (code == lir_cmp_l2i) {
2041 Label done;
2042 __ cmp(left->as_register_lo(), right->as_register_lo());
2043 __ mov(dst->as_register(), (u_int64_t)-1L);
2044 __ br(Assembler::LT, done);
2045 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2046 __ bind(done);
2047 } else {
2048 ShouldNotReachHere();
2049 }
2050 }
2051
2052
2053 void LIR_Assembler::align_call(LIR_Code code) { }
2054
2055
2056 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2057 address call = __ trampoline_call(Address(op->addr(), rtype));
2058 if (call == NULL) {
2059 bailout("trampoline stub overflow");
2060 return;
2061 }
2062 add_call_info(code_offset(), op->info());
2063 }
2064
2065
2066 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2067 address call = __ ic_call(op->addr());
2068 if (call == NULL) {
2069 bailout("trampoline stub overflow");
2070 return;
2071 }
2072 add_call_info(code_offset(), op->info());
2073 }
2074
2075
2076 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2077 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2078 ShouldNotReachHere();
2079 }
2080
2081
2082 void LIR_Assembler::emit_static_call_stub() {
2083 address call_pc = __ pc();
2084 address stub = __ start_a_stub(call_stub_size);
2085 if (stub == NULL) {
2086 bailout("static call stub overflow");
2087 return;
2088 }
2089
2090 int start = __ offset();
2091
2092 __ relocate(static_stub_Relocation::spec(call_pc));
2093 __ mov_metadata(rmethod, (Metadata*)NULL);
2094 __ movptr(rscratch1, 0);
2095 __ br(rscratch1);
2096
2097 assert(__ offset() - start <= call_stub_size, "stub too big");
2098 __ end_a_stub();
2099 }
2100
2101
2102 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2103 assert(exceptionOop->as_register() == r0, "must match");
2104 assert(exceptionPC->as_register() == r3, "must match");
2105
2106 // exception object is not added to oop map by LinearScan
2107 // (LinearScan assumes that no oops are in fixed registers)
2108 info->add_register_oop(exceptionOop);
2109 Runtime1::StubID unwind_id;
2110
2111 // get current pc information
2112 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2113 int pc_for_athrow_offset = __ offset();
2114 InternalAddress pc_for_athrow(__ pc());
2115 __ adr(exceptionPC->as_register(), pc_for_athrow);
2116 add_call_info(pc_for_athrow_offset, info); // for exception handler
2117
2118 __ verify_not_null_oop(r0);
2119 // search an exception handler (r0: exception oop, r3: throwing pc)
2120 if (compilation()->has_fpu_code()) {
2121 unwind_id = Runtime1::handle_exception_id;
2122 } else {
2123 unwind_id = Runtime1::handle_exception_nofpu_id;
2124 }
2125 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2126
2127 // FIXME: enough room for two byte trap ????
2128 __ nop();
2129 }
2130
2131
2132 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2133 assert(exceptionOop->as_register() == r0, "must match");
2134
2135 __ b(_unwind_handler_entry);
2136 }
2137
2138
2139 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2140 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2141 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2142
2143 switch (left->type()) {
2144 case T_INT: {
2145 switch (code) {
2146 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2147 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2148 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2149 default:
2150 ShouldNotReachHere();
2151 break;
2152 }
2153 break;
2154 case T_LONG:
2155 case T_ADDRESS:
2156 case T_OBJECT:
2157 switch (code) {
2158 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2159 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2160 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2161 default:
2162 ShouldNotReachHere();
2163 break;
2164 }
2165 break;
2166 default:
2167 ShouldNotReachHere();
2168 break;
2169 }
2170 }
2171 }
2172
2173
2174 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2175 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2176 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2177
2178 switch (left->type()) {
2179 case T_INT: {
2180 switch (code) {
2181 case lir_shl: __ lslw (dreg, lreg, count); break;
2182 case lir_shr: __ asrw (dreg, lreg, count); break;
2183 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2184 default:
2185 ShouldNotReachHere();
2186 break;
2187 }
2188 break;
2189 case T_LONG:
2190 case T_ADDRESS:
2191 case T_OBJECT:
2192 switch (code) {
2193 case lir_shl: __ lsl (dreg, lreg, count); break;
2194 case lir_shr: __ asr (dreg, lreg, count); break;
2195 case lir_ushr: __ lsr (dreg, lreg, count); break;
2196 default:
2197 ShouldNotReachHere();
2198 break;
2199 }
2200 break;
2201 default:
2202 ShouldNotReachHere();
2203 break;
2204 }
2205 }
2206 }
2207
2208
2209 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2210 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2211 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2212 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2213 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2214 }
2215
2216
2217 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2218 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2219 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2220 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2221 __ mov (rscratch1, c);
2222 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2223 }
2224
2225
2226 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2227 ShouldNotReachHere();
2228 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2229 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2230 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2231 __ lea(rscratch1, __ constant_oop_address(o));
2232 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2233 }
2234
2235
2236 // This code replaces a call to arraycopy; no exception may
2237 // be thrown in this code, they must be thrown in the System.arraycopy
2238 // activation frame; we could save some checks if this would not be the case
2239 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2240 ciArrayKlass* default_type = op->expected_type();
2241 Register src = op->src()->as_register();
2242 Register dst = op->dst()->as_register();
2243 Register src_pos = op->src_pos()->as_register();
2244 Register dst_pos = op->dst_pos()->as_register();
2245 Register length = op->length()->as_register();
2246 Register tmp = op->tmp()->as_register();
2247
2248 CodeStub* stub = op->stub();
2249 int flags = op->flags();
2250 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2251 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2252
2253 // if we don't know anything, just go through the generic arraycopy
2254 if (default_type == NULL // || basic_type == T_OBJECT
2255 ) {
2256 Label done;
2257 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2258
2259 // Save the arguments in case the generic arraycopy fails and we
2260 // have to fall back to the JNI stub
2261 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2262 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2263 __ str(src, Address(sp, 4*BytesPerWord));
2264
2265 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2266 address copyfunc_addr = StubRoutines::generic_arraycopy();
2267
2268 // The arguments are in java calling convention so we shift them
2269 // to C convention
2270 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2271 __ mov(c_rarg0, j_rarg0);
2272 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2273 __ mov(c_rarg1, j_rarg1);
2274 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2275 __ mov(c_rarg2, j_rarg2);
2276 assert_different_registers(c_rarg3, j_rarg4);
2277 __ mov(c_rarg3, j_rarg3);
2278 __ mov(c_rarg4, j_rarg4);
2279 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2280 __ mov(rscratch1, RuntimeAddress(C_entry));
2281 __ blr(rscratch1);
2282 } else {
2283 #ifndef PRODUCT
2284 if (PrintC1Statistics) {
2285 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2286 }
2287 #endif
2288 __ far_call(RuntimeAddress(copyfunc_addr));
2289 }
2290
2291 __ cbz(r0, *stub->continuation());
2292
2293 // Reload values from the stack so they are where the stub
2294 // expects them.
2295 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2296 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2297 __ ldr(src, Address(sp, 4*BytesPerWord));
2298
2299 if (copyfunc_addr != NULL) {
2300 // r0 is -1^K where K == partial copied count
2301 __ eonw(rscratch1, r0, zr);
2302 // adjust length down and src/end pos up by partial copied count
2303 __ subw(length, length, rscratch1);
2304 __ addw(src_pos, src_pos, rscratch1);
2305 __ addw(dst_pos, dst_pos, rscratch1);
2306 }
2307 __ b(*stub->entry());
2308
2309 __ bind(*stub->continuation());
2310 return;
2311 }
2312
2313 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2314
2315 int elem_size = type2aelembytes(basic_type);
2316 int shift_amount;
2317 int scale = exact_log2(elem_size);
2318
2319 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2320 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2321 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2322 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2323
2324 // test for NULL
2325 if (flags & LIR_OpArrayCopy::src_null_check) {
2326 __ cbz(src, *stub->entry());
2327 }
2328 if (flags & LIR_OpArrayCopy::dst_null_check) {
2329 __ cbz(dst, *stub->entry());
2330 }
2331
2332 // If the compiler was not able to prove that exact type of the source or the destination
2333 // of the arraycopy is an array type, check at runtime if the source or the destination is
2334 // an instance type.
2335 if (flags & LIR_OpArrayCopy::type_check) {
2336 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2337 __ load_klass(tmp, dst);
2338 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2339 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2340 __ br(Assembler::GE, *stub->entry());
2341 }
2342
2343 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2344 __ load_klass(tmp, src);
2345 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2346 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2347 __ br(Assembler::GE, *stub->entry());
2348 }
2349 }
2350
2351 // check if negative
2352 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2353 __ cmpw(src_pos, 0);
2354 __ br(Assembler::LT, *stub->entry());
2355 }
2356 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2357 __ cmpw(dst_pos, 0);
2358 __ br(Assembler::LT, *stub->entry());
2359 }
2360
2361 if (flags & LIR_OpArrayCopy::length_positive_check) {
2362 __ cmpw(length, 0);
2363 __ br(Assembler::LT, *stub->entry());
2364 }
2365
2366 if (flags & LIR_OpArrayCopy::src_range_check) {
2367 __ addw(tmp, src_pos, length);
2368 __ ldrw(rscratch1, src_length_addr);
2369 __ cmpw(tmp, rscratch1);
2370 __ br(Assembler::HI, *stub->entry());
2371 }
2372 if (flags & LIR_OpArrayCopy::dst_range_check) {
2373 __ addw(tmp, dst_pos, length);
2374 __ ldrw(rscratch1, dst_length_addr);
2375 __ cmpw(tmp, rscratch1);
2376 __ br(Assembler::HI, *stub->entry());
2377 }
2378
2379 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2380 // length_positive_check if the source of our length operand is an
2381 // arraylength. However, that arraylength might be zero, and the
2382 // stub that we're about to call contains an assertion that count !=
2383 // 0 . So we make this check purely in order not to trigger an
2384 // assertion failure.
2385 __ cbzw(length, *stub->continuation());
2386
2387 if (flags & LIR_OpArrayCopy::type_check) {
2388 // We don't know the array types are compatible
2389 if (basic_type != T_OBJECT) {
2390 // Simple test for basic type arrays
2391 if (UseCompressedClassPointers) {
2392 __ ldrw(tmp, src_klass_addr);
2393 __ ldrw(rscratch1, dst_klass_addr);
2394 __ cmpw(tmp, rscratch1);
2395 } else {
2396 __ ldr(tmp, src_klass_addr);
2397 __ ldr(rscratch1, dst_klass_addr);
2398 __ cmp(tmp, rscratch1);
2399 }
2400 __ br(Assembler::NE, *stub->entry());
2401 } else {
2402 // For object arrays, if src is a sub class of dst then we can
2403 // safely do the copy.
2404 Label cont, slow;
2405
2406 #define PUSH(r1, r2) \
2407 stp(r1, r2, __ pre(sp, -2 * wordSize));
2408
2409 #define POP(r1, r2) \
2410 ldp(r1, r2, __ post(sp, 2 * wordSize));
2411
2412 __ PUSH(src, dst);
2413
2414 __ load_klass(src, src);
2415 __ load_klass(dst, dst);
2416
2417 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2418
2419 __ PUSH(src, dst);
2420 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2421 __ POP(src, dst);
2422
2423 __ cbnz(src, cont);
2424
2425 __ bind(slow);
2426 __ POP(src, dst);
2427
2428 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2429 if (copyfunc_addr != NULL) { // use stub if available
2430 // src is not a sub class of dst so we have to do a
2431 // per-element check.
2432
2433 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2434 if ((flags & mask) != mask) {
2435 // Check that at least both of them object arrays.
2436 assert(flags & mask, "one of the two should be known to be an object array");
2437
2438 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2439 __ load_klass(tmp, src);
2440 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2441 __ load_klass(tmp, dst);
2442 }
2443 int lh_offset = in_bytes(Klass::layout_helper_offset());
2444 Address klass_lh_addr(tmp, lh_offset);
2445 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2446 __ ldrw(rscratch1, klass_lh_addr);
2447 __ mov(rscratch2, objArray_lh);
2448 __ eorw(rscratch1, rscratch1, rscratch2);
2449 __ cbnzw(rscratch1, *stub->entry());
2450 }
2451
2452 // Spill because stubs can use any register they like and it's
2453 // easier to restore just those that we care about.
2454 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2455 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2456 __ str(src, Address(sp, 4*BytesPerWord));
2457
2458 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2459 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2460 assert_different_registers(c_rarg0, dst, dst_pos, length);
2461 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2462 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2463 assert_different_registers(c_rarg1, dst, length);
2464 __ uxtw(c_rarg2, length);
2465 assert_different_registers(c_rarg2, dst);
2466
2467 __ load_klass(c_rarg4, dst);
2468 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2469 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2470 __ far_call(RuntimeAddress(copyfunc_addr));
2471
2472 #ifndef PRODUCT
2473 if (PrintC1Statistics) {
2474 Label failed;
2475 __ cbnz(r0, failed);
2476 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2477 __ bind(failed);
2478 }
2479 #endif
2480
2481 __ cbz(r0, *stub->continuation());
2482
2483 #ifndef PRODUCT
2484 if (PrintC1Statistics) {
2485 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2486 }
2487 #endif
2488 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2489
2490 // Restore previously spilled arguments
2491 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2492 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2493 __ ldr(src, Address(sp, 4*BytesPerWord));
2494
2495 // return value is -1^K where K is partial copied count
2496 __ eonw(rscratch1, r0, zr);
2497 // adjust length down and src/end pos up by partial copied count
2498 __ subw(length, length, rscratch1);
2499 __ addw(src_pos, src_pos, rscratch1);
2500 __ addw(dst_pos, dst_pos, rscratch1);
2501 }
2502
2503 __ b(*stub->entry());
2504
2505 __ bind(cont);
2506 __ POP(src, dst);
2507 }
2508 }
2509
2510 #ifdef ASSERT
2511 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2512 // Sanity check the known type with the incoming class. For the
2513 // primitive case the types must match exactly with src.klass and
2514 // dst.klass each exactly matching the default type. For the
2515 // object array case, if no type check is needed then either the
2516 // dst type is exactly the expected type and the src type is a
2517 // subtype which we can't check or src is the same array as dst
2518 // but not necessarily exactly of type default_type.
2519 Label known_ok, halt;
2520 __ mov_metadata(tmp, default_type->constant_encoding());
2521 #ifdef _LP64
2522 if (UseCompressedClassPointers) {
2523 __ encode_klass_not_null(tmp);
2524 }
2525 #endif
2526
2527 if (basic_type != T_OBJECT) {
2528
2529 if (UseCompressedClassPointers) {
2530 __ ldrw(rscratch1, dst_klass_addr);
2531 __ cmpw(tmp, rscratch1);
2532 } else {
2533 __ ldr(rscratch1, dst_klass_addr);
2534 __ cmp(tmp, rscratch1);
2535 }
2536 __ br(Assembler::NE, halt);
2537 if (UseCompressedClassPointers) {
2538 __ ldrw(rscratch1, src_klass_addr);
2539 __ cmpw(tmp, rscratch1);
2540 } else {
2541 __ ldr(rscratch1, src_klass_addr);
2542 __ cmp(tmp, rscratch1);
2543 }
2544 __ br(Assembler::EQ, known_ok);
2545 } else {
2546 if (UseCompressedClassPointers) {
2547 __ ldrw(rscratch1, dst_klass_addr);
2548 __ cmpw(tmp, rscratch1);
2549 } else {
2550 __ ldr(rscratch1, dst_klass_addr);
2551 __ cmp(tmp, rscratch1);
2552 }
2553 __ br(Assembler::EQ, known_ok);
2554 __ cmp(src, dst);
2555 __ br(Assembler::EQ, known_ok);
2556 }
2557 __ bind(halt);
2558 __ stop("incorrect type information in arraycopy");
2559 __ bind(known_ok);
2560 }
2561 #endif
2562
2563 #ifndef PRODUCT
2564 if (PrintC1Statistics) {
2565 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2566 }
2567 #endif
2568
2569 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2570 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2571 assert_different_registers(c_rarg0, dst, dst_pos, length);
2572 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2573 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2574 assert_different_registers(c_rarg1, dst, length);
2575 __ uxtw(c_rarg2, length);
2576 assert_different_registers(c_rarg2, dst);
2577
2578 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2579 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2580 const char *name;
2581 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2582
2583 CodeBlob *cb = CodeCache::find_blob(entry);
2584 if (cb) {
2585 __ far_call(RuntimeAddress(entry));
2586 } else {
2587 __ call_VM_leaf(entry, 3);
2588 }
2589
2590 __ bind(*stub->continuation());
2591 }
2592
2593
2594
2595
2596 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2597 Register obj = op->obj_opr()->as_register(); // may not be an oop
2598 Register hdr = op->hdr_opr()->as_register();
2599 Register lock = op->lock_opr()->as_register();
2600 if (!UseFastLocking) {
2601 __ b(*op->stub()->entry());
2602 } else if (op->code() == lir_lock) {
2603 Register scratch = noreg;
2604 if (UseBiasedLocking) {
2605 scratch = op->scratch_opr()->as_register();
2606 }
2607 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2608 // add debug info for NullPointerException only if one is possible
2609 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2610 if (op->info() != NULL) {
2611 add_debug_info_for_null_check(null_check_offset, op->info());
2612 }
2613 // done
2614 } else if (op->code() == lir_unlock) {
2615 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2616 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2617 } else {
2618 Unimplemented();
2619 }
2620 __ bind(*op->stub()->continuation());
2621 }
2622
2623
2624 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2625 ciMethod* method = op->profiled_method();
2626 int bci = op->profiled_bci();
2627 ciMethod* callee = op->profiled_callee();
2628
2629 // Update counter for all call types
2630 ciMethodData* md = method->method_data_or_null();
2631 assert(md != NULL, "Sanity");
2632 ciProfileData* data = md->bci_to_data(bci);
2633 assert(data->is_CounterData(), "need CounterData for calls");
2634 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2635 Register mdo = op->mdo()->as_register();
2636 __ mov_metadata(mdo, md->constant_encoding());
2637 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2638 Bytecodes::Code bc = method->java_code_at_bci(bci);
2639 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2640 // Perform additional virtual call profiling for invokevirtual and
2641 // invokeinterface bytecodes
2642 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2643 !callee_is_static && // required for optimized MH invokes
2644 C1ProfileVirtualCalls) {
2645 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2646 Register recv = op->recv()->as_register();
2647 assert_different_registers(mdo, recv);
2648 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2649 ciKlass* known_klass = op->known_holder();
2650 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2651 // We know the type that will be seen at this call site; we can
2652 // statically update the MethodData* rather than needing to do
2653 // dynamic tests on the receiver type
2654
2655 // NOTE: we should probably put a lock around this search to
2656 // avoid collisions by concurrent compilations
2657 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2658 uint i;
2659 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2660 ciKlass* receiver = vc_data->receiver(i);
2661 if (known_klass->equals(receiver)) {
2662 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2663 __ addptr(data_addr, DataLayout::counter_increment);
2664 return;
2665 }
2666 }
2667
2668 // Receiver type not found in profile data; select an empty slot
2669
2670 // Note that this is less efficient than it should be because it
2671 // always does a write to the receiver part of the
2672 // VirtualCallData rather than just the first time
2673 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2674 ciKlass* receiver = vc_data->receiver(i);
2675 if (receiver == NULL) {
2676 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2677 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2678 __ lea(rscratch2, recv_addr);
2679 __ str(rscratch1, Address(rscratch2));
2680 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2681 __ addptr(data_addr, DataLayout::counter_increment);
2682 return;
2683 }
2684 }
2685 } else {
2686 __ load_klass(recv, recv);
2687 Label update_done;
2688 type_profile_helper(mdo, md, data, recv, &update_done);
2689 // Receiver did not match any saved receiver and there is no empty row for it.
2690 // Increment total counter to indicate polymorphic case.
2691 __ addptr(counter_addr, DataLayout::counter_increment);
2692
2693 __ bind(update_done);
2694 }
2695 } else {
2696 // Static call
2697 __ addptr(counter_addr, DataLayout::counter_increment);
2698 }
2699 }
2700
2701
2702 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2703 Unimplemented();
2704 }
2705
2706
2707 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2708 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2709 }
2710
2711 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2712 assert(op->crc()->is_single_cpu(), "crc must be register");
2713 assert(op->val()->is_single_cpu(), "byte value must be register");
2714 assert(op->result_opr()->is_single_cpu(), "result must be register");
2715 Register crc = op->crc()->as_register();
2716 Register val = op->val()->as_register();
2717 Register res = op->result_opr()->as_register();
2718
2719 assert_different_registers(val, crc, res);
2720 unsigned long offset;
2721 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2722 if (offset) __ add(res, res, offset);
2723
2724 __ ornw(crc, zr, crc); // ~crc
2725 __ update_byte_crc32(crc, val, res);
2726 __ ornw(res, zr, crc); // ~crc
2727 }
2728
2729 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2730 COMMENT("emit_profile_type {");
2731 Register obj = op->obj()->as_register();
2732 Register tmp = op->tmp()->as_pointer_register();
2733 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2734 ciKlass* exact_klass = op->exact_klass();
2735 intptr_t current_klass = op->current_klass();
2736 bool not_null = op->not_null();
2737 bool no_conflict = op->no_conflict();
2738
2739 Label update, next, none;
2740
2741 bool do_null = !not_null;
2742 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2743 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2744
2745 assert(do_null || do_update, "why are we here?");
2746 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2747 assert(mdo_addr.base() != rscratch1, "wrong register");
2748
2749 __ verify_oop(obj);
2750
2751 if (tmp != obj) {
2752 __ mov(tmp, obj);
2753 }
2754 if (do_null) {
2755 __ cbnz(tmp, update);
2756 if (!TypeEntries::was_null_seen(current_klass)) {
2757 __ ldr(rscratch2, mdo_addr);
2758 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2759 __ str(rscratch2, mdo_addr);
2760 }
2761 if (do_update) {
2762 #ifndef ASSERT
2763 __ b(next);
2764 }
2765 #else
2766 __ b(next);
2767 }
2768 } else {
2769 __ cbnz(tmp, update);
2770 __ stop("unexpected null obj");
2771 #endif
2772 }
2773
2774 __ bind(update);
2775
2776 if (do_update) {
2777 #ifdef ASSERT
2778 if (exact_klass != NULL) {
2779 Label ok;
2780 __ load_klass(tmp, tmp);
2781 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2782 __ eor(rscratch1, tmp, rscratch1);
2783 __ cbz(rscratch1, ok);
2784 __ stop("exact klass and actual klass differ");
2785 __ bind(ok);
2786 }
2787 #endif
2788 if (!no_conflict) {
2789 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2790 if (exact_klass != NULL) {
2791 __ mov_metadata(tmp, exact_klass->constant_encoding());
2792 } else {
2793 __ load_klass(tmp, tmp);
2794 }
2795
2796 __ ldr(rscratch2, mdo_addr);
2797 __ eor(tmp, tmp, rscratch2);
2798 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2799 // klass seen before, nothing to do. The unknown bit may have been
2800 // set already but no need to check.
2801 __ cbz(rscratch1, next);
2802
2803 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2804 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2805
2806 if (TypeEntries::is_type_none(current_klass)) {
2807 __ cbz(rscratch2, none);
2808 __ cmp(rscratch2, TypeEntries::null_seen);
2809 __ br(Assembler::EQ, none);
2810 // There is a chance that the checks above (re-reading profiling
2811 // data from memory) fail if another thread has just set the
2812 // profiling to this obj's klass
2813 __ dmb(Assembler::ISHLD);
2814 __ ldr(rscratch2, mdo_addr);
2815 __ eor(tmp, tmp, rscratch2);
2816 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2817 __ cbz(rscratch1, next);
2818 }
2819 } else {
2820 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2821 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2822
2823 __ ldr(tmp, mdo_addr);
2824 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2825 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2826 }
2827
2828 // different than before. Cannot keep accurate profile.
2829 __ ldr(rscratch2, mdo_addr);
2830 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2831 __ str(rscratch2, mdo_addr);
2832
2833 if (TypeEntries::is_type_none(current_klass)) {
2834 __ b(next);
2835
2836 __ bind(none);
2837 // first time here. Set profile type.
2838 __ str(tmp, mdo_addr);
2839 }
2840 } else {
2841 // There's a single possible klass at this profile point
2842 assert(exact_klass != NULL, "should be");
2843 if (TypeEntries::is_type_none(current_klass)) {
2844 __ mov_metadata(tmp, exact_klass->constant_encoding());
2845 __ ldr(rscratch2, mdo_addr);
2846 __ eor(tmp, tmp, rscratch2);
2847 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2848 __ cbz(rscratch1, next);
2849 #ifdef ASSERT
2850 {
2851 Label ok;
2852 __ ldr(rscratch1, mdo_addr);
2853 __ cbz(rscratch1, ok);
2854 __ cmp(rscratch1, TypeEntries::null_seen);
2855 __ br(Assembler::EQ, ok);
2856 // may have been set by another thread
2857 __ dmb(Assembler::ISHLD);
2858 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2859 __ ldr(rscratch2, mdo_addr);
2860 __ eor(rscratch2, rscratch1, rscratch2);
2861 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2862 __ cbz(rscratch2, ok);
2863
2864 __ stop("unexpected profiling mismatch");
2865 __ bind(ok);
2866 }
2867 #endif
2868 // first time here. Set profile type.
2869 __ ldr(tmp, mdo_addr);
2870 } else {
2871 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2872 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2873
2874 __ ldr(tmp, mdo_addr);
2875 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2876 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2877
2878 __ orr(tmp, tmp, TypeEntries::type_unknown);
2879 __ str(tmp, mdo_addr);
2880 // FIXME: Write barrier needed here?
2881 }
2882 }
2883
2884 __ bind(next);
2885 }
2886 COMMENT("} emit_profile_type");
2887 }
2888
2889
2890 void LIR_Assembler::align_backward_branch_target() {
2891 }
2892
2893
2894 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2895 if (left->is_single_cpu()) {
2896 assert(dest->is_single_cpu(), "expect single result reg");
2897 __ negw(dest->as_register(), left->as_register());
2898 } else if (left->is_double_cpu()) {
2899 assert(dest->is_double_cpu(), "expect double result reg");
2900 __ neg(dest->as_register_lo(), left->as_register_lo());
2901 } else if (left->is_single_fpu()) {
2902 assert(dest->is_single_fpu(), "expect single float result reg");
2903 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2904 } else {
2905 assert(left->is_double_fpu(), "expect double float operand reg");
2906 assert(dest->is_double_fpu(), "expect double float result reg");
2907 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2908 }
2909 }
2910
2911
2912 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2913 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2914 }
2915
2916
2917 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2918 assert(!tmp->is_valid(), "don't need temporary");
2919
2920 CodeBlob *cb = CodeCache::find_blob(dest);
2921 if (cb) {
2922 __ far_call(RuntimeAddress(dest));
2923 } else {
2924 __ mov(rscratch1, RuntimeAddress(dest));
2925 __ blr(rscratch1);
2926 }
2927
2928 if (info != NULL) {
2929 add_call_info_here(info);
2930 }
2931 __ maybe_isb();
2932 }
2933
2934 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2935 if (dest->is_address() || src->is_address()) {
2936 move_op(src, dest, type, lir_patch_none, info,
2937 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2938 } else {
2939 ShouldNotReachHere();
2940 }
2941 }
2942
2943 #ifdef ASSERT
2944 // emit run-time assertion
2945 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2946 assert(op->code() == lir_assert, "must be");
2947
2948 if (op->in_opr1()->is_valid()) {
2949 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2950 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2951 } else {
2952 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2953 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2954 }
2955
2956 Label ok;
2957 if (op->condition() != lir_cond_always) {
2958 Assembler::Condition acond = Assembler::AL;
2959 switch (op->condition()) {
2960 case lir_cond_equal: acond = Assembler::EQ; break;
2961 case lir_cond_notEqual: acond = Assembler::NE; break;
2962 case lir_cond_less: acond = Assembler::LT; break;
2963 case lir_cond_lessEqual: acond = Assembler::LE; break;
2964 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2965 case lir_cond_greater: acond = Assembler::GT; break;
2966 case lir_cond_belowEqual: acond = Assembler::LS; break;
2967 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2968 default: ShouldNotReachHere();
2969 }
2970 __ br(acond, ok);
2971 }
2972 if (op->halt()) {
2973 const char* str = __ code_string(op->msg());
2974 __ stop(str);
2975 } else {
2976 breakpoint();
2977 }
2978 __ bind(ok);
2979 }
2980 #endif
2981
2982 #ifndef PRODUCT
2983 #define COMMENT(x) do { __ block_comment(x); } while (0)
2984 #else
2985 #define COMMENT(x)
2986 #endif
2987
2988 void LIR_Assembler::membar() {
2989 COMMENT("membar");
2990 __ membar(MacroAssembler::AnyAny);
2991 }
2992
2993 void LIR_Assembler::membar_acquire() {
2994 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2995 }
2996
2997 void LIR_Assembler::membar_release() {
2998 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2999 }
3000
3001 void LIR_Assembler::membar_loadload() {
3002 __ membar(Assembler::LoadLoad);
3003 }
3004
3005 void LIR_Assembler::membar_storestore() {
3006 __ membar(MacroAssembler::StoreStore);
3007 }
3008
3009 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3010
3011 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3012
3013 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3014 __ mov(result_reg->as_register(), rthread);
3015 }
3016
3017
3018 void LIR_Assembler::peephole(LIR_List *lir) {
3019 #if 0
3020 if (tableswitch_count >= max_tableswitches)
3021 return;
3022
3023 /*
3024 This finite-state automaton recognizes sequences of compare-and-
3025 branch instructions. We will turn them into a tableswitch. You
3026 could argue that C1 really shouldn't be doing this sort of
3027 optimization, but without it the code is really horrible.
3028 */
3029
3030 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3031 int first_key, last_key = -2147483648;
3032 int next_key = 0;
3033 int start_insn = -1;
3034 int last_insn = -1;
3035 Register reg = noreg;
3036 LIR_Opr reg_opr;
3037 state = start_s;
3038
3039 LIR_OpList* inst = lir->instructions_list();
3040 for (int i = 0; i < inst->length(); i++) {
3041 LIR_Op* op = inst->at(i);
3042 switch (state) {
3043 case start_s:
3044 first_key = -1;
3045 start_insn = i;
3046 switch (op->code()) {
3047 case lir_cmp:
3048 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3049 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3050 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3051 && opr2->is_constant()
3052 && opr2->type() == T_INT) {
3053 reg_opr = opr1;
3054 reg = opr1->as_register();
3055 first_key = opr2->as_constant_ptr()->as_jint();
3056 next_key = first_key + 1;
3057 state = cmp_s;
3058 goto next_state;
3059 }
3060 break;
3061 }
3062 break;
3063 case cmp_s:
3064 switch (op->code()) {
3065 case lir_branch:
3066 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3067 state = beq_s;
3068 last_insn = i;
3069 goto next_state;
3070 }
3071 }
3072 state = start_s;
3073 break;
3074 case beq_s:
3075 switch (op->code()) {
3076 case lir_cmp: {
3077 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3078 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3079 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3080 && opr1->as_register() == reg
3081 && opr2->is_constant()
3082 && opr2->type() == T_INT
3083 && opr2->as_constant_ptr()->as_jint() == next_key) {
3084 last_key = next_key;
3085 next_key++;
3086 state = cmp_s;
3087 goto next_state;
3088 }
3089 }
3090 }
3091 last_key = next_key;
3092 state = start_s;
3093 break;
3094 default:
3095 assert(false, "impossible state");
3096 }
3097 if (state == start_s) {
3098 if (first_key < last_key - 5L && reg != noreg) {
3099 {
3100 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3101 // reg->encoding(),
3102 // start_insn, first_key, last_key);
3103 // for (int i = 0; i < inst->length(); i++) {
3104 // inst->at(i)->print();
3105 // tty->print("\n");
3106 // }
3107 // tty->print("\n");
3108 }
3109
3110 struct tableswitch *sw = &switches[tableswitch_count];
3111 sw->_insn_index = start_insn, sw->_first_key = first_key,
3112 sw->_last_key = last_key, sw->_reg = reg;
3113 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3114 {
3115 // Insert the new table of branches
3116 int offset = last_insn;
3117 for (int n = first_key; n < last_key; n++) {
3118 inst->insert_before
3119 (last_insn + 1,
3120 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3121 inst->at(offset)->as_OpBranch()->label()));
3122 offset -= 2, i++;
3123 }
3124 }
3125 // Delete all the old compare-and-branch instructions
3126 for (int n = first_key; n < last_key; n++) {
3127 inst->remove_at(start_insn);
3128 inst->remove_at(start_insn);
3129 }
3130 // Insert the tableswitch instruction
3131 inst->insert_before(start_insn,
3132 new LIR_Op2(lir_cmp, lir_cond_always,
3133 LIR_OprFact::intConst(tableswitch_count),
3134 reg_opr));
3135 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3136 tableswitch_count++;
3137 }
3138 reg = noreg;
3139 last_key = -2147483648;
3140 }
3141 next_state:
3142 ;
3143 }
3144 #endif
3145 }
3146
3147 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3148 Address addr = as_Address(src->as_address_ptr());
3149 BasicType type = src->type();
3150 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3151
3152 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3153 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3154
3155 switch(type) {
3156 case T_INT:
3157 xchg = &MacroAssembler::atomic_xchgalw;
3158 add = &MacroAssembler::atomic_addalw;
3159 break;
3160 case T_LONG:
3161 xchg = &MacroAssembler::atomic_xchgal;
3162 add = &MacroAssembler::atomic_addal;
3163 break;
3164 case T_OBJECT:
3165 case T_ARRAY:
3166 if (UseCompressedOops) {
3167 xchg = &MacroAssembler::atomic_xchgalw;
3168 add = &MacroAssembler::atomic_addalw;
3169 } else {
3170 xchg = &MacroAssembler::atomic_xchgal;
3171 add = &MacroAssembler::atomic_addal;
3172 }
3173 break;
3174 default:
3175 ShouldNotReachHere();
3176 xchg = &MacroAssembler::atomic_xchgal;
3177 add = &MacroAssembler::atomic_addal; // unreachable
3178 }
3179
3180 switch (code) {
3181 case lir_xadd:
3182 {
3183 RegisterOrConstant inc;
3184 Register tmp = as_reg(tmp_op);
3185 Register dst = as_reg(dest);
3186 if (data->is_constant()) {
3187 inc = RegisterOrConstant(as_long(data));
3188 assert_different_registers(dst, addr.base(), tmp,
3189 rscratch1, rscratch2);
3190 } else {
3191 inc = RegisterOrConstant(as_reg(data));
3192 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3193 rscratch1, rscratch2);
3194 }
3195 __ lea(tmp, addr);
3196 (_masm->*add)(dst, inc, tmp);
3197 break;
3198 }
3199 case lir_xchg:
3200 {
3201 Register tmp = tmp_op->as_register();
3202 Register obj = as_reg(data);
3203 Register dst = as_reg(dest);
3204 if (is_oop && UseCompressedOops) {
3205 __ encode_heap_oop(rscratch2, obj);
3206 obj = rscratch2;
3207 }
3208 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3209 __ lea(tmp, addr);
3210 (_masm->*xchg)(dst, obj, tmp);
3211 if (is_oop && UseCompressedOops) {
3212 __ decode_heap_oop(dst);
3213 }
3214 }
3215 break;
3216 default:
3217 ShouldNotReachHere();
3218 }
3219 __ membar(__ AnyAny);
3220 }
3221
3222 #undef __