1 /*
2 * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "gc/shenandoah/brooksPointer.hpp"
37 #include "gc/shenandoah/shenandoahHeap.hpp"
38 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
39 #include "gc/shared/barrierSet.hpp"
40 #include "gc/shared/cardTableModRefBS.hpp"
41 #include "gc/shared/collectedHeap.hpp"
42 #include "nativeInst_aarch64.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "vmreg_aarch64.inline.hpp"
46
47
48
49 #ifndef PRODUCT
50 #define COMMENT(x) do { __ block_comment(x); } while (0)
51 #else
52 #define COMMENT(x)
53 #endif
54
55 NEEDS_CLEANUP // remove this definitions ?
56 const Register IC_Klass = rscratch2; // where the IC klass is cached
57 const Register SYNC_header = r0; // synchronization header
58 const Register SHIFT_count = r0; // where count for shift operations must be
59
60 #define __ _masm->
61
62
63 static void select_different_registers(Register preserve,
64 Register extra,
65 Register &tmp1,
66 Register &tmp2) {
67 if (tmp1 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp1 = extra;
70 } else if (tmp2 == preserve) {
71 assert_different_registers(tmp1, tmp2, extra);
72 tmp2 = extra;
73 }
74 assert_different_registers(preserve, tmp1, tmp2);
75 }
76
77
78
79 static void select_different_registers(Register preserve,
80 Register extra,
81 Register &tmp1,
82 Register &tmp2,
83 Register &tmp3) {
84 if (tmp1 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp1 = extra;
87 } else if (tmp2 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp2 = extra;
90 } else if (tmp3 == preserve) {
91 assert_different_registers(tmp1, tmp2, tmp3, extra);
92 tmp3 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2, tmp3);
95 }
96
97
98 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
99
100
101 LIR_Opr LIR_Assembler::receiverOpr() {
102 return FrameMap::receiver_opr;
103 }
104
105 LIR_Opr LIR_Assembler::osrBufferPointer() {
106 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
107 }
108
109 //--------------fpu register translations-----------------------
110
111
112 address LIR_Assembler::float_constant(float f) {
113 address const_addr = __ float_constant(f);
114 if (const_addr == NULL) {
115 bailout("const section overflow");
116 return __ code()->consts()->start();
117 } else {
118 return const_addr;
119 }
120 }
121
122
123 address LIR_Assembler::double_constant(double d) {
124 address const_addr = __ double_constant(d);
125 if (const_addr == NULL) {
126 bailout("const section overflow");
127 return __ code()->consts()->start();
128 } else {
129 return const_addr;
130 }
131 }
132
133 address LIR_Assembler::int_constant(jlong n) {
134 address const_addr = __ long_constant(n);
135 if (const_addr == NULL) {
136 bailout("const section overflow");
137 return __ code()->consts()->start();
138 } else {
139 return const_addr;
140 }
141 }
142
143 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
144
145 void LIR_Assembler::reset_FPU() { Unimplemented(); }
146
147 void LIR_Assembler::fpop() { Unimplemented(); }
148
149 void LIR_Assembler::fxch(int i) { Unimplemented(); }
150
151 void LIR_Assembler::fld(int i) { Unimplemented(); }
152
153 void LIR_Assembler::ffree(int i) { Unimplemented(); }
154
155 void LIR_Assembler::breakpoint() { Unimplemented(); }
156
157 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
158
159 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
160
161 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
162 //-------------------------------------------
163
164 static Register as_reg(LIR_Opr op) {
165 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
166 }
167
168 static jlong as_long(LIR_Opr data) {
169 jlong result;
170 switch (data->type()) {
171 case T_INT:
172 result = (data->as_jint());
173 break;
174 case T_LONG:
175 result = (data->as_jlong());
176 break;
177 default:
178 ShouldNotReachHere();
179 result = 0; // unreachable
180 }
181 return result;
182 }
183
184 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
185 Register base = addr->base()->as_pointer_register();
186 LIR_Opr opr = addr->index();
187 if (opr->is_cpu_register()) {
188 Register index;
189 if (opr->is_single_cpu())
190 index = opr->as_register();
191 else
192 index = opr->as_register_lo();
193 assert(addr->disp() == 0, "must be");
194 switch(opr->type()) {
195 case T_INT:
196 return Address(base, index, Address::sxtw(addr->scale()));
197 case T_LONG:
198 return Address(base, index, Address::lsl(addr->scale()));
199 default:
200 ShouldNotReachHere();
201 }
202 } else {
203 intptr_t addr_offset = intptr_t(addr->disp());
204 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
205 return Address(base, addr_offset, Address::lsl(addr->scale()));
206 else {
207 __ mov(tmp, addr_offset);
208 return Address(base, tmp, Address::lsl(addr->scale()));
209 }
210 }
211 return Address();
212 }
213
214 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
215 ShouldNotReachHere();
216 return Address();
217 }
218
219 Address LIR_Assembler::as_Address(LIR_Address* addr) {
220 return as_Address(addr, rscratch1);
221 }
222
223 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
224 return as_Address(addr, rscratch1); // Ouch
225 // FIXME: This needs to be much more clever. See x86.
226 }
227
228
229 void LIR_Assembler::osr_entry() {
230 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
231 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
232 ValueStack* entry_state = osr_entry->state();
233 int number_of_locks = entry_state->locks_size();
234
235 // we jump here if osr happens with the interpreter
236 // state set up to continue at the beginning of the
237 // loop that triggered osr - in particular, we have
238 // the following registers setup:
239 //
240 // r2: osr buffer
241 //
242
243 // build frame
244 ciMethod* m = compilation()->method();
245 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
246
247 // OSR buffer is
248 //
249 // locals[nlocals-1..0]
250 // monitors[0..number_of_locks]
251 //
252 // locals is a direct copy of the interpreter frame so in the osr buffer
253 // so first slot in the local array is the last local from the interpreter
254 // and last slot is local[0] (receiver) from the interpreter
255 //
256 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
257 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
258 // in the interpreter frame (the method lock if a sync method)
259
260 // Initialize monitors in the compiled activation.
261 // r2: pointer to osr buffer
262 //
263 // All other registers are dead at this point and the locals will be
264 // copied into place by code emitted in the IR.
265
266 Register OSR_buf = osrBufferPointer()->as_pointer_register();
267 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
268 int monitor_offset = BytesPerWord * method()->max_locals() +
269 (2 * BytesPerWord) * (number_of_locks - 1);
270 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
271 // the OSR buffer using 2 word entries: first the lock and then
272 // the oop.
273 for (int i = 0; i < number_of_locks; i++) {
274 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
275 #ifdef ASSERT
276 // verify the interpreter's monitor has a non-null object
277 {
278 Label L;
279 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
280 __ cbnz(rscratch1, L);
281 __ stop("locked object is NULL");
282 __ bind(L);
283 }
284 #endif
285 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
286 __ str(r19, frame_map()->address_for_monitor_lock(i));
287 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
288 __ str(r19, frame_map()->address_for_monitor_object(i));
289 }
290 }
291 }
292
293
294 // inline cache check; done before the frame is built.
295 int LIR_Assembler::check_icache() {
296 Register receiver = FrameMap::receiver_opr->as_register();
297 Register ic_klass = IC_Klass;
298 int start_offset = __ offset();
299 __ inline_cache_check(receiver, ic_klass);
300
301 // if icache check fails, then jump to runtime routine
302 // Note: RECEIVER must still contain the receiver!
303 Label dont;
304 __ br(Assembler::EQ, dont);
305 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
306
307 // We align the verified entry point unless the method body
308 // (including its inline cache check) will fit in a single 64-byte
309 // icache line.
310 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
311 // force alignment after the cache check.
312 __ align(CodeEntryAlignment);
313 }
314
315 __ bind(dont);
316 return start_offset;
317 }
318
319
320 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
321 if (o == NULL) {
322 __ mov(reg, zr);
323 } else {
324 __ movoop(reg, o, /*immediate*/true);
325 }
326 }
327
328 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
329 address target = NULL;
330 relocInfo::relocType reloc_type = relocInfo::none;
331
332 switch (patching_id(info)) {
333 case PatchingStub::access_field_id:
334 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
335 reloc_type = relocInfo::section_word_type;
336 break;
337 case PatchingStub::load_klass_id:
338 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
339 reloc_type = relocInfo::metadata_type;
340 break;
341 case PatchingStub::load_mirror_id:
342 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
343 reloc_type = relocInfo::oop_type;
344 break;
345 case PatchingStub::load_appendix_id:
346 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
347 reloc_type = relocInfo::oop_type;
348 break;
349 default: ShouldNotReachHere();
350 }
351
352 __ far_call(RuntimeAddress(target));
353 add_call_info_here(info);
354 }
355
356 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
357 deoptimize_trap(info);
358 }
359
360
361 // This specifies the rsp decrement needed to build the frame
362 int LIR_Assembler::initial_frame_size_in_bytes() const {
363 // if rounding, must let FrameMap know!
364
365 // The frame_map records size in slots (32bit word)
366
367 // subtract two words to account for return address and link
368 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
369 }
370
371
372 int LIR_Assembler::emit_exception_handler() {
373 // if the last instruction is a call (typically to do a throw which
374 // is coming at the end after block reordering) the return address
375 // must still point into the code area in order to avoid assertion
376 // failures when searching for the corresponding bci => add a nop
377 // (was bug 5/14/1999 - gri)
378 __ nop();
379
380 // generate code for exception handler
381 address handler_base = __ start_a_stub(exception_handler_size);
382 if (handler_base == NULL) {
383 // not enough space left for the handler
384 bailout("exception handler overflow");
385 return -1;
386 }
387
388 int offset = code_offset();
389
390 // the exception oop and pc are in r0, and r3
391 // no other registers need to be preserved, so invalidate them
392 __ invalidate_registers(false, true, true, false, true, true);
393
394 // check that there is really an exception
395 __ verify_not_null_oop(r0);
396
397 // search an exception handler (r0: exception oop, r3: throwing pc)
398 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here();
399 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
400 __ end_a_stub();
401
402 return offset;
403 }
404
405
406 // Emit the code to remove the frame from the stack in the exception
407 // unwind path.
408 int LIR_Assembler::emit_unwind_handler() {
409 #ifndef PRODUCT
410 if (CommentedAssembly) {
411 _masm->block_comment("Unwind handler");
412 }
413 #endif
414
415 int offset = code_offset();
416
417 // Fetch the exception from TLS and clear out exception related thread state
418 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
419 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
420 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
421
422 __ bind(_unwind_handler_entry);
423 __ verify_not_null_oop(r0);
424 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
425 __ mov(r19, r0); // Preserve the exception
426 }
427
428 // Preform needed unlocking
429 MonitorExitStub* stub = NULL;
430 if (method()->is_synchronized()) {
431 monitor_address(0, FrameMap::r0_opr);
432 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
433 __ unlock_object(r5, r4, r0, *stub->entry());
434 __ bind(*stub->continuation());
435 }
436
437 if (compilation()->env()->dtrace_method_probes()) {
438 __ call_Unimplemented();
439 #if 0
440 __ movptr(Address(rsp, 0), rax);
441 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
442 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
443 #endif
444 }
445
446 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
447 __ mov(r0, r19); // Restore the exception
448 }
449
450 // remove the activation and dispatch to the unwind handler
451 __ block_comment("remove_frame and dispatch to the unwind handler");
452 __ remove_frame(initial_frame_size_in_bytes());
453 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
454
455 // Emit the slow path assembly
456 if (stub != NULL) {
457 stub->emit_code(this);
458 }
459
460 return offset;
461 }
462
463
464 int LIR_Assembler::emit_deopt_handler() {
465 // if the last instruction is a call (typically to do a throw which
466 // is coming at the end after block reordering) the return address
467 // must still point into the code area in order to avoid assertion
468 // failures when searching for the corresponding bci => add a nop
469 // (was bug 5/14/1999 - gri)
470 __ nop();
471
472 // generate code for exception handler
473 address handler_base = __ start_a_stub(deopt_handler_size);
474 if (handler_base == NULL) {
475 // not enough space left for the handler
476 bailout("deopt handler overflow");
477 return -1;
478 }
479
480 int offset = code_offset();
481
482 __ adr(lr, pc());
483 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
484 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
485 __ end_a_stub();
486
487 return offset;
488 }
489
490 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
491 _masm->code_section()->relocate(adr, relocInfo::poll_type);
492 int pc_offset = code_offset();
493 flush_debug_info(pc_offset);
494 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
495 if (info->exception_handlers() != NULL) {
496 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
497 }
498 }
499
500 // Rather than take a segfault when the polling page is protected,
501 // explicitly check for a safepoint in progress and if there is one,
502 // fake a call to the handler as if a segfault had been caught.
503 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
504 __ mov(rscratch1, SafepointSynchronize::address_of_state());
505 __ ldrb(rscratch1, Address(rscratch1));
506 Label nope, poll;
507 __ cbz(rscratch1, nope);
508 __ block_comment("safepoint");
509 __ enter();
510 __ push(0x3, sp); // r0 & r1
511 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
512 __ adr(r0, poll);
513 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
514 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
515 __ blrt(rscratch1, 1, 0, 1);
516 __ maybe_isb();
517 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
518 __ mov(rscratch1, r0);
519 __ pop(0x3, sp); // r0 & r1
520 __ leave();
521 __ br(rscratch1);
522 address polling_page(os::get_polling_page());
523 assert(os::is_poll_address(polling_page), "should be");
524 unsigned long off;
525 __ adrp(rscratch1, Address(polling_page, rtype), off);
526 __ bind(poll);
527 if (info)
528 add_debug_info_for_branch(info); // This isn't just debug info:
529 // it's the oop map
530 else
531 __ code_section()->relocate(pc(), rtype);
532 __ ldrw(zr, Address(rscratch1, off));
533 __ bind(nope);
534 }
535
536 void LIR_Assembler::return_op(LIR_Opr result) {
537 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
538 // Pop the stack before the safepoint code
539 __ remove_frame(initial_frame_size_in_bytes());
540 address polling_page(os::get_polling_page());
541 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
542 __ ret(lr);
543 }
544
545 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
546 address polling_page(os::get_polling_page());
547 guarantee(info != NULL, "Shouldn't be NULL");
548 assert(os::is_poll_address(polling_page), "should be");
549 unsigned long off;
550 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
551 assert(off == 0, "must be");
552 add_debug_info_for_branch(info); // This isn't just debug info:
553 // it's the oop map
554 __ read_polling_page(rscratch1, relocInfo::poll_type);
555 return __ offset();
556 }
557
558
559 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
560 if (from_reg == r31_sp)
561 from_reg = sp;
562 if (to_reg == r31_sp)
563 to_reg = sp;
564 __ mov(to_reg, from_reg);
565 }
566
567 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
568
569
570 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
571 assert(src->is_constant(), "should not call otherwise");
572 assert(dest->is_register(), "should not call otherwise");
573 LIR_Const* c = src->as_constant_ptr();
574
575 switch (c->type()) {
576 case T_INT: {
577 assert(patch_code == lir_patch_none, "no patching handled here");
578 __ movw(dest->as_register(), c->as_jint());
579 break;
580 }
581
582 case T_ADDRESS: {
583 assert(patch_code == lir_patch_none, "no patching handled here");
584 __ mov(dest->as_register(), c->as_jint());
585 break;
586 }
587
588 case T_LONG: {
589 assert(patch_code == lir_patch_none, "no patching handled here");
590 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
591 break;
592 }
593
594 case T_OBJECT: {
595 if (patch_code == lir_patch_none) {
596 jobject2reg(c->as_jobject(), dest->as_register());
597 } else {
598 jobject2reg_with_patching(dest->as_register(), info);
599 }
600 break;
601 }
602
603 case T_METADATA: {
604 if (patch_code != lir_patch_none) {
605 klass2reg_with_patching(dest->as_register(), info);
606 } else {
607 __ mov_metadata(dest->as_register(), c->as_metadata());
608 }
609 break;
610 }
611
612 case T_FLOAT: {
613 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
614 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
615 } else {
616 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
617 __ ldrs(dest->as_float_reg(), Address(rscratch1));
618 }
619 break;
620 }
621
622 case T_DOUBLE: {
623 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
624 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
625 } else {
626 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
627 __ ldrd(dest->as_double_reg(), Address(rscratch1));
628 }
629 break;
630 }
631
632 default:
633 ShouldNotReachHere();
634 }
635 }
636
637 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
638 LIR_Const* c = src->as_constant_ptr();
639 switch (c->type()) {
640 case T_OBJECT:
641 {
642 if (! c->as_jobject())
643 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
644 else {
645 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
646 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
647 }
648 }
649 break;
650 case T_ADDRESS:
651 {
652 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
653 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
654 }
655 case T_INT:
656 case T_FLOAT:
657 {
658 Register reg = zr;
659 if (c->as_jint_bits() == 0)
660 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
661 else {
662 __ movw(rscratch1, c->as_jint_bits());
663 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
664 }
665 }
666 break;
667 case T_LONG:
668 case T_DOUBLE:
669 {
670 Register reg = zr;
671 if (c->as_jlong_bits() == 0)
672 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
673 lo_word_offset_in_bytes));
674 else {
675 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
676 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
677 lo_word_offset_in_bytes));
678 }
679 }
680 break;
681 default:
682 ShouldNotReachHere();
683 }
684 }
685
686 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
687 assert(src->is_constant(), "should not call otherwise");
688 LIR_Const* c = src->as_constant_ptr();
689 LIR_Address* to_addr = dest->as_address_ptr();
690
691 void (Assembler::* insn)(Register Rt, const Address &adr);
692
693 switch (type) {
694 case T_ADDRESS:
695 assert(c->as_jint() == 0, "should be");
696 insn = &Assembler::str;
697 break;
698 case T_LONG:
699 assert(c->as_jlong() == 0, "should be");
700 insn = &Assembler::str;
701 break;
702 case T_INT:
703 assert(c->as_jint() == 0, "should be");
704 insn = &Assembler::strw;
705 break;
706 case T_OBJECT:
707 case T_ARRAY:
708 assert(c->as_jobject() == 0, "should be");
709 if (UseCompressedOops && !wide) {
710 insn = &Assembler::strw;
711 } else {
712 insn = &Assembler::str;
713 }
714 break;
715 case T_CHAR:
716 case T_SHORT:
717 assert(c->as_jint() == 0, "should be");
718 insn = &Assembler::strh;
719 break;
720 case T_BOOLEAN:
721 case T_BYTE:
722 assert(c->as_jint() == 0, "should be");
723 insn = &Assembler::strb;
724 break;
725 default:
726 ShouldNotReachHere();
727 insn = &Assembler::str; // unreachable
728 }
729
730 if (info) add_debug_info_for_null_check_here(info);
731 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
732 }
733
734 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
735 assert(src->is_register(), "should not call otherwise");
736 assert(dest->is_register(), "should not call otherwise");
737
738 // move between cpu-registers
739 if (dest->is_single_cpu()) {
740 if (src->type() == T_LONG) {
741 // Can do LONG -> OBJECT
742 move_regs(src->as_register_lo(), dest->as_register());
743 return;
744 }
745 assert(src->is_single_cpu(), "must match");
746 if (src->type() == T_OBJECT) {
747 __ verify_oop(src->as_register());
748 }
749 move_regs(src->as_register(), dest->as_register());
750
751 } else if (dest->is_double_cpu()) {
752 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
753 // Surprising to me but we can see move of a long to t_object
754 __ verify_oop(src->as_register());
755 move_regs(src->as_register(), dest->as_register_lo());
756 return;
757 }
758 assert(src->is_double_cpu(), "must match");
759 Register f_lo = src->as_register_lo();
760 Register f_hi = src->as_register_hi();
761 Register t_lo = dest->as_register_lo();
762 Register t_hi = dest->as_register_hi();
763 assert(f_hi == f_lo, "must be same");
764 assert(t_hi == t_lo, "must be same");
765 move_regs(f_lo, t_lo);
766
767 } else if (dest->is_single_fpu()) {
768 __ fmovs(dest->as_float_reg(), src->as_float_reg());
769
770 } else if (dest->is_double_fpu()) {
771 __ fmovd(dest->as_double_reg(), src->as_double_reg());
772
773 } else {
774 ShouldNotReachHere();
775 }
776 }
777
778 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
779 if (src->is_single_cpu()) {
780 if (type == T_ARRAY || type == T_OBJECT) {
781 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
782 __ verify_oop(src->as_register());
783 } else if (type == T_METADATA || type == T_DOUBLE) {
784 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
785 } else {
786 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
787 }
788
789 } else if (src->is_double_cpu()) {
790 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
791 __ str(src->as_register_lo(), dest_addr_LO);
792
793 } else if (src->is_single_fpu()) {
794 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
795 __ strs(src->as_float_reg(), dest_addr);
796
797 } else if (src->is_double_fpu()) {
798 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
799 __ strd(src->as_double_reg(), dest_addr);
800
801 } else {
802 ShouldNotReachHere();
803 }
804
805 }
806
807
808 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
809 LIR_Address* to_addr = dest->as_address_ptr();
810 PatchingStub* patch = NULL;
811 Register compressed_src = rscratch1;
812
813 if (patch_code != lir_patch_none) {
814 deoptimize_trap(info);
815 return;
816 }
817
818 if (type == T_ARRAY || type == T_OBJECT) {
819 __ verify_oop(src->as_register());
820
821 __ shenandoah_store_check(as_Address(to_addr), src->as_register());
822
823 if (UseCompressedOops && !wide) {
824 __ encode_heap_oop(compressed_src, src->as_register());
825 } else {
826 compressed_src = src->as_register();
827 }
828 } else {
829 __ shenandoah_store_addr_check(to_addr->base()->as_pointer_register());
830 }
831
832 int null_check_here = code_offset();
833 switch (type) {
834 case T_FLOAT: {
835 __ strs(src->as_float_reg(), as_Address(to_addr));
836 break;
837 }
838
839 case T_DOUBLE: {
840 __ strd(src->as_double_reg(), as_Address(to_addr));
841 break;
842 }
843
844 case T_ARRAY: // fall through
845 case T_OBJECT: // fall through
846 if (UseCompressedOops && !wide) {
847 __ strw(compressed_src, as_Address(to_addr, rscratch2));
848 } else {
849 __ str(compressed_src, as_Address(to_addr));
850 }
851 break;
852 case T_METADATA:
853 // We get here to store a method pointer to the stack to pass to
854 // a dtrace runtime call. This can't work on 64 bit with
855 // compressed klass ptrs: T_METADATA can be a compressed klass
856 // ptr or a 64 bit method pointer.
857 ShouldNotReachHere();
858 __ str(src->as_register(), as_Address(to_addr));
859 break;
860 case T_ADDRESS:
861 __ str(src->as_register(), as_Address(to_addr));
862 break;
863 case T_INT:
864 __ strw(src->as_register(), as_Address(to_addr));
865 break;
866
867 case T_LONG: {
868 __ str(src->as_register_lo(), as_Address_lo(to_addr));
869 break;
870 }
871
872 case T_BYTE: // fall through
873 case T_BOOLEAN: {
874 __ strb(src->as_register(), as_Address(to_addr));
875 break;
876 }
877
878 case T_CHAR: // fall through
879 case T_SHORT:
880 __ strh(src->as_register(), as_Address(to_addr));
881 break;
882
883 default:
884 ShouldNotReachHere();
885 }
886 if (info != NULL) {
887 add_debug_info_for_null_check(null_check_here, info);
888 }
889 }
890
891
892 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
893 assert(src->is_stack(), "should not call otherwise");
894 assert(dest->is_register(), "should not call otherwise");
895
896 if (dest->is_single_cpu()) {
897 if (type == T_ARRAY || type == T_OBJECT) {
898 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
899 __ verify_oop(dest->as_register());
900 } else if (type == T_METADATA) {
901 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
902 } else {
903 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
904 }
905
906 } else if (dest->is_double_cpu()) {
907 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
908 __ ldr(dest->as_register_lo(), src_addr_LO);
909
910 } else if (dest->is_single_fpu()) {
911 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
912 __ ldrs(dest->as_float_reg(), src_addr);
913
914 } else if (dest->is_double_fpu()) {
915 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
916 __ ldrd(dest->as_double_reg(), src_addr);
917
918 } else {
919 ShouldNotReachHere();
920 }
921 }
922
923
924 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
925 address target = NULL;
926 relocInfo::relocType reloc_type = relocInfo::none;
927
928 switch (patching_id(info)) {
929 case PatchingStub::access_field_id:
930 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
931 reloc_type = relocInfo::section_word_type;
932 break;
933 case PatchingStub::load_klass_id:
934 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
935 reloc_type = relocInfo::metadata_type;
936 break;
937 case PatchingStub::load_mirror_id:
938 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
939 reloc_type = relocInfo::oop_type;
940 break;
941 case PatchingStub::load_appendix_id:
942 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
943 reloc_type = relocInfo::oop_type;
944 break;
945 default: ShouldNotReachHere();
946 }
947
948 __ far_call(RuntimeAddress(target));
949 add_call_info_here(info);
950 }
951
952 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
953
954 LIR_Opr temp;
955 if (type == T_LONG || type == T_DOUBLE)
956 temp = FrameMap::rscratch1_long_opr;
957 else
958 temp = FrameMap::rscratch1_opr;
959
960 stack2reg(src, temp, src->type());
961 reg2stack(temp, dest, dest->type(), false);
962 }
963
964
965 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
966 LIR_Address* addr = src->as_address_ptr();
967 LIR_Address* from_addr = src->as_address_ptr();
968
969 if (addr->base()->type() == T_OBJECT) {
970 __ verify_oop(addr->base()->as_pointer_register());
971 }
972
973 if (patch_code != lir_patch_none) {
974 deoptimize_trap(info);
975 return;
976 }
977
978 if (info != NULL) {
979 add_debug_info_for_null_check_here(info);
980 }
981 int null_check_here = code_offset();
982 switch (type) {
983 case T_FLOAT: {
984 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
985 break;
986 }
987
988 case T_DOUBLE: {
989 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
990 break;
991 }
992
993 case T_ARRAY: // fall through
994 case T_OBJECT: // fall through
995 if (UseCompressedOops && !wide) {
996 __ ldrw(dest->as_register(), as_Address(from_addr));
997 } else {
998 __ ldr(dest->as_register(), as_Address(from_addr));
999 }
1000 break;
1001 case T_METADATA:
1002 // We get here to store a method pointer to the stack to pass to
1003 // a dtrace runtime call. This can't work on 64 bit with
1004 // compressed klass ptrs: T_METADATA can be a compressed klass
1005 // ptr or a 64 bit method pointer.
1006 ShouldNotReachHere();
1007 __ ldr(dest->as_register(), as_Address(from_addr));
1008 break;
1009 case T_ADDRESS:
1010 // FIXME: OMG this is a horrible kludge. Any offset from an
1011 // address that matches klass_offset_in_bytes() will be loaded
1012 // as a word, not a long.
1013 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1014 __ ldrw(dest->as_register(), as_Address(from_addr));
1015 } else {
1016 __ ldr(dest->as_register(), as_Address(from_addr));
1017 }
1018 break;
1019 case T_INT:
1020 __ ldrw(dest->as_register(), as_Address(from_addr));
1021 break;
1022
1023 case T_LONG: {
1024 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1025 break;
1026 }
1027
1028 case T_BYTE:
1029 __ ldrsb(dest->as_register(), as_Address(from_addr));
1030 break;
1031 case T_BOOLEAN: {
1032 __ ldrb(dest->as_register(), as_Address(from_addr));
1033 break;
1034 }
1035
1036 case T_CHAR:
1037 __ ldrh(dest->as_register(), as_Address(from_addr));
1038 break;
1039 case T_SHORT:
1040 __ ldrsh(dest->as_register(), as_Address(from_addr));
1041 break;
1042
1043 default:
1044 ShouldNotReachHere();
1045 }
1046
1047 if (type == T_ARRAY || type == T_OBJECT) {
1048 if (UseCompressedOops && !wide) {
1049 __ decode_heap_oop(dest->as_register());
1050 }
1051 __ verify_oop(dest->as_register());
1052 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1053 if (UseCompressedClassPointers) {
1054 __ decode_klass_not_null(dest->as_register());
1055 }
1056 }
1057 }
1058
1059
1060 int LIR_Assembler::array_element_size(BasicType type) const {
1061 int elem_size = type2aelembytes(type);
1062 return exact_log2(elem_size);
1063 }
1064
1065 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1066 Register Rdividend = op->in_opr1()->as_register();
1067 Register Rdivisor = op->in_opr2()->as_register();
1068 Register Rscratch = op->in_opr3()->as_register();
1069 Register Rresult = op->result_opr()->as_register();
1070 int divisor = -1;
1071
1072 /*
1073 TODO: For some reason, using the Rscratch that gets passed in is
1074 not possible because the register allocator does not see the tmp reg
1075 as used, and assignes it the same register as Rdividend. We use rscratch1
1076 instead.
1077
1078 assert(Rdividend != Rscratch, "");
1079 assert(Rdivisor != Rscratch, "");
1080 */
1081
1082 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1083 // convert division by a power of two into some shifts and logical operations
1084 }
1085
1086 if (op->code() == lir_irem) {
1087 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1088 } else if (op->code() == lir_idiv) {
1089 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1090 } else
1091 ShouldNotReachHere();
1092 }
1093
1094 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1095 #ifdef ASSERT
1096 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1097 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1098 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1099 #endif
1100
1101 if (op->cond() == lir_cond_always) {
1102 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1103 __ b(*(op->label()));
1104 } else {
1105 Assembler::Condition acond;
1106 if (op->code() == lir_cond_float_branch) {
1107 bool is_unordered = (op->ublock() == op->block());
1108 // Assembler::EQ does not permit unordered branches, so we add
1109 // another branch here. Likewise, Assembler::NE does not permit
1110 // ordered branches.
1111 if (is_unordered && op->cond() == lir_cond_equal
1112 || !is_unordered && op->cond() == lir_cond_notEqual)
1113 __ br(Assembler::VS, *(op->ublock()->label()));
1114 switch(op->cond()) {
1115 case lir_cond_equal: acond = Assembler::EQ; break;
1116 case lir_cond_notEqual: acond = Assembler::NE; break;
1117 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1118 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1119 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1120 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1121 default: ShouldNotReachHere();
1122 acond = Assembler::EQ; // unreachable
1123 }
1124 } else {
1125 switch (op->cond()) {
1126 case lir_cond_equal: acond = Assembler::EQ; break;
1127 case lir_cond_notEqual: acond = Assembler::NE; break;
1128 case lir_cond_less: acond = Assembler::LT; break;
1129 case lir_cond_lessEqual: acond = Assembler::LE; break;
1130 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1131 case lir_cond_greater: acond = Assembler::GT; break;
1132 case lir_cond_belowEqual: acond = Assembler::LS; break;
1133 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1134 default: ShouldNotReachHere();
1135 acond = Assembler::EQ; // unreachable
1136 }
1137 }
1138 __ br(acond,*(op->label()));
1139 }
1140 }
1141
1142
1143 void LIR_Assembler::emit_opShenandoahWriteBarrier(LIR_OpShenandoahWriteBarrier* op) {
1144 Label done;
1145 Register obj = op->in_opr()->as_register();
1146 Register res = op->result_opr()->as_register();
1147 Register tmp1 = op->tmp1_opr()->as_register();
1148 Register tmp2 = op->tmp2_opr()->as_register();
1149 assert_different_registers(res, tmp1, tmp2);
1150
1151 __ block_comment("Shenandoah write barrier {");
1152
1153 // Check for null.
1154 if (op->need_null_check()) {
1155 if (res != obj) __ mov(res, obj);
1156 __ cbz(res, done);
1157 }
1158
1159 // Check for evacuation-in-progress
1160 Address evacuation_in_progress
1161 = Address(rthread, in_bytes(JavaThread::evacuation_in_progress_offset()));
1162 __ ldrb(rscratch2, evacuation_in_progress);
1163 __ membar(Assembler::LoadLoad);
1164
1165 // The read-barrier.
1166 __ ldr(res, Address(obj, BrooksPointer::byte_offset()));
1167
1168 __ cbzw(rscratch2, done);
1169
1170 // Check for object in collection set.
1171 __ lsr(tmp1, res, ShenandoahHeapRegion::RegionSizeShift);
1172 __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
1173 __ ldrb(tmp2, Address(tmp2, tmp1));
1174 __ tbz(tmp2, 0, done);
1175
1176 RegSet to_save = RegSet::of(r0, r1) - res;
1177 __ push(to_save, sp);
1178 __ mov(r1, res);
1179 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::shenandoah_write_barrier_slow_id)));
1180 __ mov(res, r0);
1181 __ pop(to_save, sp);
1182
1183 __ block_comment("} Shenandoah write barrier");
1184 __ bind(done);
1185 }
1186
1187
1188 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1189 LIR_Opr src = op->in_opr();
1190 LIR_Opr dest = op->result_opr();
1191
1192 switch (op->bytecode()) {
1193 case Bytecodes::_i2f:
1194 {
1195 __ scvtfws(dest->as_float_reg(), src->as_register());
1196 break;
1197 }
1198 case Bytecodes::_i2d:
1199 {
1200 __ scvtfwd(dest->as_double_reg(), src->as_register());
1201 break;
1202 }
1203 case Bytecodes::_l2d:
1204 {
1205 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1206 break;
1207 }
1208 case Bytecodes::_l2f:
1209 {
1210 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1211 break;
1212 }
1213 case Bytecodes::_f2d:
1214 {
1215 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1216 break;
1217 }
1218 case Bytecodes::_d2f:
1219 {
1220 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1221 break;
1222 }
1223 case Bytecodes::_i2c:
1224 {
1225 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1226 break;
1227 }
1228 case Bytecodes::_i2l:
1229 {
1230 __ sxtw(dest->as_register_lo(), src->as_register());
1231 break;
1232 }
1233 case Bytecodes::_i2s:
1234 {
1235 __ sxth(dest->as_register(), src->as_register());
1236 break;
1237 }
1238 case Bytecodes::_i2b:
1239 {
1240 __ sxtb(dest->as_register(), src->as_register());
1241 break;
1242 }
1243 case Bytecodes::_l2i:
1244 {
1245 _masm->block_comment("FIXME: This could be a no-op");
1246 __ uxtw(dest->as_register(), src->as_register_lo());
1247 break;
1248 }
1249 case Bytecodes::_d2l:
1250 {
1251 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1252 break;
1253 }
1254 case Bytecodes::_f2i:
1255 {
1256 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1257 break;
1258 }
1259 case Bytecodes::_f2l:
1260 {
1261 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1262 break;
1263 }
1264 case Bytecodes::_d2i:
1265 {
1266 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1267 break;
1268 }
1269 default: ShouldNotReachHere();
1270 }
1271 }
1272
1273 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1274 if (op->init_check()) {
1275 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1276 InstanceKlass::init_state_offset()));
1277 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1278 add_debug_info_for_null_check_here(op->stub()->info());
1279 __ br(Assembler::NE, *op->stub()->entry());
1280 }
1281 __ allocate_object(op->obj()->as_register(),
1282 op->tmp1()->as_register(),
1283 op->tmp2()->as_register(),
1284 op->header_size(),
1285 op->object_size(),
1286 op->klass()->as_register(),
1287 *op->stub()->entry());
1288 __ bind(*op->stub()->continuation());
1289 }
1290
1291 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1292 Register len = op->len()->as_register();
1293 __ uxtw(len, len);
1294
1295 if (UseSlowPath ||
1296 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1297 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1298 __ b(*op->stub()->entry());
1299 } else {
1300 Register tmp1 = op->tmp1()->as_register();
1301 Register tmp2 = op->tmp2()->as_register();
1302 Register tmp3 = op->tmp3()->as_register();
1303 if (len == tmp1) {
1304 tmp1 = tmp3;
1305 } else if (len == tmp2) {
1306 tmp2 = tmp3;
1307 } else if (len == tmp3) {
1308 // everything is ok
1309 } else {
1310 __ mov(tmp3, len);
1311 }
1312 __ allocate_array(op->obj()->as_register(),
1313 len,
1314 tmp1,
1315 tmp2,
1316 arrayOopDesc::header_size(op->type()),
1317 array_element_size(op->type()),
1318 op->klass()->as_register(),
1319 *op->stub()->entry());
1320 }
1321 __ bind(*op->stub()->continuation());
1322 }
1323
1324 void LIR_Assembler::type_profile_helper(Register mdo,
1325 ciMethodData *md, ciProfileData *data,
1326 Register recv, Label* update_done) {
1327 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1328 Label next_test;
1329 // See if the receiver is receiver[n].
1330 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1331 __ ldr(rscratch1, Address(rscratch2));
1332 __ cmp(recv, rscratch1);
1333 __ br(Assembler::NE, next_test);
1334 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1335 __ addptr(data_addr, DataLayout::counter_increment);
1336 __ b(*update_done);
1337 __ bind(next_test);
1338 }
1339
1340 // Didn't find receiver; find next empty slot and fill it in
1341 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1342 Label next_test;
1343 __ lea(rscratch2,
1344 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1345 Address recv_addr(rscratch2);
1346 __ ldr(rscratch1, recv_addr);
1347 __ cbnz(rscratch1, next_test);
1348 __ str(recv, recv_addr);
1349 __ mov(rscratch1, DataLayout::counter_increment);
1350 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1351 __ str(rscratch1, Address(rscratch2));
1352 __ b(*update_done);
1353 __ bind(next_test);
1354 }
1355 }
1356
1357 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1358 // we always need a stub for the failure case.
1359 CodeStub* stub = op->stub();
1360 Register obj = op->object()->as_register();
1361 Register k_RInfo = op->tmp1()->as_register();
1362 Register klass_RInfo = op->tmp2()->as_register();
1363 Register dst = op->result_opr()->as_register();
1364 ciKlass* k = op->klass();
1365 Register Rtmp1 = noreg;
1366
1367 // check if it needs to be profiled
1368 ciMethodData* md;
1369 ciProfileData* data;
1370
1371 const bool should_profile = op->should_profile();
1372
1373 if (should_profile) {
1374 ciMethod* method = op->profiled_method();
1375 assert(method != NULL, "Should have method");
1376 int bci = op->profiled_bci();
1377 md = method->method_data_or_null();
1378 assert(md != NULL, "Sanity");
1379 data = md->bci_to_data(bci);
1380 assert(data != NULL, "need data for type check");
1381 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1382 }
1383 Label profile_cast_success, profile_cast_failure;
1384 Label *success_target = should_profile ? &profile_cast_success : success;
1385 Label *failure_target = should_profile ? &profile_cast_failure : failure;
1386
1387 if (obj == k_RInfo) {
1388 k_RInfo = dst;
1389 } else if (obj == klass_RInfo) {
1390 klass_RInfo = dst;
1391 }
1392 if (k->is_loaded() && !UseCompressedClassPointers) {
1393 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1394 } else {
1395 Rtmp1 = op->tmp3()->as_register();
1396 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1397 }
1398
1399 assert_different_registers(obj, k_RInfo, klass_RInfo);
1400
1401 if (should_profile) {
1402 Label not_null;
1403 __ cbnz(obj, not_null);
1404 // Object is null; update MDO and exit
1405 Register mdo = klass_RInfo;
1406 __ mov_metadata(mdo, md->constant_encoding());
1407 Address data_addr
1408 = __ form_address(rscratch2, mdo,
1409 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1410 LogBytesPerWord);
1411 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1412 __ ldr(rscratch1, data_addr);
1413 __ orr(rscratch1, rscratch1, header_bits);
1414 __ str(rscratch1, data_addr);
1415 __ b(*obj_is_null);
1416 __ bind(not_null);
1417 } else {
1418 __ cbz(obj, *obj_is_null);
1419 }
1420
1421 if (!k->is_loaded()) {
1422 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1423 } else {
1424 __ mov_metadata(k_RInfo, k->constant_encoding());
1425 }
1426 __ verify_oop(obj);
1427
1428 if (op->fast_check()) {
1429 // get object class
1430 // not a safepoint as obj null check happens earlier
1431 __ load_klass(rscratch1, obj);
1432 __ cmp( rscratch1, k_RInfo);
1433
1434 __ br(Assembler::NE, *failure_target);
1435 // successful cast, fall through to profile or jump
1436 } else {
1437 // get object class
1438 // not a safepoint as obj null check happens earlier
1439 __ load_klass(klass_RInfo, obj);
1440 if (k->is_loaded()) {
1441 // See if we get an immediate positive hit
1442 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1443 __ cmp(k_RInfo, rscratch1);
1444 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1445 __ br(Assembler::NE, *failure_target);
1446 // successful cast, fall through to profile or jump
1447 } else {
1448 // See if we get an immediate positive hit
1449 __ br(Assembler::EQ, *success_target);
1450 // check for self
1451 __ cmp(klass_RInfo, k_RInfo);
1452 __ br(Assembler::EQ, *success_target);
1453
1454 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1455 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1456 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1457 // result is a boolean
1458 __ cbzw(klass_RInfo, *failure_target);
1459 // successful cast, fall through to profile or jump
1460 }
1461 } else {
1462 // perform the fast part of the checking logic
1463 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1464 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1465 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1466 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1467 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1468 // result is a boolean
1469 __ cbz(k_RInfo, *failure_target);
1470 // successful cast, fall through to profile or jump
1471 }
1472 }
1473 if (should_profile) {
1474 Register mdo = klass_RInfo, recv = k_RInfo;
1475 __ bind(profile_cast_success);
1476 __ mov_metadata(mdo, md->constant_encoding());
1477 __ load_klass(recv, obj);
1478 Label update_done;
1479 type_profile_helper(mdo, md, data, recv, success);
1480 __ b(*success);
1481
1482 __ bind(profile_cast_failure);
1483 __ mov_metadata(mdo, md->constant_encoding());
1484 Address counter_addr
1485 = __ form_address(rscratch2, mdo,
1486 md->byte_offset_of_slot(data, CounterData::count_offset()),
1487 LogBytesPerWord);
1488 __ ldr(rscratch1, counter_addr);
1489 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1490 __ str(rscratch1, counter_addr);
1491 __ b(*failure);
1492 }
1493 __ b(*success);
1494 }
1495
1496
1497 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1498 const bool should_profile = op->should_profile();
1499
1500 LIR_Code code = op->code();
1501 if (code == lir_store_check) {
1502 Register value = op->object()->as_register();
1503 Register array = op->array()->as_register();
1504 Register k_RInfo = op->tmp1()->as_register();
1505 Register klass_RInfo = op->tmp2()->as_register();
1506 Register Rtmp1 = op->tmp3()->as_register();
1507
1508 CodeStub* stub = op->stub();
1509
1510 // check if it needs to be profiled
1511 ciMethodData* md;
1512 ciProfileData* data;
1513
1514 if (should_profile) {
1515 ciMethod* method = op->profiled_method();
1516 assert(method != NULL, "Should have method");
1517 int bci = op->profiled_bci();
1518 md = method->method_data_or_null();
1519 assert(md != NULL, "Sanity");
1520 data = md->bci_to_data(bci);
1521 assert(data != NULL, "need data for type check");
1522 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1523 }
1524 Label profile_cast_success, profile_cast_failure, done;
1525 Label *success_target = should_profile ? &profile_cast_success : &done;
1526 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1527
1528 if (should_profile) {
1529 Label not_null;
1530 __ cbnz(value, not_null);
1531 // Object is null; update MDO and exit
1532 Register mdo = klass_RInfo;
1533 __ mov_metadata(mdo, md->constant_encoding());
1534 Address data_addr
1535 = __ form_address(rscratch2, mdo,
1536 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1537 LogBytesPerInt);
1538 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1539 __ ldrw(rscratch1, data_addr);
1540 __ orrw(rscratch1, rscratch1, header_bits);
1541 __ strw(rscratch1, data_addr);
1542 __ b(done);
1543 __ bind(not_null);
1544 } else {
1545 __ cbz(value, done);
1546 }
1547
1548 add_debug_info_for_null_check_here(op->info_for_exception());
1549 __ load_klass(k_RInfo, array);
1550 __ load_klass(klass_RInfo, value);
1551
1552 // get instance klass (it's already uncompressed)
1553 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1554 // perform the fast part of the checking logic
1555 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1556 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1557 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1558 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1559 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1560 // result is a boolean
1561 __ cbzw(k_RInfo, *failure_target);
1562 // fall through to the success case
1563
1564 if (should_profile) {
1565 Register mdo = klass_RInfo, recv = k_RInfo;
1566 __ bind(profile_cast_success);
1567 __ mov_metadata(mdo, md->constant_encoding());
1568 __ load_klass(recv, value);
1569 Label update_done;
1570 type_profile_helper(mdo, md, data, recv, &done);
1571 __ b(done);
1572
1573 __ bind(profile_cast_failure);
1574 __ mov_metadata(mdo, md->constant_encoding());
1575 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1576 __ lea(rscratch2, counter_addr);
1577 __ ldr(rscratch1, Address(rscratch2));
1578 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1579 __ str(rscratch1, Address(rscratch2));
1580 __ b(*stub->entry());
1581 }
1582
1583 __ bind(done);
1584 } else if (code == lir_checkcast) {
1585 Register obj = op->object()->as_register();
1586 Register dst = op->result_opr()->as_register();
1587 Label success;
1588 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1589 __ bind(success);
1590 if (dst != obj) {
1591 __ mov(dst, obj);
1592 }
1593 } else if (code == lir_instanceof) {
1594 Register obj = op->object()->as_register();
1595 Register dst = op->result_opr()->as_register();
1596 Label success, failure, done;
1597 emit_typecheck_helper(op, &success, &failure, &failure);
1598 __ bind(failure);
1599 __ mov(dst, zr);
1600 __ b(done);
1601 __ bind(success);
1602 __ mov(dst, 1);
1603 __ bind(done);
1604 } else {
1605 ShouldNotReachHere();
1606 }
1607 }
1608
1609 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1610 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1611 __ cset(rscratch1, Assembler::NE);
1612 __ membar(__ AnyAny);
1613 }
1614
1615 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1616 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1617 __ cset(rscratch1, Assembler::NE);
1618 __ membar(__ AnyAny);
1619 }
1620
1621
1622 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1623 Register addr = as_reg(op->addr());
1624 Register newval = as_reg(op->new_value());
1625 Register cmpval = as_reg(op->cmp_value());
1626 Register res = op->result_opr()->as_register();
1627
1628 if (op->code() == lir_cas_obj) {
1629 assert(op->tmp1()->is_valid(), "must be");
1630 Register t1 = op->tmp1()->as_register();
1631 if (UseCompressedOops) {
1632 if (UseShenandoahGC) {
1633 __ encode_heap_oop(t1, cmpval);
1634 cmpval = t1;
1635 assert(op->tmp2()->is_valid(), "must be");
1636 Register t2 = op->tmp2()->as_register();
1637 __ encode_heap_oop(t2, newval);
1638 newval = t2;
1639 __ cmpxchg_oop_shenandoah(res, addr, cmpval, newval, true, true, true);
1640 } else {
1641 __ encode_heap_oop(t1, cmpval);
1642 cmpval = t1;
1643 __ encode_heap_oop(rscratch2, newval);
1644 newval = rscratch2;
1645 casw(addr, newval, cmpval);
1646 __ eorw (res, r8, 1);
1647 }
1648 } else {
1649 if (UseShenandoahGC) {
1650 __ cmpxchg_oop_shenandoah(res, addr, cmpval, newval, false, true, true);
1651 } else {
1652 casl(addr, newval, cmpval);
1653 __ eorw (res, r8, 1);
1654 }
1655 }
1656 } else if (op->code() == lir_cas_int) {
1657 casw(addr, newval, cmpval);
1658 __ eorw (res, r8, 1);
1659 } else {
1660 casl(addr, newval, cmpval);
1661 __ eorw (res, r8, 1);
1662 }
1663 }
1664
1665
1666 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1667
1668 Assembler::Condition acond, ncond;
1669 switch (condition) {
1670 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1671 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1672 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1673 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1674 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1675 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1676 case lir_cond_belowEqual:
1677 case lir_cond_aboveEqual:
1678 default: ShouldNotReachHere();
1679 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable
1680 }
1681
1682 assert(result->is_single_cpu() || result->is_double_cpu(),
1683 "expect single register for result");
1684 if (opr1->is_constant() && opr2->is_constant()
1685 && opr1->type() == T_INT && opr2->type() == T_INT) {
1686 jint val1 = opr1->as_jint();
1687 jint val2 = opr2->as_jint();
1688 if (val1 == 0 && val2 == 1) {
1689 __ cset(result->as_register(), ncond);
1690 return;
1691 } else if (val1 == 1 && val2 == 0) {
1692 __ cset(result->as_register(), acond);
1693 return;
1694 }
1695 }
1696
1697 if (opr1->is_constant() && opr2->is_constant()
1698 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1699 jlong val1 = opr1->as_jlong();
1700 jlong val2 = opr2->as_jlong();
1701 if (val1 == 0 && val2 == 1) {
1702 __ cset(result->as_register_lo(), ncond);
1703 return;
1704 } else if (val1 == 1 && val2 == 0) {
1705 __ cset(result->as_register_lo(), acond);
1706 return;
1707 }
1708 }
1709
1710 if (opr1->is_stack()) {
1711 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1712 opr1 = FrameMap::rscratch1_opr;
1713 } else if (opr1->is_constant()) {
1714 LIR_Opr tmp
1715 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1716 const2reg(opr1, tmp, lir_patch_none, NULL);
1717 opr1 = tmp;
1718 }
1719
1720 if (opr2->is_stack()) {
1721 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1722 opr2 = FrameMap::rscratch2_opr;
1723 } else if (opr2->is_constant()) {
1724 LIR_Opr tmp
1725 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1726 const2reg(opr2, tmp, lir_patch_none, NULL);
1727 opr2 = tmp;
1728 }
1729
1730 if (result->type() == T_LONG)
1731 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1732 else
1733 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1734 }
1735
1736 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1737 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1738
1739 if (left->is_single_cpu()) {
1740 Register lreg = left->as_register();
1741 Register dreg = as_reg(dest);
1742
1743 if (right->is_single_cpu()) {
1744 // cpu register - cpu register
1745
1746 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1747 "should be");
1748 Register rreg = right->as_register();
1749 switch (code) {
1750 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1751 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1752 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1753 default: ShouldNotReachHere();
1754 }
1755
1756 } else if (right->is_double_cpu()) {
1757 Register rreg = right->as_register_lo();
1758 // single_cpu + double_cpu: can happen with obj+long
1759 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1760 switch (code) {
1761 case lir_add: __ add(dreg, lreg, rreg); break;
1762 case lir_sub: __ sub(dreg, lreg, rreg); break;
1763 default: ShouldNotReachHere();
1764 }
1765 } else if (right->is_constant()) {
1766 // cpu register - constant
1767 jlong c;
1768
1769 // FIXME. This is fugly: we really need to factor all this logic.
1770 switch(right->type()) {
1771 case T_LONG:
1772 c = right->as_constant_ptr()->as_jlong();
1773 break;
1774 case T_INT:
1775 case T_ADDRESS:
1776 c = right->as_constant_ptr()->as_jint();
1777 break;
1778 default:
1779 ShouldNotReachHere();
1780 c = 0; // unreachable
1781 break;
1782 }
1783
1784 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1785 if (c == 0 && dreg == lreg) {
1786 COMMENT("effective nop elided");
1787 return;
1788 }
1789 switch(left->type()) {
1790 case T_INT:
1791 switch (code) {
1792 case lir_add: __ addw(dreg, lreg, c); break;
1793 case lir_sub: __ subw(dreg, lreg, c); break;
1794 default: ShouldNotReachHere();
1795 }
1796 break;
1797 case T_OBJECT:
1798 case T_ADDRESS:
1799 switch (code) {
1800 case lir_add: __ add(dreg, lreg, c); break;
1801 case lir_sub: __ sub(dreg, lreg, c); break;
1802 default: ShouldNotReachHere();
1803 }
1804 break;
1805 ShouldNotReachHere();
1806 }
1807 } else {
1808 ShouldNotReachHere();
1809 }
1810
1811 } else if (left->is_double_cpu()) {
1812 Register lreg_lo = left->as_register_lo();
1813
1814 if (right->is_double_cpu()) {
1815 // cpu register - cpu register
1816 Register rreg_lo = right->as_register_lo();
1817 switch (code) {
1818 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1819 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1820 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1821 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1822 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1823 default:
1824 ShouldNotReachHere();
1825 }
1826
1827 } else if (right->is_constant()) {
1828 jlong c = right->as_constant_ptr()->as_jlong_bits();
1829 Register dreg = as_reg(dest);
1830 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1831 if (c == 0 && dreg == lreg_lo) {
1832 COMMENT("effective nop elided");
1833 return;
1834 }
1835 switch (code) {
1836 case lir_add: __ add(dreg, lreg_lo, c); break;
1837 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1838 default:
1839 ShouldNotReachHere();
1840 }
1841 } else {
1842 ShouldNotReachHere();
1843 }
1844 } else if (left->is_single_fpu()) {
1845 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1846 switch (code) {
1847 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1848 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1849 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1850 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1851 default:
1852 ShouldNotReachHere();
1853 }
1854 } else if (left->is_double_fpu()) {
1855 if (right->is_double_fpu()) {
1856 // cpu register - cpu register
1857 switch (code) {
1858 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1859 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1860 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1861 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1862 default:
1863 ShouldNotReachHere();
1864 }
1865 } else {
1866 if (right->is_constant()) {
1867 ShouldNotReachHere();
1868 }
1869 ShouldNotReachHere();
1870 }
1871 } else if (left->is_single_stack() || left->is_address()) {
1872 assert(left == dest, "left and dest must be equal");
1873 ShouldNotReachHere();
1874 } else {
1875 ShouldNotReachHere();
1876 }
1877 }
1878
1879 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1880
1881
1882 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1883 switch(code) {
1884 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1885 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1886 default : ShouldNotReachHere();
1887 }
1888 }
1889
1890 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1891
1892 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1893 Register Rleft = left->is_single_cpu() ? left->as_register() :
1894 left->as_register_lo();
1895 if (dst->is_single_cpu()) {
1896 Register Rdst = dst->as_register();
1897 if (right->is_constant()) {
1898 switch (code) {
1899 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1900 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1901 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1902 default: ShouldNotReachHere(); break;
1903 }
1904 } else {
1905 Register Rright = right->is_single_cpu() ? right->as_register() :
1906 right->as_register_lo();
1907 switch (code) {
1908 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1909 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1910 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1911 default: ShouldNotReachHere(); break;
1912 }
1913 }
1914 } else {
1915 Register Rdst = dst->as_register_lo();
1916 if (right->is_constant()) {
1917 switch (code) {
1918 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1919 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1920 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1921 default: ShouldNotReachHere(); break;
1922 }
1923 } else {
1924 Register Rright = right->is_single_cpu() ? right->as_register() :
1925 right->as_register_lo();
1926 switch (code) {
1927 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1928 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1929 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1930 default: ShouldNotReachHere(); break;
1931 }
1932 }
1933 }
1934 }
1935
1936
1937
1938 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1939
1940
1941 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1942 if (opr1->is_constant() && opr2->is_single_cpu()) {
1943 // tableswitch
1944 Register reg = as_reg(opr2);
1945 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1946 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1947 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1948 Register reg1 = as_reg(opr1);
1949 if (opr2->is_single_cpu()) {
1950 // cpu register - cpu register
1951 Register reg2 = opr2->as_register();
1952 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1953 __ cmp(reg1, reg2);
1954 if (UseShenandoahGC) {
1955 oopDesc::bs()->asm_acmp_barrier(masm(), reg1, reg2);
1956 }
1957 } else {
1958 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1959 __ cmpw(reg1, reg2);
1960 }
1961 return;
1962 }
1963 if (opr2->is_double_cpu()) {
1964 // cpu register - cpu register
1965 Register reg2 = opr2->as_register_lo();
1966 __ cmp(reg1, reg2);
1967 return;
1968 }
1969
1970 if (opr2->is_constant()) {
1971 jlong imm;
1972 switch(opr2->type()) {
1973 case T_LONG:
1974 imm = opr2->as_constant_ptr()->as_jlong();
1975 break;
1976 case T_INT:
1977 case T_ADDRESS:
1978 imm = opr2->as_constant_ptr()->as_jint();
1979 break;
1980 case T_OBJECT:
1981 case T_ARRAY:
1982 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1983 break;
1984 default:
1985 ShouldNotReachHere();
1986 imm = 0; // unreachable
1987 break;
1988 }
1989
1990 if (opr2->type() == T_OBJECT || opr2->type() == T_ARRAY) {
1991 jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
1992 __ cmp(reg1, rscratch1);
1993 if (UseShenandoahGC
1994 && (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY)) {
1995 oopDesc::bs()->asm_acmp_barrier(masm(), reg1, rscratch1);
1996 }
1997 return;
1998 }
1999
2000 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
2001 if (type2aelembytes(opr1->type()) <= 4)
2002 __ cmpw(reg1, imm);
2003 else
2004 __ cmp(reg1, imm);
2005 return;
2006 } else {
2007 __ mov(rscratch1, imm);
2008 if (type2aelembytes(opr1->type()) <= 4)
2009 __ cmpw(reg1, rscratch1);
2010 else
2011 __ cmp(reg1, rscratch1);
2012 return;
2013 }
2014 } else
2015 ShouldNotReachHere();
2016 } else if (opr1->is_single_fpu()) {
2017 FloatRegister reg1 = opr1->as_float_reg();
2018 assert(opr2->is_single_fpu(), "expect single float register");
2019 FloatRegister reg2 = opr2->as_float_reg();
2020 __ fcmps(reg1, reg2);
2021 } else if (opr1->is_double_fpu()) {
2022 FloatRegister reg1 = opr1->as_double_reg();
2023 assert(opr2->is_double_fpu(), "expect double float register");
2024 FloatRegister reg2 = opr2->as_double_reg();
2025 __ fcmpd(reg1, reg2);
2026 } else {
2027 ShouldNotReachHere();
2028 }
2029 }
2030
2031 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2032 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2033 bool is_unordered_less = (code == lir_ucmp_fd2i);
2034 if (left->is_single_fpu()) {
2035 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2036 } else if (left->is_double_fpu()) {
2037 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2038 } else {
2039 ShouldNotReachHere();
2040 }
2041 } else if (code == lir_cmp_l2i) {
2042 Label done;
2043 __ cmp(left->as_register_lo(), right->as_register_lo());
2044 __ mov(dst->as_register(), (u_int64_t)-1L);
2045 __ br(Assembler::LT, done);
2046 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2047 __ bind(done);
2048 } else {
2049 ShouldNotReachHere();
2050 }
2051 }
2052
2053
2054 void LIR_Assembler::align_call(LIR_Code code) { }
2055
2056
2057 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2058 address call = __ trampoline_call(Address(op->addr(), rtype));
2059 if (call == NULL) {
2060 bailout("trampoline stub overflow");
2061 return;
2062 }
2063 add_call_info(code_offset(), op->info());
2064 }
2065
2066
2067 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2068 address call = __ ic_call(op->addr());
2069 if (call == NULL) {
2070 bailout("trampoline stub overflow");
2071 return;
2072 }
2073 add_call_info(code_offset(), op->info());
2074 }
2075
2076
2077 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2078 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2079 ShouldNotReachHere();
2080 }
2081
2082
2083 void LIR_Assembler::emit_static_call_stub() {
2084 address call_pc = __ pc();
2085 address stub = __ start_a_stub(call_stub_size);
2086 if (stub == NULL) {
2087 bailout("static call stub overflow");
2088 return;
2089 }
2090
2091 int start = __ offset();
2092
2093 __ relocate(static_stub_Relocation::spec(call_pc));
2094 __ mov_metadata(rmethod, (Metadata*)NULL);
2095 __ movptr(rscratch1, 0);
2096 __ br(rscratch1);
2097
2098 assert(__ offset() - start <= call_stub_size, "stub too big");
2099 __ end_a_stub();
2100 }
2101
2102
2103 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2104 assert(exceptionOop->as_register() == r0, "must match");
2105 assert(exceptionPC->as_register() == r3, "must match");
2106
2107 // exception object is not added to oop map by LinearScan
2108 // (LinearScan assumes that no oops are in fixed registers)
2109 info->add_register_oop(exceptionOop);
2110 Runtime1::StubID unwind_id;
2111
2112 // get current pc information
2113 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2114 int pc_for_athrow_offset = __ offset();
2115 InternalAddress pc_for_athrow(__ pc());
2116 __ adr(exceptionPC->as_register(), pc_for_athrow);
2117 add_call_info(pc_for_athrow_offset, info); // for exception handler
2118
2119 __ verify_not_null_oop(r0);
2120 // search an exception handler (r0: exception oop, r3: throwing pc)
2121 if (compilation()->has_fpu_code()) {
2122 unwind_id = Runtime1::handle_exception_id;
2123 } else {
2124 unwind_id = Runtime1::handle_exception_nofpu_id;
2125 }
2126 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2127
2128 // FIXME: enough room for two byte trap ????
2129 __ nop();
2130 }
2131
2132
2133 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2134 assert(exceptionOop->as_register() == r0, "must match");
2135
2136 __ b(_unwind_handler_entry);
2137 }
2138
2139
2140 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2141 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2142 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2143
2144 switch (left->type()) {
2145 case T_INT: {
2146 switch (code) {
2147 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2148 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2149 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2150 default:
2151 ShouldNotReachHere();
2152 break;
2153 }
2154 break;
2155 case T_LONG:
2156 case T_ADDRESS:
2157 case T_OBJECT:
2158 switch (code) {
2159 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2160 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2161 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2162 default:
2163 ShouldNotReachHere();
2164 break;
2165 }
2166 break;
2167 default:
2168 ShouldNotReachHere();
2169 break;
2170 }
2171 }
2172 }
2173
2174
2175 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2176 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2177 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2178
2179 switch (left->type()) {
2180 case T_INT: {
2181 switch (code) {
2182 case lir_shl: __ lslw (dreg, lreg, count); break;
2183 case lir_shr: __ asrw (dreg, lreg, count); break;
2184 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2185 default:
2186 ShouldNotReachHere();
2187 break;
2188 }
2189 break;
2190 case T_LONG:
2191 case T_ADDRESS:
2192 case T_OBJECT:
2193 switch (code) {
2194 case lir_shl: __ lsl (dreg, lreg, count); break;
2195 case lir_shr: __ asr (dreg, lreg, count); break;
2196 case lir_ushr: __ lsr (dreg, lreg, count); break;
2197 default:
2198 ShouldNotReachHere();
2199 break;
2200 }
2201 break;
2202 default:
2203 ShouldNotReachHere();
2204 break;
2205 }
2206 }
2207 }
2208
2209
2210 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2211 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2212 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2213 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2214 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2215 }
2216
2217
2218 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2219 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2220 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2221 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2222 __ mov (rscratch1, c);
2223 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2224 }
2225
2226
2227 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2228 ShouldNotReachHere();
2229 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2230 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2231 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2232 __ lea(rscratch1, __ constant_oop_address(o));
2233 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2234 }
2235
2236
2237 // This code replaces a call to arraycopy; no exception may
2238 // be thrown in this code, they must be thrown in the System.arraycopy
2239 // activation frame; we could save some checks if this would not be the case
2240 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2241 ciArrayKlass* default_type = op->expected_type();
2242 Register src = op->src()->as_register();
2243 Register dst = op->dst()->as_register();
2244 Register src_pos = op->src_pos()->as_register();
2245 Register dst_pos = op->dst_pos()->as_register();
2246 Register length = op->length()->as_register();
2247 Register tmp = op->tmp()->as_register();
2248
2249 CodeStub* stub = op->stub();
2250 int flags = op->flags();
2251 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2252 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2253
2254 // if we don't know anything, just go through the generic arraycopy
2255 if (default_type == NULL // || basic_type == T_OBJECT
2256 ) {
2257 Label done;
2258 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2259
2260 // Save the arguments in case the generic arraycopy fails and we
2261 // have to fall back to the JNI stub
2262 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2263 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2264 __ str(src, Address(sp, 4*BytesPerWord));
2265
2266 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2267 address copyfunc_addr = StubRoutines::generic_arraycopy();
2268
2269 // The arguments are in java calling convention so we shift them
2270 // to C convention
2271 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2272 __ mov(c_rarg0, j_rarg0);
2273 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2274 __ mov(c_rarg1, j_rarg1);
2275 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2276 __ mov(c_rarg2, j_rarg2);
2277 assert_different_registers(c_rarg3, j_rarg4);
2278 __ mov(c_rarg3, j_rarg3);
2279 __ mov(c_rarg4, j_rarg4);
2280 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2281 __ mov(rscratch1, RuntimeAddress(C_entry));
2282 __ blrt(rscratch1, 5, 0, 1);
2283 } else {
2284 #ifndef PRODUCT
2285 if (PrintC1Statistics) {
2286 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2287 }
2288 #endif
2289 __ far_call(RuntimeAddress(copyfunc_addr));
2290 }
2291
2292 __ cbz(r0, *stub->continuation());
2293
2294 // Reload values from the stack so they are where the stub
2295 // expects them.
2296 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2297 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2298 __ ldr(src, Address(sp, 4*BytesPerWord));
2299
2300 if (copyfunc_addr != NULL) {
2301 // r0 is -1^K where K == partial copied count
2302 __ eonw(rscratch1, r0, 0);
2303 // adjust length down and src/end pos up by partial copied count
2304 __ subw(length, length, rscratch1);
2305 __ addw(src_pos, src_pos, rscratch1);
2306 __ addw(dst_pos, dst_pos, rscratch1);
2307 }
2308 __ b(*stub->entry());
2309
2310 __ bind(*stub->continuation());
2311 return;
2312 }
2313
2314 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2315
2316 int elem_size = type2aelembytes(basic_type);
2317 int shift_amount;
2318 int scale = exact_log2(elem_size);
2319
2320 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2321 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2322 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2323 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2324
2325 // test for NULL
2326 if (flags & LIR_OpArrayCopy::src_null_check) {
2327 __ cbz(src, *stub->entry());
2328 }
2329 if (flags & LIR_OpArrayCopy::dst_null_check) {
2330 __ cbz(dst, *stub->entry());
2331 }
2332
2333 // check if negative
2334 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2335 __ cmpw(src_pos, 0);
2336 __ br(Assembler::LT, *stub->entry());
2337 }
2338 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2339 __ cmpw(dst_pos, 0);
2340 __ br(Assembler::LT, *stub->entry());
2341 }
2342
2343 if (flags & LIR_OpArrayCopy::length_positive_check) {
2344 __ cmpw(length, 0);
2345 __ br(Assembler::LT, *stub->entry());
2346 }
2347
2348 if (flags & LIR_OpArrayCopy::src_range_check) {
2349 __ addw(tmp, src_pos, length);
2350 __ ldrw(rscratch1, src_length_addr);
2351 __ cmpw(tmp, rscratch1);
2352 __ br(Assembler::HI, *stub->entry());
2353 }
2354 if (flags & LIR_OpArrayCopy::dst_range_check) {
2355 __ addw(tmp, dst_pos, length);
2356 __ ldrw(rscratch1, dst_length_addr);
2357 __ cmpw(tmp, rscratch1);
2358 __ br(Assembler::HI, *stub->entry());
2359 }
2360
2361 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2362 // length_positive_check if the source of our length operand is an
2363 // arraylength. However, that arraylength might be zero, and the
2364 // stub that we're about to call contains an assertion that count !=
2365 // 0 . So we make this check purely in order not to trigger an
2366 // assertion failure.
2367 __ cbzw(length, *stub->continuation());
2368
2369 if (flags & LIR_OpArrayCopy::type_check) {
2370 // We don't know the array types are compatible
2371 if (basic_type != T_OBJECT) {
2372 // Simple test for basic type arrays
2373 if (UseCompressedClassPointers) {
2374 __ ldrw(tmp, src_klass_addr);
2375 __ ldrw(rscratch1, dst_klass_addr);
2376 __ cmpw(tmp, rscratch1);
2377 } else {
2378 __ ldr(tmp, src_klass_addr);
2379 __ ldr(rscratch1, dst_klass_addr);
2380 __ cmp(tmp, rscratch1);
2381 }
2382 __ br(Assembler::NE, *stub->entry());
2383 } else {
2384 // For object arrays, if src is a sub class of dst then we can
2385 // safely do the copy.
2386 Label cont, slow;
2387
2388 #define PUSH(r1, r2) \
2389 stp(r1, r2, __ pre(sp, -2 * wordSize));
2390
2391 #define POP(r1, r2) \
2392 ldp(r1, r2, __ post(sp, 2 * wordSize));
2393
2394 __ PUSH(src, dst);
2395
2396 __ load_klass(src, src);
2397 __ load_klass(dst, dst);
2398
2399 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2400
2401 __ PUSH(src, dst);
2402 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2403 __ POP(src, dst);
2404
2405 __ cbnz(src, cont);
2406
2407 __ bind(slow);
2408 __ POP(src, dst);
2409
2410 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2411 if (copyfunc_addr != NULL) { // use stub if available
2412 // src is not a sub class of dst so we have to do a
2413 // per-element check.
2414
2415 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2416 if ((flags & mask) != mask) {
2417 // Check that at least both of them object arrays.
2418 assert(flags & mask, "one of the two should be known to be an object array");
2419
2420 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2421 __ load_klass(tmp, src);
2422 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2423 __ load_klass(tmp, dst);
2424 }
2425 int lh_offset = in_bytes(Klass::layout_helper_offset());
2426 Address klass_lh_addr(tmp, lh_offset);
2427 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2428 __ ldrw(rscratch1, klass_lh_addr);
2429 __ mov(rscratch2, objArray_lh);
2430 __ eorw(rscratch1, rscratch1, rscratch2);
2431 __ cbnzw(rscratch1, *stub->entry());
2432 }
2433
2434 // Spill because stubs can use any register they like and it's
2435 // easier to restore just those that we care about.
2436 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2437 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2438 __ str(src, Address(sp, 4*BytesPerWord));
2439
2440 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2441 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2442 assert_different_registers(c_rarg0, dst, dst_pos, length);
2443 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2444 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2445 assert_different_registers(c_rarg1, dst, length);
2446 __ uxtw(c_rarg2, length);
2447 assert_different_registers(c_rarg2, dst);
2448
2449 __ load_klass(c_rarg4, dst);
2450 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2451 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2452 __ far_call(RuntimeAddress(copyfunc_addr));
2453
2454 #ifndef PRODUCT
2455 if (PrintC1Statistics) {
2456 Label failed;
2457 __ cbnz(r0, failed);
2458 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2459 __ bind(failed);
2460 }
2461 #endif
2462
2463 __ cbz(r0, *stub->continuation());
2464
2465 #ifndef PRODUCT
2466 if (PrintC1Statistics) {
2467 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2468 }
2469 #endif
2470 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2471
2472 // Restore previously spilled arguments
2473 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2474 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2475 __ ldr(src, Address(sp, 4*BytesPerWord));
2476
2477 // return value is -1^K where K is partial copied count
2478 __ eonw(rscratch1, r0, zr);
2479 // adjust length down and src/end pos up by partial copied count
2480 __ subw(length, length, rscratch1);
2481 __ addw(src_pos, src_pos, rscratch1);
2482 __ addw(dst_pos, dst_pos, rscratch1);
2483 }
2484
2485 __ b(*stub->entry());
2486
2487 __ bind(cont);
2488 __ POP(src, dst);
2489 }
2490 }
2491
2492 #ifdef ASSERT
2493 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2494 // Sanity check the known type with the incoming class. For the
2495 // primitive case the types must match exactly with src.klass and
2496 // dst.klass each exactly matching the default type. For the
2497 // object array case, if no type check is needed then either the
2498 // dst type is exactly the expected type and the src type is a
2499 // subtype which we can't check or src is the same array as dst
2500 // but not necessarily exactly of type default_type.
2501 Label known_ok, halt;
2502 __ mov_metadata(tmp, default_type->constant_encoding());
2503 if (UseCompressedClassPointers) {
2504 __ encode_klass_not_null(tmp);
2505 }
2506
2507 if (basic_type != T_OBJECT) {
2508
2509 if (UseCompressedClassPointers) {
2510 __ ldrw(rscratch1, dst_klass_addr);
2511 __ cmpw(tmp, rscratch1);
2512 } else {
2513 __ ldr(rscratch1, dst_klass_addr);
2514 __ cmp(tmp, rscratch1);
2515 }
2516 __ br(Assembler::NE, halt);
2517 if (UseCompressedClassPointers) {
2518 __ ldrw(rscratch1, src_klass_addr);
2519 __ cmpw(tmp, rscratch1);
2520 } else {
2521 __ ldr(rscratch1, src_klass_addr);
2522 __ cmp(tmp, rscratch1);
2523 }
2524 __ br(Assembler::EQ, known_ok);
2525 } else {
2526 if (UseCompressedClassPointers) {
2527 __ ldrw(rscratch1, dst_klass_addr);
2528 __ cmpw(tmp, rscratch1);
2529 } else {
2530 __ ldr(rscratch1, dst_klass_addr);
2531 __ cmp(tmp, rscratch1);
2532 }
2533 __ br(Assembler::EQ, known_ok);
2534 __ cmp(src, dst);
2535 __ br(Assembler::EQ, known_ok);
2536 }
2537 __ bind(halt);
2538 __ stop("incorrect type information in arraycopy");
2539 __ bind(known_ok);
2540 }
2541 #endif
2542
2543 #ifndef PRODUCT
2544 if (PrintC1Statistics) {
2545 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2546 }
2547 #endif
2548
2549 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2550 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2551 assert_different_registers(c_rarg0, dst, dst_pos, length);
2552 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2553 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2554 assert_different_registers(c_rarg1, dst, length);
2555 __ uxtw(c_rarg2, length);
2556 assert_different_registers(c_rarg2, dst);
2557
2558 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2559 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2560 const char *name;
2561 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2562
2563 CodeBlob *cb = CodeCache::find_blob(entry);
2564 if (cb) {
2565 __ far_call(RuntimeAddress(entry));
2566 } else {
2567 __ call_VM_leaf(entry, 3);
2568 }
2569
2570 __ bind(*stub->continuation());
2571 }
2572
2573
2574
2575
2576 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2577 Register obj = op->obj_opr()->as_register(); // may not be an oop
2578 Register hdr = op->hdr_opr()->as_register();
2579 Register lock = op->lock_opr()->as_register();
2580 if (!UseFastLocking) {
2581 __ b(*op->stub()->entry());
2582 } else if (op->code() == lir_lock) {
2583 Register scratch = noreg;
2584 if (UseBiasedLocking) {
2585 scratch = op->scratch_opr()->as_register();
2586 }
2587 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2588 // add debug info for NullPointerException only if one is possible
2589 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2590 if (op->info() != NULL) {
2591 add_debug_info_for_null_check(null_check_offset, op->info());
2592 }
2593 // done
2594 } else if (op->code() == lir_unlock) {
2595 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2596 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2597 } else {
2598 Unimplemented();
2599 }
2600 __ bind(*op->stub()->continuation());
2601 }
2602
2603
2604 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2605 ciMethod* method = op->profiled_method();
2606 int bci = op->profiled_bci();
2607 ciMethod* callee = op->profiled_callee();
2608
2609 // Update counter for all call types
2610 ciMethodData* md = method->method_data_or_null();
2611 assert(md != NULL, "Sanity");
2612 ciProfileData* data = md->bci_to_data(bci);
2613 assert(data->is_CounterData(), "need CounterData for calls");
2614 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2615 Register mdo = op->mdo()->as_register();
2616 __ mov_metadata(mdo, md->constant_encoding());
2617 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2618 Bytecodes::Code bc = method->java_code_at_bci(bci);
2619 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2620 // Perform additional virtual call profiling for invokevirtual and
2621 // invokeinterface bytecodes
2622 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2623 !callee_is_static && // required for optimized MH invokes
2624 C1ProfileVirtualCalls) {
2625 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2626 Register recv = op->recv()->as_register();
2627 assert_different_registers(mdo, recv);
2628 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2629 ciKlass* known_klass = op->known_holder();
2630 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2631 // We know the type that will be seen at this call site; we can
2632 // statically update the MethodData* rather than needing to do
2633 // dynamic tests on the receiver type
2634
2635 // NOTE: we should probably put a lock around this search to
2636 // avoid collisions by concurrent compilations
2637 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2638 uint i;
2639 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2640 ciKlass* receiver = vc_data->receiver(i);
2641 if (known_klass->equals(receiver)) {
2642 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2643 __ addptr(data_addr, DataLayout::counter_increment);
2644 return;
2645 }
2646 }
2647
2648 // Receiver type not found in profile data; select an empty slot
2649
2650 // Note that this is less efficient than it should be because it
2651 // always does a write to the receiver part of the
2652 // VirtualCallData rather than just the first time
2653 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2654 ciKlass* receiver = vc_data->receiver(i);
2655 if (receiver == NULL) {
2656 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2657 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2658 __ lea(rscratch2, recv_addr);
2659 __ str(rscratch1, Address(rscratch2));
2660 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2661 __ addptr(data_addr, DataLayout::counter_increment);
2662 return;
2663 }
2664 }
2665 } else {
2666 __ load_klass(recv, recv);
2667 Label update_done;
2668 type_profile_helper(mdo, md, data, recv, &update_done);
2669 // Receiver did not match any saved receiver and there is no empty row for it.
2670 // Increment total counter to indicate polymorphic case.
2671 __ addptr(counter_addr, DataLayout::counter_increment);
2672
2673 __ bind(update_done);
2674 }
2675 } else {
2676 // Static call
2677 __ addptr(counter_addr, DataLayout::counter_increment);
2678 }
2679 }
2680
2681
2682 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2683 Unimplemented();
2684 }
2685
2686
2687 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2688 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2689 }
2690
2691 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2692 assert(op->crc()->is_single_cpu(), "crc must be register");
2693 assert(op->val()->is_single_cpu(), "byte value must be register");
2694 assert(op->result_opr()->is_single_cpu(), "result must be register");
2695 Register crc = op->crc()->as_register();
2696 Register val = op->val()->as_register();
2697 Register res = op->result_opr()->as_register();
2698
2699 assert_different_registers(val, crc, res);
2700 unsigned long offset;
2701 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2702 if (offset) __ add(res, res, offset);
2703
2704 __ ornw(crc, zr, crc); // ~crc
2705 __ update_byte_crc32(crc, val, res);
2706 __ ornw(res, zr, crc); // ~crc
2707 }
2708
2709 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2710 COMMENT("emit_profile_type {");
2711 Register obj = op->obj()->as_register();
2712 Register tmp = op->tmp()->as_pointer_register();
2713 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2714 ciKlass* exact_klass = op->exact_klass();
2715 intptr_t current_klass = op->current_klass();
2716 bool not_null = op->not_null();
2717 bool no_conflict = op->no_conflict();
2718
2719 Label update, next, none;
2720
2721 bool do_null = !not_null;
2722 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2723 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2724
2725 assert(do_null || do_update, "why are we here?");
2726 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2727 assert(mdo_addr.base() != rscratch1, "wrong register");
2728
2729 __ verify_oop(obj);
2730
2731 if (tmp != obj) {
2732 __ mov(tmp, obj);
2733 }
2734 if (do_null) {
2735 __ cbnz(tmp, update);
2736 if (!TypeEntries::was_null_seen(current_klass)) {
2737 __ ldr(rscratch2, mdo_addr);
2738 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2739 __ str(rscratch2, mdo_addr);
2740 }
2741 if (do_update) {
2742 #ifndef ASSERT
2743 __ b(next);
2744 }
2745 #else
2746 __ b(next);
2747 }
2748 } else {
2749 __ cbnz(tmp, update);
2750 __ stop("unexpected null obj");
2751 #endif
2752 }
2753
2754 __ bind(update);
2755
2756 if (do_update) {
2757 #ifdef ASSERT
2758 if (exact_klass != NULL) {
2759 Label ok;
2760 __ load_klass(tmp, tmp);
2761 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2762 __ eor(rscratch1, tmp, rscratch1);
2763 __ cbz(rscratch1, ok);
2764 __ stop("exact klass and actual klass differ");
2765 __ bind(ok);
2766 }
2767 #endif
2768 if (!no_conflict) {
2769 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2770 if (exact_klass != NULL) {
2771 __ mov_metadata(tmp, exact_klass->constant_encoding());
2772 } else {
2773 __ load_klass(tmp, tmp);
2774 }
2775
2776 __ ldr(rscratch2, mdo_addr);
2777 __ eor(tmp, tmp, rscratch2);
2778 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2779 // klass seen before, nothing to do. The unknown bit may have been
2780 // set already but no need to check.
2781 __ cbz(rscratch1, next);
2782
2783 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2784 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2785
2786 if (TypeEntries::is_type_none(current_klass)) {
2787 __ cbz(rscratch2, none);
2788 __ cmp(rscratch2, TypeEntries::null_seen);
2789 __ br(Assembler::EQ, none);
2790 // There is a chance that the checks above (re-reading profiling
2791 // data from memory) fail if another thread has just set the
2792 // profiling to this obj's klass
2793 __ dmb(Assembler::ISHLD);
2794 __ ldr(rscratch2, mdo_addr);
2795 __ eor(tmp, tmp, rscratch2);
2796 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2797 __ cbz(rscratch1, next);
2798 }
2799 } else {
2800 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2801 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2802
2803 __ ldr(tmp, mdo_addr);
2804 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2805 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2806 }
2807
2808 // different than before. Cannot keep accurate profile.
2809 __ ldr(rscratch2, mdo_addr);
2810 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2811 __ str(rscratch2, mdo_addr);
2812
2813 if (TypeEntries::is_type_none(current_klass)) {
2814 __ b(next);
2815
2816 __ bind(none);
2817 // first time here. Set profile type.
2818 __ str(tmp, mdo_addr);
2819 }
2820 } else {
2821 // There's a single possible klass at this profile point
2822 assert(exact_klass != NULL, "should be");
2823 if (TypeEntries::is_type_none(current_klass)) {
2824 __ mov_metadata(tmp, exact_klass->constant_encoding());
2825 __ ldr(rscratch2, mdo_addr);
2826 __ eor(tmp, tmp, rscratch2);
2827 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2828 __ cbz(rscratch1, next);
2829 #ifdef ASSERT
2830 {
2831 Label ok;
2832 __ ldr(rscratch1, mdo_addr);
2833 __ cbz(rscratch1, ok);
2834 __ cmp(rscratch1, TypeEntries::null_seen);
2835 __ br(Assembler::EQ, ok);
2836 // may have been set by another thread
2837 __ dmb(Assembler::ISHLD);
2838 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2839 __ ldr(rscratch2, mdo_addr);
2840 __ eor(rscratch2, rscratch1, rscratch2);
2841 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2842 __ cbz(rscratch2, ok);
2843
2844 __ stop("unexpected profiling mismatch");
2845 __ bind(ok);
2846 }
2847 #endif
2848 // first time here. Set profile type.
2849 __ ldr(tmp, mdo_addr);
2850 } else {
2851 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2852 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2853
2854 __ ldr(tmp, mdo_addr);
2855 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2856 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2857
2858 __ orr(tmp, tmp, TypeEntries::type_unknown);
2859 __ str(tmp, mdo_addr);
2860 // FIXME: Write barrier needed here?
2861 }
2862 }
2863
2864 __ bind(next);
2865 }
2866 COMMENT("} emit_profile_type");
2867 }
2868
2869
2870 void LIR_Assembler::align_backward_branch_target() {
2871 }
2872
2873
2874 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2875 if (left->is_single_cpu()) {
2876 assert(dest->is_single_cpu(), "expect single result reg");
2877 __ negw(dest->as_register(), left->as_register());
2878 } else if (left->is_double_cpu()) {
2879 assert(dest->is_double_cpu(), "expect double result reg");
2880 __ neg(dest->as_register_lo(), left->as_register_lo());
2881 } else if (left->is_single_fpu()) {
2882 assert(dest->is_single_fpu(), "expect single float result reg");
2883 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2884 } else {
2885 assert(left->is_double_fpu(), "expect double float operand reg");
2886 assert(dest->is_double_fpu(), "expect double float result reg");
2887 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2888 }
2889 }
2890
2891
2892 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2893 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2894 }
2895
2896
2897 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2898 assert(!tmp->is_valid(), "don't need temporary");
2899
2900 CodeBlob *cb = CodeCache::find_blob(dest);
2901 if (cb) {
2902 __ far_call(RuntimeAddress(dest));
2903 } else {
2904 __ mov(rscratch1, RuntimeAddress(dest));
2905 int len = args->length();
2906 int type = 0;
2907 if (! result->is_illegal()) {
2908 switch (result->type()) {
2909 case T_VOID:
2910 type = 0;
2911 break;
2912 case T_INT:
2913 case T_LONG:
2914 case T_OBJECT:
2915 type = 1;
2916 break;
2917 case T_FLOAT:
2918 type = 2;
2919 break;
2920 case T_DOUBLE:
2921 type = 3;
2922 break;
2923 default:
2924 ShouldNotReachHere();
2925 break;
2926 }
2927 }
2928 int num_gpargs = 0;
2929 int num_fpargs = 0;
2930 for (int i = 0; i < args->length(); i++) {
2931 LIR_Opr arg = args->at(i);
2932 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2933 num_fpargs++;
2934 } else {
2935 num_gpargs++;
2936 }
2937 }
2938 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2939 }
2940
2941 if (info != NULL) {
2942 add_call_info_here(info);
2943 }
2944 __ maybe_isb();
2945 }
2946
2947 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2948 if (dest->is_address() || src->is_address()) {
2949 move_op(src, dest, type, lir_patch_none, info,
2950 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2951 } else {
2952 ShouldNotReachHere();
2953 }
2954 }
2955
2956 #ifdef ASSERT
2957 // emit run-time assertion
2958 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2959 assert(op->code() == lir_assert, "must be");
2960
2961 if (op->in_opr1()->is_valid()) {
2962 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2963 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2964 } else {
2965 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2966 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2967 }
2968
2969 Label ok;
2970 if (op->condition() != lir_cond_always) {
2971 Assembler::Condition acond = Assembler::AL;
2972 switch (op->condition()) {
2973 case lir_cond_equal: acond = Assembler::EQ; break;
2974 case lir_cond_notEqual: acond = Assembler::NE; break;
2975 case lir_cond_less: acond = Assembler::LT; break;
2976 case lir_cond_lessEqual: acond = Assembler::LE; break;
2977 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2978 case lir_cond_greater: acond = Assembler::GT; break;
2979 case lir_cond_belowEqual: acond = Assembler::LS; break;
2980 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2981 default: ShouldNotReachHere();
2982 }
2983 __ br(acond, ok);
2984 }
2985 if (op->halt()) {
2986 const char* str = __ code_string(op->msg());
2987 __ stop(str);
2988 } else {
2989 breakpoint();
2990 }
2991 __ bind(ok);
2992 }
2993 #endif
2994
2995 #ifndef PRODUCT
2996 #define COMMENT(x) do { __ block_comment(x); } while (0)
2997 #else
2998 #define COMMENT(x)
2999 #endif
3000
3001 void LIR_Assembler::membar() {
3002 COMMENT("membar");
3003 __ membar(MacroAssembler::AnyAny);
3004 }
3005
3006 void LIR_Assembler::membar_acquire() {
3007 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3008 }
3009
3010 void LIR_Assembler::membar_release() {
3011 __ membar(Assembler::LoadStore|Assembler::StoreStore);
3012 }
3013
3014 void LIR_Assembler::membar_loadload() {
3015 __ membar(Assembler::LoadLoad);
3016 }
3017
3018 void LIR_Assembler::membar_storestore() {
3019 __ membar(MacroAssembler::StoreStore);
3020 }
3021
3022 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3023
3024 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3025
3026 void LIR_Assembler::on_spin_wait() {
3027 Unimplemented();
3028 }
3029
3030 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3031 __ mov(result_reg->as_register(), rthread);
3032 }
3033
3034
3035 void LIR_Assembler::peephole(LIR_List *lir) {
3036 #if 0
3037 if (tableswitch_count >= max_tableswitches)
3038 return;
3039
3040 /*
3041 This finite-state automaton recognizes sequences of compare-and-
3042 branch instructions. We will turn them into a tableswitch. You
3043 could argue that C1 really shouldn't be doing this sort of
3044 optimization, but without it the code is really horrible.
3045 */
3046
3047 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3048 int first_key, last_key = -2147483648;
3049 int next_key = 0;
3050 int start_insn = -1;
3051 int last_insn = -1;
3052 Register reg = noreg;
3053 LIR_Opr reg_opr;
3054 state = start_s;
3055
3056 LIR_OpList* inst = lir->instructions_list();
3057 for (int i = 0; i < inst->length(); i++) {
3058 LIR_Op* op = inst->at(i);
3059 switch (state) {
3060 case start_s:
3061 first_key = -1;
3062 start_insn = i;
3063 switch (op->code()) {
3064 case lir_cmp:
3065 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3066 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3067 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3068 && opr2->is_constant()
3069 && opr2->type() == T_INT) {
3070 reg_opr = opr1;
3071 reg = opr1->as_register();
3072 first_key = opr2->as_constant_ptr()->as_jint();
3073 next_key = first_key + 1;
3074 state = cmp_s;
3075 goto next_state;
3076 }
3077 break;
3078 }
3079 break;
3080 case cmp_s:
3081 switch (op->code()) {
3082 case lir_branch:
3083 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3084 state = beq_s;
3085 last_insn = i;
3086 goto next_state;
3087 }
3088 }
3089 state = start_s;
3090 break;
3091 case beq_s:
3092 switch (op->code()) {
3093 case lir_cmp: {
3094 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3095 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3096 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3097 && opr1->as_register() == reg
3098 && opr2->is_constant()
3099 && opr2->type() == T_INT
3100 && opr2->as_constant_ptr()->as_jint() == next_key) {
3101 last_key = next_key;
3102 next_key++;
3103 state = cmp_s;
3104 goto next_state;
3105 }
3106 }
3107 }
3108 last_key = next_key;
3109 state = start_s;
3110 break;
3111 default:
3112 assert(false, "impossible state");
3113 }
3114 if (state == start_s) {
3115 if (first_key < last_key - 5L && reg != noreg) {
3116 {
3117 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3118 // reg->encoding(),
3119 // start_insn, first_key, last_key);
3120 // for (int i = 0; i < inst->length(); i++) {
3121 // inst->at(i)->print();
3122 // tty->print("\n");
3123 // }
3124 // tty->print("\n");
3125 }
3126
3127 struct tableswitch *sw = &switches[tableswitch_count];
3128 sw->_insn_index = start_insn, sw->_first_key = first_key,
3129 sw->_last_key = last_key, sw->_reg = reg;
3130 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3131 {
3132 // Insert the new table of branches
3133 int offset = last_insn;
3134 for (int n = first_key; n < last_key; n++) {
3135 inst->insert_before
3136 (last_insn + 1,
3137 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3138 inst->at(offset)->as_OpBranch()->label()));
3139 offset -= 2, i++;
3140 }
3141 }
3142 // Delete all the old compare-and-branch instructions
3143 for (int n = first_key; n < last_key; n++) {
3144 inst->remove_at(start_insn);
3145 inst->remove_at(start_insn);
3146 }
3147 // Insert the tableswitch instruction
3148 inst->insert_before(start_insn,
3149 new LIR_Op2(lir_cmp, lir_cond_always,
3150 LIR_OprFact::intConst(tableswitch_count),
3151 reg_opr));
3152 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3153 tableswitch_count++;
3154 }
3155 reg = noreg;
3156 last_key = -2147483648;
3157 }
3158 next_state:
3159 ;
3160 }
3161 #endif
3162 }
3163
3164 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3165 Address addr = as_Address(src->as_address_ptr(), noreg);
3166 BasicType type = src->type();
3167 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3168
3169 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3170 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3171
3172 switch(type) {
3173 case T_INT:
3174 xchg = &MacroAssembler::atomic_xchgalw;
3175 add = &MacroAssembler::atomic_addalw;
3176 break;
3177 case T_LONG:
3178 xchg = &MacroAssembler::atomic_xchgal;
3179 add = &MacroAssembler::atomic_addal;
3180 break;
3181 case T_OBJECT:
3182 case T_ARRAY:
3183 if (UseCompressedOops) {
3184 xchg = &MacroAssembler::atomic_xchgalw;
3185 add = &MacroAssembler::atomic_addalw;
3186 } else {
3187 xchg = &MacroAssembler::atomic_xchgal;
3188 add = &MacroAssembler::atomic_addal;
3189 }
3190 break;
3191 default:
3192 ShouldNotReachHere();
3193 xchg = &MacroAssembler::atomic_xchgal;
3194 add = &MacroAssembler::atomic_addal; // unreachable
3195 }
3196
3197 switch (code) {
3198 case lir_xadd:
3199 {
3200 RegisterOrConstant inc;
3201 Register tmp = as_reg(tmp_op);
3202 Register dst = as_reg(dest);
3203 if (data->is_constant()) {
3204 inc = RegisterOrConstant(as_long(data));
3205 assert_different_registers(dst, addr.base(), tmp,
3206 rscratch1, rscratch2);
3207 } else {
3208 inc = RegisterOrConstant(as_reg(data));
3209 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3210 rscratch1, rscratch2);
3211 }
3212 __ lea(tmp, addr);
3213 (_masm->*add)(dst, inc, tmp);
3214 break;
3215 }
3216 case lir_xchg:
3217 {
3218 Register tmp = tmp_op->as_register();
3219 Register obj = as_reg(data);
3220 Register dst = as_reg(dest);
3221 if (is_oop && UseCompressedOops) {
3222 __ encode_heap_oop(rscratch2, obj);
3223 obj = rscratch2;
3224 }
3225 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3226 __ lea(tmp, addr);
3227 (_masm->*xchg)(dst, obj, tmp);
3228 if (is_oop && UseCompressedOops) {
3229 __ decode_heap_oop(dst);
3230 }
3231 }
3232 break;
3233 default:
3234 ShouldNotReachHere();
3235 }
3236 __ membar(__ AnyAny);
3237 }
3238
3239 #undef __