1 /*
2 * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "gc/shenandoah/brooksPointer.hpp"
37 #include "gc/shenandoah/shenandoahHeap.hpp"
38 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
39 #include "gc/shared/barrierSet.hpp"
40 #include "gc/shared/cardTableModRefBS.hpp"
41 #include "gc/shared/collectedHeap.hpp"
42 #include "nativeInst_aarch64.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "vmreg_aarch64.inline.hpp"
46
47
48
49 #ifndef PRODUCT
50 #define COMMENT(x) do { __ block_comment(x); } while (0)
51 #else
52 #define COMMENT(x)
53 #endif
54
55 NEEDS_CLEANUP // remove this definitions ?
56 const Register IC_Klass = rscratch2; // where the IC klass is cached
57 const Register SYNC_header = r0; // synchronization header
58 const Register SHIFT_count = r0; // where count for shift operations must be
59
60 #define __ _masm->
61
62
63 static void select_different_registers(Register preserve,
64 Register extra,
65 Register &tmp1,
66 Register &tmp2) {
67 if (tmp1 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp1 = extra;
70 } else if (tmp2 == preserve) {
71 assert_different_registers(tmp1, tmp2, extra);
72 tmp2 = extra;
73 }
74 assert_different_registers(preserve, tmp1, tmp2);
75 }
76
77
78
79 static void select_different_registers(Register preserve,
80 Register extra,
81 Register &tmp1,
82 Register &tmp2,
83 Register &tmp3) {
84 if (tmp1 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp1 = extra;
87 } else if (tmp2 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp2 = extra;
90 } else if (tmp3 == preserve) {
91 assert_different_registers(tmp1, tmp2, tmp3, extra);
92 tmp3 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2, tmp3);
95 }
96
97
98 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
99
100
101 LIR_Opr LIR_Assembler::receiverOpr() {
102 return FrameMap::receiver_opr;
103 }
104
105 LIR_Opr LIR_Assembler::osrBufferPointer() {
106 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
107 }
108
109 //--------------fpu register translations-----------------------
110
111
112 address LIR_Assembler::float_constant(float f) {
113 address const_addr = __ float_constant(f);
114 if (const_addr == NULL) {
115 bailout("const section overflow");
116 return __ code()->consts()->start();
117 } else {
118 return const_addr;
119 }
120 }
121
122
123 address LIR_Assembler::double_constant(double d) {
124 address const_addr = __ double_constant(d);
125 if (const_addr == NULL) {
126 bailout("const section overflow");
127 return __ code()->consts()->start();
128 } else {
129 return const_addr;
130 }
131 }
132
133 address LIR_Assembler::int_constant(jlong n) {
134 address const_addr = __ long_constant(n);
135 if (const_addr == NULL) {
136 bailout("const section overflow");
137 return __ code()->consts()->start();
138 } else {
139 return const_addr;
140 }
141 }
142
143 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
144
145 void LIR_Assembler::reset_FPU() { Unimplemented(); }
146
147 void LIR_Assembler::fpop() { Unimplemented(); }
148
149 void LIR_Assembler::fxch(int i) { Unimplemented(); }
150
151 void LIR_Assembler::fld(int i) { Unimplemented(); }
152
153 void LIR_Assembler::ffree(int i) { Unimplemented(); }
154
155 void LIR_Assembler::breakpoint() { Unimplemented(); }
156
157 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
158
159 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
160
161 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
162 //-------------------------------------------
163
164 static Register as_reg(LIR_Opr op) {
165 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
166 }
167
168 static jlong as_long(LIR_Opr data) {
169 jlong result;
170 switch (data->type()) {
171 case T_INT:
172 result = (data->as_jint());
173 break;
174 case T_LONG:
175 result = (data->as_jlong());
176 break;
177 default:
178 ShouldNotReachHere();
179 result = 0; // unreachable
180 }
181 return result;
182 }
183
184 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
185 Register base = addr->base()->as_pointer_register();
186 LIR_Opr opr = addr->index();
187 if (opr->is_cpu_register()) {
188 Register index;
189 if (opr->is_single_cpu())
190 index = opr->as_register();
191 else
192 index = opr->as_register_lo();
193 assert(addr->disp() == 0, "must be");
194 switch(opr->type()) {
195 case T_INT:
196 return Address(base, index, Address::sxtw(addr->scale()));
197 case T_LONG:
198 return Address(base, index, Address::lsl(addr->scale()));
199 default:
200 ShouldNotReachHere();
201 }
202 } else {
203 intptr_t addr_offset = intptr_t(addr->disp());
204 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
205 return Address(base, addr_offset, Address::lsl(addr->scale()));
206 else {
207 __ mov(tmp, addr_offset);
208 return Address(base, tmp, Address::lsl(addr->scale()));
209 }
210 }
211 return Address();
212 }
213
214 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
215 ShouldNotReachHere();
216 return Address();
217 }
218
219 Address LIR_Assembler::as_Address(LIR_Address* addr) {
220 return as_Address(addr, rscratch1);
221 }
222
223 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
224 return as_Address(addr, rscratch1); // Ouch
225 // FIXME: This needs to be much more clever. See x86.
226 }
227
228
229 void LIR_Assembler::osr_entry() {
230 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
231 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
232 ValueStack* entry_state = osr_entry->state();
233 int number_of_locks = entry_state->locks_size();
234
235 // we jump here if osr happens with the interpreter
236 // state set up to continue at the beginning of the
237 // loop that triggered osr - in particular, we have
238 // the following registers setup:
239 //
240 // r2: osr buffer
241 //
242
243 // build frame
244 ciMethod* m = compilation()->method();
245 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
246
247 // OSR buffer is
248 //
249 // locals[nlocals-1..0]
250 // monitors[0..number_of_locks]
251 //
252 // locals is a direct copy of the interpreter frame so in the osr buffer
253 // so first slot in the local array is the last local from the interpreter
254 // and last slot is local[0] (receiver) from the interpreter
255 //
256 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
257 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
258 // in the interpreter frame (the method lock if a sync method)
259
260 // Initialize monitors in the compiled activation.
261 // r2: pointer to osr buffer
262 //
263 // All other registers are dead at this point and the locals will be
264 // copied into place by code emitted in the IR.
265
266 Register OSR_buf = osrBufferPointer()->as_pointer_register();
267 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
268 int monitor_offset = BytesPerWord * method()->max_locals() +
269 (2 * BytesPerWord) * (number_of_locks - 1);
270 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
271 // the OSR buffer using 2 word entries: first the lock and then
272 // the oop.
273 for (int i = 0; i < number_of_locks; i++) {
274 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
275 #ifdef ASSERT
276 // verify the interpreter's monitor has a non-null object
277 {
278 Label L;
279 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
280 __ cbnz(rscratch1, L);
281 __ stop("locked object is NULL");
282 __ bind(L);
283 }
284 #endif
285 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
286 __ str(r19, frame_map()->address_for_monitor_lock(i));
287 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
288 __ str(r19, frame_map()->address_for_monitor_object(i));
289 }
290 }
291 }
292
293
294 // inline cache check; done before the frame is built.
295 int LIR_Assembler::check_icache() {
296 Register receiver = FrameMap::receiver_opr->as_register();
297 Register ic_klass = IC_Klass;
298 int start_offset = __ offset();
299 __ inline_cache_check(receiver, ic_klass);
300
301 // if icache check fails, then jump to runtime routine
302 // Note: RECEIVER must still contain the receiver!
303 Label dont;
304 __ br(Assembler::EQ, dont);
305 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
306
307 // We align the verified entry point unless the method body
308 // (including its inline cache check) will fit in a single 64-byte
309 // icache line.
310 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
311 // force alignment after the cache check.
312 __ align(CodeEntryAlignment);
313 }
314
315 __ bind(dont);
316 return start_offset;
317 }
318
319
320 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
321 if (o == NULL) {
322 __ mov(reg, zr);
323 } else {
324 __ movoop(reg, o, /*immediate*/true);
325 }
326 }
327
328 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
329 address target = NULL;
330 relocInfo::relocType reloc_type = relocInfo::none;
331
332 switch (patching_id(info)) {
333 case PatchingStub::access_field_id:
334 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
335 reloc_type = relocInfo::section_word_type;
336 break;
337 case PatchingStub::load_klass_id:
338 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
339 reloc_type = relocInfo::metadata_type;
340 break;
341 case PatchingStub::load_mirror_id:
342 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
343 reloc_type = relocInfo::oop_type;
344 break;
345 case PatchingStub::load_appendix_id:
346 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
347 reloc_type = relocInfo::oop_type;
348 break;
349 default: ShouldNotReachHere();
350 }
351
352 __ far_call(RuntimeAddress(target));
353 add_call_info_here(info);
354 }
355
356 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
357 deoptimize_trap(info);
358 }
359
360
361 // This specifies the rsp decrement needed to build the frame
362 int LIR_Assembler::initial_frame_size_in_bytes() const {
363 // if rounding, must let FrameMap know!
364
365 // The frame_map records size in slots (32bit word)
366
367 // subtract two words to account for return address and link
368 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
369 }
370
371
372 int LIR_Assembler::emit_exception_handler() {
373 // if the last instruction is a call (typically to do a throw which
374 // is coming at the end after block reordering) the return address
375 // must still point into the code area in order to avoid assertion
376 // failures when searching for the corresponding bci => add a nop
377 // (was bug 5/14/1999 - gri)
378 __ nop();
379
380 // generate code for exception handler
381 address handler_base = __ start_a_stub(exception_handler_size);
382 if (handler_base == NULL) {
383 // not enough space left for the handler
384 bailout("exception handler overflow");
385 return -1;
386 }
387
388 int offset = code_offset();
389
390 // the exception oop and pc are in r0, and r3
391 // no other registers need to be preserved, so invalidate them
392 __ invalidate_registers(false, true, true, false, true, true);
393
394 // check that there is really an exception
395 __ verify_not_null_oop(r0);
396
397 // search an exception handler (r0: exception oop, r3: throwing pc)
398 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here();
399 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
400 __ end_a_stub();
401
402 return offset;
403 }
404
405
406 // Emit the code to remove the frame from the stack in the exception
407 // unwind path.
408 int LIR_Assembler::emit_unwind_handler() {
409 #ifndef PRODUCT
410 if (CommentedAssembly) {
411 _masm->block_comment("Unwind handler");
412 }
413 #endif
414
415 int offset = code_offset();
416
417 // Fetch the exception from TLS and clear out exception related thread state
418 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
419 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
420 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
421
422 __ bind(_unwind_handler_entry);
423 __ verify_not_null_oop(r0);
424 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
425 __ mov(r19, r0); // Preserve the exception
426 }
427
428 // Preform needed unlocking
429 MonitorExitStub* stub = NULL;
430 if (method()->is_synchronized()) {
431 monitor_address(0, FrameMap::r0_opr);
432 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
433 __ unlock_object(r5, r4, r0, *stub->entry());
434 __ bind(*stub->continuation());
435 }
436
437 if (compilation()->env()->dtrace_method_probes()) {
438 __ call_Unimplemented();
439 #if 0
440 __ movptr(Address(rsp, 0), rax);
441 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
442 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
443 #endif
444 }
445
446 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
447 __ mov(r0, r19); // Restore the exception
448 }
449
450 // remove the activation and dispatch to the unwind handler
451 __ block_comment("remove_frame and dispatch to the unwind handler");
452 __ remove_frame(initial_frame_size_in_bytes());
453 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
454
455 // Emit the slow path assembly
456 if (stub != NULL) {
457 stub->emit_code(this);
458 }
459
460 return offset;
461 }
462
463
464 int LIR_Assembler::emit_deopt_handler() {
465 // if the last instruction is a call (typically to do a throw which
466 // is coming at the end after block reordering) the return address
467 // must still point into the code area in order to avoid assertion
468 // failures when searching for the corresponding bci => add a nop
469 // (was bug 5/14/1999 - gri)
470 __ nop();
471
472 // generate code for exception handler
473 address handler_base = __ start_a_stub(deopt_handler_size);
474 if (handler_base == NULL) {
475 // not enough space left for the handler
476 bailout("deopt handler overflow");
477 return -1;
478 }
479
480 int offset = code_offset();
481
482 __ adr(lr, pc());
483 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
484 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
485 __ end_a_stub();
486
487 return offset;
488 }
489
490 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
491 _masm->code_section()->relocate(adr, relocInfo::poll_type);
492 int pc_offset = code_offset();
493 flush_debug_info(pc_offset);
494 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
495 if (info->exception_handlers() != NULL) {
496 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
497 }
498 }
499
500 // Rather than take a segfault when the polling page is protected,
501 // explicitly check for a safepoint in progress and if there is one,
502 // fake a call to the handler as if a segfault had been caught.
503 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
504 __ mov(rscratch1, SafepointSynchronize::address_of_state());
505 __ ldrb(rscratch1, Address(rscratch1));
506 Label nope, poll;
507 __ cbz(rscratch1, nope);
508 __ block_comment("safepoint");
509 __ enter();
510 __ push(0x3, sp); // r0 & r1
511 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
512 __ adr(r0, poll);
513 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
514 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
515 __ blrt(rscratch1, 1, 0, 1);
516 __ maybe_isb();
517 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
518 __ mov(rscratch1, r0);
519 __ pop(0x3, sp); // r0 & r1
520 __ leave();
521 __ br(rscratch1);
522 address polling_page(os::get_polling_page());
523 assert(os::is_poll_address(polling_page), "should be");
524 unsigned long off;
525 __ adrp(rscratch1, Address(polling_page, rtype), off);
526 __ bind(poll);
527 if (info)
528 add_debug_info_for_branch(info); // This isn't just debug info:
529 // it's the oop map
530 else
531 __ code_section()->relocate(pc(), rtype);
532 __ ldrw(zr, Address(rscratch1, off));
533 __ bind(nope);
534 }
535
536 void LIR_Assembler::return_op(LIR_Opr result) {
537 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
538 // Pop the stack before the safepoint code
539 __ remove_frame(initial_frame_size_in_bytes());
540 address polling_page(os::get_polling_page());
541 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
542 __ ret(lr);
543 }
544
545 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
546 address polling_page(os::get_polling_page());
547 guarantee(info != NULL, "Shouldn't be NULL");
548 assert(os::is_poll_address(polling_page), "should be");
549 unsigned long off;
550 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
551 assert(off == 0, "must be");
552 add_debug_info_for_branch(info); // This isn't just debug info:
553 // it's the oop map
554 __ read_polling_page(rscratch1, relocInfo::poll_type);
555 return __ offset();
556 }
557
558
559 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
560 if (from_reg == r31_sp)
561 from_reg = sp;
562 if (to_reg == r31_sp)
563 to_reg = sp;
564 __ mov(to_reg, from_reg);
565 }
566
567 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
568
569
570 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
571 assert(src->is_constant(), "should not call otherwise");
572 assert(dest->is_register(), "should not call otherwise");
573 LIR_Const* c = src->as_constant_ptr();
574
575 switch (c->type()) {
576 case T_INT: {
577 assert(patch_code == lir_patch_none, "no patching handled here");
578 __ movw(dest->as_register(), c->as_jint());
579 break;
580 }
581
582 case T_ADDRESS: {
583 assert(patch_code == lir_patch_none, "no patching handled here");
584 __ mov(dest->as_register(), c->as_jint());
585 break;
586 }
587
588 case T_LONG: {
589 assert(patch_code == lir_patch_none, "no patching handled here");
590 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
591 break;
592 }
593
594 case T_OBJECT: {
595 if (patch_code == lir_patch_none) {
596 jobject2reg(c->as_jobject(), dest->as_register());
597 } else {
598 jobject2reg_with_patching(dest->as_register(), info);
599 }
600 break;
601 }
602
603 case T_METADATA: {
604 if (patch_code != lir_patch_none) {
605 klass2reg_with_patching(dest->as_register(), info);
606 } else {
607 __ mov_metadata(dest->as_register(), c->as_metadata());
608 }
609 break;
610 }
611
612 case T_FLOAT: {
613 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
614 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
615 } else {
616 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
617 __ ldrs(dest->as_float_reg(), Address(rscratch1));
618 }
619 break;
620 }
621
622 case T_DOUBLE: {
623 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
624 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
625 } else {
626 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
627 __ ldrd(dest->as_double_reg(), Address(rscratch1));
628 }
629 break;
630 }
631
632 default:
633 ShouldNotReachHere();
634 }
635 }
636
637 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
638 LIR_Const* c = src->as_constant_ptr();
639 switch (c->type()) {
640 case T_OBJECT:
641 {
642 if (! c->as_jobject())
643 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
644 else {
645 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
646 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
647 }
648 }
649 break;
650 case T_ADDRESS:
651 {
652 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
653 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
654 }
655 case T_INT:
656 case T_FLOAT:
657 {
658 Register reg = zr;
659 if (c->as_jint_bits() == 0)
660 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
661 else {
662 __ movw(rscratch1, c->as_jint_bits());
663 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
664 }
665 }
666 break;
667 case T_LONG:
668 case T_DOUBLE:
669 {
670 Register reg = zr;
671 if (c->as_jlong_bits() == 0)
672 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
673 lo_word_offset_in_bytes));
674 else {
675 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
676 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
677 lo_word_offset_in_bytes));
678 }
679 }
680 break;
681 default:
682 ShouldNotReachHere();
683 }
684 }
685
686 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
687 assert(src->is_constant(), "should not call otherwise");
688 LIR_Const* c = src->as_constant_ptr();
689 LIR_Address* to_addr = dest->as_address_ptr();
690
691 void (Assembler::* insn)(Register Rt, const Address &adr);
692
693 switch (type) {
694 case T_ADDRESS:
695 assert(c->as_jint() == 0, "should be");
696 insn = &Assembler::str;
697 break;
698 case T_LONG:
699 assert(c->as_jlong() == 0, "should be");
700 insn = &Assembler::str;
701 break;
702 case T_INT:
703 assert(c->as_jint() == 0, "should be");
704 insn = &Assembler::strw;
705 break;
706 case T_OBJECT:
707 case T_ARRAY:
708 assert(c->as_jobject() == 0, "should be");
709 if (UseCompressedOops && !wide) {
710 insn = &Assembler::strw;
711 } else {
712 insn = &Assembler::str;
713 }
714 break;
715 case T_CHAR:
716 case T_SHORT:
717 assert(c->as_jint() == 0, "should be");
718 insn = &Assembler::strh;
719 break;
720 case T_BOOLEAN:
721 case T_BYTE:
722 assert(c->as_jint() == 0, "should be");
723 insn = &Assembler::strb;
724 break;
725 default:
726 ShouldNotReachHere();
727 insn = &Assembler::str; // unreachable
728 }
729
730 if (info) add_debug_info_for_null_check_here(info);
731 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
732 }
733
734 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
735 assert(src->is_register(), "should not call otherwise");
736 assert(dest->is_register(), "should not call otherwise");
737
738 // move between cpu-registers
739 if (dest->is_single_cpu()) {
740 if (src->type() == T_LONG) {
741 // Can do LONG -> OBJECT
742 move_regs(src->as_register_lo(), dest->as_register());
743 return;
744 }
745 assert(src->is_single_cpu(), "must match");
746 if (src->type() == T_OBJECT) {
747 __ verify_oop(src->as_register());
748 }
749 move_regs(src->as_register(), dest->as_register());
750
751 } else if (dest->is_double_cpu()) {
752 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
753 // Surprising to me but we can see move of a long to t_object
754 __ verify_oop(src->as_register());
755 move_regs(src->as_register(), dest->as_register_lo());
756 return;
757 }
758 assert(src->is_double_cpu(), "must match");
759 Register f_lo = src->as_register_lo();
760 Register f_hi = src->as_register_hi();
761 Register t_lo = dest->as_register_lo();
762 Register t_hi = dest->as_register_hi();
763 assert(f_hi == f_lo, "must be same");
764 assert(t_hi == t_lo, "must be same");
765 move_regs(f_lo, t_lo);
766
767 } else if (dest->is_single_fpu()) {
768 __ fmovs(dest->as_float_reg(), src->as_float_reg());
769
770 } else if (dest->is_double_fpu()) {
771 __ fmovd(dest->as_double_reg(), src->as_double_reg());
772
773 } else {
774 ShouldNotReachHere();
775 }
776 }
777
778 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
779 if (src->is_single_cpu()) {
780 if (type == T_ARRAY || type == T_OBJECT) {
781 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
782 __ verify_oop(src->as_register());
783 } else if (type == T_METADATA || type == T_DOUBLE) {
784 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
785 } else {
786 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
787 }
788
789 } else if (src->is_double_cpu()) {
790 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
791 __ str(src->as_register_lo(), dest_addr_LO);
792
793 } else if (src->is_single_fpu()) {
794 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
795 __ strs(src->as_float_reg(), dest_addr);
796
797 } else if (src->is_double_fpu()) {
798 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
799 __ strd(src->as_double_reg(), dest_addr);
800
801 } else {
802 ShouldNotReachHere();
803 }
804
805 }
806
807
808 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
809 LIR_Address* to_addr = dest->as_address_ptr();
810 PatchingStub* patch = NULL;
811 Register compressed_src = rscratch1;
812
813 if (patch_code != lir_patch_none) {
814 deoptimize_trap(info);
815 return;
816 }
817
818 if (type == T_ARRAY || type == T_OBJECT) {
819 __ verify_oop(src->as_register());
820
821 if (UseCompressedOops && !wide) {
822 __ encode_heap_oop(compressed_src, src->as_register());
823 } else {
824 compressed_src = src->as_register();
825 }
826 }
827
828 int null_check_here = code_offset();
829 switch (type) {
830 case T_FLOAT: {
831 __ strs(src->as_float_reg(), as_Address(to_addr));
832 break;
833 }
834
835 case T_DOUBLE: {
836 __ strd(src->as_double_reg(), as_Address(to_addr));
837 break;
838 }
839
840 case T_ARRAY: // fall through
841 case T_OBJECT: // fall through
842 if (UseCompressedOops && !wide) {
843 __ strw(compressed_src, as_Address(to_addr, rscratch2));
844 } else {
845 __ shenandoah_store_check(compressed_src, as_Address(to_addr));
846 __ str(compressed_src, as_Address(to_addr));
847 }
848 break;
849 case T_METADATA:
850 // We get here to store a method pointer to the stack to pass to
851 // a dtrace runtime call. This can't work on 64 bit with
852 // compressed klass ptrs: T_METADATA can be a compressed klass
853 // ptr or a 64 bit method pointer.
854 ShouldNotReachHere();
855 __ str(src->as_register(), as_Address(to_addr));
856 break;
857 case T_ADDRESS:
858 __ str(src->as_register(), as_Address(to_addr));
859 break;
860 case T_INT:
861 __ strw(src->as_register(), as_Address(to_addr));
862 break;
863
864 case T_LONG: {
865 __ str(src->as_register_lo(), as_Address_lo(to_addr));
866 break;
867 }
868
869 case T_BYTE: // fall through
870 case T_BOOLEAN: {
871 __ strb(src->as_register(), as_Address(to_addr));
872 break;
873 }
874
875 case T_CHAR: // fall through
876 case T_SHORT:
877 __ strh(src->as_register(), as_Address(to_addr));
878 break;
879
880 default:
881 ShouldNotReachHere();
882 }
883 if (info != NULL) {
884 add_debug_info_for_null_check(null_check_here, info);
885 }
886 }
887
888
889 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
890 assert(src->is_stack(), "should not call otherwise");
891 assert(dest->is_register(), "should not call otherwise");
892
893 if (dest->is_single_cpu()) {
894 if (type == T_ARRAY || type == T_OBJECT) {
895 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
896 __ verify_oop(dest->as_register());
897 } else if (type == T_METADATA) {
898 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
899 } else {
900 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
901 }
902
903 } else if (dest->is_double_cpu()) {
904 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
905 __ ldr(dest->as_register_lo(), src_addr_LO);
906
907 } else if (dest->is_single_fpu()) {
908 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
909 __ ldrs(dest->as_float_reg(), src_addr);
910
911 } else if (dest->is_double_fpu()) {
912 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
913 __ ldrd(dest->as_double_reg(), src_addr);
914
915 } else {
916 ShouldNotReachHere();
917 }
918 }
919
920
921 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
922 address target = NULL;
923 relocInfo::relocType reloc_type = relocInfo::none;
924
925 switch (patching_id(info)) {
926 case PatchingStub::access_field_id:
927 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
928 reloc_type = relocInfo::section_word_type;
929 break;
930 case PatchingStub::load_klass_id:
931 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
932 reloc_type = relocInfo::metadata_type;
933 break;
934 case PatchingStub::load_mirror_id:
935 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
936 reloc_type = relocInfo::oop_type;
937 break;
938 case PatchingStub::load_appendix_id:
939 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
940 reloc_type = relocInfo::oop_type;
941 break;
942 default: ShouldNotReachHere();
943 }
944
945 __ far_call(RuntimeAddress(target));
946 add_call_info_here(info);
947 }
948
949 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
950
951 LIR_Opr temp;
952 if (type == T_LONG || type == T_DOUBLE)
953 temp = FrameMap::rscratch1_long_opr;
954 else
955 temp = FrameMap::rscratch1_opr;
956
957 stack2reg(src, temp, src->type());
958 reg2stack(temp, dest, dest->type(), false);
959 }
960
961
962 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
963 LIR_Address* addr = src->as_address_ptr();
964 LIR_Address* from_addr = src->as_address_ptr();
965
966 if (addr->base()->type() == T_OBJECT) {
967 __ verify_oop(addr->base()->as_pointer_register());
968 }
969
970 if (patch_code != lir_patch_none) {
971 deoptimize_trap(info);
972 return;
973 }
974
975 if (info != NULL) {
976 add_debug_info_for_null_check_here(info);
977 }
978 int null_check_here = code_offset();
979 switch (type) {
980 case T_FLOAT: {
981 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
982 break;
983 }
984
985 case T_DOUBLE: {
986 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
987 break;
988 }
989
990 case T_ARRAY: // fall through
991 case T_OBJECT: // fall through
992 if (UseCompressedOops && !wide) {
993 __ ldrw(dest->as_register(), as_Address(from_addr));
994 } else {
995 __ ldr(dest->as_register(), as_Address(from_addr));
996 }
997 break;
998 case T_METADATA:
999 // We get here to store a method pointer to the stack to pass to
1000 // a dtrace runtime call. This can't work on 64 bit with
1001 // compressed klass ptrs: T_METADATA can be a compressed klass
1002 // ptr or a 64 bit method pointer.
1003 ShouldNotReachHere();
1004 __ ldr(dest->as_register(), as_Address(from_addr));
1005 break;
1006 case T_ADDRESS:
1007 // FIXME: OMG this is a horrible kludge. Any offset from an
1008 // address that matches klass_offset_in_bytes() will be loaded
1009 // as a word, not a long.
1010 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1011 __ ldrw(dest->as_register(), as_Address(from_addr));
1012 } else {
1013 __ ldr(dest->as_register(), as_Address(from_addr));
1014 }
1015 break;
1016 case T_INT:
1017 __ ldrw(dest->as_register(), as_Address(from_addr));
1018 break;
1019
1020 case T_LONG: {
1021 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1022 break;
1023 }
1024
1025 case T_BYTE:
1026 __ ldrsb(dest->as_register(), as_Address(from_addr));
1027 break;
1028 case T_BOOLEAN: {
1029 __ ldrb(dest->as_register(), as_Address(from_addr));
1030 break;
1031 }
1032
1033 case T_CHAR:
1034 __ ldrh(dest->as_register(), as_Address(from_addr));
1035 break;
1036 case T_SHORT:
1037 __ ldrsh(dest->as_register(), as_Address(from_addr));
1038 break;
1039
1040 default:
1041 ShouldNotReachHere();
1042 }
1043
1044 if (type == T_ARRAY || type == T_OBJECT) {
1045 if (UseCompressedOops && !wide) {
1046 __ decode_heap_oop(dest->as_register());
1047 }
1048 __ verify_oop(dest->as_register());
1049 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1050 if (UseCompressedClassPointers) {
1051 __ decode_klass_not_null(dest->as_register());
1052 }
1053 }
1054 }
1055
1056
1057 int LIR_Assembler::array_element_size(BasicType type) const {
1058 int elem_size = type2aelembytes(type);
1059 return exact_log2(elem_size);
1060 }
1061
1062 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1063 Register Rdividend = op->in_opr1()->as_register();
1064 Register Rdivisor = op->in_opr2()->as_register();
1065 Register Rscratch = op->in_opr3()->as_register();
1066 Register Rresult = op->result_opr()->as_register();
1067 int divisor = -1;
1068
1069 /*
1070 TODO: For some reason, using the Rscratch that gets passed in is
1071 not possible because the register allocator does not see the tmp reg
1072 as used, and assignes it the same register as Rdividend. We use rscratch1
1073 instead.
1074
1075 assert(Rdividend != Rscratch, "");
1076 assert(Rdivisor != Rscratch, "");
1077 */
1078
1079 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1080 // convert division by a power of two into some shifts and logical operations
1081 }
1082
1083 if (op->code() == lir_irem) {
1084 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1085 } else if (op->code() == lir_idiv) {
1086 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1087 } else
1088 ShouldNotReachHere();
1089 }
1090
1091 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1092 #ifdef ASSERT
1093 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1094 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1095 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1096 #endif
1097
1098 if (op->cond() == lir_cond_always) {
1099 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1100 __ b(*(op->label()));
1101 } else {
1102 Assembler::Condition acond;
1103 if (op->code() == lir_cond_float_branch) {
1104 bool is_unordered = (op->ublock() == op->block());
1105 // Assembler::EQ does not permit unordered branches, so we add
1106 // another branch here. Likewise, Assembler::NE does not permit
1107 // ordered branches.
1108 if (is_unordered && op->cond() == lir_cond_equal
1109 || !is_unordered && op->cond() == lir_cond_notEqual)
1110 __ br(Assembler::VS, *(op->ublock()->label()));
1111 switch(op->cond()) {
1112 case lir_cond_equal: acond = Assembler::EQ; break;
1113 case lir_cond_notEqual: acond = Assembler::NE; break;
1114 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1115 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1116 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1117 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1118 default: ShouldNotReachHere();
1119 acond = Assembler::EQ; // unreachable
1120 }
1121 } else {
1122 switch (op->cond()) {
1123 case lir_cond_equal: acond = Assembler::EQ; break;
1124 case lir_cond_notEqual: acond = Assembler::NE; break;
1125 case lir_cond_less: acond = Assembler::LT; break;
1126 case lir_cond_lessEqual: acond = Assembler::LE; break;
1127 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1128 case lir_cond_greater: acond = Assembler::GT; break;
1129 case lir_cond_belowEqual: acond = Assembler::LS; break;
1130 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1131 default: ShouldNotReachHere();
1132 acond = Assembler::EQ; // unreachable
1133 }
1134 }
1135 __ br(acond,*(op->label()));
1136 }
1137 }
1138
1139
1140 void LIR_Assembler::emit_opShenandoahWriteBarrier(LIR_OpShenandoahWriteBarrier* op) {
1141 Label done;
1142 Register obj = op->in_opr()->as_register();
1143 Register res = op->result_opr()->as_register();
1144 Register tmp1 = op->tmp1_opr()->as_register();
1145 Register tmp2 = op->tmp2_opr()->as_register();
1146 assert_different_registers(res, tmp1, tmp2);
1147
1148 __ block_comment("Shenandoah write barrier {");
1149
1150 // Check for null.
1151 if (op->need_null_check()) {
1152 if (res != obj) __ mov(res, obj);
1153 __ cbz(res, done);
1154 }
1155
1156 // Check for evacuation-in-progress
1157 Address evacuation_in_progress
1158 = Address(rthread, in_bytes(JavaThread::evacuation_in_progress_offset()));
1159 __ ldrb(rscratch2, evacuation_in_progress);
1160 __ membar(Assembler::LoadLoad);
1161
1162 // The read-barrier.
1163 __ ldr(res, Address(obj, BrooksPointer::byte_offset()));
1164
1165 __ cbzw(rscratch2, done);
1166
1167 // Check for object in collection set.
1168 __ lsr(tmp1, res, ShenandoahHeapRegion::RegionSizeShift);
1169 __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
1170 __ ldrb(tmp2, Address(tmp2, tmp1));
1171 __ tbz(tmp2, 0, done);
1172
1173 RegSet to_save = RegSet::of(r0, r1) - res;
1174 __ push(to_save, sp);
1175 __ mov(r1, res);
1176 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::shenandoah_write_barrier_slow_id)));
1177 __ mov(res, r0);
1178 __ pop(to_save, sp);
1179
1180 __ block_comment("} Shenandoah write barrier");
1181 __ bind(done);
1182 }
1183
1184
1185 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1186 LIR_Opr src = op->in_opr();
1187 LIR_Opr dest = op->result_opr();
1188
1189 switch (op->bytecode()) {
1190 case Bytecodes::_i2f:
1191 {
1192 __ scvtfws(dest->as_float_reg(), src->as_register());
1193 break;
1194 }
1195 case Bytecodes::_i2d:
1196 {
1197 __ scvtfwd(dest->as_double_reg(), src->as_register());
1198 break;
1199 }
1200 case Bytecodes::_l2d:
1201 {
1202 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1203 break;
1204 }
1205 case Bytecodes::_l2f:
1206 {
1207 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1208 break;
1209 }
1210 case Bytecodes::_f2d:
1211 {
1212 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1213 break;
1214 }
1215 case Bytecodes::_d2f:
1216 {
1217 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1218 break;
1219 }
1220 case Bytecodes::_i2c:
1221 {
1222 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1223 break;
1224 }
1225 case Bytecodes::_i2l:
1226 {
1227 __ sxtw(dest->as_register_lo(), src->as_register());
1228 break;
1229 }
1230 case Bytecodes::_i2s:
1231 {
1232 __ sxth(dest->as_register(), src->as_register());
1233 break;
1234 }
1235 case Bytecodes::_i2b:
1236 {
1237 __ sxtb(dest->as_register(), src->as_register());
1238 break;
1239 }
1240 case Bytecodes::_l2i:
1241 {
1242 _masm->block_comment("FIXME: This could be a no-op");
1243 __ uxtw(dest->as_register(), src->as_register_lo());
1244 break;
1245 }
1246 case Bytecodes::_d2l:
1247 {
1248 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1249 break;
1250 }
1251 case Bytecodes::_f2i:
1252 {
1253 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1254 break;
1255 }
1256 case Bytecodes::_f2l:
1257 {
1258 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1259 break;
1260 }
1261 case Bytecodes::_d2i:
1262 {
1263 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1264 break;
1265 }
1266 default: ShouldNotReachHere();
1267 }
1268 }
1269
1270 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1271 if (op->init_check()) {
1272 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1273 InstanceKlass::init_state_offset()));
1274 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1275 add_debug_info_for_null_check_here(op->stub()->info());
1276 __ br(Assembler::NE, *op->stub()->entry());
1277 }
1278 __ allocate_object(op->obj()->as_register(),
1279 op->tmp1()->as_register(),
1280 op->tmp2()->as_register(),
1281 op->header_size(),
1282 op->object_size(),
1283 op->klass()->as_register(),
1284 *op->stub()->entry());
1285 __ bind(*op->stub()->continuation());
1286 }
1287
1288 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1289 Register len = op->len()->as_register();
1290 __ uxtw(len, len);
1291
1292 if (UseSlowPath ||
1293 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1294 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1295 __ b(*op->stub()->entry());
1296 } else {
1297 Register tmp1 = op->tmp1()->as_register();
1298 Register tmp2 = op->tmp2()->as_register();
1299 Register tmp3 = op->tmp3()->as_register();
1300 if (len == tmp1) {
1301 tmp1 = tmp3;
1302 } else if (len == tmp2) {
1303 tmp2 = tmp3;
1304 } else if (len == tmp3) {
1305 // everything is ok
1306 } else {
1307 __ mov(tmp3, len);
1308 }
1309 __ allocate_array(op->obj()->as_register(),
1310 len,
1311 tmp1,
1312 tmp2,
1313 arrayOopDesc::header_size(op->type()),
1314 array_element_size(op->type()),
1315 op->klass()->as_register(),
1316 *op->stub()->entry());
1317 }
1318 __ bind(*op->stub()->continuation());
1319 }
1320
1321 void LIR_Assembler::type_profile_helper(Register mdo,
1322 ciMethodData *md, ciProfileData *data,
1323 Register recv, Label* update_done) {
1324 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1325 Label next_test;
1326 // See if the receiver is receiver[n].
1327 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1328 __ ldr(rscratch1, Address(rscratch2));
1329 __ cmp(recv, rscratch1);
1330 __ br(Assembler::NE, next_test);
1331 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1332 __ addptr(data_addr, DataLayout::counter_increment);
1333 __ b(*update_done);
1334 __ bind(next_test);
1335 }
1336
1337 // Didn't find receiver; find next empty slot and fill it in
1338 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1339 Label next_test;
1340 __ lea(rscratch2,
1341 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1342 Address recv_addr(rscratch2);
1343 __ ldr(rscratch1, recv_addr);
1344 __ cbnz(rscratch1, next_test);
1345 __ str(recv, recv_addr);
1346 __ mov(rscratch1, DataLayout::counter_increment);
1347 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1348 __ str(rscratch1, Address(rscratch2));
1349 __ b(*update_done);
1350 __ bind(next_test);
1351 }
1352 }
1353
1354 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1355 // we always need a stub for the failure case.
1356 CodeStub* stub = op->stub();
1357 Register obj = op->object()->as_register();
1358 Register k_RInfo = op->tmp1()->as_register();
1359 Register klass_RInfo = op->tmp2()->as_register();
1360 Register dst = op->result_opr()->as_register();
1361 ciKlass* k = op->klass();
1362 Register Rtmp1 = noreg;
1363
1364 // check if it needs to be profiled
1365 ciMethodData* md;
1366 ciProfileData* data;
1367
1368 const bool should_profile = op->should_profile();
1369
1370 if (should_profile) {
1371 ciMethod* method = op->profiled_method();
1372 assert(method != NULL, "Should have method");
1373 int bci = op->profiled_bci();
1374 md = method->method_data_or_null();
1375 assert(md != NULL, "Sanity");
1376 data = md->bci_to_data(bci);
1377 assert(data != NULL, "need data for type check");
1378 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1379 }
1380 Label profile_cast_success, profile_cast_failure;
1381 Label *success_target = should_profile ? &profile_cast_success : success;
1382 Label *failure_target = should_profile ? &profile_cast_failure : failure;
1383
1384 if (obj == k_RInfo) {
1385 k_RInfo = dst;
1386 } else if (obj == klass_RInfo) {
1387 klass_RInfo = dst;
1388 }
1389 if (k->is_loaded() && !UseCompressedClassPointers) {
1390 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1391 } else {
1392 Rtmp1 = op->tmp3()->as_register();
1393 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1394 }
1395
1396 assert_different_registers(obj, k_RInfo, klass_RInfo);
1397
1398 if (should_profile) {
1399 Label not_null;
1400 __ cbnz(obj, not_null);
1401 // Object is null; update MDO and exit
1402 Register mdo = klass_RInfo;
1403 __ mov_metadata(mdo, md->constant_encoding());
1404 Address data_addr
1405 = __ form_address(rscratch2, mdo,
1406 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1407 LogBytesPerWord);
1408 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1409 __ ldr(rscratch1, data_addr);
1410 __ orr(rscratch1, rscratch1, header_bits);
1411 __ str(rscratch1, data_addr);
1412 __ b(*obj_is_null);
1413 __ bind(not_null);
1414 } else {
1415 __ cbz(obj, *obj_is_null);
1416 }
1417
1418 if (!k->is_loaded()) {
1419 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1420 } else {
1421 __ mov_metadata(k_RInfo, k->constant_encoding());
1422 }
1423 __ verify_oop(obj);
1424
1425 if (op->fast_check()) {
1426 // get object class
1427 // not a safepoint as obj null check happens earlier
1428 __ load_klass(rscratch1, obj);
1429 __ cmp( rscratch1, k_RInfo);
1430
1431 __ br(Assembler::NE, *failure_target);
1432 // successful cast, fall through to profile or jump
1433 } else {
1434 // get object class
1435 // not a safepoint as obj null check happens earlier
1436 __ load_klass(klass_RInfo, obj);
1437 if (k->is_loaded()) {
1438 // See if we get an immediate positive hit
1439 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1440 __ cmp(k_RInfo, rscratch1);
1441 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1442 __ br(Assembler::NE, *failure_target);
1443 // successful cast, fall through to profile or jump
1444 } else {
1445 // See if we get an immediate positive hit
1446 __ br(Assembler::EQ, *success_target);
1447 // check for self
1448 __ cmp(klass_RInfo, k_RInfo);
1449 __ br(Assembler::EQ, *success_target);
1450
1451 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1452 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1453 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1454 // result is a boolean
1455 __ cbzw(klass_RInfo, *failure_target);
1456 // successful cast, fall through to profile or jump
1457 }
1458 } else {
1459 // perform the fast part of the checking logic
1460 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1461 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1462 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1463 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1464 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1465 // result is a boolean
1466 __ cbz(k_RInfo, *failure_target);
1467 // successful cast, fall through to profile or jump
1468 }
1469 }
1470 if (should_profile) {
1471 Register mdo = klass_RInfo, recv = k_RInfo;
1472 __ bind(profile_cast_success);
1473 __ mov_metadata(mdo, md->constant_encoding());
1474 __ load_klass(recv, obj);
1475 Label update_done;
1476 type_profile_helper(mdo, md, data, recv, success);
1477 __ b(*success);
1478
1479 __ bind(profile_cast_failure);
1480 __ mov_metadata(mdo, md->constant_encoding());
1481 Address counter_addr
1482 = __ form_address(rscratch2, mdo,
1483 md->byte_offset_of_slot(data, CounterData::count_offset()),
1484 LogBytesPerWord);
1485 __ ldr(rscratch1, counter_addr);
1486 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1487 __ str(rscratch1, counter_addr);
1488 __ b(*failure);
1489 }
1490 __ b(*success);
1491 }
1492
1493
1494 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1495 const bool should_profile = op->should_profile();
1496
1497 LIR_Code code = op->code();
1498 if (code == lir_store_check) {
1499 Register value = op->object()->as_register();
1500 Register array = op->array()->as_register();
1501 Register k_RInfo = op->tmp1()->as_register();
1502 Register klass_RInfo = op->tmp2()->as_register();
1503 Register Rtmp1 = op->tmp3()->as_register();
1504
1505 CodeStub* stub = op->stub();
1506
1507 // check if it needs to be profiled
1508 ciMethodData* md;
1509 ciProfileData* data;
1510
1511 if (should_profile) {
1512 ciMethod* method = op->profiled_method();
1513 assert(method != NULL, "Should have method");
1514 int bci = op->profiled_bci();
1515 md = method->method_data_or_null();
1516 assert(md != NULL, "Sanity");
1517 data = md->bci_to_data(bci);
1518 assert(data != NULL, "need data for type check");
1519 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1520 }
1521 Label profile_cast_success, profile_cast_failure, done;
1522 Label *success_target = should_profile ? &profile_cast_success : &done;
1523 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1524
1525 if (should_profile) {
1526 Label not_null;
1527 __ cbnz(value, not_null);
1528 // Object is null; update MDO and exit
1529 Register mdo = klass_RInfo;
1530 __ mov_metadata(mdo, md->constant_encoding());
1531 Address data_addr
1532 = __ form_address(rscratch2, mdo,
1533 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1534 LogBytesPerInt);
1535 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1536 __ ldrw(rscratch1, data_addr);
1537 __ orrw(rscratch1, rscratch1, header_bits);
1538 __ strw(rscratch1, data_addr);
1539 __ b(done);
1540 __ bind(not_null);
1541 } else {
1542 __ cbz(value, done);
1543 }
1544
1545 add_debug_info_for_null_check_here(op->info_for_exception());
1546 __ load_klass(k_RInfo, array);
1547 __ load_klass(klass_RInfo, value);
1548
1549 // get instance klass (it's already uncompressed)
1550 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1551 // perform the fast part of the checking logic
1552 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1553 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1554 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1555 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1556 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1557 // result is a boolean
1558 __ cbzw(k_RInfo, *failure_target);
1559 // fall through to the success case
1560
1561 if (should_profile) {
1562 Register mdo = klass_RInfo, recv = k_RInfo;
1563 __ bind(profile_cast_success);
1564 __ mov_metadata(mdo, md->constant_encoding());
1565 __ load_klass(recv, value);
1566 Label update_done;
1567 type_profile_helper(mdo, md, data, recv, &done);
1568 __ b(done);
1569
1570 __ bind(profile_cast_failure);
1571 __ mov_metadata(mdo, md->constant_encoding());
1572 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1573 __ lea(rscratch2, counter_addr);
1574 __ ldr(rscratch1, Address(rscratch2));
1575 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1576 __ str(rscratch1, Address(rscratch2));
1577 __ b(*stub->entry());
1578 }
1579
1580 __ bind(done);
1581 } else if (code == lir_checkcast) {
1582 Register obj = op->object()->as_register();
1583 Register dst = op->result_opr()->as_register();
1584 Label success;
1585 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1586 __ bind(success);
1587 if (dst != obj) {
1588 __ mov(dst, obj);
1589 }
1590 } else if (code == lir_instanceof) {
1591 Register obj = op->object()->as_register();
1592 Register dst = op->result_opr()->as_register();
1593 Label success, failure, done;
1594 emit_typecheck_helper(op, &success, &failure, &failure);
1595 __ bind(failure);
1596 __ mov(dst, zr);
1597 __ b(done);
1598 __ bind(success);
1599 __ mov(dst, 1);
1600 __ bind(done);
1601 } else {
1602 ShouldNotReachHere();
1603 }
1604 }
1605
1606 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1607 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1608 __ cset(rscratch1, Assembler::NE);
1609 __ membar(__ AnyAny);
1610 }
1611
1612 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1613 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1614 __ cset(rscratch1, Assembler::NE);
1615 __ membar(__ AnyAny);
1616 }
1617
1618
1619 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1620 Register addr = as_reg(op->addr());
1621 Register newval = as_reg(op->new_value());
1622 Register cmpval = as_reg(op->cmp_value());
1623 Register res = op->result_opr()->as_register();
1624
1625 if (op->code() == lir_cas_obj) {
1626 assert(op->tmp1()->is_valid(), "must be");
1627 Register t1 = op->tmp1()->as_register();
1628 if (UseCompressedOops) {
1629 if (UseShenandoahGC) {
1630 __ encode_heap_oop(t1, cmpval);
1631 cmpval = t1;
1632 assert(op->tmp2()->is_valid(), "must be");
1633 Register t2 = op->tmp2()->as_register();
1634 __ encode_heap_oop(t2, newval);
1635 newval = t2;
1636 __ cmpxchg_oop_shenandoah(res, addr, cmpval, newval, true, true, true);
1637 } else {
1638 __ encode_heap_oop(t1, cmpval);
1639 cmpval = t1;
1640 __ encode_heap_oop(rscratch2, newval);
1641 newval = rscratch2;
1642 casw(addr, newval, cmpval);
1643 __ eorw (res, r8, 1);
1644 }
1645 } else {
1646 if (UseShenandoahGC) {
1647 __ cmpxchg_oop_shenandoah(res, addr, cmpval, newval, false, true, true);
1648 } else {
1649 casl(addr, newval, cmpval);
1650 __ eorw (res, r8, 1);
1651 }
1652 }
1653 } else if (op->code() == lir_cas_int) {
1654 casw(addr, newval, cmpval);
1655 __ eorw (res, r8, 1);
1656 } else {
1657 casl(addr, newval, cmpval);
1658 __ eorw (res, r8, 1);
1659 }
1660 }
1661
1662
1663 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1664
1665 Assembler::Condition acond, ncond;
1666 switch (condition) {
1667 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1668 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1669 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1670 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1671 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1672 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1673 case lir_cond_belowEqual:
1674 case lir_cond_aboveEqual:
1675 default: ShouldNotReachHere();
1676 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable
1677 }
1678
1679 assert(result->is_single_cpu() || result->is_double_cpu(),
1680 "expect single register for result");
1681 if (opr1->is_constant() && opr2->is_constant()
1682 && opr1->type() == T_INT && opr2->type() == T_INT) {
1683 jint val1 = opr1->as_jint();
1684 jint val2 = opr2->as_jint();
1685 if (val1 == 0 && val2 == 1) {
1686 __ cset(result->as_register(), ncond);
1687 return;
1688 } else if (val1 == 1 && val2 == 0) {
1689 __ cset(result->as_register(), acond);
1690 return;
1691 }
1692 }
1693
1694 if (opr1->is_constant() && opr2->is_constant()
1695 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1696 jlong val1 = opr1->as_jlong();
1697 jlong val2 = opr2->as_jlong();
1698 if (val1 == 0 && val2 == 1) {
1699 __ cset(result->as_register_lo(), ncond);
1700 return;
1701 } else if (val1 == 1 && val2 == 0) {
1702 __ cset(result->as_register_lo(), acond);
1703 return;
1704 }
1705 }
1706
1707 if (opr1->is_stack()) {
1708 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1709 opr1 = FrameMap::rscratch1_opr;
1710 } else if (opr1->is_constant()) {
1711 LIR_Opr tmp
1712 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1713 const2reg(opr1, tmp, lir_patch_none, NULL);
1714 opr1 = tmp;
1715 }
1716
1717 if (opr2->is_stack()) {
1718 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1719 opr2 = FrameMap::rscratch2_opr;
1720 } else if (opr2->is_constant()) {
1721 LIR_Opr tmp
1722 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1723 const2reg(opr2, tmp, lir_patch_none, NULL);
1724 opr2 = tmp;
1725 }
1726
1727 if (result->type() == T_LONG)
1728 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1729 else
1730 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1731 }
1732
1733 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1734 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1735
1736 if (left->is_single_cpu()) {
1737 Register lreg = left->as_register();
1738 Register dreg = as_reg(dest);
1739
1740 if (right->is_single_cpu()) {
1741 // cpu register - cpu register
1742
1743 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1744 "should be");
1745 Register rreg = right->as_register();
1746 switch (code) {
1747 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1748 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1749 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1750 default: ShouldNotReachHere();
1751 }
1752
1753 } else if (right->is_double_cpu()) {
1754 Register rreg = right->as_register_lo();
1755 // single_cpu + double_cpu: can happen with obj+long
1756 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1757 switch (code) {
1758 case lir_add: __ add(dreg, lreg, rreg); break;
1759 case lir_sub: __ sub(dreg, lreg, rreg); break;
1760 default: ShouldNotReachHere();
1761 }
1762 } else if (right->is_constant()) {
1763 // cpu register - constant
1764 jlong c;
1765
1766 // FIXME. This is fugly: we really need to factor all this logic.
1767 switch(right->type()) {
1768 case T_LONG:
1769 c = right->as_constant_ptr()->as_jlong();
1770 break;
1771 case T_INT:
1772 case T_ADDRESS:
1773 c = right->as_constant_ptr()->as_jint();
1774 break;
1775 default:
1776 ShouldNotReachHere();
1777 c = 0; // unreachable
1778 break;
1779 }
1780
1781 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1782 if (c == 0 && dreg == lreg) {
1783 COMMENT("effective nop elided");
1784 return;
1785 }
1786 switch(left->type()) {
1787 case T_INT:
1788 switch (code) {
1789 case lir_add: __ addw(dreg, lreg, c); break;
1790 case lir_sub: __ subw(dreg, lreg, c); break;
1791 default: ShouldNotReachHere();
1792 }
1793 break;
1794 case T_OBJECT:
1795 case T_ADDRESS:
1796 switch (code) {
1797 case lir_add: __ add(dreg, lreg, c); break;
1798 case lir_sub: __ sub(dreg, lreg, c); break;
1799 default: ShouldNotReachHere();
1800 }
1801 break;
1802 ShouldNotReachHere();
1803 }
1804 } else {
1805 ShouldNotReachHere();
1806 }
1807
1808 } else if (left->is_double_cpu()) {
1809 Register lreg_lo = left->as_register_lo();
1810
1811 if (right->is_double_cpu()) {
1812 // cpu register - cpu register
1813 Register rreg_lo = right->as_register_lo();
1814 switch (code) {
1815 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1816 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1817 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1818 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1819 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1820 default:
1821 ShouldNotReachHere();
1822 }
1823
1824 } else if (right->is_constant()) {
1825 jlong c = right->as_constant_ptr()->as_jlong_bits();
1826 Register dreg = as_reg(dest);
1827 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1828 if (c == 0 && dreg == lreg_lo) {
1829 COMMENT("effective nop elided");
1830 return;
1831 }
1832 switch (code) {
1833 case lir_add: __ add(dreg, lreg_lo, c); break;
1834 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1835 default:
1836 ShouldNotReachHere();
1837 }
1838 } else {
1839 ShouldNotReachHere();
1840 }
1841 } else if (left->is_single_fpu()) {
1842 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1843 switch (code) {
1844 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1845 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1846 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1847 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1848 default:
1849 ShouldNotReachHere();
1850 }
1851 } else if (left->is_double_fpu()) {
1852 if (right->is_double_fpu()) {
1853 // cpu register - cpu register
1854 switch (code) {
1855 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1856 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1857 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1858 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1859 default:
1860 ShouldNotReachHere();
1861 }
1862 } else {
1863 if (right->is_constant()) {
1864 ShouldNotReachHere();
1865 }
1866 ShouldNotReachHere();
1867 }
1868 } else if (left->is_single_stack() || left->is_address()) {
1869 assert(left == dest, "left and dest must be equal");
1870 ShouldNotReachHere();
1871 } else {
1872 ShouldNotReachHere();
1873 }
1874 }
1875
1876 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1877
1878
1879 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1880 switch(code) {
1881 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1882 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1883 default : ShouldNotReachHere();
1884 }
1885 }
1886
1887 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1888
1889 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1890 Register Rleft = left->is_single_cpu() ? left->as_register() :
1891 left->as_register_lo();
1892 if (dst->is_single_cpu()) {
1893 Register Rdst = dst->as_register();
1894 if (right->is_constant()) {
1895 switch (code) {
1896 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1897 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1898 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1899 default: ShouldNotReachHere(); break;
1900 }
1901 } else {
1902 Register Rright = right->is_single_cpu() ? right->as_register() :
1903 right->as_register_lo();
1904 switch (code) {
1905 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1906 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1907 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1908 default: ShouldNotReachHere(); break;
1909 }
1910 }
1911 } else {
1912 Register Rdst = dst->as_register_lo();
1913 if (right->is_constant()) {
1914 switch (code) {
1915 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1916 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1917 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1918 default: ShouldNotReachHere(); break;
1919 }
1920 } else {
1921 Register Rright = right->is_single_cpu() ? right->as_register() :
1922 right->as_register_lo();
1923 switch (code) {
1924 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1925 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1926 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1927 default: ShouldNotReachHere(); break;
1928 }
1929 }
1930 }
1931 }
1932
1933
1934
1935 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1936
1937
1938 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1939 if (opr1->is_constant() && opr2->is_single_cpu()) {
1940 // tableswitch
1941 Register reg = as_reg(opr2);
1942 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1943 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1944 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1945 Register reg1 = as_reg(opr1);
1946 if (opr2->is_single_cpu()) {
1947 // cpu register - cpu register
1948 Register reg2 = opr2->as_register();
1949 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1950 __ cmp(reg1, reg2);
1951 if (UseShenandoahGC) {
1952 oopDesc::bs()->asm_acmp_barrier(masm(), reg1, reg2);
1953 }
1954 } else {
1955 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1956 __ cmpw(reg1, reg2);
1957 }
1958 return;
1959 }
1960 if (opr2->is_double_cpu()) {
1961 // cpu register - cpu register
1962 Register reg2 = opr2->as_register_lo();
1963 __ cmp(reg1, reg2);
1964 return;
1965 }
1966
1967 if (opr2->is_constant()) {
1968 jlong imm;
1969 switch(opr2->type()) {
1970 case T_LONG:
1971 imm = opr2->as_constant_ptr()->as_jlong();
1972 break;
1973 case T_INT:
1974 case T_ADDRESS:
1975 imm = opr2->as_constant_ptr()->as_jint();
1976 break;
1977 case T_OBJECT:
1978 case T_ARRAY:
1979 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1980 break;
1981 default:
1982 ShouldNotReachHere();
1983 imm = 0; // unreachable
1984 break;
1985 }
1986
1987 if (opr2->type() == T_OBJECT || opr2->type() == T_ARRAY) {
1988 jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
1989 __ cmp(reg1, rscratch1);
1990 if (UseShenandoahGC
1991 && (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY)) {
1992 oopDesc::bs()->asm_acmp_barrier(masm(), reg1, rscratch1);
1993 }
1994 return;
1995 }
1996
1997 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1998 if (type2aelembytes(opr1->type()) <= 4)
1999 __ cmpw(reg1, imm);
2000 else
2001 __ cmp(reg1, imm);
2002 return;
2003 } else {
2004 __ mov(rscratch1, imm);
2005 if (type2aelembytes(opr1->type()) <= 4)
2006 __ cmpw(reg1, rscratch1);
2007 else
2008 __ cmp(reg1, rscratch1);
2009 return;
2010 }
2011 } else
2012 ShouldNotReachHere();
2013 } else if (opr1->is_single_fpu()) {
2014 FloatRegister reg1 = opr1->as_float_reg();
2015 assert(opr2->is_single_fpu(), "expect single float register");
2016 FloatRegister reg2 = opr2->as_float_reg();
2017 __ fcmps(reg1, reg2);
2018 } else if (opr1->is_double_fpu()) {
2019 FloatRegister reg1 = opr1->as_double_reg();
2020 assert(opr2->is_double_fpu(), "expect double float register");
2021 FloatRegister reg2 = opr2->as_double_reg();
2022 __ fcmpd(reg1, reg2);
2023 } else {
2024 ShouldNotReachHere();
2025 }
2026 }
2027
2028 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2029 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2030 bool is_unordered_less = (code == lir_ucmp_fd2i);
2031 if (left->is_single_fpu()) {
2032 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2033 } else if (left->is_double_fpu()) {
2034 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2035 } else {
2036 ShouldNotReachHere();
2037 }
2038 } else if (code == lir_cmp_l2i) {
2039 Label done;
2040 __ cmp(left->as_register_lo(), right->as_register_lo());
2041 __ mov(dst->as_register(), (u_int64_t)-1L);
2042 __ br(Assembler::LT, done);
2043 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2044 __ bind(done);
2045 } else {
2046 ShouldNotReachHere();
2047 }
2048 }
2049
2050
2051 void LIR_Assembler::align_call(LIR_Code code) { }
2052
2053
2054 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2055 address call = __ trampoline_call(Address(op->addr(), rtype));
2056 if (call == NULL) {
2057 bailout("trampoline stub overflow");
2058 return;
2059 }
2060 add_call_info(code_offset(), op->info());
2061 }
2062
2063
2064 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2065 address call = __ ic_call(op->addr());
2066 if (call == NULL) {
2067 bailout("trampoline stub overflow");
2068 return;
2069 }
2070 add_call_info(code_offset(), op->info());
2071 }
2072
2073
2074 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2075 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2076 ShouldNotReachHere();
2077 }
2078
2079
2080 void LIR_Assembler::emit_static_call_stub() {
2081 address call_pc = __ pc();
2082 address stub = __ start_a_stub(call_stub_size);
2083 if (stub == NULL) {
2084 bailout("static call stub overflow");
2085 return;
2086 }
2087
2088 int start = __ offset();
2089
2090 __ relocate(static_stub_Relocation::spec(call_pc));
2091 __ mov_metadata(rmethod, (Metadata*)NULL);
2092 __ movptr(rscratch1, 0);
2093 __ br(rscratch1);
2094
2095 assert(__ offset() - start <= call_stub_size, "stub too big");
2096 __ end_a_stub();
2097 }
2098
2099
2100 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2101 assert(exceptionOop->as_register() == r0, "must match");
2102 assert(exceptionPC->as_register() == r3, "must match");
2103
2104 // exception object is not added to oop map by LinearScan
2105 // (LinearScan assumes that no oops are in fixed registers)
2106 info->add_register_oop(exceptionOop);
2107 Runtime1::StubID unwind_id;
2108
2109 // get current pc information
2110 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2111 int pc_for_athrow_offset = __ offset();
2112 InternalAddress pc_for_athrow(__ pc());
2113 __ adr(exceptionPC->as_register(), pc_for_athrow);
2114 add_call_info(pc_for_athrow_offset, info); // for exception handler
2115
2116 __ verify_not_null_oop(r0);
2117 // search an exception handler (r0: exception oop, r3: throwing pc)
2118 if (compilation()->has_fpu_code()) {
2119 unwind_id = Runtime1::handle_exception_id;
2120 } else {
2121 unwind_id = Runtime1::handle_exception_nofpu_id;
2122 }
2123 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2124
2125 // FIXME: enough room for two byte trap ????
2126 __ nop();
2127 }
2128
2129
2130 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2131 assert(exceptionOop->as_register() == r0, "must match");
2132
2133 __ b(_unwind_handler_entry);
2134 }
2135
2136
2137 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2138 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2139 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2140
2141 switch (left->type()) {
2142 case T_INT: {
2143 switch (code) {
2144 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2145 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2146 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2147 default:
2148 ShouldNotReachHere();
2149 break;
2150 }
2151 break;
2152 case T_LONG:
2153 case T_ADDRESS:
2154 case T_OBJECT:
2155 switch (code) {
2156 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2157 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2158 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2159 default:
2160 ShouldNotReachHere();
2161 break;
2162 }
2163 break;
2164 default:
2165 ShouldNotReachHere();
2166 break;
2167 }
2168 }
2169 }
2170
2171
2172 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2173 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2174 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2175
2176 switch (left->type()) {
2177 case T_INT: {
2178 switch (code) {
2179 case lir_shl: __ lslw (dreg, lreg, count); break;
2180 case lir_shr: __ asrw (dreg, lreg, count); break;
2181 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2182 default:
2183 ShouldNotReachHere();
2184 break;
2185 }
2186 break;
2187 case T_LONG:
2188 case T_ADDRESS:
2189 case T_OBJECT:
2190 switch (code) {
2191 case lir_shl: __ lsl (dreg, lreg, count); break;
2192 case lir_shr: __ asr (dreg, lreg, count); break;
2193 case lir_ushr: __ lsr (dreg, lreg, count); break;
2194 default:
2195 ShouldNotReachHere();
2196 break;
2197 }
2198 break;
2199 default:
2200 ShouldNotReachHere();
2201 break;
2202 }
2203 }
2204 }
2205
2206
2207 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2208 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2209 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2210 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2211 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2212 }
2213
2214
2215 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2216 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2217 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2218 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2219 __ mov (rscratch1, c);
2220 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2221 }
2222
2223
2224 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2225 ShouldNotReachHere();
2226 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2227 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2228 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2229 __ lea(rscratch1, __ constant_oop_address(o));
2230 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2231 }
2232
2233
2234 // This code replaces a call to arraycopy; no exception may
2235 // be thrown in this code, they must be thrown in the System.arraycopy
2236 // activation frame; we could save some checks if this would not be the case
2237 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2238 ciArrayKlass* default_type = op->expected_type();
2239 Register src = op->src()->as_register();
2240 Register dst = op->dst()->as_register();
2241 Register src_pos = op->src_pos()->as_register();
2242 Register dst_pos = op->dst_pos()->as_register();
2243 Register length = op->length()->as_register();
2244 Register tmp = op->tmp()->as_register();
2245
2246 CodeStub* stub = op->stub();
2247 int flags = op->flags();
2248 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2249 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2250
2251 // if we don't know anything, just go through the generic arraycopy
2252 if (default_type == NULL // || basic_type == T_OBJECT
2253 ) {
2254 Label done;
2255 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2256
2257 // Save the arguments in case the generic arraycopy fails and we
2258 // have to fall back to the JNI stub
2259 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2260 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2261 __ str(src, Address(sp, 4*BytesPerWord));
2262
2263 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2264 address copyfunc_addr = StubRoutines::generic_arraycopy();
2265
2266 // The arguments are in java calling convention so we shift them
2267 // to C convention
2268 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2269 __ mov(c_rarg0, j_rarg0);
2270 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2271 __ mov(c_rarg1, j_rarg1);
2272 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2273 __ mov(c_rarg2, j_rarg2);
2274 assert_different_registers(c_rarg3, j_rarg4);
2275 __ mov(c_rarg3, j_rarg3);
2276 __ mov(c_rarg4, j_rarg4);
2277 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2278 __ mov(rscratch1, RuntimeAddress(C_entry));
2279 __ blrt(rscratch1, 5, 0, 1);
2280 } else {
2281 #ifndef PRODUCT
2282 if (PrintC1Statistics) {
2283 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2284 }
2285 #endif
2286 __ far_call(RuntimeAddress(copyfunc_addr));
2287 }
2288
2289 __ cbz(r0, *stub->continuation());
2290
2291 // Reload values from the stack so they are where the stub
2292 // expects them.
2293 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2294 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2295 __ ldr(src, Address(sp, 4*BytesPerWord));
2296
2297 if (copyfunc_addr != NULL) {
2298 // r0 is -1^K where K == partial copied count
2299 __ eonw(rscratch1, r0, 0);
2300 // adjust length down and src/end pos up by partial copied count
2301 __ subw(length, length, rscratch1);
2302 __ addw(src_pos, src_pos, rscratch1);
2303 __ addw(dst_pos, dst_pos, rscratch1);
2304 }
2305 __ b(*stub->entry());
2306
2307 __ bind(*stub->continuation());
2308 return;
2309 }
2310
2311 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2312
2313 int elem_size = type2aelembytes(basic_type);
2314 int shift_amount;
2315 int scale = exact_log2(elem_size);
2316
2317 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2318 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2319 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2320 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2321
2322 // test for NULL
2323 if (flags & LIR_OpArrayCopy::src_null_check) {
2324 __ cbz(src, *stub->entry());
2325 }
2326 if (flags & LIR_OpArrayCopy::dst_null_check) {
2327 __ cbz(dst, *stub->entry());
2328 }
2329
2330 // check if negative
2331 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2332 __ cmpw(src_pos, 0);
2333 __ br(Assembler::LT, *stub->entry());
2334 }
2335 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2336 __ cmpw(dst_pos, 0);
2337 __ br(Assembler::LT, *stub->entry());
2338 }
2339
2340 if (flags & LIR_OpArrayCopy::length_positive_check) {
2341 __ cmpw(length, 0);
2342 __ br(Assembler::LT, *stub->entry());
2343 }
2344
2345 if (flags & LIR_OpArrayCopy::src_range_check) {
2346 __ addw(tmp, src_pos, length);
2347 __ ldrw(rscratch1, src_length_addr);
2348 __ cmpw(tmp, rscratch1);
2349 __ br(Assembler::HI, *stub->entry());
2350 }
2351 if (flags & LIR_OpArrayCopy::dst_range_check) {
2352 __ addw(tmp, dst_pos, length);
2353 __ ldrw(rscratch1, dst_length_addr);
2354 __ cmpw(tmp, rscratch1);
2355 __ br(Assembler::HI, *stub->entry());
2356 }
2357
2358 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2359 // length_positive_check if the source of our length operand is an
2360 // arraylength. However, that arraylength might be zero, and the
2361 // stub that we're about to call contains an assertion that count !=
2362 // 0 . So we make this check purely in order not to trigger an
2363 // assertion failure.
2364 __ cbzw(length, *stub->continuation());
2365
2366 if (flags & LIR_OpArrayCopy::type_check) {
2367 // We don't know the array types are compatible
2368 if (basic_type != T_OBJECT) {
2369 // Simple test for basic type arrays
2370 if (UseCompressedClassPointers) {
2371 __ ldrw(tmp, src_klass_addr);
2372 __ ldrw(rscratch1, dst_klass_addr);
2373 __ cmpw(tmp, rscratch1);
2374 } else {
2375 __ ldr(tmp, src_klass_addr);
2376 __ ldr(rscratch1, dst_klass_addr);
2377 __ cmp(tmp, rscratch1);
2378 }
2379 __ br(Assembler::NE, *stub->entry());
2380 } else {
2381 // For object arrays, if src is a sub class of dst then we can
2382 // safely do the copy.
2383 Label cont, slow;
2384
2385 #define PUSH(r1, r2) \
2386 stp(r1, r2, __ pre(sp, -2 * wordSize));
2387
2388 #define POP(r1, r2) \
2389 ldp(r1, r2, __ post(sp, 2 * wordSize));
2390
2391 __ PUSH(src, dst);
2392
2393 __ load_klass(src, src);
2394 __ load_klass(dst, dst);
2395
2396 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2397
2398 __ PUSH(src, dst);
2399 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2400 __ POP(src, dst);
2401
2402 __ cbnz(src, cont);
2403
2404 __ bind(slow);
2405 __ POP(src, dst);
2406
2407 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2408 if (copyfunc_addr != NULL) { // use stub if available
2409 // src is not a sub class of dst so we have to do a
2410 // per-element check.
2411
2412 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2413 if ((flags & mask) != mask) {
2414 // Check that at least both of them object arrays.
2415 assert(flags & mask, "one of the two should be known to be an object array");
2416
2417 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2418 __ load_klass(tmp, src);
2419 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2420 __ load_klass(tmp, dst);
2421 }
2422 int lh_offset = in_bytes(Klass::layout_helper_offset());
2423 Address klass_lh_addr(tmp, lh_offset);
2424 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2425 __ ldrw(rscratch1, klass_lh_addr);
2426 __ mov(rscratch2, objArray_lh);
2427 __ eorw(rscratch1, rscratch1, rscratch2);
2428 __ cbnzw(rscratch1, *stub->entry());
2429 }
2430
2431 // Spill because stubs can use any register they like and it's
2432 // easier to restore just those that we care about.
2433 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2434 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2435 __ str(src, Address(sp, 4*BytesPerWord));
2436
2437 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2438 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2439 assert_different_registers(c_rarg0, dst, dst_pos, length);
2440 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2441 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2442 assert_different_registers(c_rarg1, dst, length);
2443 __ uxtw(c_rarg2, length);
2444 assert_different_registers(c_rarg2, dst);
2445
2446 __ load_klass(c_rarg4, dst);
2447 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2448 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2449 __ far_call(RuntimeAddress(copyfunc_addr));
2450
2451 #ifndef PRODUCT
2452 if (PrintC1Statistics) {
2453 Label failed;
2454 __ cbnz(r0, failed);
2455 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2456 __ bind(failed);
2457 }
2458 #endif
2459
2460 __ cbz(r0, *stub->continuation());
2461
2462 #ifndef PRODUCT
2463 if (PrintC1Statistics) {
2464 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2465 }
2466 #endif
2467 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2468
2469 // Restore previously spilled arguments
2470 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2471 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2472 __ ldr(src, Address(sp, 4*BytesPerWord));
2473
2474 // return value is -1^K where K is partial copied count
2475 __ eonw(rscratch1, r0, zr);
2476 // adjust length down and src/end pos up by partial copied count
2477 __ subw(length, length, rscratch1);
2478 __ addw(src_pos, src_pos, rscratch1);
2479 __ addw(dst_pos, dst_pos, rscratch1);
2480 }
2481
2482 __ b(*stub->entry());
2483
2484 __ bind(cont);
2485 __ POP(src, dst);
2486 }
2487 }
2488
2489 #ifdef ASSERT
2490 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2491 // Sanity check the known type with the incoming class. For the
2492 // primitive case the types must match exactly with src.klass and
2493 // dst.klass each exactly matching the default type. For the
2494 // object array case, if no type check is needed then either the
2495 // dst type is exactly the expected type and the src type is a
2496 // subtype which we can't check or src is the same array as dst
2497 // but not necessarily exactly of type default_type.
2498 Label known_ok, halt;
2499 __ mov_metadata(tmp, default_type->constant_encoding());
2500 if (UseCompressedClassPointers) {
2501 __ encode_klass_not_null(tmp);
2502 }
2503
2504 if (basic_type != T_OBJECT) {
2505
2506 if (UseCompressedClassPointers) {
2507 __ ldrw(rscratch1, dst_klass_addr);
2508 __ cmpw(tmp, rscratch1);
2509 } else {
2510 __ ldr(rscratch1, dst_klass_addr);
2511 __ cmp(tmp, rscratch1);
2512 }
2513 __ br(Assembler::NE, halt);
2514 if (UseCompressedClassPointers) {
2515 __ ldrw(rscratch1, src_klass_addr);
2516 __ cmpw(tmp, rscratch1);
2517 } else {
2518 __ ldr(rscratch1, src_klass_addr);
2519 __ cmp(tmp, rscratch1);
2520 }
2521 __ br(Assembler::EQ, known_ok);
2522 } else {
2523 if (UseCompressedClassPointers) {
2524 __ ldrw(rscratch1, dst_klass_addr);
2525 __ cmpw(tmp, rscratch1);
2526 } else {
2527 __ ldr(rscratch1, dst_klass_addr);
2528 __ cmp(tmp, rscratch1);
2529 }
2530 __ br(Assembler::EQ, known_ok);
2531 __ cmp(src, dst);
2532 __ br(Assembler::EQ, known_ok);
2533 }
2534 __ bind(halt);
2535 __ stop("incorrect type information in arraycopy");
2536 __ bind(known_ok);
2537 }
2538 #endif
2539
2540 #ifndef PRODUCT
2541 if (PrintC1Statistics) {
2542 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2543 }
2544 #endif
2545
2546 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2547 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2548 assert_different_registers(c_rarg0, dst, dst_pos, length);
2549 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2550 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2551 assert_different_registers(c_rarg1, dst, length);
2552 __ uxtw(c_rarg2, length);
2553 assert_different_registers(c_rarg2, dst);
2554
2555 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2556 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2557 const char *name;
2558 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2559
2560 CodeBlob *cb = CodeCache::find_blob(entry);
2561 if (cb) {
2562 __ far_call(RuntimeAddress(entry));
2563 } else {
2564 __ call_VM_leaf(entry, 3);
2565 }
2566
2567 __ bind(*stub->continuation());
2568 }
2569
2570
2571
2572
2573 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2574 Register obj = op->obj_opr()->as_register(); // may not be an oop
2575 Register hdr = op->hdr_opr()->as_register();
2576 Register lock = op->lock_opr()->as_register();
2577 if (!UseFastLocking) {
2578 __ b(*op->stub()->entry());
2579 } else if (op->code() == lir_lock) {
2580 Register scratch = noreg;
2581 if (UseBiasedLocking) {
2582 scratch = op->scratch_opr()->as_register();
2583 }
2584 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2585 // add debug info for NullPointerException only if one is possible
2586 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2587 if (op->info() != NULL) {
2588 add_debug_info_for_null_check(null_check_offset, op->info());
2589 }
2590 // done
2591 } else if (op->code() == lir_unlock) {
2592 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2593 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2594 } else {
2595 Unimplemented();
2596 }
2597 __ bind(*op->stub()->continuation());
2598 }
2599
2600
2601 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2602 ciMethod* method = op->profiled_method();
2603 int bci = op->profiled_bci();
2604 ciMethod* callee = op->profiled_callee();
2605
2606 // Update counter for all call types
2607 ciMethodData* md = method->method_data_or_null();
2608 assert(md != NULL, "Sanity");
2609 ciProfileData* data = md->bci_to_data(bci);
2610 assert(data->is_CounterData(), "need CounterData for calls");
2611 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2612 Register mdo = op->mdo()->as_register();
2613 __ mov_metadata(mdo, md->constant_encoding());
2614 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2615 Bytecodes::Code bc = method->java_code_at_bci(bci);
2616 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2617 // Perform additional virtual call profiling for invokevirtual and
2618 // invokeinterface bytecodes
2619 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2620 !callee_is_static && // required for optimized MH invokes
2621 C1ProfileVirtualCalls) {
2622 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2623 Register recv = op->recv()->as_register();
2624 assert_different_registers(mdo, recv);
2625 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2626 ciKlass* known_klass = op->known_holder();
2627 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2628 // We know the type that will be seen at this call site; we can
2629 // statically update the MethodData* rather than needing to do
2630 // dynamic tests on the receiver type
2631
2632 // NOTE: we should probably put a lock around this search to
2633 // avoid collisions by concurrent compilations
2634 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2635 uint i;
2636 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2637 ciKlass* receiver = vc_data->receiver(i);
2638 if (known_klass->equals(receiver)) {
2639 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2640 __ addptr(data_addr, DataLayout::counter_increment);
2641 return;
2642 }
2643 }
2644
2645 // Receiver type not found in profile data; select an empty slot
2646
2647 // Note that this is less efficient than it should be because it
2648 // always does a write to the receiver part of the
2649 // VirtualCallData rather than just the first time
2650 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2651 ciKlass* receiver = vc_data->receiver(i);
2652 if (receiver == NULL) {
2653 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2654 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2655 __ lea(rscratch2, recv_addr);
2656 __ str(rscratch1, Address(rscratch2));
2657 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2658 __ addptr(data_addr, DataLayout::counter_increment);
2659 return;
2660 }
2661 }
2662 } else {
2663 __ load_klass(recv, recv);
2664 Label update_done;
2665 type_profile_helper(mdo, md, data, recv, &update_done);
2666 // Receiver did not match any saved receiver and there is no empty row for it.
2667 // Increment total counter to indicate polymorphic case.
2668 __ addptr(counter_addr, DataLayout::counter_increment);
2669
2670 __ bind(update_done);
2671 }
2672 } else {
2673 // Static call
2674 __ addptr(counter_addr, DataLayout::counter_increment);
2675 }
2676 }
2677
2678
2679 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2680 Unimplemented();
2681 }
2682
2683
2684 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2685 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2686 }
2687
2688 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2689 assert(op->crc()->is_single_cpu(), "crc must be register");
2690 assert(op->val()->is_single_cpu(), "byte value must be register");
2691 assert(op->result_opr()->is_single_cpu(), "result must be register");
2692 Register crc = op->crc()->as_register();
2693 Register val = op->val()->as_register();
2694 Register res = op->result_opr()->as_register();
2695
2696 assert_different_registers(val, crc, res);
2697 unsigned long offset;
2698 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2699 if (offset) __ add(res, res, offset);
2700
2701 __ ornw(crc, zr, crc); // ~crc
2702 __ update_byte_crc32(crc, val, res);
2703 __ ornw(res, zr, crc); // ~crc
2704 }
2705
2706 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2707 COMMENT("emit_profile_type {");
2708 Register obj = op->obj()->as_register();
2709 Register tmp = op->tmp()->as_pointer_register();
2710 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2711 ciKlass* exact_klass = op->exact_klass();
2712 intptr_t current_klass = op->current_klass();
2713 bool not_null = op->not_null();
2714 bool no_conflict = op->no_conflict();
2715
2716 Label update, next, none;
2717
2718 bool do_null = !not_null;
2719 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2720 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2721
2722 assert(do_null || do_update, "why are we here?");
2723 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2724 assert(mdo_addr.base() != rscratch1, "wrong register");
2725
2726 __ verify_oop(obj);
2727
2728 if (tmp != obj) {
2729 __ mov(tmp, obj);
2730 }
2731 if (do_null) {
2732 __ cbnz(tmp, update);
2733 if (!TypeEntries::was_null_seen(current_klass)) {
2734 __ ldr(rscratch2, mdo_addr);
2735 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2736 __ str(rscratch2, mdo_addr);
2737 }
2738 if (do_update) {
2739 #ifndef ASSERT
2740 __ b(next);
2741 }
2742 #else
2743 __ b(next);
2744 }
2745 } else {
2746 __ cbnz(tmp, update);
2747 __ stop("unexpected null obj");
2748 #endif
2749 }
2750
2751 __ bind(update);
2752
2753 if (do_update) {
2754 #ifdef ASSERT
2755 if (exact_klass != NULL) {
2756 Label ok;
2757 __ load_klass(tmp, tmp);
2758 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2759 __ eor(rscratch1, tmp, rscratch1);
2760 __ cbz(rscratch1, ok);
2761 __ stop("exact klass and actual klass differ");
2762 __ bind(ok);
2763 }
2764 #endif
2765 if (!no_conflict) {
2766 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2767 if (exact_klass != NULL) {
2768 __ mov_metadata(tmp, exact_klass->constant_encoding());
2769 } else {
2770 __ load_klass(tmp, tmp);
2771 }
2772
2773 __ ldr(rscratch2, mdo_addr);
2774 __ eor(tmp, tmp, rscratch2);
2775 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2776 // klass seen before, nothing to do. The unknown bit may have been
2777 // set already but no need to check.
2778 __ cbz(rscratch1, next);
2779
2780 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2781 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2782
2783 if (TypeEntries::is_type_none(current_klass)) {
2784 __ cbz(rscratch2, none);
2785 __ cmp(rscratch2, TypeEntries::null_seen);
2786 __ br(Assembler::EQ, none);
2787 // There is a chance that the checks above (re-reading profiling
2788 // data from memory) fail if another thread has just set the
2789 // profiling to this obj's klass
2790 __ dmb(Assembler::ISHLD);
2791 __ ldr(rscratch2, mdo_addr);
2792 __ eor(tmp, tmp, rscratch2);
2793 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2794 __ cbz(rscratch1, next);
2795 }
2796 } else {
2797 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2798 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2799
2800 __ ldr(tmp, mdo_addr);
2801 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2802 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2803 }
2804
2805 // different than before. Cannot keep accurate profile.
2806 __ ldr(rscratch2, mdo_addr);
2807 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2808 __ str(rscratch2, mdo_addr);
2809
2810 if (TypeEntries::is_type_none(current_klass)) {
2811 __ b(next);
2812
2813 __ bind(none);
2814 // first time here. Set profile type.
2815 __ str(tmp, mdo_addr);
2816 }
2817 } else {
2818 // There's a single possible klass at this profile point
2819 assert(exact_klass != NULL, "should be");
2820 if (TypeEntries::is_type_none(current_klass)) {
2821 __ mov_metadata(tmp, exact_klass->constant_encoding());
2822 __ ldr(rscratch2, mdo_addr);
2823 __ eor(tmp, tmp, rscratch2);
2824 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2825 __ cbz(rscratch1, next);
2826 #ifdef ASSERT
2827 {
2828 Label ok;
2829 __ ldr(rscratch1, mdo_addr);
2830 __ cbz(rscratch1, ok);
2831 __ cmp(rscratch1, TypeEntries::null_seen);
2832 __ br(Assembler::EQ, ok);
2833 // may have been set by another thread
2834 __ dmb(Assembler::ISHLD);
2835 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2836 __ ldr(rscratch2, mdo_addr);
2837 __ eor(rscratch2, rscratch1, rscratch2);
2838 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2839 __ cbz(rscratch2, ok);
2840
2841 __ stop("unexpected profiling mismatch");
2842 __ bind(ok);
2843 }
2844 #endif
2845 // first time here. Set profile type.
2846 __ ldr(tmp, mdo_addr);
2847 } else {
2848 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2849 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2850
2851 __ ldr(tmp, mdo_addr);
2852 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2853 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2854
2855 __ orr(tmp, tmp, TypeEntries::type_unknown);
2856 __ str(tmp, mdo_addr);
2857 // FIXME: Write barrier needed here?
2858 }
2859 }
2860
2861 __ bind(next);
2862 }
2863 COMMENT("} emit_profile_type");
2864 }
2865
2866
2867 void LIR_Assembler::align_backward_branch_target() {
2868 }
2869
2870
2871 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2872 if (left->is_single_cpu()) {
2873 assert(dest->is_single_cpu(), "expect single result reg");
2874 __ negw(dest->as_register(), left->as_register());
2875 } else if (left->is_double_cpu()) {
2876 assert(dest->is_double_cpu(), "expect double result reg");
2877 __ neg(dest->as_register_lo(), left->as_register_lo());
2878 } else if (left->is_single_fpu()) {
2879 assert(dest->is_single_fpu(), "expect single float result reg");
2880 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2881 } else {
2882 assert(left->is_double_fpu(), "expect double float operand reg");
2883 assert(dest->is_double_fpu(), "expect double float result reg");
2884 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2885 }
2886 }
2887
2888
2889 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2890 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2891 }
2892
2893
2894 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2895 assert(!tmp->is_valid(), "don't need temporary");
2896
2897 CodeBlob *cb = CodeCache::find_blob(dest);
2898 if (cb) {
2899 __ far_call(RuntimeAddress(dest));
2900 } else {
2901 __ mov(rscratch1, RuntimeAddress(dest));
2902 int len = args->length();
2903 int type = 0;
2904 if (! result->is_illegal()) {
2905 switch (result->type()) {
2906 case T_VOID:
2907 type = 0;
2908 break;
2909 case T_INT:
2910 case T_LONG:
2911 case T_OBJECT:
2912 type = 1;
2913 break;
2914 case T_FLOAT:
2915 type = 2;
2916 break;
2917 case T_DOUBLE:
2918 type = 3;
2919 break;
2920 default:
2921 ShouldNotReachHere();
2922 break;
2923 }
2924 }
2925 int num_gpargs = 0;
2926 int num_fpargs = 0;
2927 for (int i = 0; i < args->length(); i++) {
2928 LIR_Opr arg = args->at(i);
2929 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2930 num_fpargs++;
2931 } else {
2932 num_gpargs++;
2933 }
2934 }
2935 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2936 }
2937
2938 if (info != NULL) {
2939 add_call_info_here(info);
2940 }
2941 __ maybe_isb();
2942 }
2943
2944 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2945 if (dest->is_address() || src->is_address()) {
2946 move_op(src, dest, type, lir_patch_none, info,
2947 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2948 } else {
2949 ShouldNotReachHere();
2950 }
2951 }
2952
2953 #ifdef ASSERT
2954 // emit run-time assertion
2955 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2956 assert(op->code() == lir_assert, "must be");
2957
2958 if (op->in_opr1()->is_valid()) {
2959 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2960 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2961 } else {
2962 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2963 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2964 }
2965
2966 Label ok;
2967 if (op->condition() != lir_cond_always) {
2968 Assembler::Condition acond = Assembler::AL;
2969 switch (op->condition()) {
2970 case lir_cond_equal: acond = Assembler::EQ; break;
2971 case lir_cond_notEqual: acond = Assembler::NE; break;
2972 case lir_cond_less: acond = Assembler::LT; break;
2973 case lir_cond_lessEqual: acond = Assembler::LE; break;
2974 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2975 case lir_cond_greater: acond = Assembler::GT; break;
2976 case lir_cond_belowEqual: acond = Assembler::LS; break;
2977 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2978 default: ShouldNotReachHere();
2979 }
2980 __ br(acond, ok);
2981 }
2982 if (op->halt()) {
2983 const char* str = __ code_string(op->msg());
2984 __ stop(str);
2985 } else {
2986 breakpoint();
2987 }
2988 __ bind(ok);
2989 }
2990 #endif
2991
2992 #ifndef PRODUCT
2993 #define COMMENT(x) do { __ block_comment(x); } while (0)
2994 #else
2995 #define COMMENT(x)
2996 #endif
2997
2998 void LIR_Assembler::membar() {
2999 COMMENT("membar");
3000 __ membar(MacroAssembler::AnyAny);
3001 }
3002
3003 void LIR_Assembler::membar_acquire() {
3004 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3005 }
3006
3007 void LIR_Assembler::membar_release() {
3008 __ membar(Assembler::LoadStore|Assembler::StoreStore);
3009 }
3010
3011 void LIR_Assembler::membar_loadload() {
3012 __ membar(Assembler::LoadLoad);
3013 }
3014
3015 void LIR_Assembler::membar_storestore() {
3016 __ membar(MacroAssembler::StoreStore);
3017 }
3018
3019 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3020
3021 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3022
3023 void LIR_Assembler::on_spin_wait() {
3024 Unimplemented();
3025 }
3026
3027 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3028 __ mov(result_reg->as_register(), rthread);
3029 }
3030
3031
3032 void LIR_Assembler::peephole(LIR_List *lir) {
3033 #if 0
3034 if (tableswitch_count >= max_tableswitches)
3035 return;
3036
3037 /*
3038 This finite-state automaton recognizes sequences of compare-and-
3039 branch instructions. We will turn them into a tableswitch. You
3040 could argue that C1 really shouldn't be doing this sort of
3041 optimization, but without it the code is really horrible.
3042 */
3043
3044 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3045 int first_key, last_key = -2147483648;
3046 int next_key = 0;
3047 int start_insn = -1;
3048 int last_insn = -1;
3049 Register reg = noreg;
3050 LIR_Opr reg_opr;
3051 state = start_s;
3052
3053 LIR_OpList* inst = lir->instructions_list();
3054 for (int i = 0; i < inst->length(); i++) {
3055 LIR_Op* op = inst->at(i);
3056 switch (state) {
3057 case start_s:
3058 first_key = -1;
3059 start_insn = i;
3060 switch (op->code()) {
3061 case lir_cmp:
3062 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3063 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3064 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3065 && opr2->is_constant()
3066 && opr2->type() == T_INT) {
3067 reg_opr = opr1;
3068 reg = opr1->as_register();
3069 first_key = opr2->as_constant_ptr()->as_jint();
3070 next_key = first_key + 1;
3071 state = cmp_s;
3072 goto next_state;
3073 }
3074 break;
3075 }
3076 break;
3077 case cmp_s:
3078 switch (op->code()) {
3079 case lir_branch:
3080 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3081 state = beq_s;
3082 last_insn = i;
3083 goto next_state;
3084 }
3085 }
3086 state = start_s;
3087 break;
3088 case beq_s:
3089 switch (op->code()) {
3090 case lir_cmp: {
3091 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3092 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3093 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3094 && opr1->as_register() == reg
3095 && opr2->is_constant()
3096 && opr2->type() == T_INT
3097 && opr2->as_constant_ptr()->as_jint() == next_key) {
3098 last_key = next_key;
3099 next_key++;
3100 state = cmp_s;
3101 goto next_state;
3102 }
3103 }
3104 }
3105 last_key = next_key;
3106 state = start_s;
3107 break;
3108 default:
3109 assert(false, "impossible state");
3110 }
3111 if (state == start_s) {
3112 if (first_key < last_key - 5L && reg != noreg) {
3113 {
3114 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3115 // reg->encoding(),
3116 // start_insn, first_key, last_key);
3117 // for (int i = 0; i < inst->length(); i++) {
3118 // inst->at(i)->print();
3119 // tty->print("\n");
3120 // }
3121 // tty->print("\n");
3122 }
3123
3124 struct tableswitch *sw = &switches[tableswitch_count];
3125 sw->_insn_index = start_insn, sw->_first_key = first_key,
3126 sw->_last_key = last_key, sw->_reg = reg;
3127 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3128 {
3129 // Insert the new table of branches
3130 int offset = last_insn;
3131 for (int n = first_key; n < last_key; n++) {
3132 inst->insert_before
3133 (last_insn + 1,
3134 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3135 inst->at(offset)->as_OpBranch()->label()));
3136 offset -= 2, i++;
3137 }
3138 }
3139 // Delete all the old compare-and-branch instructions
3140 for (int n = first_key; n < last_key; n++) {
3141 inst->remove_at(start_insn);
3142 inst->remove_at(start_insn);
3143 }
3144 // Insert the tableswitch instruction
3145 inst->insert_before(start_insn,
3146 new LIR_Op2(lir_cmp, lir_cond_always,
3147 LIR_OprFact::intConst(tableswitch_count),
3148 reg_opr));
3149 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3150 tableswitch_count++;
3151 }
3152 reg = noreg;
3153 last_key = -2147483648;
3154 }
3155 next_state:
3156 ;
3157 }
3158 #endif
3159 }
3160
3161 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3162 Address addr = as_Address(src->as_address_ptr(), noreg);
3163 BasicType type = src->type();
3164 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3165
3166 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3167 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3168
3169 switch(type) {
3170 case T_INT:
3171 xchg = &MacroAssembler::atomic_xchgalw;
3172 add = &MacroAssembler::atomic_addalw;
3173 break;
3174 case T_LONG:
3175 xchg = &MacroAssembler::atomic_xchgal;
3176 add = &MacroAssembler::atomic_addal;
3177 break;
3178 case T_OBJECT:
3179 case T_ARRAY:
3180 if (UseCompressedOops) {
3181 xchg = &MacroAssembler::atomic_xchgalw;
3182 add = &MacroAssembler::atomic_addalw;
3183 } else {
3184 xchg = &MacroAssembler::atomic_xchgal;
3185 add = &MacroAssembler::atomic_addal;
3186 }
3187 break;
3188 default:
3189 ShouldNotReachHere();
3190 xchg = &MacroAssembler::atomic_xchgal;
3191 add = &MacroAssembler::atomic_addal; // unreachable
3192 }
3193
3194 switch (code) {
3195 case lir_xadd:
3196 {
3197 RegisterOrConstant inc;
3198 Register tmp = as_reg(tmp_op);
3199 Register dst = as_reg(dest);
3200 if (data->is_constant()) {
3201 inc = RegisterOrConstant(as_long(data));
3202 assert_different_registers(dst, addr.base(), tmp,
3203 rscratch1, rscratch2);
3204 } else {
3205 inc = RegisterOrConstant(as_reg(data));
3206 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3207 rscratch1, rscratch2);
3208 }
3209 __ lea(tmp, addr);
3210 (_masm->*add)(dst, inc, tmp);
3211 break;
3212 }
3213 case lir_xchg:
3214 {
3215 Register tmp = tmp_op->as_register();
3216 Register obj = as_reg(data);
3217 Register dst = as_reg(dest);
3218 if (is_oop && UseCompressedOops) {
3219 __ encode_heap_oop(rscratch2, obj);
3220 obj = rscratch2;
3221 }
3222 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3223 __ lea(tmp, addr);
3224 (_masm->*xchg)(dst, obj, tmp);
3225 if (is_oop && UseCompressedOops) {
3226 __ decode_heap_oop(dst);
3227 }
3228 }
3229 break;
3230 default:
3231 ShouldNotReachHere();
3232 }
3233 __ membar(__ AnyAny);
3234 }
3235
3236 #undef __