1 /*
2 * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "gc/shared/barrierSet.hpp"
37 #include "gc/shared/cardTableModRefBS.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "vmreg_aarch64.inline.hpp"
43
44
45
46 #ifndef PRODUCT
47 #define COMMENT(x) do { __ block_comment(x); } while (0)
48 #else
49 #define COMMENT(x)
50 #endif
51
52 NEEDS_CLEANUP // remove this definitions ?
53 const Register IC_Klass = rscratch2; // where the IC klass is cached
54 const Register SYNC_header = r0; // synchronization header
55 const Register SHIFT_count = r0; // where count for shift operations must be
56
57 #define __ _masm->
58
59
60 static void select_different_registers(Register preserve,
61 Register extra,
62 Register &tmp1,
63 Register &tmp2) {
64 if (tmp1 == preserve) {
65 assert_different_registers(tmp1, tmp2, extra);
66 tmp1 = extra;
67 } else if (tmp2 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp2 = extra;
70 }
71 assert_different_registers(preserve, tmp1, tmp2);
72 }
73
74
75
76 static void select_different_registers(Register preserve,
77 Register extra,
78 Register &tmp1,
79 Register &tmp2,
80 Register &tmp3) {
81 if (tmp1 == preserve) {
82 assert_different_registers(tmp1, tmp2, tmp3, extra);
83 tmp1 = extra;
84 } else if (tmp2 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp2 = extra;
87 } else if (tmp3 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp3 = extra;
90 }
91 assert_different_registers(preserve, tmp1, tmp2, tmp3);
92 }
93
94
95 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
96
97
98 LIR_Opr LIR_Assembler::receiverOpr() {
99 return FrameMap::receiver_opr;
100 }
101
102 LIR_Opr LIR_Assembler::osrBufferPointer() {
103 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
104 }
105
106 //--------------fpu register translations-----------------------
107
108
109 address LIR_Assembler::float_constant(float f) {
110 address const_addr = __ float_constant(f);
111 if (const_addr == NULL) {
112 bailout("const section overflow");
113 return __ code()->consts()->start();
114 } else {
115 return const_addr;
116 }
117 }
118
119
120 address LIR_Assembler::double_constant(double d) {
121 address const_addr = __ double_constant(d);
122 if (const_addr == NULL) {
123 bailout("const section overflow");
124 return __ code()->consts()->start();
125 } else {
126 return const_addr;
127 }
128 }
129
130 address LIR_Assembler::int_constant(jlong n) {
131 address const_addr = __ long_constant(n);
132 if (const_addr == NULL) {
133 bailout("const section overflow");
134 return __ code()->consts()->start();
135 } else {
136 return const_addr;
137 }
138 }
139
140 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
141
142 void LIR_Assembler::reset_FPU() { Unimplemented(); }
143
144 void LIR_Assembler::fpop() { Unimplemented(); }
145
146 void LIR_Assembler::fxch(int i) { Unimplemented(); }
147
148 void LIR_Assembler::fld(int i) { Unimplemented(); }
149
150 void LIR_Assembler::ffree(int i) { Unimplemented(); }
151
152 void LIR_Assembler::breakpoint() { Unimplemented(); }
153
154 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
155
156 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
157
158 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
159 //-------------------------------------------
160
161 static Register as_reg(LIR_Opr op) {
162 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
163 }
164
165 static jlong as_long(LIR_Opr data) {
166 jlong result;
167 switch (data->type()) {
168 case T_INT:
169 result = (data->as_jint());
170 break;
171 case T_LONG:
172 result = (data->as_jlong());
173 break;
174 default:
175 ShouldNotReachHere();
176 result = 0; // unreachable
177 }
178 return result;
179 }
180
181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
182 Register base = addr->base()->as_pointer_register();
183 LIR_Opr opr = addr->index();
184 if (opr->is_cpu_register()) {
185 Register index;
186 if (opr->is_single_cpu())
187 index = opr->as_register();
188 else
189 index = opr->as_register_lo();
190 assert(addr->disp() == 0, "must be");
191 switch(opr->type()) {
192 case T_INT:
193 return Address(base, index, Address::sxtw(addr->scale()));
194 case T_LONG:
195 return Address(base, index, Address::lsl(addr->scale()));
196 default:
197 ShouldNotReachHere();
198 }
199 } else {
200 intptr_t addr_offset = intptr_t(addr->disp());
201 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
202 return Address(base, addr_offset, Address::lsl(addr->scale()));
203 else {
204 __ mov(tmp, addr_offset);
205 return Address(base, tmp, Address::lsl(addr->scale()));
206 }
207 }
208 return Address();
209 }
210
211 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
212 ShouldNotReachHere();
213 return Address();
214 }
215
216 Address LIR_Assembler::as_Address(LIR_Address* addr) {
217 return as_Address(addr, rscratch1);
218 }
219
220 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
221 return as_Address(addr, rscratch1); // Ouch
222 // FIXME: This needs to be much more clever. See x86.
223 }
224
225
226 void LIR_Assembler::osr_entry() {
227 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
228 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
229 ValueStack* entry_state = osr_entry->state();
230 int number_of_locks = entry_state->locks_size();
231
232 // we jump here if osr happens with the interpreter
233 // state set up to continue at the beginning of the
234 // loop that triggered osr - in particular, we have
235 // the following registers setup:
236 //
237 // r2: osr buffer
238 //
239
240 // build frame
241 ciMethod* m = compilation()->method();
242 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
243
244 // OSR buffer is
245 //
246 // locals[nlocals-1..0]
247 // monitors[0..number_of_locks]
248 //
249 // locals is a direct copy of the interpreter frame so in the osr buffer
250 // so first slot in the local array is the last local from the interpreter
251 // and last slot is local[0] (receiver) from the interpreter
252 //
253 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
254 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
255 // in the interpreter frame (the method lock if a sync method)
256
257 // Initialize monitors in the compiled activation.
258 // r2: pointer to osr buffer
259 //
260 // All other registers are dead at this point and the locals will be
261 // copied into place by code emitted in the IR.
262
263 Register OSR_buf = osrBufferPointer()->as_pointer_register();
264 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
265 int monitor_offset = BytesPerWord * method()->max_locals() +
266 (2 * BytesPerWord) * (number_of_locks - 1);
267 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
268 // the OSR buffer using 2 word entries: first the lock and then
269 // the oop.
270 for (int i = 0; i < number_of_locks; i++) {
271 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
272 #ifdef ASSERT
273 // verify the interpreter's monitor has a non-null object
274 {
275 Label L;
276 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
277 __ cbnz(rscratch1, L);
278 __ stop("locked object is NULL");
279 __ bind(L);
280 }
281 #endif
282 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
283 __ str(r19, frame_map()->address_for_monitor_lock(i));
284 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
285 __ str(r19, frame_map()->address_for_monitor_object(i));
286 }
287 }
288 }
289
290
291 // inline cache check; done before the frame is built.
292 int LIR_Assembler::check_icache() {
293 Register receiver = FrameMap::receiver_opr->as_register();
294 Register ic_klass = IC_Klass;
295 int start_offset = __ offset();
296 __ inline_cache_check(receiver, ic_klass);
297
298 // if icache check fails, then jump to runtime routine
299 // Note: RECEIVER must still contain the receiver!
300 Label dont;
301 __ br(Assembler::EQ, dont);
302 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
303
304 // We align the verified entry point unless the method body
305 // (including its inline cache check) will fit in a single 64-byte
306 // icache line.
307 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
308 // force alignment after the cache check.
309 __ align(CodeEntryAlignment);
310 }
311
312 __ bind(dont);
313 return start_offset;
314 }
315
316
317 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
318 if (o == NULL) {
319 __ mov(reg, zr);
320 } else {
321 __ movoop(reg, o, /*immediate*/true);
322 }
323 }
324
325 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
326 address target = NULL;
327 relocInfo::relocType reloc_type = relocInfo::none;
328
329 switch (patching_id(info)) {
330 case PatchingStub::access_field_id:
331 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
332 reloc_type = relocInfo::section_word_type;
333 break;
334 case PatchingStub::load_klass_id:
335 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
336 reloc_type = relocInfo::metadata_type;
337 break;
338 case PatchingStub::load_mirror_id:
339 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
340 reloc_type = relocInfo::oop_type;
341 break;
342 case PatchingStub::load_appendix_id:
343 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
344 reloc_type = relocInfo::oop_type;
345 break;
346 default: ShouldNotReachHere();
347 }
348
349 __ far_call(RuntimeAddress(target));
350 add_call_info_here(info);
351 }
352
353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
354 deoptimize_trap(info);
355 }
356
357
358 // This specifies the rsp decrement needed to build the frame
359 int LIR_Assembler::initial_frame_size_in_bytes() const {
360 // if rounding, must let FrameMap know!
361
362 // The frame_map records size in slots (32bit word)
363
364 // subtract two words to account for return address and link
365 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
366 }
367
368
369 int LIR_Assembler::emit_exception_handler() {
370 // if the last instruction is a call (typically to do a throw which
371 // is coming at the end after block reordering) the return address
372 // must still point into the code area in order to avoid assertion
373 // failures when searching for the corresponding bci => add a nop
374 // (was bug 5/14/1999 - gri)
375 __ nop();
376
377 // generate code for exception handler
378 address handler_base = __ start_a_stub(exception_handler_size());
379 if (handler_base == NULL) {
380 // not enough space left for the handler
381 bailout("exception handler overflow");
382 return -1;
383 }
384
385 int offset = code_offset();
386
387 // the exception oop and pc are in r0, and r3
388 // no other registers need to be preserved, so invalidate them
389 __ invalidate_registers(false, true, true, false, true, true);
390
391 // check that there is really an exception
392 __ verify_not_null_oop(r0);
393
394 // search an exception handler (r0: exception oop, r3: throwing pc)
395 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here();
396 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
397 __ end_a_stub();
398
399 return offset;
400 }
401
402
403 // Emit the code to remove the frame from the stack in the exception
404 // unwind path.
405 int LIR_Assembler::emit_unwind_handler() {
406 #ifndef PRODUCT
407 if (CommentedAssembly) {
408 _masm->block_comment("Unwind handler");
409 }
410 #endif
411
412 int offset = code_offset();
413
414 // Fetch the exception from TLS and clear out exception related thread state
415 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
417 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
418
419 __ bind(_unwind_handler_entry);
420 __ verify_not_null_oop(r0);
421 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
422 __ mov(r19, r0); // Preserve the exception
423 }
424
425 // Preform needed unlocking
426 MonitorExitStub* stub = NULL;
427 if (method()->is_synchronized()) {
428 monitor_address(0, FrameMap::r0_opr);
429 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
430 __ unlock_object(r5, r4, r0, *stub->entry());
431 __ bind(*stub->continuation());
432 }
433
434 if (compilation()->env()->dtrace_method_probes()) {
435 __ call_Unimplemented();
436 #if 0
437 __ movptr(Address(rsp, 0), rax);
438 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
439 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
440 #endif
441 }
442
443 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
444 __ mov(r0, r19); // Restore the exception
445 }
446
447 // remove the activation and dispatch to the unwind handler
448 __ block_comment("remove_frame and dispatch to the unwind handler");
449 __ remove_frame(initial_frame_size_in_bytes());
450 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
451
452 // Emit the slow path assembly
453 if (stub != NULL) {
454 stub->emit_code(this);
455 }
456
457 return offset;
458 }
459
460
461 int LIR_Assembler::emit_deopt_handler() {
462 // if the last instruction is a call (typically to do a throw which
463 // is coming at the end after block reordering) the return address
464 // must still point into the code area in order to avoid assertion
465 // failures when searching for the corresponding bci => add a nop
466 // (was bug 5/14/1999 - gri)
467 __ nop();
468
469 // generate code for exception handler
470 address handler_base = __ start_a_stub(deopt_handler_size());
471 if (handler_base == NULL) {
472 // not enough space left for the handler
473 bailout("deopt handler overflow");
474 return -1;
475 }
476
477 int offset = code_offset();
478
479 __ adr(lr, pc());
480 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
481 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
482 __ end_a_stub();
483
484 return offset;
485 }
486
487 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
488 _masm->code_section()->relocate(adr, relocInfo::poll_type);
489 int pc_offset = code_offset();
490 flush_debug_info(pc_offset);
491 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
492 if (info->exception_handlers() != NULL) {
493 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
494 }
495 }
496
497 // Rather than take a segfault when the polling page is protected,
498 // explicitly check for a safepoint in progress and if there is one,
499 // fake a call to the handler as if a segfault had been caught.
500 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
501 __ mov(rscratch1, SafepointSynchronize::address_of_state());
502 __ ldrb(rscratch1, Address(rscratch1));
503 Label nope, poll;
504 __ cbz(rscratch1, nope);
505 __ block_comment("safepoint");
506 __ enter();
507 __ push(0x3, sp); // r0 & r1
508 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
509 __ adr(r0, poll);
510 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
511 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
512 __ blrt(rscratch1, 1, 0, 1);
513 __ maybe_isb();
514 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
515 __ mov(rscratch1, r0);
516 __ pop(0x3, sp); // r0 & r1
517 __ leave();
518 __ br(rscratch1);
519 address polling_page(os::get_polling_page());
520 assert(os::is_poll_address(polling_page), "should be");
521 unsigned long off;
522 __ adrp(rscratch1, Address(polling_page, rtype), off);
523 __ bind(poll);
524 if (info)
525 add_debug_info_for_branch(info); // This isn't just debug info:
526 // it's the oop map
527 else
528 __ code_section()->relocate(pc(), rtype);
529 __ ldrw(zr, Address(rscratch1, off));
530 __ bind(nope);
531 }
532
533 void LIR_Assembler::return_op(LIR_Opr result) {
534 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
535
536 // Pop the stack before the safepoint code
537 __ remove_frame(initial_frame_size_in_bytes());
538
539 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
540 __ reserved_stack_check();
541 }
542
543 address polling_page(os::get_polling_page());
544 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
545 __ ret(lr);
546 }
547
548 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
549 address polling_page(os::get_polling_page());
550 guarantee(info != NULL, "Shouldn't be NULL");
551 assert(os::is_poll_address(polling_page), "should be");
552 unsigned long off;
553 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
554 assert(off == 0, "must be");
555 add_debug_info_for_branch(info); // This isn't just debug info:
556 // it's the oop map
557 __ read_polling_page(rscratch1, relocInfo::poll_type);
558 return __ offset();
559 }
560
561
562 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
563 if (from_reg == r31_sp)
564 from_reg = sp;
565 if (to_reg == r31_sp)
566 to_reg = sp;
567 __ mov(to_reg, from_reg);
568 }
569
570 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
571
572
573 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
574 assert(src->is_constant(), "should not call otherwise");
575 assert(dest->is_register(), "should not call otherwise");
576 LIR_Const* c = src->as_constant_ptr();
577
578 switch (c->type()) {
579 case T_INT: {
580 assert(patch_code == lir_patch_none, "no patching handled here");
581 __ movw(dest->as_register(), c->as_jint());
582 break;
583 }
584
585 case T_ADDRESS: {
586 assert(patch_code == lir_patch_none, "no patching handled here");
587 __ mov(dest->as_register(), c->as_jint());
588 break;
589 }
590
591 case T_LONG: {
592 assert(patch_code == lir_patch_none, "no patching handled here");
593 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
594 break;
595 }
596
597 case T_OBJECT: {
598 if (patch_code == lir_patch_none) {
599 jobject2reg(c->as_jobject(), dest->as_register());
600 } else {
601 jobject2reg_with_patching(dest->as_register(), info);
602 }
603 break;
604 }
605
606 case T_METADATA: {
607 if (patch_code != lir_patch_none) {
608 klass2reg_with_patching(dest->as_register(), info);
609 } else {
610 __ mov_metadata(dest->as_register(), c->as_metadata());
611 }
612 break;
613 }
614
615 case T_FLOAT: {
616 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
617 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
618 } else {
619 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
620 __ ldrs(dest->as_float_reg(), Address(rscratch1));
621 }
622 break;
623 }
624
625 case T_DOUBLE: {
626 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
627 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
628 } else {
629 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
630 __ ldrd(dest->as_double_reg(), Address(rscratch1));
631 }
632 break;
633 }
634
635 default:
636 ShouldNotReachHere();
637 }
638 }
639
640 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
641 LIR_Const* c = src->as_constant_ptr();
642 switch (c->type()) {
643 case T_OBJECT:
644 {
645 if (! c->as_jobject())
646 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
647 else {
648 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
649 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
650 }
651 }
652 break;
653 case T_ADDRESS:
654 {
655 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
656 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
657 }
658 case T_INT:
659 case T_FLOAT:
660 {
661 Register reg = zr;
662 if (c->as_jint_bits() == 0)
663 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
664 else {
665 __ movw(rscratch1, c->as_jint_bits());
666 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
667 }
668 }
669 break;
670 case T_LONG:
671 case T_DOUBLE:
672 {
673 Register reg = zr;
674 if (c->as_jlong_bits() == 0)
675 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
676 lo_word_offset_in_bytes));
677 else {
678 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
679 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
680 lo_word_offset_in_bytes));
681 }
682 }
683 break;
684 default:
685 ShouldNotReachHere();
686 }
687 }
688
689 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
690 assert(src->is_constant(), "should not call otherwise");
691 LIR_Const* c = src->as_constant_ptr();
692 LIR_Address* to_addr = dest->as_address_ptr();
693
694 void (Assembler::* insn)(Register Rt, const Address &adr);
695
696 switch (type) {
697 case T_ADDRESS:
698 assert(c->as_jint() == 0, "should be");
699 insn = &Assembler::str;
700 break;
701 case T_LONG:
702 assert(c->as_jlong() == 0, "should be");
703 insn = &Assembler::str;
704 break;
705 case T_INT:
706 assert(c->as_jint() == 0, "should be");
707 insn = &Assembler::strw;
708 break;
709 case T_OBJECT:
710 case T_ARRAY:
711 assert(c->as_jobject() == 0, "should be");
712 if (UseCompressedOops && !wide) {
713 insn = &Assembler::strw;
714 } else {
715 insn = &Assembler::str;
716 }
717 break;
718 case T_CHAR:
719 case T_SHORT:
720 assert(c->as_jint() == 0, "should be");
721 insn = &Assembler::strh;
722 break;
723 case T_BOOLEAN:
724 case T_BYTE:
725 assert(c->as_jint() == 0, "should be");
726 insn = &Assembler::strb;
727 break;
728 default:
729 ShouldNotReachHere();
730 insn = &Assembler::str; // unreachable
731 }
732
733 if (info) add_debug_info_for_null_check_here(info);
734 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
735 }
736
737 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
738 assert(src->is_register(), "should not call otherwise");
739 assert(dest->is_register(), "should not call otherwise");
740
741 // move between cpu-registers
742 if (dest->is_single_cpu()) {
743 if (src->type() == T_LONG) {
744 // Can do LONG -> OBJECT
745 move_regs(src->as_register_lo(), dest->as_register());
746 return;
747 }
748 assert(src->is_single_cpu(), "must match");
749 if (src->type() == T_OBJECT) {
750 __ verify_oop(src->as_register());
751 }
752 move_regs(src->as_register(), dest->as_register());
753
754 } else if (dest->is_double_cpu()) {
755 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
756 // Surprising to me but we can see move of a long to t_object
757 __ verify_oop(src->as_register());
758 move_regs(src->as_register(), dest->as_register_lo());
759 return;
760 }
761 assert(src->is_double_cpu(), "must match");
762 Register f_lo = src->as_register_lo();
763 Register f_hi = src->as_register_hi();
764 Register t_lo = dest->as_register_lo();
765 Register t_hi = dest->as_register_hi();
766 assert(f_hi == f_lo, "must be same");
767 assert(t_hi == t_lo, "must be same");
768 move_regs(f_lo, t_lo);
769
770 } else if (dest->is_single_fpu()) {
771 __ fmovs(dest->as_float_reg(), src->as_float_reg());
772
773 } else if (dest->is_double_fpu()) {
774 __ fmovd(dest->as_double_reg(), src->as_double_reg());
775
776 } else {
777 ShouldNotReachHere();
778 }
779 }
780
781 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
782 if (src->is_single_cpu()) {
783 if (type == T_ARRAY || type == T_OBJECT) {
784 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
785 __ verify_oop(src->as_register());
786 } else if (type == T_METADATA || type == T_DOUBLE) {
787 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
788 } else {
789 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
790 }
791
792 } else if (src->is_double_cpu()) {
793 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
794 __ str(src->as_register_lo(), dest_addr_LO);
795
796 } else if (src->is_single_fpu()) {
797 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
798 __ strs(src->as_float_reg(), dest_addr);
799
800 } else if (src->is_double_fpu()) {
801 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
802 __ strd(src->as_double_reg(), dest_addr);
803
804 } else {
805 ShouldNotReachHere();
806 }
807
808 }
809
810
811 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
812 LIR_Address* to_addr = dest->as_address_ptr();
813 PatchingStub* patch = NULL;
814 Register compressed_src = rscratch1;
815
816 if (patch_code != lir_patch_none) {
817 deoptimize_trap(info);
818 return;
819 }
820
821 if (type == T_ARRAY || type == T_OBJECT) {
822 __ verify_oop(src->as_register());
823
824 if (UseCompressedOops && !wide) {
825 __ encode_heap_oop(compressed_src, src->as_register());
826 } else {
827 compressed_src = src->as_register();
828 }
829 }
830
831 int null_check_here = code_offset();
832 switch (type) {
833 case T_FLOAT: {
834 __ strs(src->as_float_reg(), as_Address(to_addr));
835 break;
836 }
837
838 case T_DOUBLE: {
839 __ strd(src->as_double_reg(), as_Address(to_addr));
840 break;
841 }
842
843 case T_ARRAY: // fall through
844 case T_OBJECT: // fall through
845 if (UseCompressedOops && !wide) {
846 __ strw(compressed_src, as_Address(to_addr, rscratch2));
847 } else {
848 __ str(compressed_src, as_Address(to_addr));
849 }
850 break;
851 case T_METADATA:
852 // We get here to store a method pointer to the stack to pass to
853 // a dtrace runtime call. This can't work on 64 bit with
854 // compressed klass ptrs: T_METADATA can be a compressed klass
855 // ptr or a 64 bit method pointer.
856 ShouldNotReachHere();
857 __ str(src->as_register(), as_Address(to_addr));
858 break;
859 case T_ADDRESS:
860 __ str(src->as_register(), as_Address(to_addr));
861 break;
862 case T_INT:
863 __ strw(src->as_register(), as_Address(to_addr));
864 break;
865
866 case T_LONG: {
867 __ str(src->as_register_lo(), as_Address_lo(to_addr));
868 break;
869 }
870
871 case T_BYTE: // fall through
872 case T_BOOLEAN: {
873 __ strb(src->as_register(), as_Address(to_addr));
874 break;
875 }
876
877 case T_CHAR: // fall through
878 case T_SHORT:
879 __ strh(src->as_register(), as_Address(to_addr));
880 break;
881
882 default:
883 ShouldNotReachHere();
884 }
885 if (info != NULL) {
886 add_debug_info_for_null_check(null_check_here, info);
887 }
888 }
889
890
891 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
892 assert(src->is_stack(), "should not call otherwise");
893 assert(dest->is_register(), "should not call otherwise");
894
895 if (dest->is_single_cpu()) {
896 if (type == T_ARRAY || type == T_OBJECT) {
897 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
898 __ verify_oop(dest->as_register());
899 } else if (type == T_METADATA) {
900 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
901 } else {
902 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
903 }
904
905 } else if (dest->is_double_cpu()) {
906 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
907 __ ldr(dest->as_register_lo(), src_addr_LO);
908
909 } else if (dest->is_single_fpu()) {
910 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
911 __ ldrs(dest->as_float_reg(), src_addr);
912
913 } else if (dest->is_double_fpu()) {
914 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
915 __ ldrd(dest->as_double_reg(), src_addr);
916
917 } else {
918 ShouldNotReachHere();
919 }
920 }
921
922
923 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
924 address target = NULL;
925 relocInfo::relocType reloc_type = relocInfo::none;
926
927 switch (patching_id(info)) {
928 case PatchingStub::access_field_id:
929 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
930 reloc_type = relocInfo::section_word_type;
931 break;
932 case PatchingStub::load_klass_id:
933 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
934 reloc_type = relocInfo::metadata_type;
935 break;
936 case PatchingStub::load_mirror_id:
937 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
938 reloc_type = relocInfo::oop_type;
939 break;
940 case PatchingStub::load_appendix_id:
941 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
942 reloc_type = relocInfo::oop_type;
943 break;
944 default: ShouldNotReachHere();
945 }
946
947 __ far_call(RuntimeAddress(target));
948 add_call_info_here(info);
949 }
950
951 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
952
953 LIR_Opr temp;
954 if (type == T_LONG || type == T_DOUBLE)
955 temp = FrameMap::rscratch1_long_opr;
956 else
957 temp = FrameMap::rscratch1_opr;
958
959 stack2reg(src, temp, src->type());
960 reg2stack(temp, dest, dest->type(), false);
961 }
962
963
964 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
965 LIR_Address* addr = src->as_address_ptr();
966 LIR_Address* from_addr = src->as_address_ptr();
967
968 if (addr->base()->type() == T_OBJECT) {
969 __ verify_oop(addr->base()->as_pointer_register());
970 }
971
972 if (patch_code != lir_patch_none) {
973 deoptimize_trap(info);
974 return;
975 }
976
977 if (info != NULL) {
978 add_debug_info_for_null_check_here(info);
979 }
980 int null_check_here = code_offset();
981 switch (type) {
982 case T_FLOAT: {
983 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
984 break;
985 }
986
987 case T_DOUBLE: {
988 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
989 break;
990 }
991
992 case T_ARRAY: // fall through
993 case T_OBJECT: // fall through
994 if (UseCompressedOops && !wide) {
995 __ ldrw(dest->as_register(), as_Address(from_addr));
996 } else {
997 __ ldr(dest->as_register(), as_Address(from_addr));
998 }
999 break;
1000 case T_METADATA:
1001 // We get here to store a method pointer to the stack to pass to
1002 // a dtrace runtime call. This can't work on 64 bit with
1003 // compressed klass ptrs: T_METADATA can be a compressed klass
1004 // ptr or a 64 bit method pointer.
1005 ShouldNotReachHere();
1006 __ ldr(dest->as_register(), as_Address(from_addr));
1007 break;
1008 case T_ADDRESS:
1009 // FIXME: OMG this is a horrible kludge. Any offset from an
1010 // address that matches klass_offset_in_bytes() will be loaded
1011 // as a word, not a long.
1012 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1013 __ ldrw(dest->as_register(), as_Address(from_addr));
1014 } else {
1015 __ ldr(dest->as_register(), as_Address(from_addr));
1016 }
1017 break;
1018 case T_INT:
1019 __ ldrw(dest->as_register(), as_Address(from_addr));
1020 break;
1021
1022 case T_LONG: {
1023 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1024 break;
1025 }
1026
1027 case T_BYTE:
1028 __ ldrsb(dest->as_register(), as_Address(from_addr));
1029 break;
1030 case T_BOOLEAN: {
1031 __ ldrb(dest->as_register(), as_Address(from_addr));
1032 break;
1033 }
1034
1035 case T_CHAR:
1036 __ ldrh(dest->as_register(), as_Address(from_addr));
1037 break;
1038 case T_SHORT:
1039 __ ldrsh(dest->as_register(), as_Address(from_addr));
1040 break;
1041
1042 default:
1043 ShouldNotReachHere();
1044 }
1045
1046 if (type == T_ARRAY || type == T_OBJECT) {
1047 if (UseCompressedOops && !wide) {
1048 __ decode_heap_oop(dest->as_register());
1049 }
1050 __ verify_oop(dest->as_register());
1051 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1052 if (UseCompressedClassPointers) {
1053 __ decode_klass_not_null(dest->as_register());
1054 }
1055 }
1056 }
1057
1058
1059 int LIR_Assembler::array_element_size(BasicType type) const {
1060 int elem_size = type2aelembytes(type);
1061 return exact_log2(elem_size);
1062 }
1063
1064 void LIR_Assembler::arithmetic_idiv(LIR_Op3* op, bool is_irem) {
1065 Register Rdividend = op->in_opr1()->as_register();
1066 Register Rdivisor = op->in_opr2()->as_register();
1067 Register Rscratch = op->in_opr3()->as_register();
1068 Register Rresult = op->result_opr()->as_register();
1069 int divisor = -1;
1070
1071 /*
1072 TODO: For some reason, using the Rscratch that gets passed in is
1073 not possible because the register allocator does not see the tmp reg
1074 as used, and assignes it the same register as Rdividend. We use rscratch1
1075 instead.
1076
1077 assert(Rdividend != Rscratch, "");
1078 assert(Rdivisor != Rscratch, "");
1079 */
1080
1081 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1082 // convert division by a power of two into some shifts and logical operations
1083 }
1084
1085 __ corrected_idivl(Rresult, Rdividend, Rdivisor, is_irem, rscratch1);
1086 }
1087
1088 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1089 switch (op->code()) {
1090 case lir_idiv:
1091 arithmetic_idiv(op, false);
1092 break;
1093 case lir_irem:
1094 arithmetic_idiv(op, true);
1095 break;
1096 case lir_fmad:
1097 __ fmaddd(op->result_opr()->as_double_reg(),
1098 op->in_opr1()->as_double_reg(),
1099 op->in_opr2()->as_double_reg(),
1100 op->in_opr3()->as_double_reg());
1101 break;
1102 case lir_fmaf:
1103 __ fmadds(op->result_opr()->as_float_reg(),
1104 op->in_opr1()->as_float_reg(),
1105 op->in_opr2()->as_float_reg(),
1106 op->in_opr3()->as_float_reg());
1107 break;
1108 default: ShouldNotReachHere(); break;
1109 }
1110 }
1111
1112 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1113 #ifdef ASSERT
1114 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1115 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1116 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1117 #endif
1118
1119 if (op->cond() == lir_cond_always) {
1120 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1121 __ b(*(op->label()));
1122 } else {
1123 Assembler::Condition acond;
1124 if (op->code() == lir_cond_float_branch) {
1125 bool is_unordered = (op->ublock() == op->block());
1126 // Assembler::EQ does not permit unordered branches, so we add
1127 // another branch here. Likewise, Assembler::NE does not permit
1128 // ordered branches.
1129 if (is_unordered && op->cond() == lir_cond_equal
1130 || !is_unordered && op->cond() == lir_cond_notEqual)
1131 __ br(Assembler::VS, *(op->ublock()->label()));
1132 switch(op->cond()) {
1133 case lir_cond_equal: acond = Assembler::EQ; break;
1134 case lir_cond_notEqual: acond = Assembler::NE; break;
1135 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1136 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1137 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1138 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1139 default: ShouldNotReachHere();
1140 acond = Assembler::EQ; // unreachable
1141 }
1142 } else {
1143 switch (op->cond()) {
1144 case lir_cond_equal: acond = Assembler::EQ; break;
1145 case lir_cond_notEqual: acond = Assembler::NE; break;
1146 case lir_cond_less: acond = Assembler::LT; break;
1147 case lir_cond_lessEqual: acond = Assembler::LE; break;
1148 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1149 case lir_cond_greater: acond = Assembler::GT; break;
1150 case lir_cond_belowEqual: acond = Assembler::LS; break;
1151 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1152 default: ShouldNotReachHere();
1153 acond = Assembler::EQ; // unreachable
1154 }
1155 }
1156 __ br(acond,*(op->label()));
1157 }
1158 }
1159
1160
1161
1162 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1163 LIR_Opr src = op->in_opr();
1164 LIR_Opr dest = op->result_opr();
1165
1166 switch (op->bytecode()) {
1167 case Bytecodes::_i2f:
1168 {
1169 __ scvtfws(dest->as_float_reg(), src->as_register());
1170 break;
1171 }
1172 case Bytecodes::_i2d:
1173 {
1174 __ scvtfwd(dest->as_double_reg(), src->as_register());
1175 break;
1176 }
1177 case Bytecodes::_l2d:
1178 {
1179 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1180 break;
1181 }
1182 case Bytecodes::_l2f:
1183 {
1184 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1185 break;
1186 }
1187 case Bytecodes::_f2d:
1188 {
1189 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1190 break;
1191 }
1192 case Bytecodes::_d2f:
1193 {
1194 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1195 break;
1196 }
1197 case Bytecodes::_i2c:
1198 {
1199 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1200 break;
1201 }
1202 case Bytecodes::_i2l:
1203 {
1204 __ sxtw(dest->as_register_lo(), src->as_register());
1205 break;
1206 }
1207 case Bytecodes::_i2s:
1208 {
1209 __ sxth(dest->as_register(), src->as_register());
1210 break;
1211 }
1212 case Bytecodes::_i2b:
1213 {
1214 __ sxtb(dest->as_register(), src->as_register());
1215 break;
1216 }
1217 case Bytecodes::_l2i:
1218 {
1219 _masm->block_comment("FIXME: This could be a no-op");
1220 __ uxtw(dest->as_register(), src->as_register_lo());
1221 break;
1222 }
1223 case Bytecodes::_d2l:
1224 {
1225 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1226 break;
1227 }
1228 case Bytecodes::_f2i:
1229 {
1230 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1231 break;
1232 }
1233 case Bytecodes::_f2l:
1234 {
1235 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1236 break;
1237 }
1238 case Bytecodes::_d2i:
1239 {
1240 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1241 break;
1242 }
1243 default: ShouldNotReachHere();
1244 }
1245 }
1246
1247 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1248 if (op->init_check()) {
1249 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1250 InstanceKlass::init_state_offset()));
1251 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1252 add_debug_info_for_null_check_here(op->stub()->info());
1253 __ br(Assembler::NE, *op->stub()->entry());
1254 }
1255 __ allocate_object(op->obj()->as_register(),
1256 op->tmp1()->as_register(),
1257 op->tmp2()->as_register(),
1258 op->header_size(),
1259 op->object_size(),
1260 op->klass()->as_register(),
1261 *op->stub()->entry());
1262 __ bind(*op->stub()->continuation());
1263 }
1264
1265 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1266 Register len = op->len()->as_register();
1267 __ uxtw(len, len);
1268
1269 if (UseSlowPath ||
1270 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1271 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1272 __ b(*op->stub()->entry());
1273 } else {
1274 Register tmp1 = op->tmp1()->as_register();
1275 Register tmp2 = op->tmp2()->as_register();
1276 Register tmp3 = op->tmp3()->as_register();
1277 if (len == tmp1) {
1278 tmp1 = tmp3;
1279 } else if (len == tmp2) {
1280 tmp2 = tmp3;
1281 } else if (len == tmp3) {
1282 // everything is ok
1283 } else {
1284 __ mov(tmp3, len);
1285 }
1286 __ allocate_array(op->obj()->as_register(),
1287 len,
1288 tmp1,
1289 tmp2,
1290 arrayOopDesc::header_size(op->type()),
1291 array_element_size(op->type()),
1292 op->klass()->as_register(),
1293 *op->stub()->entry());
1294 }
1295 __ bind(*op->stub()->continuation());
1296 }
1297
1298 void LIR_Assembler::type_profile_helper(Register mdo,
1299 ciMethodData *md, ciProfileData *data,
1300 Register recv, Label* update_done) {
1301 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1302 Label next_test;
1303 // See if the receiver is receiver[n].
1304 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1305 __ ldr(rscratch1, Address(rscratch2));
1306 __ cmp(recv, rscratch1);
1307 __ br(Assembler::NE, next_test);
1308 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1309 __ addptr(data_addr, DataLayout::counter_increment);
1310 __ b(*update_done);
1311 __ bind(next_test);
1312 }
1313
1314 // Didn't find receiver; find next empty slot and fill it in
1315 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1316 Label next_test;
1317 __ lea(rscratch2,
1318 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1319 Address recv_addr(rscratch2);
1320 __ ldr(rscratch1, recv_addr);
1321 __ cbnz(rscratch1, next_test);
1322 __ str(recv, recv_addr);
1323 __ mov(rscratch1, DataLayout::counter_increment);
1324 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1325 __ str(rscratch1, Address(rscratch2));
1326 __ b(*update_done);
1327 __ bind(next_test);
1328 }
1329 }
1330
1331 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1332 // we always need a stub for the failure case.
1333 CodeStub* stub = op->stub();
1334 Register obj = op->object()->as_register();
1335 Register k_RInfo = op->tmp1()->as_register();
1336 Register klass_RInfo = op->tmp2()->as_register();
1337 Register dst = op->result_opr()->as_register();
1338 ciKlass* k = op->klass();
1339 Register Rtmp1 = noreg;
1340
1341 // check if it needs to be profiled
1342 ciMethodData* md;
1343 ciProfileData* data;
1344
1345 const bool should_profile = op->should_profile();
1346
1347 if (should_profile) {
1348 ciMethod* method = op->profiled_method();
1349 assert(method != NULL, "Should have method");
1350 int bci = op->profiled_bci();
1351 md = method->method_data_or_null();
1352 assert(md != NULL, "Sanity");
1353 data = md->bci_to_data(bci);
1354 assert(data != NULL, "need data for type check");
1355 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1356 }
1357 Label profile_cast_success, profile_cast_failure;
1358 Label *success_target = should_profile ? &profile_cast_success : success;
1359 Label *failure_target = should_profile ? &profile_cast_failure : failure;
1360
1361 if (obj == k_RInfo) {
1362 k_RInfo = dst;
1363 } else if (obj == klass_RInfo) {
1364 klass_RInfo = dst;
1365 }
1366 if (k->is_loaded() && !UseCompressedClassPointers) {
1367 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1368 } else {
1369 Rtmp1 = op->tmp3()->as_register();
1370 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1371 }
1372
1373 assert_different_registers(obj, k_RInfo, klass_RInfo);
1374
1375 if (should_profile) {
1376 Label not_null;
1377 __ cbnz(obj, not_null);
1378 // Object is null; update MDO and exit
1379 Register mdo = klass_RInfo;
1380 __ mov_metadata(mdo, md->constant_encoding());
1381 Address data_addr
1382 = __ form_address(rscratch2, mdo,
1383 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1384 LogBytesPerWord);
1385 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1386 __ ldr(rscratch1, data_addr);
1387 __ orr(rscratch1, rscratch1, header_bits);
1388 __ str(rscratch1, data_addr);
1389 __ b(*obj_is_null);
1390 __ bind(not_null);
1391 } else {
1392 __ cbz(obj, *obj_is_null);
1393 }
1394
1395 if (!k->is_loaded()) {
1396 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1397 } else {
1398 __ mov_metadata(k_RInfo, k->constant_encoding());
1399 }
1400 __ verify_oop(obj);
1401
1402 if (op->fast_check()) {
1403 // get object class
1404 // not a safepoint as obj null check happens earlier
1405 __ load_klass(rscratch1, obj);
1406 __ cmp( rscratch1, k_RInfo);
1407
1408 __ br(Assembler::NE, *failure_target);
1409 // successful cast, fall through to profile or jump
1410 } else {
1411 // get object class
1412 // not a safepoint as obj null check happens earlier
1413 __ load_klass(klass_RInfo, obj);
1414 if (k->is_loaded()) {
1415 // See if we get an immediate positive hit
1416 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1417 __ cmp(k_RInfo, rscratch1);
1418 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1419 __ br(Assembler::NE, *failure_target);
1420 // successful cast, fall through to profile or jump
1421 } else {
1422 // See if we get an immediate positive hit
1423 __ br(Assembler::EQ, *success_target);
1424 // check for self
1425 __ cmp(klass_RInfo, k_RInfo);
1426 __ br(Assembler::EQ, *success_target);
1427
1428 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1429 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1430 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1431 // result is a boolean
1432 __ cbzw(klass_RInfo, *failure_target);
1433 // successful cast, fall through to profile or jump
1434 }
1435 } else {
1436 // perform the fast part of the checking logic
1437 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1438 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1439 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1440 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1441 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1442 // result is a boolean
1443 __ cbz(k_RInfo, *failure_target);
1444 // successful cast, fall through to profile or jump
1445 }
1446 }
1447 if (should_profile) {
1448 Register mdo = klass_RInfo, recv = k_RInfo;
1449 __ bind(profile_cast_success);
1450 __ mov_metadata(mdo, md->constant_encoding());
1451 __ load_klass(recv, obj);
1452 Label update_done;
1453 type_profile_helper(mdo, md, data, recv, success);
1454 __ b(*success);
1455
1456 __ bind(profile_cast_failure);
1457 __ mov_metadata(mdo, md->constant_encoding());
1458 Address counter_addr
1459 = __ form_address(rscratch2, mdo,
1460 md->byte_offset_of_slot(data, CounterData::count_offset()),
1461 LogBytesPerWord);
1462 __ ldr(rscratch1, counter_addr);
1463 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1464 __ str(rscratch1, counter_addr);
1465 __ b(*failure);
1466 }
1467 __ b(*success);
1468 }
1469
1470
1471 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1472 const bool should_profile = op->should_profile();
1473
1474 LIR_Code code = op->code();
1475 if (code == lir_store_check) {
1476 Register value = op->object()->as_register();
1477 Register array = op->array()->as_register();
1478 Register k_RInfo = op->tmp1()->as_register();
1479 Register klass_RInfo = op->tmp2()->as_register();
1480 Register Rtmp1 = op->tmp3()->as_register();
1481
1482 CodeStub* stub = op->stub();
1483
1484 // check if it needs to be profiled
1485 ciMethodData* md;
1486 ciProfileData* data;
1487
1488 if (should_profile) {
1489 ciMethod* method = op->profiled_method();
1490 assert(method != NULL, "Should have method");
1491 int bci = op->profiled_bci();
1492 md = method->method_data_or_null();
1493 assert(md != NULL, "Sanity");
1494 data = md->bci_to_data(bci);
1495 assert(data != NULL, "need data for type check");
1496 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1497 }
1498 Label profile_cast_success, profile_cast_failure, done;
1499 Label *success_target = should_profile ? &profile_cast_success : &done;
1500 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1501
1502 if (should_profile) {
1503 Label not_null;
1504 __ cbnz(value, not_null);
1505 // Object is null; update MDO and exit
1506 Register mdo = klass_RInfo;
1507 __ mov_metadata(mdo, md->constant_encoding());
1508 Address data_addr
1509 = __ form_address(rscratch2, mdo,
1510 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1511 LogBytesPerInt);
1512 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1513 __ ldrw(rscratch1, data_addr);
1514 __ orrw(rscratch1, rscratch1, header_bits);
1515 __ strw(rscratch1, data_addr);
1516 __ b(done);
1517 __ bind(not_null);
1518 } else {
1519 __ cbz(value, done);
1520 }
1521
1522 add_debug_info_for_null_check_here(op->info_for_exception());
1523 __ load_klass(k_RInfo, array);
1524 __ load_klass(klass_RInfo, value);
1525
1526 // get instance klass (it's already uncompressed)
1527 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1528 // perform the fast part of the checking logic
1529 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1530 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1531 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1532 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1533 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1534 // result is a boolean
1535 __ cbzw(k_RInfo, *failure_target);
1536 // fall through to the success case
1537
1538 if (should_profile) {
1539 Register mdo = klass_RInfo, recv = k_RInfo;
1540 __ bind(profile_cast_success);
1541 __ mov_metadata(mdo, md->constant_encoding());
1542 __ load_klass(recv, value);
1543 Label update_done;
1544 type_profile_helper(mdo, md, data, recv, &done);
1545 __ b(done);
1546
1547 __ bind(profile_cast_failure);
1548 __ mov_metadata(mdo, md->constant_encoding());
1549 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1550 __ lea(rscratch2, counter_addr);
1551 __ ldr(rscratch1, Address(rscratch2));
1552 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1553 __ str(rscratch1, Address(rscratch2));
1554 __ b(*stub->entry());
1555 }
1556
1557 __ bind(done);
1558 } else if (code == lir_checkcast) {
1559 Register obj = op->object()->as_register();
1560 Register dst = op->result_opr()->as_register();
1561 Label success;
1562 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1563 __ bind(success);
1564 if (dst != obj) {
1565 __ mov(dst, obj);
1566 }
1567 } else if (code == lir_instanceof) {
1568 Register obj = op->object()->as_register();
1569 Register dst = op->result_opr()->as_register();
1570 Label success, failure, done;
1571 emit_typecheck_helper(op, &success, &failure, &failure);
1572 __ bind(failure);
1573 __ mov(dst, zr);
1574 __ b(done);
1575 __ bind(success);
1576 __ mov(dst, 1);
1577 __ bind(done);
1578 } else {
1579 ShouldNotReachHere();
1580 }
1581 }
1582
1583 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1584 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1585 __ cset(rscratch1, Assembler::NE);
1586 __ membar(__ AnyAny);
1587 }
1588
1589 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1590 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1591 __ cset(rscratch1, Assembler::NE);
1592 __ membar(__ AnyAny);
1593 }
1594
1595
1596 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1597 assert(VM_Version::supports_cx8(), "wrong machine");
1598 Register addr = as_reg(op->addr());
1599 Register newval = as_reg(op->new_value());
1600 Register cmpval = as_reg(op->cmp_value());
1601 Label succeed, fail, around;
1602
1603 if (op->code() == lir_cas_obj) {
1604 if (UseCompressedOops) {
1605 Register t1 = op->tmp1()->as_register();
1606 assert(op->tmp1()->is_valid(), "must be");
1607 __ encode_heap_oop(t1, cmpval);
1608 cmpval = t1;
1609 __ encode_heap_oop(rscratch2, newval);
1610 newval = rscratch2;
1611 casw(addr, newval, cmpval);
1612 } else {
1613 casl(addr, newval, cmpval);
1614 }
1615 } else if (op->code() == lir_cas_int) {
1616 casw(addr, newval, cmpval);
1617 } else {
1618 casl(addr, newval, cmpval);
1619 }
1620 }
1621
1622
1623 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1624
1625 Assembler::Condition acond, ncond;
1626 switch (condition) {
1627 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1628 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1629 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1630 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1631 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1632 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1633 case lir_cond_belowEqual:
1634 case lir_cond_aboveEqual:
1635 default: ShouldNotReachHere();
1636 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable
1637 }
1638
1639 assert(result->is_single_cpu() || result->is_double_cpu(),
1640 "expect single register for result");
1641 if (opr1->is_constant() && opr2->is_constant()
1642 && opr1->type() == T_INT && opr2->type() == T_INT) {
1643 jint val1 = opr1->as_jint();
1644 jint val2 = opr2->as_jint();
1645 if (val1 == 0 && val2 == 1) {
1646 __ cset(result->as_register(), ncond);
1647 return;
1648 } else if (val1 == 1 && val2 == 0) {
1649 __ cset(result->as_register(), acond);
1650 return;
1651 }
1652 }
1653
1654 if (opr1->is_constant() && opr2->is_constant()
1655 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1656 jlong val1 = opr1->as_jlong();
1657 jlong val2 = opr2->as_jlong();
1658 if (val1 == 0 && val2 == 1) {
1659 __ cset(result->as_register_lo(), ncond);
1660 return;
1661 } else if (val1 == 1 && val2 == 0) {
1662 __ cset(result->as_register_lo(), acond);
1663 return;
1664 }
1665 }
1666
1667 if (opr1->is_stack()) {
1668 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1669 opr1 = FrameMap::rscratch1_opr;
1670 } else if (opr1->is_constant()) {
1671 LIR_Opr tmp
1672 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1673 const2reg(opr1, tmp, lir_patch_none, NULL);
1674 opr1 = tmp;
1675 }
1676
1677 if (opr2->is_stack()) {
1678 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1679 opr2 = FrameMap::rscratch2_opr;
1680 } else if (opr2->is_constant()) {
1681 LIR_Opr tmp
1682 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1683 const2reg(opr2, tmp, lir_patch_none, NULL);
1684 opr2 = tmp;
1685 }
1686
1687 if (result->type() == T_LONG)
1688 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1689 else
1690 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1691 }
1692
1693 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1694 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1695
1696 if (left->is_single_cpu()) {
1697 Register lreg = left->as_register();
1698 Register dreg = as_reg(dest);
1699
1700 if (right->is_single_cpu()) {
1701 // cpu register - cpu register
1702
1703 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1704 "should be");
1705 Register rreg = right->as_register();
1706 switch (code) {
1707 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1708 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1709 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1710 default: ShouldNotReachHere();
1711 }
1712
1713 } else if (right->is_double_cpu()) {
1714 Register rreg = right->as_register_lo();
1715 // single_cpu + double_cpu: can happen with obj+long
1716 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1717 switch (code) {
1718 case lir_add: __ add(dreg, lreg, rreg); break;
1719 case lir_sub: __ sub(dreg, lreg, rreg); break;
1720 default: ShouldNotReachHere();
1721 }
1722 } else if (right->is_constant()) {
1723 // cpu register - constant
1724 jlong c;
1725
1726 // FIXME. This is fugly: we really need to factor all this logic.
1727 switch(right->type()) {
1728 case T_LONG:
1729 c = right->as_constant_ptr()->as_jlong();
1730 break;
1731 case T_INT:
1732 case T_ADDRESS:
1733 c = right->as_constant_ptr()->as_jint();
1734 break;
1735 default:
1736 ShouldNotReachHere();
1737 c = 0; // unreachable
1738 break;
1739 }
1740
1741 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1742 if (c == 0 && dreg == lreg) {
1743 COMMENT("effective nop elided");
1744 return;
1745 }
1746 switch(left->type()) {
1747 case T_INT:
1748 switch (code) {
1749 case lir_add: __ addw(dreg, lreg, c); break;
1750 case lir_sub: __ subw(dreg, lreg, c); break;
1751 default: ShouldNotReachHere();
1752 }
1753 break;
1754 case T_OBJECT:
1755 case T_ADDRESS:
1756 switch (code) {
1757 case lir_add: __ add(dreg, lreg, c); break;
1758 case lir_sub: __ sub(dreg, lreg, c); break;
1759 default: ShouldNotReachHere();
1760 }
1761 break;
1762 ShouldNotReachHere();
1763 }
1764 } else {
1765 ShouldNotReachHere();
1766 }
1767
1768 } else if (left->is_double_cpu()) {
1769 Register lreg_lo = left->as_register_lo();
1770
1771 if (right->is_double_cpu()) {
1772 // cpu register - cpu register
1773 Register rreg_lo = right->as_register_lo();
1774 switch (code) {
1775 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1776 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1777 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1778 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1779 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1780 default:
1781 ShouldNotReachHere();
1782 }
1783
1784 } else if (right->is_constant()) {
1785 jlong c = right->as_constant_ptr()->as_jlong_bits();
1786 Register dreg = as_reg(dest);
1787 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1788 if (c == 0 && dreg == lreg_lo) {
1789 COMMENT("effective nop elided");
1790 return;
1791 }
1792 switch (code) {
1793 case lir_add: __ add(dreg, lreg_lo, c); break;
1794 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1795 default:
1796 ShouldNotReachHere();
1797 }
1798 } else {
1799 ShouldNotReachHere();
1800 }
1801 } else if (left->is_single_fpu()) {
1802 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1803 switch (code) {
1804 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1805 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1806 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1807 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1808 default:
1809 ShouldNotReachHere();
1810 }
1811 } else if (left->is_double_fpu()) {
1812 if (right->is_double_fpu()) {
1813 // cpu register - cpu register
1814 switch (code) {
1815 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1816 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1817 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1818 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1819 default:
1820 ShouldNotReachHere();
1821 }
1822 } else {
1823 if (right->is_constant()) {
1824 ShouldNotReachHere();
1825 }
1826 ShouldNotReachHere();
1827 }
1828 } else if (left->is_single_stack() || left->is_address()) {
1829 assert(left == dest, "left and dest must be equal");
1830 ShouldNotReachHere();
1831 } else {
1832 ShouldNotReachHere();
1833 }
1834 }
1835
1836 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1837
1838
1839 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1840 switch(code) {
1841 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1842 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1843 default : ShouldNotReachHere();
1844 }
1845 }
1846
1847 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1848
1849 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1850 Register Rleft = left->is_single_cpu() ? left->as_register() :
1851 left->as_register_lo();
1852 if (dst->is_single_cpu()) {
1853 Register Rdst = dst->as_register();
1854 if (right->is_constant()) {
1855 switch (code) {
1856 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1857 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1858 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1859 default: ShouldNotReachHere(); break;
1860 }
1861 } else {
1862 Register Rright = right->is_single_cpu() ? right->as_register() :
1863 right->as_register_lo();
1864 switch (code) {
1865 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1866 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1867 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1868 default: ShouldNotReachHere(); break;
1869 }
1870 }
1871 } else {
1872 Register Rdst = dst->as_register_lo();
1873 if (right->is_constant()) {
1874 switch (code) {
1875 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1876 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1877 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1878 default: ShouldNotReachHere(); break;
1879 }
1880 } else {
1881 Register Rright = right->is_single_cpu() ? right->as_register() :
1882 right->as_register_lo();
1883 switch (code) {
1884 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1885 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1886 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1887 default: ShouldNotReachHere(); break;
1888 }
1889 }
1890 }
1891 }
1892
1893
1894
1895 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1896
1897
1898 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1899 if (opr1->is_constant() && opr2->is_single_cpu()) {
1900 // tableswitch
1901 Register reg = as_reg(opr2);
1902 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1903 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1904 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1905 Register reg1 = as_reg(opr1);
1906 if (opr2->is_single_cpu()) {
1907 // cpu register - cpu register
1908 Register reg2 = opr2->as_register();
1909 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1910 __ cmp(reg1, reg2);
1911 } else {
1912 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1913 __ cmpw(reg1, reg2);
1914 }
1915 return;
1916 }
1917 if (opr2->is_double_cpu()) {
1918 // cpu register - cpu register
1919 Register reg2 = opr2->as_register_lo();
1920 __ cmp(reg1, reg2);
1921 return;
1922 }
1923
1924 if (opr2->is_constant()) {
1925 bool is_32bit = false; // width of register operand
1926 jlong imm;
1927
1928 switch(opr2->type()) {
1929 case T_INT:
1930 imm = opr2->as_constant_ptr()->as_jint();
1931 is_32bit = true;
1932 break;
1933 case T_LONG:
1934 imm = opr2->as_constant_ptr()->as_jlong();
1935 break;
1936 case T_ADDRESS:
1937 imm = opr2->as_constant_ptr()->as_jint();
1938 break;
1939 case T_OBJECT:
1940 case T_ARRAY:
1941 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1942 break;
1943 default:
1944 ShouldNotReachHere();
1945 imm = 0; // unreachable
1946 break;
1947 }
1948
1949 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1950 if (is_32bit)
1951 __ cmpw(reg1, imm);
1952 else
1953 __ cmp(reg1, imm);
1954 return;
1955 } else {
1956 __ mov(rscratch1, imm);
1957 if (is_32bit)
1958 __ cmpw(reg1, rscratch1);
1959 else
1960 __ cmp(reg1, rscratch1);
1961 return;
1962 }
1963 } else
1964 ShouldNotReachHere();
1965 } else if (opr1->is_single_fpu()) {
1966 FloatRegister reg1 = opr1->as_float_reg();
1967 assert(opr2->is_single_fpu(), "expect single float register");
1968 FloatRegister reg2 = opr2->as_float_reg();
1969 __ fcmps(reg1, reg2);
1970 } else if (opr1->is_double_fpu()) {
1971 FloatRegister reg1 = opr1->as_double_reg();
1972 assert(opr2->is_double_fpu(), "expect double float register");
1973 FloatRegister reg2 = opr2->as_double_reg();
1974 __ fcmpd(reg1, reg2);
1975 } else {
1976 ShouldNotReachHere();
1977 }
1978 }
1979
1980 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1981 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1982 bool is_unordered_less = (code == lir_ucmp_fd2i);
1983 if (left->is_single_fpu()) {
1984 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1985 } else if (left->is_double_fpu()) {
1986 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1987 } else {
1988 ShouldNotReachHere();
1989 }
1990 } else if (code == lir_cmp_l2i) {
1991 Label done;
1992 __ cmp(left->as_register_lo(), right->as_register_lo());
1993 __ mov(dst->as_register(), (u_int64_t)-1L);
1994 __ br(Assembler::LT, done);
1995 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1996 __ bind(done);
1997 } else {
1998 ShouldNotReachHere();
1999 }
2000 }
2001
2002
2003 void LIR_Assembler::align_call(LIR_Code code) { }
2004
2005
2006 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2007 address call = __ trampoline_call(Address(op->addr(), rtype));
2008 if (call == NULL) {
2009 bailout("trampoline stub overflow");
2010 return;
2011 }
2012 add_call_info(code_offset(), op->info());
2013 }
2014
2015
2016 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2017 address call = __ ic_call(op->addr());
2018 if (call == NULL) {
2019 bailout("trampoline stub overflow");
2020 return;
2021 }
2022 add_call_info(code_offset(), op->info());
2023 }
2024
2025
2026 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2027 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2028 ShouldNotReachHere();
2029 }
2030
2031
2032 void LIR_Assembler::emit_static_call_stub() {
2033 address call_pc = __ pc();
2034 address stub = __ start_a_stub(call_stub_size());
2035 if (stub == NULL) {
2036 bailout("static call stub overflow");
2037 return;
2038 }
2039
2040 int start = __ offset();
2041
2042 __ relocate(static_stub_Relocation::spec(call_pc));
2043 __ mov_metadata(rmethod, (Metadata*)NULL);
2044 __ movptr(rscratch1, 0);
2045 __ br(rscratch1);
2046
2047 assert(__ offset() - start <= call_stub_size(), "stub too big");
2048 __ end_a_stub();
2049 }
2050
2051
2052 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2053 assert(exceptionOop->as_register() == r0, "must match");
2054 assert(exceptionPC->as_register() == r3, "must match");
2055
2056 // exception object is not added to oop map by LinearScan
2057 // (LinearScan assumes that no oops are in fixed registers)
2058 info->add_register_oop(exceptionOop);
2059 Runtime1::StubID unwind_id;
2060
2061 // get current pc information
2062 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2063 int pc_for_athrow_offset = __ offset();
2064 InternalAddress pc_for_athrow(__ pc());
2065 __ adr(exceptionPC->as_register(), pc_for_athrow);
2066 add_call_info(pc_for_athrow_offset, info); // for exception handler
2067
2068 __ verify_not_null_oop(r0);
2069 // search an exception handler (r0: exception oop, r3: throwing pc)
2070 if (compilation()->has_fpu_code()) {
2071 unwind_id = Runtime1::handle_exception_id;
2072 } else {
2073 unwind_id = Runtime1::handle_exception_nofpu_id;
2074 }
2075 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2076
2077 // FIXME: enough room for two byte trap ????
2078 __ nop();
2079 }
2080
2081
2082 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2083 assert(exceptionOop->as_register() == r0, "must match");
2084
2085 __ b(_unwind_handler_entry);
2086 }
2087
2088
2089 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2090 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2091 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2092
2093 switch (left->type()) {
2094 case T_INT: {
2095 switch (code) {
2096 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2097 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2098 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2099 default:
2100 ShouldNotReachHere();
2101 break;
2102 }
2103 break;
2104 case T_LONG:
2105 case T_ADDRESS:
2106 case T_OBJECT:
2107 switch (code) {
2108 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2109 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2110 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2111 default:
2112 ShouldNotReachHere();
2113 break;
2114 }
2115 break;
2116 default:
2117 ShouldNotReachHere();
2118 break;
2119 }
2120 }
2121 }
2122
2123
2124 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2125 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2126 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2127
2128 switch (left->type()) {
2129 case T_INT: {
2130 switch (code) {
2131 case lir_shl: __ lslw (dreg, lreg, count); break;
2132 case lir_shr: __ asrw (dreg, lreg, count); break;
2133 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2134 default:
2135 ShouldNotReachHere();
2136 break;
2137 }
2138 break;
2139 case T_LONG:
2140 case T_ADDRESS:
2141 case T_OBJECT:
2142 switch (code) {
2143 case lir_shl: __ lsl (dreg, lreg, count); break;
2144 case lir_shr: __ asr (dreg, lreg, count); break;
2145 case lir_ushr: __ lsr (dreg, lreg, count); break;
2146 default:
2147 ShouldNotReachHere();
2148 break;
2149 }
2150 break;
2151 default:
2152 ShouldNotReachHere();
2153 break;
2154 }
2155 }
2156 }
2157
2158
2159 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2160 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2161 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2162 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2163 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2164 }
2165
2166
2167 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2168 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2169 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2170 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2171 __ mov (rscratch1, c);
2172 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2173 }
2174
2175
2176 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2177 ShouldNotReachHere();
2178 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2179 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2180 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2181 __ lea(rscratch1, __ constant_oop_address(o));
2182 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2183 }
2184
2185
2186 // This code replaces a call to arraycopy; no exception may
2187 // be thrown in this code, they must be thrown in the System.arraycopy
2188 // activation frame; we could save some checks if this would not be the case
2189 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2190 ciArrayKlass* default_type = op->expected_type();
2191 Register src = op->src()->as_register();
2192 Register dst = op->dst()->as_register();
2193 Register src_pos = op->src_pos()->as_register();
2194 Register dst_pos = op->dst_pos()->as_register();
2195 Register length = op->length()->as_register();
2196 Register tmp = op->tmp()->as_register();
2197
2198 CodeStub* stub = op->stub();
2199 int flags = op->flags();
2200 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2201 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2202
2203 // if we don't know anything, just go through the generic arraycopy
2204 if (default_type == NULL // || basic_type == T_OBJECT
2205 ) {
2206 Label done;
2207 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2208
2209 // Save the arguments in case the generic arraycopy fails and we
2210 // have to fall back to the JNI stub
2211 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2212 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2213 __ str(src, Address(sp, 4*BytesPerWord));
2214
2215 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2216 address copyfunc_addr = StubRoutines::generic_arraycopy();
2217
2218 // The arguments are in java calling convention so we shift them
2219 // to C convention
2220 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2221 __ mov(c_rarg0, j_rarg0);
2222 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2223 __ mov(c_rarg1, j_rarg1);
2224 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2225 __ mov(c_rarg2, j_rarg2);
2226 assert_different_registers(c_rarg3, j_rarg4);
2227 __ mov(c_rarg3, j_rarg3);
2228 __ mov(c_rarg4, j_rarg4);
2229 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2230 __ mov(rscratch1, RuntimeAddress(C_entry));
2231 __ blrt(rscratch1, 5, 0, 1);
2232 } else {
2233 #ifndef PRODUCT
2234 if (PrintC1Statistics) {
2235 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2236 }
2237 #endif
2238 __ far_call(RuntimeAddress(copyfunc_addr));
2239 }
2240
2241 __ cbz(r0, *stub->continuation());
2242
2243 // Reload values from the stack so they are where the stub
2244 // expects them.
2245 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2246 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2247 __ ldr(src, Address(sp, 4*BytesPerWord));
2248
2249 if (copyfunc_addr != NULL) {
2250 // r0 is -1^K where K == partial copied count
2251 __ eonw(rscratch1, r0, 0);
2252 // adjust length down and src/end pos up by partial copied count
2253 __ subw(length, length, rscratch1);
2254 __ addw(src_pos, src_pos, rscratch1);
2255 __ addw(dst_pos, dst_pos, rscratch1);
2256 }
2257 __ b(*stub->entry());
2258
2259 __ bind(*stub->continuation());
2260 return;
2261 }
2262
2263 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2264
2265 int elem_size = type2aelembytes(basic_type);
2266 int shift_amount;
2267 int scale = exact_log2(elem_size);
2268
2269 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2270 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2271 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2272 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2273
2274 // test for NULL
2275 if (flags & LIR_OpArrayCopy::src_null_check) {
2276 __ cbz(src, *stub->entry());
2277 }
2278 if (flags & LIR_OpArrayCopy::dst_null_check) {
2279 __ cbz(dst, *stub->entry());
2280 }
2281
2282 // If the compiler was not able to prove that exact type of the source or the destination
2283 // of the arraycopy is an array type, check at runtime if the source or the destination is
2284 // an instance type.
2285 if (flags & LIR_OpArrayCopy::type_check) {
2286 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2287 __ load_klass(tmp, dst);
2288 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2289 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2290 __ br(Assembler::GE, *stub->entry());
2291 }
2292
2293 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2294 __ load_klass(tmp, src);
2295 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2296 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2297 __ br(Assembler::GE, *stub->entry());
2298 }
2299 }
2300
2301 // check if negative
2302 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2303 __ cmpw(src_pos, 0);
2304 __ br(Assembler::LT, *stub->entry());
2305 }
2306 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2307 __ cmpw(dst_pos, 0);
2308 __ br(Assembler::LT, *stub->entry());
2309 }
2310
2311 if (flags & LIR_OpArrayCopy::length_positive_check) {
2312 __ cmpw(length, 0);
2313 __ br(Assembler::LT, *stub->entry());
2314 }
2315
2316 if (flags & LIR_OpArrayCopy::src_range_check) {
2317 __ addw(tmp, src_pos, length);
2318 __ ldrw(rscratch1, src_length_addr);
2319 __ cmpw(tmp, rscratch1);
2320 __ br(Assembler::HI, *stub->entry());
2321 }
2322 if (flags & LIR_OpArrayCopy::dst_range_check) {
2323 __ addw(tmp, dst_pos, length);
2324 __ ldrw(rscratch1, dst_length_addr);
2325 __ cmpw(tmp, rscratch1);
2326 __ br(Assembler::HI, *stub->entry());
2327 }
2328
2329 if (flags & LIR_OpArrayCopy::type_check) {
2330 // We don't know the array types are compatible
2331 if (basic_type != T_OBJECT) {
2332 // Simple test for basic type arrays
2333 if (UseCompressedClassPointers) {
2334 __ ldrw(tmp, src_klass_addr);
2335 __ ldrw(rscratch1, dst_klass_addr);
2336 __ cmpw(tmp, rscratch1);
2337 } else {
2338 __ ldr(tmp, src_klass_addr);
2339 __ ldr(rscratch1, dst_klass_addr);
2340 __ cmp(tmp, rscratch1);
2341 }
2342 __ br(Assembler::NE, *stub->entry());
2343 } else {
2344 // For object arrays, if src is a sub class of dst then we can
2345 // safely do the copy.
2346 Label cont, slow;
2347
2348 #define PUSH(r1, r2) \
2349 stp(r1, r2, __ pre(sp, -2 * wordSize));
2350
2351 #define POP(r1, r2) \
2352 ldp(r1, r2, __ post(sp, 2 * wordSize));
2353
2354 __ PUSH(src, dst);
2355
2356 __ load_klass(src, src);
2357 __ load_klass(dst, dst);
2358
2359 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2360
2361 __ PUSH(src, dst);
2362 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2363 __ POP(src, dst);
2364
2365 __ cbnz(src, cont);
2366
2367 __ bind(slow);
2368 __ POP(src, dst);
2369
2370 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2371 if (copyfunc_addr != NULL) { // use stub if available
2372 // src is not a sub class of dst so we have to do a
2373 // per-element check.
2374
2375 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2376 if ((flags & mask) != mask) {
2377 // Check that at least both of them object arrays.
2378 assert(flags & mask, "one of the two should be known to be an object array");
2379
2380 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2381 __ load_klass(tmp, src);
2382 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2383 __ load_klass(tmp, dst);
2384 }
2385 int lh_offset = in_bytes(Klass::layout_helper_offset());
2386 Address klass_lh_addr(tmp, lh_offset);
2387 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2388 __ ldrw(rscratch1, klass_lh_addr);
2389 __ mov(rscratch2, objArray_lh);
2390 __ eorw(rscratch1, rscratch1, rscratch2);
2391 __ cbnzw(rscratch1, *stub->entry());
2392 }
2393
2394 // Spill because stubs can use any register they like and it's
2395 // easier to restore just those that we care about.
2396 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2397 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2398 __ str(src, Address(sp, 4*BytesPerWord));
2399
2400 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2401 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2402 assert_different_registers(c_rarg0, dst, dst_pos, length);
2403 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2404 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2405 assert_different_registers(c_rarg1, dst, length);
2406 __ uxtw(c_rarg2, length);
2407 assert_different_registers(c_rarg2, dst);
2408
2409 __ load_klass(c_rarg4, dst);
2410 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2411 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2412 __ far_call(RuntimeAddress(copyfunc_addr));
2413
2414 #ifndef PRODUCT
2415 if (PrintC1Statistics) {
2416 Label failed;
2417 __ cbnz(r0, failed);
2418 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2419 __ bind(failed);
2420 }
2421 #endif
2422
2423 __ cbz(r0, *stub->continuation());
2424
2425 #ifndef PRODUCT
2426 if (PrintC1Statistics) {
2427 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2428 }
2429 #endif
2430 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2431
2432 // Restore previously spilled arguments
2433 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2434 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2435 __ ldr(src, Address(sp, 4*BytesPerWord));
2436
2437 // return value is -1^K where K is partial copied count
2438 __ eonw(rscratch1, r0, zr);
2439 // adjust length down and src/end pos up by partial copied count
2440 __ subw(length, length, rscratch1);
2441 __ addw(src_pos, src_pos, rscratch1);
2442 __ addw(dst_pos, dst_pos, rscratch1);
2443 }
2444
2445 __ b(*stub->entry());
2446
2447 __ bind(cont);
2448 __ POP(src, dst);
2449 }
2450 }
2451
2452 #ifdef ASSERT
2453 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2454 // Sanity check the known type with the incoming class. For the
2455 // primitive case the types must match exactly with src.klass and
2456 // dst.klass each exactly matching the default type. For the
2457 // object array case, if no type check is needed then either the
2458 // dst type is exactly the expected type and the src type is a
2459 // subtype which we can't check or src is the same array as dst
2460 // but not necessarily exactly of type default_type.
2461 Label known_ok, halt;
2462 __ mov_metadata(tmp, default_type->constant_encoding());
2463 if (UseCompressedClassPointers) {
2464 __ encode_klass_not_null(tmp);
2465 }
2466
2467 if (basic_type != T_OBJECT) {
2468
2469 if (UseCompressedClassPointers) {
2470 __ ldrw(rscratch1, dst_klass_addr);
2471 __ cmpw(tmp, rscratch1);
2472 } else {
2473 __ ldr(rscratch1, dst_klass_addr);
2474 __ cmp(tmp, rscratch1);
2475 }
2476 __ br(Assembler::NE, halt);
2477 if (UseCompressedClassPointers) {
2478 __ ldrw(rscratch1, src_klass_addr);
2479 __ cmpw(tmp, rscratch1);
2480 } else {
2481 __ ldr(rscratch1, src_klass_addr);
2482 __ cmp(tmp, rscratch1);
2483 }
2484 __ br(Assembler::EQ, known_ok);
2485 } else {
2486 if (UseCompressedClassPointers) {
2487 __ ldrw(rscratch1, dst_klass_addr);
2488 __ cmpw(tmp, rscratch1);
2489 } else {
2490 __ ldr(rscratch1, dst_klass_addr);
2491 __ cmp(tmp, rscratch1);
2492 }
2493 __ br(Assembler::EQ, known_ok);
2494 __ cmp(src, dst);
2495 __ br(Assembler::EQ, known_ok);
2496 }
2497 __ bind(halt);
2498 __ stop("incorrect type information in arraycopy");
2499 __ bind(known_ok);
2500 }
2501 #endif
2502
2503 #ifndef PRODUCT
2504 if (PrintC1Statistics) {
2505 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2506 }
2507 #endif
2508
2509 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2510 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2511 assert_different_registers(c_rarg0, dst, dst_pos, length);
2512 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2513 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2514 assert_different_registers(c_rarg1, dst, length);
2515 __ uxtw(c_rarg2, length);
2516 assert_different_registers(c_rarg2, dst);
2517
2518 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2519 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2520 const char *name;
2521 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2522
2523 CodeBlob *cb = CodeCache::find_blob(entry);
2524 if (cb) {
2525 __ far_call(RuntimeAddress(entry));
2526 } else {
2527 __ call_VM_leaf(entry, 3);
2528 }
2529
2530 __ bind(*stub->continuation());
2531 }
2532
2533
2534
2535
2536 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2537 Register obj = op->obj_opr()->as_register(); // may not be an oop
2538 Register hdr = op->hdr_opr()->as_register();
2539 Register lock = op->lock_opr()->as_register();
2540 if (!UseFastLocking) {
2541 __ b(*op->stub()->entry());
2542 } else if (op->code() == lir_lock) {
2543 Register scratch = noreg;
2544 if (UseBiasedLocking) {
2545 scratch = op->scratch_opr()->as_register();
2546 }
2547 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2548 // add debug info for NullPointerException only if one is possible
2549 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2550 if (op->info() != NULL) {
2551 add_debug_info_for_null_check(null_check_offset, op->info());
2552 }
2553 // done
2554 } else if (op->code() == lir_unlock) {
2555 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2556 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2557 } else {
2558 Unimplemented();
2559 }
2560 __ bind(*op->stub()->continuation());
2561 }
2562
2563
2564 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2565 ciMethod* method = op->profiled_method();
2566 int bci = op->profiled_bci();
2567 ciMethod* callee = op->profiled_callee();
2568
2569 // Update counter for all call types
2570 ciMethodData* md = method->method_data_or_null();
2571 assert(md != NULL, "Sanity");
2572 ciProfileData* data = md->bci_to_data(bci);
2573 assert(data->is_CounterData(), "need CounterData for calls");
2574 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2575 Register mdo = op->mdo()->as_register();
2576 __ mov_metadata(mdo, md->constant_encoding());
2577 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2578 Bytecodes::Code bc = method->java_code_at_bci(bci);
2579 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2580 // Perform additional virtual call profiling for invokevirtual and
2581 // invokeinterface bytecodes
2582 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2583 !callee_is_static && // required for optimized MH invokes
2584 C1ProfileVirtualCalls) {
2585 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2586 Register recv = op->recv()->as_register();
2587 assert_different_registers(mdo, recv);
2588 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2589 ciKlass* known_klass = op->known_holder();
2590 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2591 // We know the type that will be seen at this call site; we can
2592 // statically update the MethodData* rather than needing to do
2593 // dynamic tests on the receiver type
2594
2595 // NOTE: we should probably put a lock around this search to
2596 // avoid collisions by concurrent compilations
2597 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2598 uint i;
2599 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2600 ciKlass* receiver = vc_data->receiver(i);
2601 if (known_klass->equals(receiver)) {
2602 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2603 __ addptr(data_addr, DataLayout::counter_increment);
2604 return;
2605 }
2606 }
2607
2608 // Receiver type not found in profile data; select an empty slot
2609
2610 // Note that this is less efficient than it should be because it
2611 // always does a write to the receiver part of the
2612 // VirtualCallData rather than just the first time
2613 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2614 ciKlass* receiver = vc_data->receiver(i);
2615 if (receiver == NULL) {
2616 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2617 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2618 __ lea(rscratch2, recv_addr);
2619 __ str(rscratch1, Address(rscratch2));
2620 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2621 __ addptr(data_addr, DataLayout::counter_increment);
2622 return;
2623 }
2624 }
2625 } else {
2626 __ load_klass(recv, recv);
2627 Label update_done;
2628 type_profile_helper(mdo, md, data, recv, &update_done);
2629 // Receiver did not match any saved receiver and there is no empty row for it.
2630 // Increment total counter to indicate polymorphic case.
2631 __ addptr(counter_addr, DataLayout::counter_increment);
2632
2633 __ bind(update_done);
2634 }
2635 } else {
2636 // Static call
2637 __ addptr(counter_addr, DataLayout::counter_increment);
2638 }
2639 }
2640
2641
2642 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2643 Unimplemented();
2644 }
2645
2646
2647 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2648 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2649 }
2650
2651 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2652 assert(op->crc()->is_single_cpu(), "crc must be register");
2653 assert(op->val()->is_single_cpu(), "byte value must be register");
2654 assert(op->result_opr()->is_single_cpu(), "result must be register");
2655 Register crc = op->crc()->as_register();
2656 Register val = op->val()->as_register();
2657 Register res = op->result_opr()->as_register();
2658
2659 assert_different_registers(val, crc, res);
2660 unsigned long offset;
2661 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2662 if (offset) __ add(res, res, offset);
2663
2664 __ ornw(crc, zr, crc); // ~crc
2665 __ update_byte_crc32(crc, val, res);
2666 __ ornw(res, zr, crc); // ~crc
2667 }
2668
2669 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2670 COMMENT("emit_profile_type {");
2671 Register obj = op->obj()->as_register();
2672 Register tmp = op->tmp()->as_pointer_register();
2673 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2674 ciKlass* exact_klass = op->exact_klass();
2675 intptr_t current_klass = op->current_klass();
2676 bool not_null = op->not_null();
2677 bool no_conflict = op->no_conflict();
2678
2679 Label update, next, none;
2680
2681 bool do_null = !not_null;
2682 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2683 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2684
2685 assert(do_null || do_update, "why are we here?");
2686 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2687 assert(mdo_addr.base() != rscratch1, "wrong register");
2688
2689 __ verify_oop(obj);
2690
2691 if (tmp != obj) {
2692 __ mov(tmp, obj);
2693 }
2694 if (do_null) {
2695 __ cbnz(tmp, update);
2696 if (!TypeEntries::was_null_seen(current_klass)) {
2697 __ ldr(rscratch2, mdo_addr);
2698 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2699 __ str(rscratch2, mdo_addr);
2700 }
2701 if (do_update) {
2702 #ifndef ASSERT
2703 __ b(next);
2704 }
2705 #else
2706 __ b(next);
2707 }
2708 } else {
2709 __ cbnz(tmp, update);
2710 __ stop("unexpected null obj");
2711 #endif
2712 }
2713
2714 __ bind(update);
2715
2716 if (do_update) {
2717 #ifdef ASSERT
2718 if (exact_klass != NULL) {
2719 Label ok;
2720 __ load_klass(tmp, tmp);
2721 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2722 __ eor(rscratch1, tmp, rscratch1);
2723 __ cbz(rscratch1, ok);
2724 __ stop("exact klass and actual klass differ");
2725 __ bind(ok);
2726 }
2727 #endif
2728 if (!no_conflict) {
2729 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2730 if (exact_klass != NULL) {
2731 __ mov_metadata(tmp, exact_klass->constant_encoding());
2732 } else {
2733 __ load_klass(tmp, tmp);
2734 }
2735
2736 __ ldr(rscratch2, mdo_addr);
2737 __ eor(tmp, tmp, rscratch2);
2738 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2739 // klass seen before, nothing to do. The unknown bit may have been
2740 // set already but no need to check.
2741 __ cbz(rscratch1, next);
2742
2743 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2744 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2745
2746 if (TypeEntries::is_type_none(current_klass)) {
2747 __ cbz(rscratch2, none);
2748 __ cmp(rscratch2, TypeEntries::null_seen);
2749 __ br(Assembler::EQ, none);
2750 // There is a chance that the checks above (re-reading profiling
2751 // data from memory) fail if another thread has just set the
2752 // profiling to this obj's klass
2753 __ dmb(Assembler::ISHLD);
2754 __ ldr(rscratch2, mdo_addr);
2755 __ eor(tmp, tmp, rscratch2);
2756 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2757 __ cbz(rscratch1, next);
2758 }
2759 } else {
2760 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2761 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2762
2763 __ ldr(tmp, mdo_addr);
2764 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2765 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2766 }
2767
2768 // different than before. Cannot keep accurate profile.
2769 __ ldr(rscratch2, mdo_addr);
2770 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2771 __ str(rscratch2, mdo_addr);
2772
2773 if (TypeEntries::is_type_none(current_klass)) {
2774 __ b(next);
2775
2776 __ bind(none);
2777 // first time here. Set profile type.
2778 __ str(tmp, mdo_addr);
2779 }
2780 } else {
2781 // There's a single possible klass at this profile point
2782 assert(exact_klass != NULL, "should be");
2783 if (TypeEntries::is_type_none(current_klass)) {
2784 __ mov_metadata(tmp, exact_klass->constant_encoding());
2785 __ ldr(rscratch2, mdo_addr);
2786 __ eor(tmp, tmp, rscratch2);
2787 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2788 __ cbz(rscratch1, next);
2789 #ifdef ASSERT
2790 {
2791 Label ok;
2792 __ ldr(rscratch1, mdo_addr);
2793 __ cbz(rscratch1, ok);
2794 __ cmp(rscratch1, TypeEntries::null_seen);
2795 __ br(Assembler::EQ, ok);
2796 // may have been set by another thread
2797 __ dmb(Assembler::ISHLD);
2798 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2799 __ ldr(rscratch2, mdo_addr);
2800 __ eor(rscratch2, rscratch1, rscratch2);
2801 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2802 __ cbz(rscratch2, ok);
2803
2804 __ stop("unexpected profiling mismatch");
2805 __ bind(ok);
2806 }
2807 #endif
2808 // first time here. Set profile type.
2809 __ ldr(tmp, mdo_addr);
2810 } else {
2811 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2812 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2813
2814 __ ldr(tmp, mdo_addr);
2815 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2816 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2817
2818 __ orr(tmp, tmp, TypeEntries::type_unknown);
2819 __ str(tmp, mdo_addr);
2820 // FIXME: Write barrier needed here?
2821 }
2822 }
2823
2824 __ bind(next);
2825 }
2826 COMMENT("} emit_profile_type");
2827 }
2828
2829
2830 void LIR_Assembler::align_backward_branch_target() {
2831 }
2832
2833
2834 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2835 if (left->is_single_cpu()) {
2836 assert(dest->is_single_cpu(), "expect single result reg");
2837 __ negw(dest->as_register(), left->as_register());
2838 } else if (left->is_double_cpu()) {
2839 assert(dest->is_double_cpu(), "expect double result reg");
2840 __ neg(dest->as_register_lo(), left->as_register_lo());
2841 } else if (left->is_single_fpu()) {
2842 assert(dest->is_single_fpu(), "expect single float result reg");
2843 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2844 } else {
2845 assert(left->is_double_fpu(), "expect double float operand reg");
2846 assert(dest->is_double_fpu(), "expect double float result reg");
2847 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2848 }
2849 }
2850
2851
2852 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2853 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2854 }
2855
2856
2857 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2858 assert(!tmp->is_valid(), "don't need temporary");
2859
2860 CodeBlob *cb = CodeCache::find_blob(dest);
2861 if (cb) {
2862 __ far_call(RuntimeAddress(dest));
2863 } else {
2864 __ mov(rscratch1, RuntimeAddress(dest));
2865 int len = args->length();
2866 int type = 0;
2867 if (! result->is_illegal()) {
2868 switch (result->type()) {
2869 case T_VOID:
2870 type = 0;
2871 break;
2872 case T_INT:
2873 case T_LONG:
2874 case T_OBJECT:
2875 type = 1;
2876 break;
2877 case T_FLOAT:
2878 type = 2;
2879 break;
2880 case T_DOUBLE:
2881 type = 3;
2882 break;
2883 default:
2884 ShouldNotReachHere();
2885 break;
2886 }
2887 }
2888 int num_gpargs = 0;
2889 int num_fpargs = 0;
2890 for (int i = 0; i < args->length(); i++) {
2891 LIR_Opr arg = args->at(i);
2892 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2893 num_fpargs++;
2894 } else {
2895 num_gpargs++;
2896 }
2897 }
2898 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2899 }
2900
2901 if (info != NULL) {
2902 add_call_info_here(info);
2903 }
2904 __ maybe_isb();
2905 }
2906
2907 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2908 if (dest->is_address() || src->is_address()) {
2909 move_op(src, dest, type, lir_patch_none, info,
2910 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2911 } else {
2912 ShouldNotReachHere();
2913 }
2914 }
2915
2916 #ifdef ASSERT
2917 // emit run-time assertion
2918 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2919 assert(op->code() == lir_assert, "must be");
2920
2921 if (op->in_opr1()->is_valid()) {
2922 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2923 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2924 } else {
2925 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2926 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2927 }
2928
2929 Label ok;
2930 if (op->condition() != lir_cond_always) {
2931 Assembler::Condition acond = Assembler::AL;
2932 switch (op->condition()) {
2933 case lir_cond_equal: acond = Assembler::EQ; break;
2934 case lir_cond_notEqual: acond = Assembler::NE; break;
2935 case lir_cond_less: acond = Assembler::LT; break;
2936 case lir_cond_lessEqual: acond = Assembler::LE; break;
2937 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2938 case lir_cond_greater: acond = Assembler::GT; break;
2939 case lir_cond_belowEqual: acond = Assembler::LS; break;
2940 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2941 default: ShouldNotReachHere();
2942 }
2943 __ br(acond, ok);
2944 }
2945 if (op->halt()) {
2946 const char* str = __ code_string(op->msg());
2947 __ stop(str);
2948 } else {
2949 breakpoint();
2950 }
2951 __ bind(ok);
2952 }
2953 #endif
2954
2955 #ifndef PRODUCT
2956 #define COMMENT(x) do { __ block_comment(x); } while (0)
2957 #else
2958 #define COMMENT(x)
2959 #endif
2960
2961 void LIR_Assembler::membar() {
2962 COMMENT("membar");
2963 __ membar(MacroAssembler::AnyAny);
2964 }
2965
2966 void LIR_Assembler::membar_acquire() {
2967 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2968 }
2969
2970 void LIR_Assembler::membar_release() {
2971 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2972 }
2973
2974 void LIR_Assembler::membar_loadload() {
2975 __ membar(Assembler::LoadLoad);
2976 }
2977
2978 void LIR_Assembler::membar_storestore() {
2979 __ membar(MacroAssembler::StoreStore);
2980 }
2981
2982 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2983
2984 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2985
2986 void LIR_Assembler::on_spin_wait() {
2987 Unimplemented();
2988 }
2989
2990 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2991 __ mov(result_reg->as_register(), rthread);
2992 }
2993
2994
2995 void LIR_Assembler::peephole(LIR_List *lir) {
2996 #if 0
2997 if (tableswitch_count >= max_tableswitches)
2998 return;
2999
3000 /*
3001 This finite-state automaton recognizes sequences of compare-and-
3002 branch instructions. We will turn them into a tableswitch. You
3003 could argue that C1 really shouldn't be doing this sort of
3004 optimization, but without it the code is really horrible.
3005 */
3006
3007 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3008 int first_key, last_key = -2147483648;
3009 int next_key = 0;
3010 int start_insn = -1;
3011 int last_insn = -1;
3012 Register reg = noreg;
3013 LIR_Opr reg_opr;
3014 state = start_s;
3015
3016 LIR_OpList* inst = lir->instructions_list();
3017 for (int i = 0; i < inst->length(); i++) {
3018 LIR_Op* op = inst->at(i);
3019 switch (state) {
3020 case start_s:
3021 first_key = -1;
3022 start_insn = i;
3023 switch (op->code()) {
3024 case lir_cmp:
3025 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3026 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3027 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3028 && opr2->is_constant()
3029 && opr2->type() == T_INT) {
3030 reg_opr = opr1;
3031 reg = opr1->as_register();
3032 first_key = opr2->as_constant_ptr()->as_jint();
3033 next_key = first_key + 1;
3034 state = cmp_s;
3035 goto next_state;
3036 }
3037 break;
3038 }
3039 break;
3040 case cmp_s:
3041 switch (op->code()) {
3042 case lir_branch:
3043 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3044 state = beq_s;
3045 last_insn = i;
3046 goto next_state;
3047 }
3048 }
3049 state = start_s;
3050 break;
3051 case beq_s:
3052 switch (op->code()) {
3053 case lir_cmp: {
3054 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3055 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3056 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3057 && opr1->as_register() == reg
3058 && opr2->is_constant()
3059 && opr2->type() == T_INT
3060 && opr2->as_constant_ptr()->as_jint() == next_key) {
3061 last_key = next_key;
3062 next_key++;
3063 state = cmp_s;
3064 goto next_state;
3065 }
3066 }
3067 }
3068 last_key = next_key;
3069 state = start_s;
3070 break;
3071 default:
3072 assert(false, "impossible state");
3073 }
3074 if (state == start_s) {
3075 if (first_key < last_key - 5L && reg != noreg) {
3076 {
3077 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3078 // reg->encoding(),
3079 // start_insn, first_key, last_key);
3080 // for (int i = 0; i < inst->length(); i++) {
3081 // inst->at(i)->print();
3082 // tty->print("\n");
3083 // }
3084 // tty->print("\n");
3085 }
3086
3087 struct tableswitch *sw = &switches[tableswitch_count];
3088 sw->_insn_index = start_insn, sw->_first_key = first_key,
3089 sw->_last_key = last_key, sw->_reg = reg;
3090 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3091 {
3092 // Insert the new table of branches
3093 int offset = last_insn;
3094 for (int n = first_key; n < last_key; n++) {
3095 inst->insert_before
3096 (last_insn + 1,
3097 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3098 inst->at(offset)->as_OpBranch()->label()));
3099 offset -= 2, i++;
3100 }
3101 }
3102 // Delete all the old compare-and-branch instructions
3103 for (int n = first_key; n < last_key; n++) {
3104 inst->remove_at(start_insn);
3105 inst->remove_at(start_insn);
3106 }
3107 // Insert the tableswitch instruction
3108 inst->insert_before(start_insn,
3109 new LIR_Op2(lir_cmp, lir_cond_always,
3110 LIR_OprFact::intConst(tableswitch_count),
3111 reg_opr));
3112 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3113 tableswitch_count++;
3114 }
3115 reg = noreg;
3116 last_key = -2147483648;
3117 }
3118 next_state:
3119 ;
3120 }
3121 #endif
3122 }
3123
3124 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3125 Address addr = as_Address(src->as_address_ptr());
3126 BasicType type = src->type();
3127 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3128
3129 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3130 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3131
3132 switch(type) {
3133 case T_INT:
3134 xchg = &MacroAssembler::atomic_xchgalw;
3135 add = &MacroAssembler::atomic_addalw;
3136 break;
3137 case T_LONG:
3138 xchg = &MacroAssembler::atomic_xchgal;
3139 add = &MacroAssembler::atomic_addal;
3140 break;
3141 case T_OBJECT:
3142 case T_ARRAY:
3143 if (UseCompressedOops) {
3144 xchg = &MacroAssembler::atomic_xchgalw;
3145 add = &MacroAssembler::atomic_addalw;
3146 } else {
3147 xchg = &MacroAssembler::atomic_xchgal;
3148 add = &MacroAssembler::atomic_addal;
3149 }
3150 break;
3151 default:
3152 ShouldNotReachHere();
3153 xchg = &MacroAssembler::atomic_xchgal;
3154 add = &MacroAssembler::atomic_addal; // unreachable
3155 }
3156
3157 switch (code) {
3158 case lir_xadd:
3159 {
3160 RegisterOrConstant inc;
3161 Register tmp = as_reg(tmp_op);
3162 Register dst = as_reg(dest);
3163 if (data->is_constant()) {
3164 inc = RegisterOrConstant(as_long(data));
3165 assert_different_registers(dst, addr.base(), tmp,
3166 rscratch1, rscratch2);
3167 } else {
3168 inc = RegisterOrConstant(as_reg(data));
3169 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3170 rscratch1, rscratch2);
3171 }
3172 __ lea(tmp, addr);
3173 (_masm->*add)(dst, inc, tmp);
3174 break;
3175 }
3176 case lir_xchg:
3177 {
3178 Register tmp = tmp_op->as_register();
3179 Register obj = as_reg(data);
3180 Register dst = as_reg(dest);
3181 if (is_oop && UseCompressedOops) {
3182 __ encode_heap_oop(rscratch2, obj);
3183 obj = rscratch2;
3184 }
3185 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3186 __ lea(tmp, addr);
3187 (_masm->*xchg)(dst, obj, tmp);
3188 if (is_oop && UseCompressedOops) {
3189 __ decode_heap_oop(dst);
3190 }
3191 }
3192 break;
3193 default:
3194 ShouldNotReachHere();
3195 }
3196 __ membar(__ AnyAny);
3197 }
3198
3199 #undef __