1 /*
2 * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "gc/shared/barrierSet.hpp"
37 #include "gc/shared/cardTableModRefBS.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "vmreg_aarch64.inline.hpp"
43
44
45
46 #ifndef PRODUCT
47 #define COMMENT(x) do { __ block_comment(x); } while (0)
48 #else
49 #define COMMENT(x)
50 #endif
51
52 NEEDS_CLEANUP // remove this definitions ?
53 const Register IC_Klass = rscratch2; // where the IC klass is cached
54 const Register SYNC_header = r0; // synchronization header
55 const Register SHIFT_count = r0; // where count for shift operations must be
56
57 #define __ _masm->
58
59
60 static void select_different_registers(Register preserve,
61 Register extra,
62 Register &tmp1,
63 Register &tmp2) {
64 if (tmp1 == preserve) {
65 assert_different_registers(tmp1, tmp2, extra);
66 tmp1 = extra;
67 } else if (tmp2 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp2 = extra;
70 }
71 assert_different_registers(preserve, tmp1, tmp2);
72 }
73
74
75
76 static void select_different_registers(Register preserve,
77 Register extra,
78 Register &tmp1,
79 Register &tmp2,
80 Register &tmp3) {
81 if (tmp1 == preserve) {
82 assert_different_registers(tmp1, tmp2, tmp3, extra);
83 tmp1 = extra;
84 } else if (tmp2 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp2 = extra;
87 } else if (tmp3 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp3 = extra;
90 }
91 assert_different_registers(preserve, tmp1, tmp2, tmp3);
92 }
93
94
95 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
96
97
98 LIR_Opr LIR_Assembler::receiverOpr() {
99 return FrameMap::receiver_opr;
100 }
101
102 LIR_Opr LIR_Assembler::osrBufferPointer() {
103 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
104 }
105
106 //--------------fpu register translations-----------------------
107
108
109 address LIR_Assembler::float_constant(float f) {
110 address const_addr = __ float_constant(f);
111 if (const_addr == NULL) {
112 bailout("const section overflow");
113 return __ code()->consts()->start();
114 } else {
115 return const_addr;
116 }
117 }
118
119
120 address LIR_Assembler::double_constant(double d) {
121 address const_addr = __ double_constant(d);
122 if (const_addr == NULL) {
123 bailout("const section overflow");
124 return __ code()->consts()->start();
125 } else {
126 return const_addr;
127 }
128 }
129
130 address LIR_Assembler::int_constant(jlong n) {
131 address const_addr = __ long_constant(n);
132 if (const_addr == NULL) {
133 bailout("const section overflow");
134 return __ code()->consts()->start();
135 } else {
136 return const_addr;
137 }
138 }
139
140 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
141
142 void LIR_Assembler::reset_FPU() { Unimplemented(); }
143
144 void LIR_Assembler::fpop() { Unimplemented(); }
145
146 void LIR_Assembler::fxch(int i) { Unimplemented(); }
147
148 void LIR_Assembler::fld(int i) { Unimplemented(); }
149
150 void LIR_Assembler::ffree(int i) { Unimplemented(); }
151
152 void LIR_Assembler::breakpoint() { Unimplemented(); }
153
154 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
155
156 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
157
158 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
159 //-------------------------------------------
160
161 static Register as_reg(LIR_Opr op) {
162 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
163 }
164
165 static jlong as_long(LIR_Opr data) {
166 jlong result;
167 switch (data->type()) {
168 case T_INT:
169 result = (data->as_jint());
170 break;
171 case T_LONG:
172 result = (data->as_jlong());
173 break;
174 default:
175 ShouldNotReachHere();
176 result = 0; // unreachable
177 }
178 return result;
179 }
180
181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
182 Register base = addr->base()->as_pointer_register();
183 LIR_Opr opr = addr->index();
184 if (opr->is_cpu_register()) {
185 Register index;
186 if (opr->is_single_cpu())
187 index = opr->as_register();
188 else
189 index = opr->as_register_lo();
190 assert(addr->disp() == 0, "must be");
191 switch(opr->type()) {
192 case T_INT:
193 return Address(base, index, Address::sxtw(addr->scale()));
194 case T_LONG:
195 return Address(base, index, Address::lsl(addr->scale()));
196 default:
197 ShouldNotReachHere();
198 }
199 } else {
200 intptr_t addr_offset = intptr_t(addr->disp());
201 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
202 return Address(base, addr_offset, Address::lsl(addr->scale()));
203 else {
204 __ mov(tmp, addr_offset);
205 return Address(base, tmp, Address::lsl(addr->scale()));
206 }
207 }
208 return Address();
209 }
210
211 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
212 ShouldNotReachHere();
213 return Address();
214 }
215
216 Address LIR_Assembler::as_Address(LIR_Address* addr) {
217 return as_Address(addr, rscratch1);
218 }
219
220 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
221 return as_Address(addr, rscratch1); // Ouch
222 // FIXME: This needs to be much more clever. See x86.
223 }
224
225
226 void LIR_Assembler::osr_entry() {
227 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
228 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
229 ValueStack* entry_state = osr_entry->state();
230 int number_of_locks = entry_state->locks_size();
231
232 // we jump here if osr happens with the interpreter
233 // state set up to continue at the beginning of the
234 // loop that triggered osr - in particular, we have
235 // the following registers setup:
236 //
237 // r2: osr buffer
238 //
239
240 // build frame
241 ciMethod* m = compilation()->method();
242 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
243
244 // OSR buffer is
245 //
246 // locals[nlocals-1..0]
247 // monitors[0..number_of_locks]
248 //
249 // locals is a direct copy of the interpreter frame so in the osr buffer
250 // so first slot in the local array is the last local from the interpreter
251 // and last slot is local[0] (receiver) from the interpreter
252 //
253 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
254 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
255 // in the interpreter frame (the method lock if a sync method)
256
257 // Initialize monitors in the compiled activation.
258 // r2: pointer to osr buffer
259 //
260 // All other registers are dead at this point and the locals will be
261 // copied into place by code emitted in the IR.
262
263 Register OSR_buf = osrBufferPointer()->as_pointer_register();
264 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
265 int monitor_offset = BytesPerWord * method()->max_locals() +
266 (2 * BytesPerWord) * (number_of_locks - 1);
267 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
268 // the OSR buffer using 2 word entries: first the lock and then
269 // the oop.
270 for (int i = 0; i < number_of_locks; i++) {
271 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
272 #ifdef ASSERT
273 // verify the interpreter's monitor has a non-null object
274 {
275 Label L;
276 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
277 __ cbnz(rscratch1, L);
278 __ stop("locked object is NULL");
279 __ bind(L);
280 }
281 #endif
282 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
283 __ str(r19, frame_map()->address_for_monitor_lock(i));
284 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
285 __ str(r19, frame_map()->address_for_monitor_object(i));
286 }
287 }
288 }
289
290
291 // inline cache check; done before the frame is built.
292 int LIR_Assembler::check_icache() {
293 Register receiver = FrameMap::receiver_opr->as_register();
294 Register ic_klass = IC_Klass;
295 int start_offset = __ offset();
296 __ inline_cache_check(receiver, ic_klass);
297
298 // if icache check fails, then jump to runtime routine
299 // Note: RECEIVER must still contain the receiver!
300 Label dont;
301 __ br(Assembler::EQ, dont);
302 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
303
304 // We align the verified entry point unless the method body
305 // (including its inline cache check) will fit in a single 64-byte
306 // icache line.
307 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
308 // force alignment after the cache check.
309 __ align(CodeEntryAlignment);
310 }
311
312 __ bind(dont);
313 return start_offset;
314 }
315
316
317 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
318 if (o == NULL) {
319 __ mov(reg, zr);
320 } else {
321 __ movoop(reg, o, /*immediate*/true);
322 }
323 }
324
325 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
326 address target = NULL;
327 relocInfo::relocType reloc_type = relocInfo::none;
328
329 switch (patching_id(info)) {
330 case PatchingStub::access_field_id:
331 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
332 reloc_type = relocInfo::section_word_type;
333 break;
334 case PatchingStub::load_klass_id:
335 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
336 reloc_type = relocInfo::metadata_type;
337 break;
338 case PatchingStub::load_mirror_id:
339 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
340 reloc_type = relocInfo::oop_type;
341 break;
342 case PatchingStub::load_appendix_id:
343 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
344 reloc_type = relocInfo::oop_type;
345 break;
346 default: ShouldNotReachHere();
347 }
348
349 __ far_call(RuntimeAddress(target));
350 add_call_info_here(info);
351 }
352
353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
354 deoptimize_trap(info);
355 }
356
357
358 // This specifies the rsp decrement needed to build the frame
359 int LIR_Assembler::initial_frame_size_in_bytes() const {
360 // if rounding, must let FrameMap know!
361
362 // The frame_map records size in slots (32bit word)
363
364 // subtract two words to account for return address and link
365 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
366 }
367
368
369 int LIR_Assembler::emit_exception_handler() {
370 // if the last instruction is a call (typically to do a throw which
371 // is coming at the end after block reordering) the return address
372 // must still point into the code area in order to avoid assertion
373 // failures when searching for the corresponding bci => add a nop
374 // (was bug 5/14/1999 - gri)
375 __ nop();
376
377 // generate code for exception handler
378 address handler_base = __ start_a_stub(exception_handler_size);
379 if (handler_base == NULL) {
380 // not enough space left for the handler
381 bailout("exception handler overflow");
382 return -1;
383 }
384
385 int offset = code_offset();
386
387 // the exception oop and pc are in r0, and r3
388 // no other registers need to be preserved, so invalidate them
389 __ invalidate_registers(false, true, true, false, true, true);
390
391 // check that there is really an exception
392 __ verify_not_null_oop(r0);
393
394 // search an exception handler (r0: exception oop, r3: throwing pc)
395 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here();
396 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
397 __ end_a_stub();
398
399 return offset;
400 }
401
402
403 // Emit the code to remove the frame from the stack in the exception
404 // unwind path.
405 int LIR_Assembler::emit_unwind_handler() {
406 #ifndef PRODUCT
407 if (CommentedAssembly) {
408 _masm->block_comment("Unwind handler");
409 }
410 #endif
411
412 int offset = code_offset();
413
414 // Fetch the exception from TLS and clear out exception related thread state
415 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
417 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
418
419 __ bind(_unwind_handler_entry);
420 __ verify_not_null_oop(r0);
421 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
422 __ mov(r19, r0); // Preserve the exception
423 }
424
425 // Preform needed unlocking
426 MonitorExitStub* stub = NULL;
427 if (method()->is_synchronized()) {
428 monitor_address(0, FrameMap::r0_opr);
429 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
430 __ unlock_object(r5, r4, r0, *stub->entry());
431 __ bind(*stub->continuation());
432 }
433
434 if (compilation()->env()->dtrace_method_probes()) {
435 __ call_Unimplemented();
436 #if 0
437 __ movptr(Address(rsp, 0), rax);
438 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
439 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
440 #endif
441 }
442
443 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
444 __ mov(r0, r19); // Restore the exception
445 }
446
447 // remove the activation and dispatch to the unwind handler
448 __ block_comment("remove_frame and dispatch to the unwind handler");
449 __ remove_frame(initial_frame_size_in_bytes());
450 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
451
452 // Emit the slow path assembly
453 if (stub != NULL) {
454 stub->emit_code(this);
455 }
456
457 return offset;
458 }
459
460
461 int LIR_Assembler::emit_deopt_handler() {
462 // if the last instruction is a call (typically to do a throw which
463 // is coming at the end after block reordering) the return address
464 // must still point into the code area in order to avoid assertion
465 // failures when searching for the corresponding bci => add a nop
466 // (was bug 5/14/1999 - gri)
467 __ nop();
468
469 // generate code for exception handler
470 address handler_base = __ start_a_stub(deopt_handler_size);
471 if (handler_base == NULL) {
472 // not enough space left for the handler
473 bailout("deopt handler overflow");
474 return -1;
475 }
476
477 int offset = code_offset();
478
479 __ adr(lr, pc());
480 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
481 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
482 __ end_a_stub();
483
484 return offset;
485 }
486
487 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
488 _masm->code_section()->relocate(adr, relocInfo::poll_type);
489 int pc_offset = code_offset();
490 flush_debug_info(pc_offset);
491 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
492 if (info->exception_handlers() != NULL) {
493 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
494 }
495 }
496
497 // Rather than take a segfault when the polling page is protected,
498 // explicitly check for a safepoint in progress and if there is one,
499 // fake a call to the handler as if a segfault had been caught.
500 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
501 __ mov(rscratch1, SafepointSynchronize::address_of_state());
502 __ ldrb(rscratch1, Address(rscratch1));
503 Label nope, poll;
504 __ cbz(rscratch1, nope);
505 __ block_comment("safepoint");
506 __ enter();
507 __ push(0x3, sp); // r0 & r1
508 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
509 __ adr(r0, poll);
510 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
511 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
512 __ blrt(rscratch1, 1, 0, 1);
513 __ maybe_isb();
514 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
515 __ mov(rscratch1, r0);
516 __ pop(0x3, sp); // r0 & r1
517 __ leave();
518 __ br(rscratch1);
519 address polling_page(os::get_polling_page());
520 assert(os::is_poll_address(polling_page), "should be");
521 unsigned long off;
522 __ adrp(rscratch1, Address(polling_page, rtype), off);
523 __ bind(poll);
524 if (info)
525 add_debug_info_for_branch(info); // This isn't just debug info:
526 // it's the oop map
527 else
528 __ code_section()->relocate(pc(), rtype);
529 __ ldrw(zr, Address(rscratch1, off));
530 __ bind(nope);
531 }
532
533 void LIR_Assembler::return_op(LIR_Opr result) {
534 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
535 // Pop the stack before the safepoint code
536 __ remove_frame(initial_frame_size_in_bytes());
537 address polling_page(os::get_polling_page());
538 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
539 __ ret(lr);
540 }
541
542 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
543 address polling_page(os::get_polling_page());
544 guarantee(info != NULL, "Shouldn't be NULL");
545 assert(os::is_poll_address(polling_page), "should be");
546 unsigned long off;
547 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
548 assert(off == 0, "must be");
549 add_debug_info_for_branch(info); // This isn't just debug info:
550 // it's the oop map
551 __ read_polling_page(rscratch1, relocInfo::poll_type);
552 return __ offset();
553 }
554
555
556 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
557 if (from_reg == r31_sp)
558 from_reg = sp;
559 if (to_reg == r31_sp)
560 to_reg = sp;
561 __ mov(to_reg, from_reg);
562 }
563
564 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
565
566
567 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
568 assert(src->is_constant(), "should not call otherwise");
569 assert(dest->is_register(), "should not call otherwise");
570 LIR_Const* c = src->as_constant_ptr();
571
572 switch (c->type()) {
573 case T_INT: {
574 assert(patch_code == lir_patch_none, "no patching handled here");
575 __ movw(dest->as_register(), c->as_jint());
576 break;
577 }
578
579 case T_ADDRESS: {
580 assert(patch_code == lir_patch_none, "no patching handled here");
581 __ mov(dest->as_register(), c->as_jint());
582 break;
583 }
584
585 case T_LONG: {
586 assert(patch_code == lir_patch_none, "no patching handled here");
587 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
588 break;
589 }
590
591 case T_OBJECT: {
592 if (patch_code == lir_patch_none) {
593 jobject2reg(c->as_jobject(), dest->as_register());
594 } else {
595 jobject2reg_with_patching(dest->as_register(), info);
596 }
597 break;
598 }
599
600 case T_METADATA: {
601 if (patch_code != lir_patch_none) {
602 klass2reg_with_patching(dest->as_register(), info);
603 } else {
604 __ mov_metadata(dest->as_register(), c->as_metadata());
605 }
606 break;
607 }
608
609 case T_FLOAT: {
610 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
611 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
612 } else {
613 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
614 __ ldrs(dest->as_float_reg(), Address(rscratch1));
615 }
616 break;
617 }
618
619 case T_DOUBLE: {
620 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
621 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
622 } else {
623 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
624 __ ldrd(dest->as_double_reg(), Address(rscratch1));
625 }
626 break;
627 }
628
629 default:
630 ShouldNotReachHere();
631 }
632 }
633
634 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
635 LIR_Const* c = src->as_constant_ptr();
636 switch (c->type()) {
637 case T_OBJECT:
638 {
639 if (! c->as_jobject())
640 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
641 else {
642 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
643 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
644 }
645 }
646 break;
647 case T_ADDRESS:
648 {
649 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
650 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
651 }
652 case T_INT:
653 case T_FLOAT:
654 {
655 Register reg = zr;
656 if (c->as_jint_bits() == 0)
657 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
658 else {
659 __ movw(rscratch1, c->as_jint_bits());
660 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
661 }
662 }
663 break;
664 case T_LONG:
665 case T_DOUBLE:
666 {
667 Register reg = zr;
668 if (c->as_jlong_bits() == 0)
669 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
670 lo_word_offset_in_bytes));
671 else {
672 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
673 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
674 lo_word_offset_in_bytes));
675 }
676 }
677 break;
678 default:
679 ShouldNotReachHere();
680 }
681 }
682
683 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
684 assert(src->is_constant(), "should not call otherwise");
685 LIR_Const* c = src->as_constant_ptr();
686 LIR_Address* to_addr = dest->as_address_ptr();
687
688 void (Assembler::* insn)(Register Rt, const Address &adr);
689
690 switch (type) {
691 case T_ADDRESS:
692 assert(c->as_jint() == 0, "should be");
693 insn = &Assembler::str;
694 break;
695 case T_LONG:
696 assert(c->as_jlong() == 0, "should be");
697 insn = &Assembler::str;
698 break;
699 case T_INT:
700 assert(c->as_jint() == 0, "should be");
701 insn = &Assembler::strw;
702 break;
703 case T_OBJECT:
704 case T_ARRAY:
705 assert(c->as_jobject() == 0, "should be");
706 if (UseCompressedOops && !wide) {
707 insn = &Assembler::strw;
708 } else {
709 insn = &Assembler::str;
710 }
711 break;
712 case T_CHAR:
713 case T_SHORT:
714 assert(c->as_jint() == 0, "should be");
715 insn = &Assembler::strh;
716 break;
717 case T_BOOLEAN:
718 case T_BYTE:
719 assert(c->as_jint() == 0, "should be");
720 insn = &Assembler::strb;
721 break;
722 default:
723 ShouldNotReachHere();
724 insn = &Assembler::str; // unreachable
725 }
726
727 if (info) add_debug_info_for_null_check_here(info);
728 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
729 }
730
731 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
732 assert(src->is_register(), "should not call otherwise");
733 assert(dest->is_register(), "should not call otherwise");
734
735 // move between cpu-registers
736 if (dest->is_single_cpu()) {
737 if (src->type() == T_LONG) {
738 // Can do LONG -> OBJECT
739 move_regs(src->as_register_lo(), dest->as_register());
740 return;
741 }
742 assert(src->is_single_cpu(), "must match");
743 if (src->type() == T_OBJECT) {
744 __ verify_oop(src->as_register());
745 }
746 move_regs(src->as_register(), dest->as_register());
747
748 } else if (dest->is_double_cpu()) {
749 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
750 // Surprising to me but we can see move of a long to t_object
751 __ verify_oop(src->as_register());
752 move_regs(src->as_register(), dest->as_register_lo());
753 return;
754 }
755 assert(src->is_double_cpu(), "must match");
756 Register f_lo = src->as_register_lo();
757 Register f_hi = src->as_register_hi();
758 Register t_lo = dest->as_register_lo();
759 Register t_hi = dest->as_register_hi();
760 assert(f_hi == f_lo, "must be same");
761 assert(t_hi == t_lo, "must be same");
762 move_regs(f_lo, t_lo);
763
764 } else if (dest->is_single_fpu()) {
765 __ fmovs(dest->as_float_reg(), src->as_float_reg());
766
767 } else if (dest->is_double_fpu()) {
768 __ fmovd(dest->as_double_reg(), src->as_double_reg());
769
770 } else {
771 ShouldNotReachHere();
772 }
773 }
774
775 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
776 if (src->is_single_cpu()) {
777 if (type == T_ARRAY || type == T_OBJECT) {
778 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
779 __ verify_oop(src->as_register());
780 } else if (type == T_METADATA || type == T_DOUBLE) {
781 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
782 } else {
783 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
784 }
785
786 } else if (src->is_double_cpu()) {
787 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
788 __ str(src->as_register_lo(), dest_addr_LO);
789
790 } else if (src->is_single_fpu()) {
791 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
792 __ strs(src->as_float_reg(), dest_addr);
793
794 } else if (src->is_double_fpu()) {
795 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
796 __ strd(src->as_double_reg(), dest_addr);
797
798 } else {
799 ShouldNotReachHere();
800 }
801
802 }
803
804
805 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
806 LIR_Address* to_addr = dest->as_address_ptr();
807 PatchingStub* patch = NULL;
808 Register compressed_src = rscratch1;
809
810 if (patch_code != lir_patch_none) {
811 deoptimize_trap(info);
812 return;
813 }
814
815 if (type == T_ARRAY || type == T_OBJECT) {
816 __ verify_oop(src->as_register());
817
818 if (UseCompressedOops && !wide) {
819 __ encode_heap_oop(compressed_src, src->as_register());
820 } else {
821 compressed_src = src->as_register();
822 }
823 }
824
825 int null_check_here = code_offset();
826 switch (type) {
827 case T_FLOAT: {
828 __ strs(src->as_float_reg(), as_Address(to_addr));
829 break;
830 }
831
832 case T_DOUBLE: {
833 __ strd(src->as_double_reg(), as_Address(to_addr));
834 break;
835 }
836
837 case T_ARRAY: // fall through
838 case T_OBJECT: // fall through
839 if (UseCompressedOops && !wide) {
840 __ strw(compressed_src, as_Address(to_addr, rscratch2));
841 } else {
842 __ str(compressed_src, as_Address(to_addr));
843 }
844 break;
845 case T_METADATA:
846 // We get here to store a method pointer to the stack to pass to
847 // a dtrace runtime call. This can't work on 64 bit with
848 // compressed klass ptrs: T_METADATA can be a compressed klass
849 // ptr or a 64 bit method pointer.
850 ShouldNotReachHere();
851 __ str(src->as_register(), as_Address(to_addr));
852 break;
853 case T_ADDRESS:
854 __ str(src->as_register(), as_Address(to_addr));
855 break;
856 case T_INT:
857 __ strw(src->as_register(), as_Address(to_addr));
858 break;
859
860 case T_LONG: {
861 __ str(src->as_register_lo(), as_Address_lo(to_addr));
862 break;
863 }
864
865 case T_BYTE: // fall through
866 case T_BOOLEAN: {
867 __ strb(src->as_register(), as_Address(to_addr));
868 break;
869 }
870
871 case T_CHAR: // fall through
872 case T_SHORT:
873 __ strh(src->as_register(), as_Address(to_addr));
874 break;
875
876 default:
877 ShouldNotReachHere();
878 }
879 if (info != NULL) {
880 add_debug_info_for_null_check(null_check_here, info);
881 }
882 }
883
884
885 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
886 assert(src->is_stack(), "should not call otherwise");
887 assert(dest->is_register(), "should not call otherwise");
888
889 if (dest->is_single_cpu()) {
890 if (type == T_ARRAY || type == T_OBJECT) {
891 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
892 __ verify_oop(dest->as_register());
893 } else if (type == T_METADATA) {
894 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
895 } else {
896 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
897 }
898
899 } else if (dest->is_double_cpu()) {
900 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
901 __ ldr(dest->as_register_lo(), src_addr_LO);
902
903 } else if (dest->is_single_fpu()) {
904 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
905 __ ldrs(dest->as_float_reg(), src_addr);
906
907 } else if (dest->is_double_fpu()) {
908 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
909 __ ldrd(dest->as_double_reg(), src_addr);
910
911 } else {
912 ShouldNotReachHere();
913 }
914 }
915
916
917 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
918 address target = NULL;
919 relocInfo::relocType reloc_type = relocInfo::none;
920
921 switch (patching_id(info)) {
922 case PatchingStub::access_field_id:
923 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
924 reloc_type = relocInfo::section_word_type;
925 break;
926 case PatchingStub::load_klass_id:
927 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
928 reloc_type = relocInfo::metadata_type;
929 break;
930 case PatchingStub::load_mirror_id:
931 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
932 reloc_type = relocInfo::oop_type;
933 break;
934 case PatchingStub::load_appendix_id:
935 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
936 reloc_type = relocInfo::oop_type;
937 break;
938 default: ShouldNotReachHere();
939 }
940
941 __ far_call(RuntimeAddress(target));
942 add_call_info_here(info);
943 }
944
945 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
946
947 LIR_Opr temp;
948 if (type == T_LONG || type == T_DOUBLE)
949 temp = FrameMap::rscratch1_long_opr;
950 else
951 temp = FrameMap::rscratch1_opr;
952
953 stack2reg(src, temp, src->type());
954 reg2stack(temp, dest, dest->type(), false);
955 }
956
957
958 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
959 LIR_Address* addr = src->as_address_ptr();
960 LIR_Address* from_addr = src->as_address_ptr();
961
962 if (addr->base()->type() == T_OBJECT) {
963 __ verify_oop(addr->base()->as_pointer_register());
964 }
965
966 if (patch_code != lir_patch_none) {
967 deoptimize_trap(info);
968 return;
969 }
970
971 if (info != NULL) {
972 add_debug_info_for_null_check_here(info);
973 }
974 int null_check_here = code_offset();
975 switch (type) {
976 case T_FLOAT: {
977 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
978 break;
979 }
980
981 case T_DOUBLE: {
982 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
983 break;
984 }
985
986 case T_ARRAY: // fall through
987 case T_OBJECT: // fall through
988 if (UseCompressedOops && !wide) {
989 __ ldrw(dest->as_register(), as_Address(from_addr));
990 } else {
991 __ ldr(dest->as_register(), as_Address(from_addr));
992 }
993 break;
994 case T_METADATA:
995 // We get here to store a method pointer to the stack to pass to
996 // a dtrace runtime call. This can't work on 64 bit with
997 // compressed klass ptrs: T_METADATA can be a compressed klass
998 // ptr or a 64 bit method pointer.
999 ShouldNotReachHere();
1000 __ ldr(dest->as_register(), as_Address(from_addr));
1001 break;
1002 case T_ADDRESS:
1003 // FIXME: OMG this is a horrible kludge. Any offset from an
1004 // address that matches klass_offset_in_bytes() will be loaded
1005 // as a word, not a long.
1006 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1007 __ ldrw(dest->as_register(), as_Address(from_addr));
1008 } else {
1009 __ ldr(dest->as_register(), as_Address(from_addr));
1010 }
1011 break;
1012 case T_INT:
1013 __ ldrw(dest->as_register(), as_Address(from_addr));
1014 break;
1015
1016 case T_LONG: {
1017 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1018 break;
1019 }
1020
1021 case T_BYTE:
1022 __ ldrsb(dest->as_register(), as_Address(from_addr));
1023 break;
1024 case T_BOOLEAN: {
1025 __ ldrb(dest->as_register(), as_Address(from_addr));
1026 break;
1027 }
1028
1029 case T_CHAR:
1030 __ ldrh(dest->as_register(), as_Address(from_addr));
1031 break;
1032 case T_SHORT:
1033 __ ldrsh(dest->as_register(), as_Address(from_addr));
1034 break;
1035
1036 default:
1037 ShouldNotReachHere();
1038 }
1039
1040 if (type == T_ARRAY || type == T_OBJECT) {
1041 if (UseCompressedOops && !wide) {
1042 __ decode_heap_oop(dest->as_register());
1043 }
1044 __ verify_oop(dest->as_register());
1045 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1046 if (UseCompressedClassPointers) {
1047 __ decode_klass_not_null(dest->as_register());
1048 }
1049 }
1050 }
1051
1052
1053 int LIR_Assembler::array_element_size(BasicType type) const {
1054 int elem_size = type2aelembytes(type);
1055 return exact_log2(elem_size);
1056 }
1057
1058 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1059 Register Rdividend = op->in_opr1()->as_register();
1060 Register Rdivisor = op->in_opr2()->as_register();
1061 Register Rscratch = op->in_opr3()->as_register();
1062 Register Rresult = op->result_opr()->as_register();
1063 int divisor = -1;
1064
1065 /*
1066 TODO: For some reason, using the Rscratch that gets passed in is
1067 not possible because the register allocator does not see the tmp reg
1068 as used, and assignes it the same register as Rdividend. We use rscratch1
1069 instead.
1070
1071 assert(Rdividend != Rscratch, "");
1072 assert(Rdivisor != Rscratch, "");
1073 */
1074
1075 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1076 // convert division by a power of two into some shifts and logical operations
1077 }
1078
1079 if (op->code() == lir_irem) {
1080 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1081 } else if (op->code() == lir_idiv) {
1082 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1083 } else
1084 ShouldNotReachHere();
1085 }
1086
1087 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1088 #ifdef ASSERT
1089 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1090 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1091 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1092 #endif
1093
1094 if (op->cond() == lir_cond_always) {
1095 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1096 __ b(*(op->label()));
1097 } else {
1098 Assembler::Condition acond;
1099 if (op->code() == lir_cond_float_branch) {
1100 bool is_unordered = (op->ublock() == op->block());
1101 // Assembler::EQ does not permit unordered branches, so we add
1102 // another branch here. Likewise, Assembler::NE does not permit
1103 // ordered branches.
1104 if (is_unordered && op->cond() == lir_cond_equal
1105 || !is_unordered && op->cond() == lir_cond_notEqual)
1106 __ br(Assembler::VS, *(op->ublock()->label()));
1107 switch(op->cond()) {
1108 case lir_cond_equal: acond = Assembler::EQ; break;
1109 case lir_cond_notEqual: acond = Assembler::NE; break;
1110 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1111 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1112 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1113 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1114 default: ShouldNotReachHere();
1115 acond = Assembler::EQ; // unreachable
1116 }
1117 } else {
1118 switch (op->cond()) {
1119 case lir_cond_equal: acond = Assembler::EQ; break;
1120 case lir_cond_notEqual: acond = Assembler::NE; break;
1121 case lir_cond_less: acond = Assembler::LT; break;
1122 case lir_cond_lessEqual: acond = Assembler::LE; break;
1123 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1124 case lir_cond_greater: acond = Assembler::GT; break;
1125 case lir_cond_belowEqual: acond = Assembler::LS; break;
1126 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1127 default: ShouldNotReachHere();
1128 acond = Assembler::EQ; // unreachable
1129 }
1130 }
1131 __ br(acond,*(op->label()));
1132 }
1133 }
1134
1135
1136
1137 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1138 LIR_Opr src = op->in_opr();
1139 LIR_Opr dest = op->result_opr();
1140
1141 switch (op->bytecode()) {
1142 case Bytecodes::_i2f:
1143 {
1144 __ scvtfws(dest->as_float_reg(), src->as_register());
1145 break;
1146 }
1147 case Bytecodes::_i2d:
1148 {
1149 __ scvtfwd(dest->as_double_reg(), src->as_register());
1150 break;
1151 }
1152 case Bytecodes::_l2d:
1153 {
1154 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1155 break;
1156 }
1157 case Bytecodes::_l2f:
1158 {
1159 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1160 break;
1161 }
1162 case Bytecodes::_f2d:
1163 {
1164 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1165 break;
1166 }
1167 case Bytecodes::_d2f:
1168 {
1169 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1170 break;
1171 }
1172 case Bytecodes::_i2c:
1173 {
1174 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1175 break;
1176 }
1177 case Bytecodes::_i2l:
1178 {
1179 __ sxtw(dest->as_register_lo(), src->as_register());
1180 break;
1181 }
1182 case Bytecodes::_i2s:
1183 {
1184 __ sxth(dest->as_register(), src->as_register());
1185 break;
1186 }
1187 case Bytecodes::_i2b:
1188 {
1189 __ sxtb(dest->as_register(), src->as_register());
1190 break;
1191 }
1192 case Bytecodes::_l2i:
1193 {
1194 _masm->block_comment("FIXME: This could be a no-op");
1195 __ uxtw(dest->as_register(), src->as_register_lo());
1196 break;
1197 }
1198 case Bytecodes::_d2l:
1199 {
1200 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1201 break;
1202 }
1203 case Bytecodes::_f2i:
1204 {
1205 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1206 break;
1207 }
1208 case Bytecodes::_f2l:
1209 {
1210 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1211 break;
1212 }
1213 case Bytecodes::_d2i:
1214 {
1215 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1216 break;
1217 }
1218 default: ShouldNotReachHere();
1219 }
1220 }
1221
1222 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1223 if (op->init_check()) {
1224 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1225 InstanceKlass::init_state_offset()));
1226 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1227 add_debug_info_for_null_check_here(op->stub()->info());
1228 __ br(Assembler::NE, *op->stub()->entry());
1229 }
1230 __ allocate_object(op->obj()->as_register(),
1231 op->tmp1()->as_register(),
1232 op->tmp2()->as_register(),
1233 op->header_size(),
1234 op->object_size(),
1235 op->klass()->as_register(),
1236 *op->stub()->entry());
1237 __ bind(*op->stub()->continuation());
1238 }
1239
1240 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1241 Register len = op->len()->as_register();
1242 __ uxtw(len, len);
1243
1244 if (UseSlowPath ||
1245 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1246 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1247 __ b(*op->stub()->entry());
1248 } else {
1249 Register tmp1 = op->tmp1()->as_register();
1250 Register tmp2 = op->tmp2()->as_register();
1251 Register tmp3 = op->tmp3()->as_register();
1252 if (len == tmp1) {
1253 tmp1 = tmp3;
1254 } else if (len == tmp2) {
1255 tmp2 = tmp3;
1256 } else if (len == tmp3) {
1257 // everything is ok
1258 } else {
1259 __ mov(tmp3, len);
1260 }
1261 __ allocate_array(op->obj()->as_register(),
1262 len,
1263 tmp1,
1264 tmp2,
1265 arrayOopDesc::header_size(op->type()),
1266 array_element_size(op->type()),
1267 op->klass()->as_register(),
1268 *op->stub()->entry());
1269 }
1270 __ bind(*op->stub()->continuation());
1271 }
1272
1273 void LIR_Assembler::type_profile_helper(Register mdo,
1274 ciMethodData *md, ciProfileData *data,
1275 Register recv, Label* update_done) {
1276 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1277 Label next_test;
1278 // See if the receiver is receiver[n].
1279 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1280 __ ldr(rscratch1, Address(rscratch2));
1281 __ cmp(recv, rscratch1);
1282 __ br(Assembler::NE, next_test);
1283 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1284 __ addptr(data_addr, DataLayout::counter_increment);
1285 __ b(*update_done);
1286 __ bind(next_test);
1287 }
1288
1289 // Didn't find receiver; find next empty slot and fill it in
1290 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1291 Label next_test;
1292 __ lea(rscratch2,
1293 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1294 Address recv_addr(rscratch2);
1295 __ ldr(rscratch1, recv_addr);
1296 __ cbnz(rscratch1, next_test);
1297 __ str(recv, recv_addr);
1298 __ mov(rscratch1, DataLayout::counter_increment);
1299 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1300 __ str(rscratch1, Address(rscratch2));
1301 __ b(*update_done);
1302 __ bind(next_test);
1303 }
1304 }
1305
1306 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1307 // we always need a stub for the failure case.
1308 CodeStub* stub = op->stub();
1309 Register obj = op->object()->as_register();
1310 Register k_RInfo = op->tmp1()->as_register();
1311 Register klass_RInfo = op->tmp2()->as_register();
1312 Register dst = op->result_opr()->as_register();
1313 ciKlass* k = op->klass();
1314 Register Rtmp1 = noreg;
1315
1316 // check if it needs to be profiled
1317 ciMethodData* md;
1318 ciProfileData* data;
1319
1320 const bool should_profile = op->should_profile();
1321
1322 if (should_profile) {
1323 ciMethod* method = op->profiled_method();
1324 assert(method != NULL, "Should have method");
1325 int bci = op->profiled_bci();
1326 md = method->method_data_or_null();
1327 assert(md != NULL, "Sanity");
1328 data = md->bci_to_data(bci);
1329 assert(data != NULL, "need data for type check");
1330 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1331 }
1332 Label profile_cast_success, profile_cast_failure;
1333 Label *success_target = should_profile ? &profile_cast_success : success;
1334 Label *failure_target = should_profile ? &profile_cast_failure : failure;
1335
1336 if (obj == k_RInfo) {
1337 k_RInfo = dst;
1338 } else if (obj == klass_RInfo) {
1339 klass_RInfo = dst;
1340 }
1341 if (k->is_loaded() && !UseCompressedClassPointers) {
1342 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1343 } else {
1344 Rtmp1 = op->tmp3()->as_register();
1345 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1346 }
1347
1348 assert_different_registers(obj, k_RInfo, klass_RInfo);
1349
1350 if (should_profile) {
1351 Label not_null;
1352 __ cbnz(obj, not_null);
1353 // Object is null; update MDO and exit
1354 Register mdo = klass_RInfo;
1355 __ mov_metadata(mdo, md->constant_encoding());
1356 Address data_addr
1357 = __ form_address(rscratch2, mdo,
1358 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1359 LogBytesPerWord);
1360 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1361 __ ldr(rscratch1, data_addr);
1362 __ orr(rscratch1, rscratch1, header_bits);
1363 __ str(rscratch1, data_addr);
1364 __ b(*obj_is_null);
1365 __ bind(not_null);
1366 } else {
1367 __ cbz(obj, *obj_is_null);
1368 }
1369
1370 if (!k->is_loaded()) {
1371 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1372 } else {
1373 __ mov_metadata(k_RInfo, k->constant_encoding());
1374 }
1375 __ verify_oop(obj);
1376
1377 if (op->fast_check()) {
1378 // get object class
1379 // not a safepoint as obj null check happens earlier
1380 __ load_klass(rscratch1, obj);
1381 __ cmp( rscratch1, k_RInfo);
1382
1383 __ br(Assembler::NE, *failure_target);
1384 // successful cast, fall through to profile or jump
1385 } else {
1386 // get object class
1387 // not a safepoint as obj null check happens earlier
1388 __ load_klass(klass_RInfo, obj);
1389 if (k->is_loaded()) {
1390 // See if we get an immediate positive hit
1391 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1392 __ cmp(k_RInfo, rscratch1);
1393 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1394 __ br(Assembler::NE, *failure_target);
1395 // successful cast, fall through to profile or jump
1396 } else {
1397 // See if we get an immediate positive hit
1398 __ br(Assembler::EQ, *success_target);
1399 // check for self
1400 __ cmp(klass_RInfo, k_RInfo);
1401 __ br(Assembler::EQ, *success_target);
1402
1403 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1404 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1405 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1406 // result is a boolean
1407 __ cbzw(klass_RInfo, *failure_target);
1408 // successful cast, fall through to profile or jump
1409 }
1410 } else {
1411 // perform the fast part of the checking logic
1412 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1413 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1414 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1415 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1416 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1417 // result is a boolean
1418 __ cbz(k_RInfo, *failure_target);
1419 // successful cast, fall through to profile or jump
1420 }
1421 }
1422 if (should_profile) {
1423 Register mdo = klass_RInfo, recv = k_RInfo;
1424 __ bind(profile_cast_success);
1425 __ mov_metadata(mdo, md->constant_encoding());
1426 __ load_klass(recv, obj);
1427 Label update_done;
1428 type_profile_helper(mdo, md, data, recv, success);
1429 __ b(*success);
1430
1431 __ bind(profile_cast_failure);
1432 __ mov_metadata(mdo, md->constant_encoding());
1433 Address counter_addr
1434 = __ form_address(rscratch2, mdo,
1435 md->byte_offset_of_slot(data, CounterData::count_offset()),
1436 LogBytesPerWord);
1437 __ ldr(rscratch1, counter_addr);
1438 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1439 __ str(rscratch1, counter_addr);
1440 __ b(*failure);
1441 }
1442 __ b(*success);
1443 }
1444
1445
1446 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1447 const bool should_profile = op->should_profile();
1448
1449 LIR_Code code = op->code();
1450 if (code == lir_store_check) {
1451 Register value = op->object()->as_register();
1452 Register array = op->array()->as_register();
1453 Register k_RInfo = op->tmp1()->as_register();
1454 Register klass_RInfo = op->tmp2()->as_register();
1455 Register Rtmp1 = op->tmp3()->as_register();
1456
1457 CodeStub* stub = op->stub();
1458
1459 // check if it needs to be profiled
1460 ciMethodData* md;
1461 ciProfileData* data;
1462
1463 if (should_profile) {
1464 ciMethod* method = op->profiled_method();
1465 assert(method != NULL, "Should have method");
1466 int bci = op->profiled_bci();
1467 md = method->method_data_or_null();
1468 assert(md != NULL, "Sanity");
1469 data = md->bci_to_data(bci);
1470 assert(data != NULL, "need data for type check");
1471 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1472 }
1473 Label profile_cast_success, profile_cast_failure, done;
1474 Label *success_target = should_profile ? &profile_cast_success : &done;
1475 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1476
1477 if (should_profile) {
1478 Label not_null;
1479 __ cbnz(value, not_null);
1480 // Object is null; update MDO and exit
1481 Register mdo = klass_RInfo;
1482 __ mov_metadata(mdo, md->constant_encoding());
1483 Address data_addr
1484 = __ form_address(rscratch2, mdo,
1485 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1486 LogBytesPerInt);
1487 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1488 __ ldrw(rscratch1, data_addr);
1489 __ orrw(rscratch1, rscratch1, header_bits);
1490 __ strw(rscratch1, data_addr);
1491 __ b(done);
1492 __ bind(not_null);
1493 } else {
1494 __ cbz(value, done);
1495 }
1496
1497 add_debug_info_for_null_check_here(op->info_for_exception());
1498 __ load_klass(k_RInfo, array);
1499 __ load_klass(klass_RInfo, value);
1500
1501 // get instance klass (it's already uncompressed)
1502 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1503 // perform the fast part of the checking logic
1504 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1505 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1506 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1507 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1508 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1509 // result is a boolean
1510 __ cbzw(k_RInfo, *failure_target);
1511 // fall through to the success case
1512
1513 if (should_profile) {
1514 Register mdo = klass_RInfo, recv = k_RInfo;
1515 __ bind(profile_cast_success);
1516 __ mov_metadata(mdo, md->constant_encoding());
1517 __ load_klass(recv, value);
1518 Label update_done;
1519 type_profile_helper(mdo, md, data, recv, &done);
1520 __ b(done);
1521
1522 __ bind(profile_cast_failure);
1523 __ mov_metadata(mdo, md->constant_encoding());
1524 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1525 __ lea(rscratch2, counter_addr);
1526 __ ldr(rscratch1, Address(rscratch2));
1527 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1528 __ str(rscratch1, Address(rscratch2));
1529 __ b(*stub->entry());
1530 }
1531
1532 __ bind(done);
1533 } else if (code == lir_checkcast) {
1534 Register obj = op->object()->as_register();
1535 Register dst = op->result_opr()->as_register();
1536 Label success;
1537 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1538 __ bind(success);
1539 if (dst != obj) {
1540 __ mov(dst, obj);
1541 }
1542 } else if (code == lir_instanceof) {
1543 Register obj = op->object()->as_register();
1544 Register dst = op->result_opr()->as_register();
1545 Label success, failure, done;
1546 emit_typecheck_helper(op, &success, &failure, &failure);
1547 __ bind(failure);
1548 __ mov(dst, zr);
1549 __ b(done);
1550 __ bind(success);
1551 __ mov(dst, 1);
1552 __ bind(done);
1553 } else {
1554 ShouldNotReachHere();
1555 }
1556 }
1557
1558 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1559 Label retry_load, nope;
1560 // flush and load exclusive from the memory location
1561 // and fail if it is not what we expect
1562 __ bind(retry_load);
1563 __ ldaxrw(rscratch1, addr);
1564 __ cmpw(rscratch1, cmpval);
1565 __ cset(rscratch1, Assembler::NE);
1566 __ br(Assembler::NE, nope);
1567 // if we store+flush with no intervening write rscratch1 wil be zero
1568 __ stlxrw(rscratch1, newval, addr);
1569 // retry so we only ever return after a load fails to compare
1570 // ensures we don't return a stale value after a failed write.
1571 __ cbnzw(rscratch1, retry_load);
1572 __ bind(nope);
1573 __ membar(__ AnyAny);
1574 }
1575
1576 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1577 Label retry_load, nope;
1578 // flush and load exclusive from the memory location
1579 // and fail if it is not what we expect
1580 __ bind(retry_load);
1581 __ ldaxr(rscratch1, addr);
1582 __ cmp(rscratch1, cmpval);
1583 __ cset(rscratch1, Assembler::NE);
1584 __ br(Assembler::NE, nope);
1585 // if we store+flush with no intervening write rscratch1 wil be zero
1586 __ stlxr(rscratch1, newval, addr);
1587 // retry so we only ever return after a load fails to compare
1588 // ensures we don't return a stale value after a failed write.
1589 __ cbnz(rscratch1, retry_load);
1590 __ bind(nope);
1591 __ membar(__ AnyAny);
1592 }
1593
1594
1595 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1596 assert(VM_Version::supports_cx8(), "wrong machine");
1597 Register addr = as_reg(op->addr());
1598 Register newval = as_reg(op->new_value());
1599 Register cmpval = as_reg(op->cmp_value());
1600 Label succeed, fail, around;
1601
1602 if (op->code() == lir_cas_obj) {
1603 if (UseCompressedOops) {
1604 Register t1 = op->tmp1()->as_register();
1605 assert(op->tmp1()->is_valid(), "must be");
1606 __ encode_heap_oop(t1, cmpval);
1607 cmpval = t1;
1608 __ encode_heap_oop(rscratch2, newval);
1609 newval = rscratch2;
1610 casw(addr, newval, cmpval);
1611 } else {
1612 casl(addr, newval, cmpval);
1613 }
1614 } else if (op->code() == lir_cas_int) {
1615 casw(addr, newval, cmpval);
1616 } else {
1617 casl(addr, newval, cmpval);
1618 }
1619 }
1620
1621
1622 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1623
1624 Assembler::Condition acond, ncond;
1625 switch (condition) {
1626 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1627 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1628 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1629 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1630 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1631 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1632 case lir_cond_belowEqual:
1633 case lir_cond_aboveEqual:
1634 default: ShouldNotReachHere();
1635 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable
1636 }
1637
1638 assert(result->is_single_cpu() || result->is_double_cpu(),
1639 "expect single register for result");
1640 if (opr1->is_constant() && opr2->is_constant()
1641 && opr1->type() == T_INT && opr2->type() == T_INT) {
1642 jint val1 = opr1->as_jint();
1643 jint val2 = opr2->as_jint();
1644 if (val1 == 0 && val2 == 1) {
1645 __ cset(result->as_register(), ncond);
1646 return;
1647 } else if (val1 == 1 && val2 == 0) {
1648 __ cset(result->as_register(), acond);
1649 return;
1650 }
1651 }
1652
1653 if (opr1->is_constant() && opr2->is_constant()
1654 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1655 jlong val1 = opr1->as_jlong();
1656 jlong val2 = opr2->as_jlong();
1657 if (val1 == 0 && val2 == 1) {
1658 __ cset(result->as_register_lo(), ncond);
1659 return;
1660 } else if (val1 == 1 && val2 == 0) {
1661 __ cset(result->as_register_lo(), acond);
1662 return;
1663 }
1664 }
1665
1666 if (opr1->is_stack()) {
1667 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1668 opr1 = FrameMap::rscratch1_opr;
1669 } else if (opr1->is_constant()) {
1670 LIR_Opr tmp
1671 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1672 const2reg(opr1, tmp, lir_patch_none, NULL);
1673 opr1 = tmp;
1674 }
1675
1676 if (opr2->is_stack()) {
1677 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1678 opr2 = FrameMap::rscratch2_opr;
1679 } else if (opr2->is_constant()) {
1680 LIR_Opr tmp
1681 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1682 const2reg(opr2, tmp, lir_patch_none, NULL);
1683 opr2 = tmp;
1684 }
1685
1686 if (result->type() == T_LONG)
1687 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1688 else
1689 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1690 }
1691
1692 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1693 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1694
1695 if (left->is_single_cpu()) {
1696 Register lreg = left->as_register();
1697 Register dreg = as_reg(dest);
1698
1699 if (right->is_single_cpu()) {
1700 // cpu register - cpu register
1701
1702 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1703 "should be");
1704 Register rreg = right->as_register();
1705 switch (code) {
1706 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1707 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1708 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1709 default: ShouldNotReachHere();
1710 }
1711
1712 } else if (right->is_double_cpu()) {
1713 Register rreg = right->as_register_lo();
1714 // single_cpu + double_cpu: can happen with obj+long
1715 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1716 switch (code) {
1717 case lir_add: __ add(dreg, lreg, rreg); break;
1718 case lir_sub: __ sub(dreg, lreg, rreg); break;
1719 default: ShouldNotReachHere();
1720 }
1721 } else if (right->is_constant()) {
1722 // cpu register - constant
1723 jlong c;
1724
1725 // FIXME. This is fugly: we really need to factor all this logic.
1726 switch(right->type()) {
1727 case T_LONG:
1728 c = right->as_constant_ptr()->as_jlong();
1729 break;
1730 case T_INT:
1731 case T_ADDRESS:
1732 c = right->as_constant_ptr()->as_jint();
1733 break;
1734 default:
1735 ShouldNotReachHere();
1736 c = 0; // unreachable
1737 break;
1738 }
1739
1740 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1741 if (c == 0 && dreg == lreg) {
1742 COMMENT("effective nop elided");
1743 return;
1744 }
1745 switch(left->type()) {
1746 case T_INT:
1747 switch (code) {
1748 case lir_add: __ addw(dreg, lreg, c); break;
1749 case lir_sub: __ subw(dreg, lreg, c); break;
1750 default: ShouldNotReachHere();
1751 }
1752 break;
1753 case T_OBJECT:
1754 case T_ADDRESS:
1755 switch (code) {
1756 case lir_add: __ add(dreg, lreg, c); break;
1757 case lir_sub: __ sub(dreg, lreg, c); break;
1758 default: ShouldNotReachHere();
1759 }
1760 break;
1761 ShouldNotReachHere();
1762 }
1763 } else {
1764 ShouldNotReachHere();
1765 }
1766
1767 } else if (left->is_double_cpu()) {
1768 Register lreg_lo = left->as_register_lo();
1769
1770 if (right->is_double_cpu()) {
1771 // cpu register - cpu register
1772 Register rreg_lo = right->as_register_lo();
1773 switch (code) {
1774 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1775 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1776 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1777 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1778 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1779 default:
1780 ShouldNotReachHere();
1781 }
1782
1783 } else if (right->is_constant()) {
1784 jlong c = right->as_constant_ptr()->as_jlong_bits();
1785 Register dreg = as_reg(dest);
1786 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1787 if (c == 0 && dreg == lreg_lo) {
1788 COMMENT("effective nop elided");
1789 return;
1790 }
1791 switch (code) {
1792 case lir_add: __ add(dreg, lreg_lo, c); break;
1793 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1794 default:
1795 ShouldNotReachHere();
1796 }
1797 } else {
1798 ShouldNotReachHere();
1799 }
1800 } else if (left->is_single_fpu()) {
1801 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1802 switch (code) {
1803 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1804 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1805 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1806 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1807 default:
1808 ShouldNotReachHere();
1809 }
1810 } else if (left->is_double_fpu()) {
1811 if (right->is_double_fpu()) {
1812 // cpu register - cpu register
1813 switch (code) {
1814 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1815 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1816 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1817 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1818 default:
1819 ShouldNotReachHere();
1820 }
1821 } else {
1822 if (right->is_constant()) {
1823 ShouldNotReachHere();
1824 }
1825 ShouldNotReachHere();
1826 }
1827 } else if (left->is_single_stack() || left->is_address()) {
1828 assert(left == dest, "left and dest must be equal");
1829 ShouldNotReachHere();
1830 } else {
1831 ShouldNotReachHere();
1832 }
1833 }
1834
1835 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1836
1837
1838 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1839 switch(code) {
1840 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1841 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1842 default : ShouldNotReachHere();
1843 }
1844 }
1845
1846 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1847
1848 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1849 Register Rleft = left->is_single_cpu() ? left->as_register() :
1850 left->as_register_lo();
1851 if (dst->is_single_cpu()) {
1852 Register Rdst = dst->as_register();
1853 if (right->is_constant()) {
1854 switch (code) {
1855 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1856 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1857 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1858 default: ShouldNotReachHere(); break;
1859 }
1860 } else {
1861 Register Rright = right->is_single_cpu() ? right->as_register() :
1862 right->as_register_lo();
1863 switch (code) {
1864 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1865 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1866 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1867 default: ShouldNotReachHere(); break;
1868 }
1869 }
1870 } else {
1871 Register Rdst = dst->as_register_lo();
1872 if (right->is_constant()) {
1873 switch (code) {
1874 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1875 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1876 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1877 default: ShouldNotReachHere(); break;
1878 }
1879 } else {
1880 Register Rright = right->is_single_cpu() ? right->as_register() :
1881 right->as_register_lo();
1882 switch (code) {
1883 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1884 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1885 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1886 default: ShouldNotReachHere(); break;
1887 }
1888 }
1889 }
1890 }
1891
1892
1893
1894 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1895
1896
1897 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1898 if (opr1->is_constant() && opr2->is_single_cpu()) {
1899 // tableswitch
1900 Register reg = as_reg(opr2);
1901 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1902 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1903 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1904 Register reg1 = as_reg(opr1);
1905 if (opr2->is_single_cpu()) {
1906 // cpu register - cpu register
1907 Register reg2 = opr2->as_register();
1908 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1909 __ cmp(reg1, reg2);
1910 } else {
1911 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1912 __ cmpw(reg1, reg2);
1913 }
1914 return;
1915 }
1916 if (opr2->is_double_cpu()) {
1917 // cpu register - cpu register
1918 Register reg2 = opr2->as_register_lo();
1919 __ cmp(reg1, reg2);
1920 return;
1921 }
1922
1923 if (opr2->is_constant()) {
1924 jlong imm;
1925 switch(opr2->type()) {
1926 case T_LONG:
1927 imm = opr2->as_constant_ptr()->as_jlong();
1928 break;
1929 case T_INT:
1930 case T_ADDRESS:
1931 imm = opr2->as_constant_ptr()->as_jint();
1932 break;
1933 case T_OBJECT:
1934 case T_ARRAY:
1935 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1936 break;
1937 default:
1938 ShouldNotReachHere();
1939 imm = 0; // unreachable
1940 break;
1941 }
1942
1943 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1944 if (type2aelembytes(opr1->type()) <= 4)
1945 __ cmpw(reg1, imm);
1946 else
1947 __ cmp(reg1, imm);
1948 return;
1949 } else {
1950 __ mov(rscratch1, imm);
1951 if (type2aelembytes(opr1->type()) <= 4)
1952 __ cmpw(reg1, rscratch1);
1953 else
1954 __ cmp(reg1, rscratch1);
1955 return;
1956 }
1957 } else
1958 ShouldNotReachHere();
1959 } else if (opr1->is_single_fpu()) {
1960 FloatRegister reg1 = opr1->as_float_reg();
1961 assert(opr2->is_single_fpu(), "expect single float register");
1962 FloatRegister reg2 = opr2->as_float_reg();
1963 __ fcmps(reg1, reg2);
1964 } else if (opr1->is_double_fpu()) {
1965 FloatRegister reg1 = opr1->as_double_reg();
1966 assert(opr2->is_double_fpu(), "expect double float register");
1967 FloatRegister reg2 = opr2->as_double_reg();
1968 __ fcmpd(reg1, reg2);
1969 } else {
1970 ShouldNotReachHere();
1971 }
1972 }
1973
1974 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1975 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1976 bool is_unordered_less = (code == lir_ucmp_fd2i);
1977 if (left->is_single_fpu()) {
1978 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1979 } else if (left->is_double_fpu()) {
1980 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1981 } else {
1982 ShouldNotReachHere();
1983 }
1984 } else if (code == lir_cmp_l2i) {
1985 Label done;
1986 __ cmp(left->as_register_lo(), right->as_register_lo());
1987 __ mov(dst->as_register(), (u_int64_t)-1L);
1988 __ br(Assembler::LT, done);
1989 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1990 __ bind(done);
1991 } else {
1992 ShouldNotReachHere();
1993 }
1994 }
1995
1996
1997 void LIR_Assembler::align_call(LIR_Code code) { }
1998
1999
2000 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2001 address call = __ trampoline_call(Address(op->addr(), rtype));
2002 if (call == NULL) {
2003 bailout("trampoline stub overflow");
2004 return;
2005 }
2006 add_call_info(code_offset(), op->info());
2007 }
2008
2009
2010 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2011 address call = __ ic_call(op->addr());
2012 if (call == NULL) {
2013 bailout("trampoline stub overflow");
2014 return;
2015 }
2016 add_call_info(code_offset(), op->info());
2017 }
2018
2019
2020 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2021 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2022 ShouldNotReachHere();
2023 }
2024
2025
2026 void LIR_Assembler::emit_static_call_stub() {
2027 address call_pc = __ pc();
2028 address stub = __ start_a_stub(call_stub_size);
2029 if (stub == NULL) {
2030 bailout("static call stub overflow");
2031 return;
2032 }
2033
2034 int start = __ offset();
2035
2036 __ relocate(static_stub_Relocation::spec(call_pc));
2037 __ mov_metadata(rmethod, (Metadata*)NULL);
2038 __ movptr(rscratch1, 0);
2039 __ br(rscratch1);
2040
2041 assert(__ offset() - start <= call_stub_size, "stub too big");
2042 __ end_a_stub();
2043 }
2044
2045
2046 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2047 assert(exceptionOop->as_register() == r0, "must match");
2048 assert(exceptionPC->as_register() == r3, "must match");
2049
2050 // exception object is not added to oop map by LinearScan
2051 // (LinearScan assumes that no oops are in fixed registers)
2052 info->add_register_oop(exceptionOop);
2053 Runtime1::StubID unwind_id;
2054
2055 // get current pc information
2056 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2057 int pc_for_athrow_offset = __ offset();
2058 InternalAddress pc_for_athrow(__ pc());
2059 __ adr(exceptionPC->as_register(), pc_for_athrow);
2060 add_call_info(pc_for_athrow_offset, info); // for exception handler
2061
2062 __ verify_not_null_oop(r0);
2063 // search an exception handler (r0: exception oop, r3: throwing pc)
2064 if (compilation()->has_fpu_code()) {
2065 unwind_id = Runtime1::handle_exception_id;
2066 } else {
2067 unwind_id = Runtime1::handle_exception_nofpu_id;
2068 }
2069 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2070
2071 // FIXME: enough room for two byte trap ????
2072 __ nop();
2073 }
2074
2075
2076 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2077 assert(exceptionOop->as_register() == r0, "must match");
2078
2079 __ b(_unwind_handler_entry);
2080 }
2081
2082
2083 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2084 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2085 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2086
2087 switch (left->type()) {
2088 case T_INT: {
2089 switch (code) {
2090 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2091 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2092 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2093 default:
2094 ShouldNotReachHere();
2095 break;
2096 }
2097 break;
2098 case T_LONG:
2099 case T_ADDRESS:
2100 case T_OBJECT:
2101 switch (code) {
2102 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2103 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2104 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2105 default:
2106 ShouldNotReachHere();
2107 break;
2108 }
2109 break;
2110 default:
2111 ShouldNotReachHere();
2112 break;
2113 }
2114 }
2115 }
2116
2117
2118 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2119 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2120 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2121
2122 switch (left->type()) {
2123 case T_INT: {
2124 switch (code) {
2125 case lir_shl: __ lslw (dreg, lreg, count); break;
2126 case lir_shr: __ asrw (dreg, lreg, count); break;
2127 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2128 default:
2129 ShouldNotReachHere();
2130 break;
2131 }
2132 break;
2133 case T_LONG:
2134 case T_ADDRESS:
2135 case T_OBJECT:
2136 switch (code) {
2137 case lir_shl: __ lsl (dreg, lreg, count); break;
2138 case lir_shr: __ asr (dreg, lreg, count); break;
2139 case lir_ushr: __ lsr (dreg, lreg, count); break;
2140 default:
2141 ShouldNotReachHere();
2142 break;
2143 }
2144 break;
2145 default:
2146 ShouldNotReachHere();
2147 break;
2148 }
2149 }
2150 }
2151
2152
2153 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2154 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2155 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2156 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2157 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2158 }
2159
2160
2161 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2162 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2163 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2164 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2165 __ mov (rscratch1, c);
2166 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2167 }
2168
2169
2170 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2171 ShouldNotReachHere();
2172 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2173 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2174 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2175 __ lea(rscratch1, __ constant_oop_address(o));
2176 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2177 }
2178
2179
2180 // This code replaces a call to arraycopy; no exception may
2181 // be thrown in this code, they must be thrown in the System.arraycopy
2182 // activation frame; we could save some checks if this would not be the case
2183 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2184 ciArrayKlass* default_type = op->expected_type();
2185 Register src = op->src()->as_register();
2186 Register dst = op->dst()->as_register();
2187 Register src_pos = op->src_pos()->as_register();
2188 Register dst_pos = op->dst_pos()->as_register();
2189 Register length = op->length()->as_register();
2190 Register tmp = op->tmp()->as_register();
2191
2192 CodeStub* stub = op->stub();
2193 int flags = op->flags();
2194 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2195 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2196
2197 // if we don't know anything, just go through the generic arraycopy
2198 if (default_type == NULL // || basic_type == T_OBJECT
2199 ) {
2200 Label done;
2201 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2202
2203 // Save the arguments in case the generic arraycopy fails and we
2204 // have to fall back to the JNI stub
2205 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2206 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2207 __ str(src, Address(sp, 4*BytesPerWord));
2208
2209 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2210 address copyfunc_addr = StubRoutines::generic_arraycopy();
2211
2212 // The arguments are in java calling convention so we shift them
2213 // to C convention
2214 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2215 __ mov(c_rarg0, j_rarg0);
2216 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2217 __ mov(c_rarg1, j_rarg1);
2218 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2219 __ mov(c_rarg2, j_rarg2);
2220 assert_different_registers(c_rarg3, j_rarg4);
2221 __ mov(c_rarg3, j_rarg3);
2222 __ mov(c_rarg4, j_rarg4);
2223 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2224 __ mov(rscratch1, RuntimeAddress(C_entry));
2225 __ blrt(rscratch1, 5, 0, 1);
2226 } else {
2227 #ifndef PRODUCT
2228 if (PrintC1Statistics) {
2229 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2230 }
2231 #endif
2232 __ far_call(RuntimeAddress(copyfunc_addr));
2233 }
2234
2235 __ cbz(r0, *stub->continuation());
2236
2237 // Reload values from the stack so they are where the stub
2238 // expects them.
2239 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2240 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2241 __ ldr(src, Address(sp, 4*BytesPerWord));
2242
2243 if (copyfunc_addr != NULL) {
2244 // r0 is -1^K where K == partial copied count
2245 __ eonw(rscratch1, r0, 0);
2246 // adjust length down and src/end pos up by partial copied count
2247 __ subw(length, length, rscratch1);
2248 __ addw(src_pos, src_pos, rscratch1);
2249 __ addw(dst_pos, dst_pos, rscratch1);
2250 }
2251 __ b(*stub->entry());
2252
2253 __ bind(*stub->continuation());
2254 return;
2255 }
2256
2257 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2258
2259 int elem_size = type2aelembytes(basic_type);
2260 int shift_amount;
2261 int scale = exact_log2(elem_size);
2262
2263 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2264 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2265 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2266 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2267
2268 // test for NULL
2269 if (flags & LIR_OpArrayCopy::src_null_check) {
2270 __ cbz(src, *stub->entry());
2271 }
2272 if (flags & LIR_OpArrayCopy::dst_null_check) {
2273 __ cbz(dst, *stub->entry());
2274 }
2275
2276 // check if negative
2277 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2278 __ cmpw(src_pos, 0);
2279 __ br(Assembler::LT, *stub->entry());
2280 }
2281 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2282 __ cmpw(dst_pos, 0);
2283 __ br(Assembler::LT, *stub->entry());
2284 }
2285
2286 if (flags & LIR_OpArrayCopy::length_positive_check) {
2287 __ cmpw(length, 0);
2288 __ br(Assembler::LT, *stub->entry());
2289 }
2290
2291 if (flags & LIR_OpArrayCopy::src_range_check) {
2292 __ addw(tmp, src_pos, length);
2293 __ ldrw(rscratch1, src_length_addr);
2294 __ cmpw(tmp, rscratch1);
2295 __ br(Assembler::HI, *stub->entry());
2296 }
2297 if (flags & LIR_OpArrayCopy::dst_range_check) {
2298 __ addw(tmp, dst_pos, length);
2299 __ ldrw(rscratch1, dst_length_addr);
2300 __ cmpw(tmp, rscratch1);
2301 __ br(Assembler::HI, *stub->entry());
2302 }
2303
2304 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2305 // length_positive_check if the source of our length operand is an
2306 // arraylength. However, that arraylength might be zero, and the
2307 // stub that we're about to call contains an assertion that count !=
2308 // 0 . So we make this check purely in order not to trigger an
2309 // assertion failure.
2310 __ cbzw(length, *stub->continuation());
2311
2312 if (flags & LIR_OpArrayCopy::type_check) {
2313 // We don't know the array types are compatible
2314 if (basic_type != T_OBJECT) {
2315 // Simple test for basic type arrays
2316 if (UseCompressedClassPointers) {
2317 __ ldrw(tmp, src_klass_addr);
2318 __ ldrw(rscratch1, dst_klass_addr);
2319 __ cmpw(tmp, rscratch1);
2320 } else {
2321 __ ldr(tmp, src_klass_addr);
2322 __ ldr(rscratch1, dst_klass_addr);
2323 __ cmp(tmp, rscratch1);
2324 }
2325 __ br(Assembler::NE, *stub->entry());
2326 } else {
2327 // For object arrays, if src is a sub class of dst then we can
2328 // safely do the copy.
2329 Label cont, slow;
2330
2331 #define PUSH(r1, r2) \
2332 stp(r1, r2, __ pre(sp, -2 * wordSize));
2333
2334 #define POP(r1, r2) \
2335 ldp(r1, r2, __ post(sp, 2 * wordSize));
2336
2337 __ PUSH(src, dst);
2338
2339 __ load_klass(src, src);
2340 __ load_klass(dst, dst);
2341
2342 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2343
2344 __ PUSH(src, dst);
2345 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2346 __ POP(src, dst);
2347
2348 __ cbnz(src, cont);
2349
2350 __ bind(slow);
2351 __ POP(src, dst);
2352
2353 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2354 if (copyfunc_addr != NULL) { // use stub if available
2355 // src is not a sub class of dst so we have to do a
2356 // per-element check.
2357
2358 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2359 if ((flags & mask) != mask) {
2360 // Check that at least both of them object arrays.
2361 assert(flags & mask, "one of the two should be known to be an object array");
2362
2363 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2364 __ load_klass(tmp, src);
2365 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2366 __ load_klass(tmp, dst);
2367 }
2368 int lh_offset = in_bytes(Klass::layout_helper_offset());
2369 Address klass_lh_addr(tmp, lh_offset);
2370 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2371 __ ldrw(rscratch1, klass_lh_addr);
2372 __ mov(rscratch2, objArray_lh);
2373 __ eorw(rscratch1, rscratch1, rscratch2);
2374 __ cbnzw(rscratch1, *stub->entry());
2375 }
2376
2377 // Spill because stubs can use any register they like and it's
2378 // easier to restore just those that we care about.
2379 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2380 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2381 __ str(src, Address(sp, 4*BytesPerWord));
2382
2383 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2384 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2385 assert_different_registers(c_rarg0, dst, dst_pos, length);
2386 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2387 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2388 assert_different_registers(c_rarg1, dst, length);
2389 __ uxtw(c_rarg2, length);
2390 assert_different_registers(c_rarg2, dst);
2391
2392 __ load_klass(c_rarg4, dst);
2393 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2394 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2395 __ far_call(RuntimeAddress(copyfunc_addr));
2396
2397 #ifndef PRODUCT
2398 if (PrintC1Statistics) {
2399 Label failed;
2400 __ cbnz(r0, failed);
2401 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2402 __ bind(failed);
2403 }
2404 #endif
2405
2406 __ cbz(r0, *stub->continuation());
2407
2408 #ifndef PRODUCT
2409 if (PrintC1Statistics) {
2410 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2411 }
2412 #endif
2413 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2414
2415 // Restore previously spilled arguments
2416 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2417 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2418 __ ldr(src, Address(sp, 4*BytesPerWord));
2419
2420 // return value is -1^K where K is partial copied count
2421 __ eonw(rscratch1, r0, zr);
2422 // adjust length down and src/end pos up by partial copied count
2423 __ subw(length, length, rscratch1);
2424 __ addw(src_pos, src_pos, rscratch1);
2425 __ addw(dst_pos, dst_pos, rscratch1);
2426 }
2427
2428 __ b(*stub->entry());
2429
2430 __ bind(cont);
2431 __ POP(src, dst);
2432 }
2433 }
2434
2435 #ifdef ASSERT
2436 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2437 // Sanity check the known type with the incoming class. For the
2438 // primitive case the types must match exactly with src.klass and
2439 // dst.klass each exactly matching the default type. For the
2440 // object array case, if no type check is needed then either the
2441 // dst type is exactly the expected type and the src type is a
2442 // subtype which we can't check or src is the same array as dst
2443 // but not necessarily exactly of type default_type.
2444 Label known_ok, halt;
2445 __ mov_metadata(tmp, default_type->constant_encoding());
2446 if (UseCompressedClassPointers) {
2447 __ encode_klass_not_null(tmp);
2448 }
2449
2450 if (basic_type != T_OBJECT) {
2451
2452 if (UseCompressedClassPointers) {
2453 __ ldrw(rscratch1, dst_klass_addr);
2454 __ cmpw(tmp, rscratch1);
2455 } else {
2456 __ ldr(rscratch1, dst_klass_addr);
2457 __ cmp(tmp, rscratch1);
2458 }
2459 __ br(Assembler::NE, halt);
2460 if (UseCompressedClassPointers) {
2461 __ ldrw(rscratch1, src_klass_addr);
2462 __ cmpw(tmp, rscratch1);
2463 } else {
2464 __ ldr(rscratch1, src_klass_addr);
2465 __ cmp(tmp, rscratch1);
2466 }
2467 __ br(Assembler::EQ, known_ok);
2468 } else {
2469 if (UseCompressedClassPointers) {
2470 __ ldrw(rscratch1, dst_klass_addr);
2471 __ cmpw(tmp, rscratch1);
2472 } else {
2473 __ ldr(rscratch1, dst_klass_addr);
2474 __ cmp(tmp, rscratch1);
2475 }
2476 __ br(Assembler::EQ, known_ok);
2477 __ cmp(src, dst);
2478 __ br(Assembler::EQ, known_ok);
2479 }
2480 __ bind(halt);
2481 __ stop("incorrect type information in arraycopy");
2482 __ bind(known_ok);
2483 }
2484 #endif
2485
2486 #ifndef PRODUCT
2487 if (PrintC1Statistics) {
2488 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2489 }
2490 #endif
2491
2492 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2493 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2494 assert_different_registers(c_rarg0, dst, dst_pos, length);
2495 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2496 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2497 assert_different_registers(c_rarg1, dst, length);
2498 __ uxtw(c_rarg2, length);
2499 assert_different_registers(c_rarg2, dst);
2500
2501 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2502 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2503 const char *name;
2504 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2505
2506 CodeBlob *cb = CodeCache::find_blob(entry);
2507 if (cb) {
2508 __ far_call(RuntimeAddress(entry));
2509 } else {
2510 __ call_VM_leaf(entry, 3);
2511 }
2512
2513 __ bind(*stub->continuation());
2514 }
2515
2516
2517
2518
2519 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2520 Register obj = op->obj_opr()->as_register(); // may not be an oop
2521 Register hdr = op->hdr_opr()->as_register();
2522 Register lock = op->lock_opr()->as_register();
2523 if (!UseFastLocking) {
2524 __ b(*op->stub()->entry());
2525 } else if (op->code() == lir_lock) {
2526 Register scratch = noreg;
2527 if (UseBiasedLocking) {
2528 scratch = op->scratch_opr()->as_register();
2529 }
2530 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2531 // add debug info for NullPointerException only if one is possible
2532 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2533 if (op->info() != NULL) {
2534 add_debug_info_for_null_check(null_check_offset, op->info());
2535 }
2536 // done
2537 } else if (op->code() == lir_unlock) {
2538 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2539 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2540 } else {
2541 Unimplemented();
2542 }
2543 __ bind(*op->stub()->continuation());
2544 }
2545
2546
2547 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2548 ciMethod* method = op->profiled_method();
2549 int bci = op->profiled_bci();
2550 ciMethod* callee = op->profiled_callee();
2551
2552 // Update counter for all call types
2553 ciMethodData* md = method->method_data_or_null();
2554 assert(md != NULL, "Sanity");
2555 ciProfileData* data = md->bci_to_data(bci);
2556 assert(data->is_CounterData(), "need CounterData for calls");
2557 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2558 Register mdo = op->mdo()->as_register();
2559 __ mov_metadata(mdo, md->constant_encoding());
2560 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2561 Bytecodes::Code bc = method->java_code_at_bci(bci);
2562 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2563 // Perform additional virtual call profiling for invokevirtual and
2564 // invokeinterface bytecodes
2565 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2566 !callee_is_static && // required for optimized MH invokes
2567 C1ProfileVirtualCalls) {
2568 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2569 Register recv = op->recv()->as_register();
2570 assert_different_registers(mdo, recv);
2571 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2572 ciKlass* known_klass = op->known_holder();
2573 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2574 // We know the type that will be seen at this call site; we can
2575 // statically update the MethodData* rather than needing to do
2576 // dynamic tests on the receiver type
2577
2578 // NOTE: we should probably put a lock around this search to
2579 // avoid collisions by concurrent compilations
2580 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2581 uint i;
2582 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2583 ciKlass* receiver = vc_data->receiver(i);
2584 if (known_klass->equals(receiver)) {
2585 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2586 __ addptr(data_addr, DataLayout::counter_increment);
2587 return;
2588 }
2589 }
2590
2591 // Receiver type not found in profile data; select an empty slot
2592
2593 // Note that this is less efficient than it should be because it
2594 // always does a write to the receiver part of the
2595 // VirtualCallData rather than just the first time
2596 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2597 ciKlass* receiver = vc_data->receiver(i);
2598 if (receiver == NULL) {
2599 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2600 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2601 __ lea(rscratch2, recv_addr);
2602 __ str(rscratch1, Address(rscratch2));
2603 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2604 __ addptr(data_addr, DataLayout::counter_increment);
2605 return;
2606 }
2607 }
2608 } else {
2609 __ load_klass(recv, recv);
2610 Label update_done;
2611 type_profile_helper(mdo, md, data, recv, &update_done);
2612 // Receiver did not match any saved receiver and there is no empty row for it.
2613 // Increment total counter to indicate polymorphic case.
2614 __ addptr(counter_addr, DataLayout::counter_increment);
2615
2616 __ bind(update_done);
2617 }
2618 } else {
2619 // Static call
2620 __ addptr(counter_addr, DataLayout::counter_increment);
2621 }
2622 }
2623
2624
2625 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2626 Unimplemented();
2627 }
2628
2629
2630 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2631 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2632 }
2633
2634 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2635 assert(op->crc()->is_single_cpu(), "crc must be register");
2636 assert(op->val()->is_single_cpu(), "byte value must be register");
2637 assert(op->result_opr()->is_single_cpu(), "result must be register");
2638 Register crc = op->crc()->as_register();
2639 Register val = op->val()->as_register();
2640 Register res = op->result_opr()->as_register();
2641
2642 assert_different_registers(val, crc, res);
2643 unsigned long offset;
2644 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2645 if (offset) __ add(res, res, offset);
2646
2647 __ ornw(crc, zr, crc); // ~crc
2648 __ update_byte_crc32(crc, val, res);
2649 __ ornw(res, zr, crc); // ~crc
2650 }
2651
2652 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2653 COMMENT("emit_profile_type {");
2654 Register obj = op->obj()->as_register();
2655 Register tmp = op->tmp()->as_pointer_register();
2656 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2657 ciKlass* exact_klass = op->exact_klass();
2658 intptr_t current_klass = op->current_klass();
2659 bool not_null = op->not_null();
2660 bool no_conflict = op->no_conflict();
2661
2662 Label update, next, none;
2663
2664 bool do_null = !not_null;
2665 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2666 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2667
2668 assert(do_null || do_update, "why are we here?");
2669 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2670 assert(mdo_addr.base() != rscratch1, "wrong register");
2671
2672 __ verify_oop(obj);
2673
2674 if (tmp != obj) {
2675 __ mov(tmp, obj);
2676 }
2677 if (do_null) {
2678 __ cbnz(tmp, update);
2679 if (!TypeEntries::was_null_seen(current_klass)) {
2680 __ ldr(rscratch2, mdo_addr);
2681 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2682 __ str(rscratch2, mdo_addr);
2683 }
2684 if (do_update) {
2685 #ifndef ASSERT
2686 __ b(next);
2687 }
2688 #else
2689 __ b(next);
2690 }
2691 } else {
2692 __ cbnz(tmp, update);
2693 __ stop("unexpected null obj");
2694 #endif
2695 }
2696
2697 __ bind(update);
2698
2699 if (do_update) {
2700 #ifdef ASSERT
2701 if (exact_klass != NULL) {
2702 Label ok;
2703 __ load_klass(tmp, tmp);
2704 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2705 __ eor(rscratch1, tmp, rscratch1);
2706 __ cbz(rscratch1, ok);
2707 __ stop("exact klass and actual klass differ");
2708 __ bind(ok);
2709 }
2710 #endif
2711 if (!no_conflict) {
2712 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2713 if (exact_klass != NULL) {
2714 __ mov_metadata(tmp, exact_klass->constant_encoding());
2715 } else {
2716 __ load_klass(tmp, tmp);
2717 }
2718
2719 __ ldr(rscratch2, mdo_addr);
2720 __ eor(tmp, tmp, rscratch2);
2721 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2722 // klass seen before, nothing to do. The unknown bit may have been
2723 // set already but no need to check.
2724 __ cbz(rscratch1, next);
2725
2726 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2727 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2728
2729 if (TypeEntries::is_type_none(current_klass)) {
2730 __ cbz(rscratch2, none);
2731 __ cmp(rscratch2, TypeEntries::null_seen);
2732 __ br(Assembler::EQ, none);
2733 // There is a chance that the checks above (re-reading profiling
2734 // data from memory) fail if another thread has just set the
2735 // profiling to this obj's klass
2736 __ dmb(Assembler::ISHLD);
2737 __ ldr(rscratch2, mdo_addr);
2738 __ eor(tmp, tmp, rscratch2);
2739 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2740 __ cbz(rscratch1, next);
2741 }
2742 } else {
2743 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2744 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2745
2746 __ ldr(tmp, mdo_addr);
2747 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2748 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2749 }
2750
2751 // different than before. Cannot keep accurate profile.
2752 __ ldr(rscratch2, mdo_addr);
2753 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2754 __ str(rscratch2, mdo_addr);
2755
2756 if (TypeEntries::is_type_none(current_klass)) {
2757 __ b(next);
2758
2759 __ bind(none);
2760 // first time here. Set profile type.
2761 __ str(tmp, mdo_addr);
2762 }
2763 } else {
2764 // There's a single possible klass at this profile point
2765 assert(exact_klass != NULL, "should be");
2766 if (TypeEntries::is_type_none(current_klass)) {
2767 __ mov_metadata(tmp, exact_klass->constant_encoding());
2768 __ ldr(rscratch2, mdo_addr);
2769 __ eor(tmp, tmp, rscratch2);
2770 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2771 __ cbz(rscratch1, next);
2772 #ifdef ASSERT
2773 {
2774 Label ok;
2775 __ ldr(rscratch1, mdo_addr);
2776 __ cbz(rscratch1, ok);
2777 __ cmp(rscratch1, TypeEntries::null_seen);
2778 __ br(Assembler::EQ, ok);
2779 // may have been set by another thread
2780 __ dmb(Assembler::ISHLD);
2781 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2782 __ ldr(rscratch2, mdo_addr);
2783 __ eor(rscratch2, rscratch1, rscratch2);
2784 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2785 __ cbz(rscratch2, ok);
2786
2787 __ stop("unexpected profiling mismatch");
2788 __ bind(ok);
2789 }
2790 #endif
2791 // first time here. Set profile type.
2792 __ ldr(tmp, mdo_addr);
2793 } else {
2794 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2795 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2796
2797 __ ldr(tmp, mdo_addr);
2798 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2799 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2800
2801 __ orr(tmp, tmp, TypeEntries::type_unknown);
2802 __ str(tmp, mdo_addr);
2803 // FIXME: Write barrier needed here?
2804 }
2805 }
2806
2807 __ bind(next);
2808 }
2809 COMMENT("} emit_profile_type");
2810 }
2811
2812
2813 void LIR_Assembler::align_backward_branch_target() {
2814 }
2815
2816
2817 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2818 if (left->is_single_cpu()) {
2819 assert(dest->is_single_cpu(), "expect single result reg");
2820 __ negw(dest->as_register(), left->as_register());
2821 } else if (left->is_double_cpu()) {
2822 assert(dest->is_double_cpu(), "expect double result reg");
2823 __ neg(dest->as_register_lo(), left->as_register_lo());
2824 } else if (left->is_single_fpu()) {
2825 assert(dest->is_single_fpu(), "expect single float result reg");
2826 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2827 } else {
2828 assert(left->is_double_fpu(), "expect double float operand reg");
2829 assert(dest->is_double_fpu(), "expect double float result reg");
2830 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2831 }
2832 }
2833
2834
2835 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2836 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2837 }
2838
2839
2840 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2841 assert(!tmp->is_valid(), "don't need temporary");
2842
2843 CodeBlob *cb = CodeCache::find_blob(dest);
2844 if (cb) {
2845 __ far_call(RuntimeAddress(dest));
2846 } else {
2847 __ mov(rscratch1, RuntimeAddress(dest));
2848 int len = args->length();
2849 int type = 0;
2850 if (! result->is_illegal()) {
2851 switch (result->type()) {
2852 case T_VOID:
2853 type = 0;
2854 break;
2855 case T_INT:
2856 case T_LONG:
2857 case T_OBJECT:
2858 type = 1;
2859 break;
2860 case T_FLOAT:
2861 type = 2;
2862 break;
2863 case T_DOUBLE:
2864 type = 3;
2865 break;
2866 default:
2867 ShouldNotReachHere();
2868 break;
2869 }
2870 }
2871 int num_gpargs = 0;
2872 int num_fpargs = 0;
2873 for (int i = 0; i < args->length(); i++) {
2874 LIR_Opr arg = args->at(i);
2875 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2876 num_fpargs++;
2877 } else {
2878 num_gpargs++;
2879 }
2880 }
2881 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2882 }
2883
2884 if (info != NULL) {
2885 add_call_info_here(info);
2886 }
2887 __ maybe_isb();
2888 }
2889
2890 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2891 if (dest->is_address() || src->is_address()) {
2892 move_op(src, dest, type, lir_patch_none, info,
2893 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2894 } else {
2895 ShouldNotReachHere();
2896 }
2897 }
2898
2899 #ifdef ASSERT
2900 // emit run-time assertion
2901 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2902 assert(op->code() == lir_assert, "must be");
2903
2904 if (op->in_opr1()->is_valid()) {
2905 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2906 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2907 } else {
2908 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2909 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2910 }
2911
2912 Label ok;
2913 if (op->condition() != lir_cond_always) {
2914 Assembler::Condition acond = Assembler::AL;
2915 switch (op->condition()) {
2916 case lir_cond_equal: acond = Assembler::EQ; break;
2917 case lir_cond_notEqual: acond = Assembler::NE; break;
2918 case lir_cond_less: acond = Assembler::LT; break;
2919 case lir_cond_lessEqual: acond = Assembler::LE; break;
2920 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2921 case lir_cond_greater: acond = Assembler::GT; break;
2922 case lir_cond_belowEqual: acond = Assembler::LS; break;
2923 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2924 default: ShouldNotReachHere();
2925 }
2926 __ br(acond, ok);
2927 }
2928 if (op->halt()) {
2929 const char* str = __ code_string(op->msg());
2930 __ stop(str);
2931 } else {
2932 breakpoint();
2933 }
2934 __ bind(ok);
2935 }
2936 #endif
2937
2938 #ifndef PRODUCT
2939 #define COMMENT(x) do { __ block_comment(x); } while (0)
2940 #else
2941 #define COMMENT(x)
2942 #endif
2943
2944 void LIR_Assembler::membar() {
2945 COMMENT("membar");
2946 __ membar(MacroAssembler::AnyAny);
2947 }
2948
2949 void LIR_Assembler::membar_acquire() {
2950 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2951 }
2952
2953 void LIR_Assembler::membar_release() {
2954 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2955 }
2956
2957 void LIR_Assembler::membar_loadload() {
2958 __ membar(Assembler::LoadLoad);
2959 }
2960
2961 void LIR_Assembler::membar_storestore() {
2962 __ membar(MacroAssembler::StoreStore);
2963 }
2964
2965 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2966
2967 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2968
2969 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2970 __ mov(result_reg->as_register(), rthread);
2971 }
2972
2973
2974 void LIR_Assembler::peephole(LIR_List *lir) {
2975 #if 0
2976 if (tableswitch_count >= max_tableswitches)
2977 return;
2978
2979 /*
2980 This finite-state automaton recognizes sequences of compare-and-
2981 branch instructions. We will turn them into a tableswitch. You
2982 could argue that C1 really shouldn't be doing this sort of
2983 optimization, but without it the code is really horrible.
2984 */
2985
2986 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2987 int first_key, last_key = -2147483648;
2988 int next_key = 0;
2989 int start_insn = -1;
2990 int last_insn = -1;
2991 Register reg = noreg;
2992 LIR_Opr reg_opr;
2993 state = start_s;
2994
2995 LIR_OpList* inst = lir->instructions_list();
2996 for (int i = 0; i < inst->length(); i++) {
2997 LIR_Op* op = inst->at(i);
2998 switch (state) {
2999 case start_s:
3000 first_key = -1;
3001 start_insn = i;
3002 switch (op->code()) {
3003 case lir_cmp:
3004 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3005 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3006 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3007 && opr2->is_constant()
3008 && opr2->type() == T_INT) {
3009 reg_opr = opr1;
3010 reg = opr1->as_register();
3011 first_key = opr2->as_constant_ptr()->as_jint();
3012 next_key = first_key + 1;
3013 state = cmp_s;
3014 goto next_state;
3015 }
3016 break;
3017 }
3018 break;
3019 case cmp_s:
3020 switch (op->code()) {
3021 case lir_branch:
3022 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3023 state = beq_s;
3024 last_insn = i;
3025 goto next_state;
3026 }
3027 }
3028 state = start_s;
3029 break;
3030 case beq_s:
3031 switch (op->code()) {
3032 case lir_cmp: {
3033 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3034 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3035 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3036 && opr1->as_register() == reg
3037 && opr2->is_constant()
3038 && opr2->type() == T_INT
3039 && opr2->as_constant_ptr()->as_jint() == next_key) {
3040 last_key = next_key;
3041 next_key++;
3042 state = cmp_s;
3043 goto next_state;
3044 }
3045 }
3046 }
3047 last_key = next_key;
3048 state = start_s;
3049 break;
3050 default:
3051 assert(false, "impossible state");
3052 }
3053 if (state == start_s) {
3054 if (first_key < last_key - 5L && reg != noreg) {
3055 {
3056 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3057 // reg->encoding(),
3058 // start_insn, first_key, last_key);
3059 // for (int i = 0; i < inst->length(); i++) {
3060 // inst->at(i)->print();
3061 // tty->print("\n");
3062 // }
3063 // tty->print("\n");
3064 }
3065
3066 struct tableswitch *sw = &switches[tableswitch_count];
3067 sw->_insn_index = start_insn, sw->_first_key = first_key,
3068 sw->_last_key = last_key, sw->_reg = reg;
3069 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3070 {
3071 // Insert the new table of branches
3072 int offset = last_insn;
3073 for (int n = first_key; n < last_key; n++) {
3074 inst->insert_before
3075 (last_insn + 1,
3076 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3077 inst->at(offset)->as_OpBranch()->label()));
3078 offset -= 2, i++;
3079 }
3080 }
3081 // Delete all the old compare-and-branch instructions
3082 for (int n = first_key; n < last_key; n++) {
3083 inst->remove_at(start_insn);
3084 inst->remove_at(start_insn);
3085 }
3086 // Insert the tableswitch instruction
3087 inst->insert_before(start_insn,
3088 new LIR_Op2(lir_cmp, lir_cond_always,
3089 LIR_OprFact::intConst(tableswitch_count),
3090 reg_opr));
3091 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3092 tableswitch_count++;
3093 }
3094 reg = noreg;
3095 last_key = -2147483648;
3096 }
3097 next_state:
3098 ;
3099 }
3100 #endif
3101 }
3102
3103 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3104 Address addr = as_Address(src->as_address_ptr(), noreg);
3105 BasicType type = src->type();
3106 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3107
3108 void (MacroAssembler::* lda)(Register Rd, Register Ra);
3109 void (MacroAssembler::* add)(Register Rd, Register Rn, RegisterOrConstant increment);
3110 void (MacroAssembler::* stl)(Register Rs, Register Rt, Register Rn);
3111
3112 switch(type) {
3113 case T_INT:
3114 lda = &MacroAssembler::ldaxrw;
3115 add = &MacroAssembler::addw;
3116 stl = &MacroAssembler::stlxrw;
3117 break;
3118 case T_LONG:
3119 lda = &MacroAssembler::ldaxr;
3120 add = &MacroAssembler::add;
3121 stl = &MacroAssembler::stlxr;
3122 break;
3123 case T_OBJECT:
3124 case T_ARRAY:
3125 if (UseCompressedOops) {
3126 lda = &MacroAssembler::ldaxrw;
3127 add = &MacroAssembler::addw;
3128 stl = &MacroAssembler::stlxrw;
3129 } else {
3130 lda = &MacroAssembler::ldaxr;
3131 add = &MacroAssembler::add;
3132 stl = &MacroAssembler::stlxr;
3133 }
3134 break;
3135 default:
3136 ShouldNotReachHere();
3137 lda = &MacroAssembler::ldaxr;
3138 add = &MacroAssembler::add;
3139 stl = &MacroAssembler::stlxr; // unreachable
3140 }
3141
3142 switch (code) {
3143 case lir_xadd:
3144 {
3145 RegisterOrConstant inc;
3146 Register tmp = as_reg(tmp_op);
3147 Register dst = as_reg(dest);
3148 if (data->is_constant()) {
3149 inc = RegisterOrConstant(as_long(data));
3150 assert_different_registers(dst, addr.base(), tmp,
3151 rscratch1, rscratch2);
3152 } else {
3153 inc = RegisterOrConstant(as_reg(data));
3154 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3155 rscratch1, rscratch2);
3156 }
3157 Label again;
3158 __ lea(tmp, addr);
3159 __ bind(again);
3160 (_masm->*lda)(dst, tmp);
3161 (_masm->*add)(rscratch1, dst, inc);
3162 (_masm->*stl)(rscratch2, rscratch1, tmp);
3163 __ cbnzw(rscratch2, again);
3164 break;
3165 }
3166 case lir_xchg:
3167 {
3168 Register tmp = tmp_op->as_register();
3169 Register obj = as_reg(data);
3170 Register dst = as_reg(dest);
3171 if (is_oop && UseCompressedOops) {
3172 __ encode_heap_oop(rscratch1, obj);
3173 obj = rscratch1;
3174 }
3175 assert_different_registers(obj, addr.base(), tmp, rscratch2, dst);
3176 Label again;
3177 __ lea(tmp, addr);
3178 __ bind(again);
3179 (_masm->*lda)(dst, tmp);
3180 (_masm->*stl)(rscratch2, obj, tmp);
3181 __ cbnzw(rscratch2, again);
3182 if (is_oop && UseCompressedOops) {
3183 __ decode_heap_oop(dst);
3184 }
3185 }
3186 break;
3187 default:
3188 ShouldNotReachHere();
3189 }
3190 __ membar(__ AnyAny);
3191 }
3192
3193 #undef __