1 /*
2 * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "gc/shared/barrierSet.hpp"
37 #include "gc/shared/cardTableModRefBS.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "vmreg_aarch64.inline.hpp"
43
44
45
46 #ifndef PRODUCT
47 #define COMMENT(x) do { __ block_comment(x); } while (0)
48 #else
49 #define COMMENT(x)
50 #endif
51
52 NEEDS_CLEANUP // remove this definitions ?
53 const Register IC_Klass = rscratch2; // where the IC klass is cached
54 const Register SYNC_header = r0; // synchronization header
55 const Register SHIFT_count = r0; // where count for shift operations must be
56
57 #define __ _masm->
58
59
60 static void select_different_registers(Register preserve,
61 Register extra,
62 Register &tmp1,
63 Register &tmp2) {
64 if (tmp1 == preserve) {
65 assert_different_registers(tmp1, tmp2, extra);
66 tmp1 = extra;
67 } else if (tmp2 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp2 = extra;
70 }
71 assert_different_registers(preserve, tmp1, tmp2);
72 }
73
74
75
76 static void select_different_registers(Register preserve,
77 Register extra,
78 Register &tmp1,
79 Register &tmp2,
80 Register &tmp3) {
81 if (tmp1 == preserve) {
82 assert_different_registers(tmp1, tmp2, tmp3, extra);
83 tmp1 = extra;
84 } else if (tmp2 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp2 = extra;
87 } else if (tmp3 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp3 = extra;
90 }
91 assert_different_registers(preserve, tmp1, tmp2, tmp3);
92 }
93
94
95 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
96
97
98 LIR_Opr LIR_Assembler::receiverOpr() {
99 return FrameMap::receiver_opr;
100 }
101
102 LIR_Opr LIR_Assembler::osrBufferPointer() {
103 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
104 }
105
106 //--------------fpu register translations-----------------------
107
108
109 address LIR_Assembler::float_constant(float f) {
110 address const_addr = __ float_constant(f);
111 if (const_addr == NULL) {
112 bailout("const section overflow");
113 return __ code()->consts()->start();
114 } else {
115 return const_addr;
116 }
117 }
118
119
120 address LIR_Assembler::double_constant(double d) {
121 address const_addr = __ double_constant(d);
122 if (const_addr == NULL) {
123 bailout("const section overflow");
124 return __ code()->consts()->start();
125 } else {
126 return const_addr;
127 }
128 }
129
130 address LIR_Assembler::int_constant(jlong n) {
131 address const_addr = __ long_constant(n);
132 if (const_addr == NULL) {
133 bailout("const section overflow");
134 return __ code()->consts()->start();
135 } else {
136 return const_addr;
137 }
138 }
139
140 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
141
142 void LIR_Assembler::reset_FPU() { Unimplemented(); }
143
144 void LIR_Assembler::fpop() { Unimplemented(); }
145
146 void LIR_Assembler::fxch(int i) { Unimplemented(); }
147
148 void LIR_Assembler::fld(int i) { Unimplemented(); }
149
150 void LIR_Assembler::ffree(int i) { Unimplemented(); }
151
152 void LIR_Assembler::breakpoint() { Unimplemented(); }
153
154 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
155
156 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
157
158 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
159 //-------------------------------------------
160
161 static Register as_reg(LIR_Opr op) {
162 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
163 }
164
165 static jlong as_long(LIR_Opr data) {
166 jlong result;
167 switch (data->type()) {
168 case T_INT:
169 result = (data->as_jint());
170 break;
171 case T_LONG:
172 result = (data->as_jlong());
173 break;
174 default:
175 ShouldNotReachHere();
176 result = 0; // unreachable
177 }
178 return result;
179 }
180
181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
182 Register base = addr->base()->as_pointer_register();
183 LIR_Opr opr = addr->index();
184 if (opr->is_cpu_register()) {
185 Register index;
186 if (opr->is_single_cpu())
187 index = opr->as_register();
188 else
189 index = opr->as_register_lo();
190 assert(addr->disp() == 0, "must be");
191 switch(opr->type()) {
192 case T_INT:
193 return Address(base, index, Address::sxtw(addr->scale()));
194 case T_LONG:
195 return Address(base, index, Address::lsl(addr->scale()));
196 default:
197 ShouldNotReachHere();
198 }
199 } else {
200 intptr_t addr_offset = intptr_t(addr->disp());
201 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
202 return Address(base, addr_offset, Address::lsl(addr->scale()));
203 else {
204 __ mov(tmp, addr_offset);
205 return Address(base, tmp, Address::lsl(addr->scale()));
206 }
207 }
208 return Address();
209 }
210
211 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
212 ShouldNotReachHere();
213 return Address();
214 }
215
216 Address LIR_Assembler::as_Address(LIR_Address* addr) {
217 return as_Address(addr, rscratch1);
218 }
219
220 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
221 return as_Address(addr, rscratch1); // Ouch
222 // FIXME: This needs to be much more clever. See x86.
223 }
224
225
226 void LIR_Assembler::osr_entry() {
227 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
228 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
229 ValueStack* entry_state = osr_entry->state();
230 int number_of_locks = entry_state->locks_size();
231
232 // we jump here if osr happens with the interpreter
233 // state set up to continue at the beginning of the
234 // loop that triggered osr - in particular, we have
235 // the following registers setup:
236 //
237 // r2: osr buffer
238 //
239
240 // build frame
241 ciMethod* m = compilation()->method();
242 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
243
244 // OSR buffer is
245 //
246 // locals[nlocals-1..0]
247 // monitors[0..number_of_locks]
248 //
249 // locals is a direct copy of the interpreter frame so in the osr buffer
250 // so first slot in the local array is the last local from the interpreter
251 // and last slot is local[0] (receiver) from the interpreter
252 //
253 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
254 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
255 // in the interpreter frame (the method lock if a sync method)
256
257 // Initialize monitors in the compiled activation.
258 // r2: pointer to osr buffer
259 //
260 // All other registers are dead at this point and the locals will be
261 // copied into place by code emitted in the IR.
262
263 Register OSR_buf = osrBufferPointer()->as_pointer_register();
264 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
265 int monitor_offset = BytesPerWord * method()->max_locals() +
266 (2 * BytesPerWord) * (number_of_locks - 1);
267 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
268 // the OSR buffer using 2 word entries: first the lock and then
269 // the oop.
270 for (int i = 0; i < number_of_locks; i++) {
271 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
272 #ifdef ASSERT
273 // verify the interpreter's monitor has a non-null object
274 {
275 Label L;
276 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
277 __ cbnz(rscratch1, L);
278 __ stop("locked object is NULL");
279 __ bind(L);
280 }
281 #endif
282 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
283 __ str(r19, frame_map()->address_for_monitor_lock(i));
284 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
285 __ str(r19, frame_map()->address_for_monitor_object(i));
286 }
287 }
288 }
289
290
291 // inline cache check; done before the frame is built.
292 int LIR_Assembler::check_icache() {
293 Register receiver = FrameMap::receiver_opr->as_register();
294 Register ic_klass = IC_Klass;
295 int start_offset = __ offset();
296 __ inline_cache_check(receiver, ic_klass);
297
298 // if icache check fails, then jump to runtime routine
299 // Note: RECEIVER must still contain the receiver!
300 Label dont;
301 __ br(Assembler::EQ, dont);
302 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
303
304 // We align the verified entry point unless the method body
305 // (including its inline cache check) will fit in a single 64-byte
306 // icache line.
307 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
308 // force alignment after the cache check.
309 __ align(CodeEntryAlignment);
310 }
311
312 __ bind(dont);
313 return start_offset;
314 }
315
316
317 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
318 if (o == NULL) {
319 __ mov(reg, zr);
320 } else {
321 __ movoop(reg, o, /*immediate*/true);
322 }
323 }
324
325 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
326 address target = NULL;
327 relocInfo::relocType reloc_type = relocInfo::none;
328
329 switch (patching_id(info)) {
330 case PatchingStub::access_field_id:
331 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
332 reloc_type = relocInfo::section_word_type;
333 break;
334 case PatchingStub::load_klass_id:
335 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
336 reloc_type = relocInfo::metadata_type;
337 break;
338 case PatchingStub::load_mirror_id:
339 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
340 reloc_type = relocInfo::oop_type;
341 break;
342 case PatchingStub::load_appendix_id:
343 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
344 reloc_type = relocInfo::oop_type;
345 break;
346 default: ShouldNotReachHere();
347 }
348
349 __ far_call(RuntimeAddress(target));
350 add_call_info_here(info);
351 }
352
353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
354 deoptimize_trap(info);
355 }
356
357
358 // This specifies the rsp decrement needed to build the frame
359 int LIR_Assembler::initial_frame_size_in_bytes() const {
360 // if rounding, must let FrameMap know!
361
362 // The frame_map records size in slots (32bit word)
363
364 // subtract two words to account for return address and link
365 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
366 }
367
368
369 int LIR_Assembler::emit_exception_handler() {
370 // if the last instruction is a call (typically to do a throw which
371 // is coming at the end after block reordering) the return address
372 // must still point into the code area in order to avoid assertion
373 // failures when searching for the corresponding bci => add a nop
374 // (was bug 5/14/1999 - gri)
375 __ nop();
376
377 // generate code for exception handler
378 address handler_base = __ start_a_stub(exception_handler_size());
379 if (handler_base == NULL) {
380 // not enough space left for the handler
381 bailout("exception handler overflow");
382 return -1;
383 }
384
385 int offset = code_offset();
386
387 // the exception oop and pc are in r0, and r3
388 // no other registers need to be preserved, so invalidate them
389 __ invalidate_registers(false, true, true, false, true, true);
390
391 // check that there is really an exception
392 __ verify_not_null_oop(r0);
393
394 // search an exception handler (r0: exception oop, r3: throwing pc)
395 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here();
396 guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
397 __ end_a_stub();
398
399 return offset;
400 }
401
402
403 // Emit the code to remove the frame from the stack in the exception
404 // unwind path.
405 int LIR_Assembler::emit_unwind_handler() {
406 #ifndef PRODUCT
407 if (CommentedAssembly) {
408 _masm->block_comment("Unwind handler");
409 }
410 #endif
411
412 int offset = code_offset();
413
414 // Fetch the exception from TLS and clear out exception related thread state
415 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
417 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
418
419 __ bind(_unwind_handler_entry);
420 __ verify_not_null_oop(r0);
421 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
422 __ mov(r19, r0); // Preserve the exception
423 }
424
425 // Preform needed unlocking
426 MonitorExitStub* stub = NULL;
427 if (method()->is_synchronized()) {
428 monitor_address(0, FrameMap::r0_opr);
429 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
430 __ unlock_object(r5, r4, r0, *stub->entry());
431 __ bind(*stub->continuation());
432 }
433
434 if (compilation()->env()->dtrace_method_probes()) {
435 __ call_Unimplemented();
436 #if 0
437 __ movptr(Address(rsp, 0), rax);
438 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
439 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
440 #endif
441 }
442
443 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
444 __ mov(r0, r19); // Restore the exception
445 }
446
447 // remove the activation and dispatch to the unwind handler
448 __ block_comment("remove_frame and dispatch to the unwind handler");
449 __ remove_frame(initial_frame_size_in_bytes());
450 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
451
452 // Emit the slow path assembly
453 if (stub != NULL) {
454 stub->emit_code(this);
455 }
456
457 return offset;
458 }
459
460
461 int LIR_Assembler::emit_deopt_handler() {
462 // if the last instruction is a call (typically to do a throw which
463 // is coming at the end after block reordering) the return address
464 // must still point into the code area in order to avoid assertion
465 // failures when searching for the corresponding bci => add a nop
466 // (was bug 5/14/1999 - gri)
467 __ nop();
468
469 // generate code for exception handler
470 address handler_base = __ start_a_stub(deopt_handler_size());
471 if (handler_base == NULL) {
472 // not enough space left for the handler
473 bailout("deopt handler overflow");
474 return -1;
475 }
476
477 int offset = code_offset();
478
479 __ adr(lr, pc());
480 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
481 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
482 __ end_a_stub();
483
484 return offset;
485 }
486
487 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
488 _masm->code_section()->relocate(adr, relocInfo::poll_type);
489 int pc_offset = code_offset();
490 flush_debug_info(pc_offset);
491 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
492 if (info->exception_handlers() != NULL) {
493 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
494 }
495 }
496
497 void LIR_Assembler::return_op(LIR_Opr result) {
498 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
499
500 // Pop the stack before the safepoint code
501 __ remove_frame(initial_frame_size_in_bytes());
502
503 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
504 __ reserved_stack_check();
505 }
506
507 address polling_page(os::get_polling_page());
508 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
509 __ ret(lr);
510 }
511
512 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
513 address polling_page(os::get_polling_page());
514 guarantee(info != NULL, "Shouldn't be NULL");
515 assert(os::is_poll_address(polling_page), "should be");
516 __ get_polling_page(rscratch1, polling_page, relocInfo::poll_type);
517 add_debug_info_for_branch(info); // This isn't just debug info:
518 // it's the oop map
519 __ read_polling_page(rscratch1, relocInfo::poll_type);
520 return __ offset();
521 }
522
523
524 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
525 if (from_reg == r31_sp)
526 from_reg = sp;
527 if (to_reg == r31_sp)
528 to_reg = sp;
529 __ mov(to_reg, from_reg);
530 }
531
532 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
533
534
535 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
536 assert(src->is_constant(), "should not call otherwise");
537 assert(dest->is_register(), "should not call otherwise");
538 LIR_Const* c = src->as_constant_ptr();
539
540 switch (c->type()) {
541 case T_INT: {
542 assert(patch_code == lir_patch_none, "no patching handled here");
543 __ movw(dest->as_register(), c->as_jint());
544 break;
545 }
546
547 case T_ADDRESS: {
548 assert(patch_code == lir_patch_none, "no patching handled here");
549 __ mov(dest->as_register(), c->as_jint());
550 break;
551 }
552
553 case T_LONG: {
554 assert(patch_code == lir_patch_none, "no patching handled here");
555 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
556 break;
557 }
558
559 case T_OBJECT: {
560 if (patch_code == lir_patch_none) {
561 jobject2reg(c->as_jobject(), dest->as_register());
562 } else {
563 jobject2reg_with_patching(dest->as_register(), info);
564 }
565 break;
566 }
567
568 case T_METADATA: {
569 if (patch_code != lir_patch_none) {
570 klass2reg_with_patching(dest->as_register(), info);
571 } else {
572 __ mov_metadata(dest->as_register(), c->as_metadata());
573 }
574 break;
575 }
576
577 case T_FLOAT: {
578 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
579 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
580 } else {
581 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
582 __ ldrs(dest->as_float_reg(), Address(rscratch1));
583 }
584 break;
585 }
586
587 case T_DOUBLE: {
588 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
589 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
590 } else {
591 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
592 __ ldrd(dest->as_double_reg(), Address(rscratch1));
593 }
594 break;
595 }
596
597 default:
598 ShouldNotReachHere();
599 }
600 }
601
602 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
603 LIR_Const* c = src->as_constant_ptr();
604 switch (c->type()) {
605 case T_OBJECT:
606 {
607 if (! c->as_jobject())
608 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
609 else {
610 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
611 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
612 }
613 }
614 break;
615 case T_ADDRESS:
616 {
617 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
618 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
619 }
620 case T_INT:
621 case T_FLOAT:
622 {
623 Register reg = zr;
624 if (c->as_jint_bits() == 0)
625 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
626 else {
627 __ movw(rscratch1, c->as_jint_bits());
628 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
629 }
630 }
631 break;
632 case T_LONG:
633 case T_DOUBLE:
634 {
635 Register reg = zr;
636 if (c->as_jlong_bits() == 0)
637 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
638 lo_word_offset_in_bytes));
639 else {
640 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
641 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
642 lo_word_offset_in_bytes));
643 }
644 }
645 break;
646 default:
647 ShouldNotReachHere();
648 }
649 }
650
651 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
652 assert(src->is_constant(), "should not call otherwise");
653 LIR_Const* c = src->as_constant_ptr();
654 LIR_Address* to_addr = dest->as_address_ptr();
655
656 void (Assembler::* insn)(Register Rt, const Address &adr);
657
658 switch (type) {
659 case T_ADDRESS:
660 assert(c->as_jint() == 0, "should be");
661 insn = &Assembler::str;
662 break;
663 case T_LONG:
664 assert(c->as_jlong() == 0, "should be");
665 insn = &Assembler::str;
666 break;
667 case T_INT:
668 assert(c->as_jint() == 0, "should be");
669 insn = &Assembler::strw;
670 break;
671 case T_OBJECT:
672 case T_ARRAY:
673 assert(c->as_jobject() == 0, "should be");
674 if (UseCompressedOops && !wide) {
675 insn = &Assembler::strw;
676 } else {
677 insn = &Assembler::str;
678 }
679 break;
680 case T_CHAR:
681 case T_SHORT:
682 assert(c->as_jint() == 0, "should be");
683 insn = &Assembler::strh;
684 break;
685 case T_BOOLEAN:
686 case T_BYTE:
687 assert(c->as_jint() == 0, "should be");
688 insn = &Assembler::strb;
689 break;
690 default:
691 ShouldNotReachHere();
692 insn = &Assembler::str; // unreachable
693 }
694
695 if (info) add_debug_info_for_null_check_here(info);
696 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
697 }
698
699 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
700 assert(src->is_register(), "should not call otherwise");
701 assert(dest->is_register(), "should not call otherwise");
702
703 // move between cpu-registers
704 if (dest->is_single_cpu()) {
705 if (src->type() == T_LONG) {
706 // Can do LONG -> OBJECT
707 move_regs(src->as_register_lo(), dest->as_register());
708 return;
709 }
710 assert(src->is_single_cpu(), "must match");
711 if (src->type() == T_OBJECT) {
712 __ verify_oop(src->as_register());
713 }
714 move_regs(src->as_register(), dest->as_register());
715
716 } else if (dest->is_double_cpu()) {
717 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
718 // Surprising to me but we can see move of a long to t_object
719 __ verify_oop(src->as_register());
720 move_regs(src->as_register(), dest->as_register_lo());
721 return;
722 }
723 assert(src->is_double_cpu(), "must match");
724 Register f_lo = src->as_register_lo();
725 Register f_hi = src->as_register_hi();
726 Register t_lo = dest->as_register_lo();
727 Register t_hi = dest->as_register_hi();
728 assert(f_hi == f_lo, "must be same");
729 assert(t_hi == t_lo, "must be same");
730 move_regs(f_lo, t_lo);
731
732 } else if (dest->is_single_fpu()) {
733 __ fmovs(dest->as_float_reg(), src->as_float_reg());
734
735 } else if (dest->is_double_fpu()) {
736 __ fmovd(dest->as_double_reg(), src->as_double_reg());
737
738 } else {
739 ShouldNotReachHere();
740 }
741 }
742
743 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
744 if (src->is_single_cpu()) {
745 if (type == T_ARRAY || type == T_OBJECT) {
746 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
747 __ verify_oop(src->as_register());
748 } else if (type == T_METADATA || type == T_DOUBLE) {
749 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
750 } else {
751 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
752 }
753
754 } else if (src->is_double_cpu()) {
755 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
756 __ str(src->as_register_lo(), dest_addr_LO);
757
758 } else if (src->is_single_fpu()) {
759 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
760 __ strs(src->as_float_reg(), dest_addr);
761
762 } else if (src->is_double_fpu()) {
763 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
764 __ strd(src->as_double_reg(), dest_addr);
765
766 } else {
767 ShouldNotReachHere();
768 }
769
770 }
771
772
773 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
774 LIR_Address* to_addr = dest->as_address_ptr();
775 PatchingStub* patch = NULL;
776 Register compressed_src = rscratch1;
777
778 if (patch_code != lir_patch_none) {
779 deoptimize_trap(info);
780 return;
781 }
782
783 if (type == T_ARRAY || type == T_OBJECT) {
784 __ verify_oop(src->as_register());
785
786 if (UseCompressedOops && !wide) {
787 __ encode_heap_oop(compressed_src, src->as_register());
788 } else {
789 compressed_src = src->as_register();
790 }
791 }
792
793 int null_check_here = code_offset();
794 switch (type) {
795 case T_FLOAT: {
796 __ strs(src->as_float_reg(), as_Address(to_addr));
797 break;
798 }
799
800 case T_DOUBLE: {
801 __ strd(src->as_double_reg(), as_Address(to_addr));
802 break;
803 }
804
805 case T_ARRAY: // fall through
806 case T_OBJECT: // fall through
807 if (UseCompressedOops && !wide) {
808 __ strw(compressed_src, as_Address(to_addr, rscratch2));
809 } else {
810 __ str(compressed_src, as_Address(to_addr));
811 }
812 break;
813 case T_METADATA:
814 // We get here to store a method pointer to the stack to pass to
815 // a dtrace runtime call. This can't work on 64 bit with
816 // compressed klass ptrs: T_METADATA can be a compressed klass
817 // ptr or a 64 bit method pointer.
818 ShouldNotReachHere();
819 __ str(src->as_register(), as_Address(to_addr));
820 break;
821 case T_ADDRESS:
822 __ str(src->as_register(), as_Address(to_addr));
823 break;
824 case T_INT:
825 __ strw(src->as_register(), as_Address(to_addr));
826 break;
827
828 case T_LONG: {
829 __ str(src->as_register_lo(), as_Address_lo(to_addr));
830 break;
831 }
832
833 case T_BYTE: // fall through
834 case T_BOOLEAN: {
835 __ strb(src->as_register(), as_Address(to_addr));
836 break;
837 }
838
839 case T_CHAR: // fall through
840 case T_SHORT:
841 __ strh(src->as_register(), as_Address(to_addr));
842 break;
843
844 default:
845 ShouldNotReachHere();
846 }
847 if (info != NULL) {
848 add_debug_info_for_null_check(null_check_here, info);
849 }
850 }
851
852
853 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
854 assert(src->is_stack(), "should not call otherwise");
855 assert(dest->is_register(), "should not call otherwise");
856
857 if (dest->is_single_cpu()) {
858 if (type == T_ARRAY || type == T_OBJECT) {
859 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
860 __ verify_oop(dest->as_register());
861 } else if (type == T_METADATA) {
862 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
863 } else {
864 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
865 }
866
867 } else if (dest->is_double_cpu()) {
868 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
869 __ ldr(dest->as_register_lo(), src_addr_LO);
870
871 } else if (dest->is_single_fpu()) {
872 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
873 __ ldrs(dest->as_float_reg(), src_addr);
874
875 } else if (dest->is_double_fpu()) {
876 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
877 __ ldrd(dest->as_double_reg(), src_addr);
878
879 } else {
880 ShouldNotReachHere();
881 }
882 }
883
884
885 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
886 address target = NULL;
887 relocInfo::relocType reloc_type = relocInfo::none;
888
889 switch (patching_id(info)) {
890 case PatchingStub::access_field_id:
891 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
892 reloc_type = relocInfo::section_word_type;
893 break;
894 case PatchingStub::load_klass_id:
895 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
896 reloc_type = relocInfo::metadata_type;
897 break;
898 case PatchingStub::load_mirror_id:
899 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
900 reloc_type = relocInfo::oop_type;
901 break;
902 case PatchingStub::load_appendix_id:
903 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
904 reloc_type = relocInfo::oop_type;
905 break;
906 default: ShouldNotReachHere();
907 }
908
909 __ far_call(RuntimeAddress(target));
910 add_call_info_here(info);
911 }
912
913 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
914
915 LIR_Opr temp;
916 if (type == T_LONG || type == T_DOUBLE)
917 temp = FrameMap::rscratch1_long_opr;
918 else
919 temp = FrameMap::rscratch1_opr;
920
921 stack2reg(src, temp, src->type());
922 reg2stack(temp, dest, dest->type(), false);
923 }
924
925
926 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
927 LIR_Address* addr = src->as_address_ptr();
928 LIR_Address* from_addr = src->as_address_ptr();
929
930 if (addr->base()->type() == T_OBJECT) {
931 __ verify_oop(addr->base()->as_pointer_register());
932 }
933
934 if (patch_code != lir_patch_none) {
935 deoptimize_trap(info);
936 return;
937 }
938
939 if (info != NULL) {
940 add_debug_info_for_null_check_here(info);
941 }
942 int null_check_here = code_offset();
943 switch (type) {
944 case T_FLOAT: {
945 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
946 break;
947 }
948
949 case T_DOUBLE: {
950 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
951 break;
952 }
953
954 case T_ARRAY: // fall through
955 case T_OBJECT: // fall through
956 if (UseCompressedOops && !wide) {
957 __ ldrw(dest->as_register(), as_Address(from_addr));
958 } else {
959 __ ldr(dest->as_register(), as_Address(from_addr));
960 }
961 break;
962 case T_METADATA:
963 // We get here to store a method pointer to the stack to pass to
964 // a dtrace runtime call. This can't work on 64 bit with
965 // compressed klass ptrs: T_METADATA can be a compressed klass
966 // ptr or a 64 bit method pointer.
967 ShouldNotReachHere();
968 __ ldr(dest->as_register(), as_Address(from_addr));
969 break;
970 case T_ADDRESS:
971 // FIXME: OMG this is a horrible kludge. Any offset from an
972 // address that matches klass_offset_in_bytes() will be loaded
973 // as a word, not a long.
974 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
975 __ ldrw(dest->as_register(), as_Address(from_addr));
976 } else {
977 __ ldr(dest->as_register(), as_Address(from_addr));
978 }
979 break;
980 case T_INT:
981 __ ldrw(dest->as_register(), as_Address(from_addr));
982 break;
983
984 case T_LONG: {
985 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
986 break;
987 }
988
989 case T_BYTE:
990 __ ldrsb(dest->as_register(), as_Address(from_addr));
991 break;
992 case T_BOOLEAN: {
993 __ ldrb(dest->as_register(), as_Address(from_addr));
994 break;
995 }
996
997 case T_CHAR:
998 __ ldrh(dest->as_register(), as_Address(from_addr));
999 break;
1000 case T_SHORT:
1001 __ ldrsh(dest->as_register(), as_Address(from_addr));
1002 break;
1003
1004 default:
1005 ShouldNotReachHere();
1006 }
1007
1008 if (type == T_ARRAY || type == T_OBJECT) {
1009 if (UseCompressedOops && !wide) {
1010 __ decode_heap_oop(dest->as_register());
1011 }
1012 __ verify_oop(dest->as_register());
1013 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1014 if (UseCompressedClassPointers) {
1015 __ decode_klass_not_null(dest->as_register());
1016 }
1017 }
1018 }
1019
1020
1021 int LIR_Assembler::array_element_size(BasicType type) const {
1022 int elem_size = type2aelembytes(type);
1023 return exact_log2(elem_size);
1024 }
1025
1026 void LIR_Assembler::arithmetic_idiv(LIR_Op3* op, bool is_irem) {
1027 Register Rdividend = op->in_opr1()->as_register();
1028 Register Rdivisor = op->in_opr2()->as_register();
1029 Register Rscratch = op->in_opr3()->as_register();
1030 Register Rresult = op->result_opr()->as_register();
1031 int divisor = -1;
1032
1033 /*
1034 TODO: For some reason, using the Rscratch that gets passed in is
1035 not possible because the register allocator does not see the tmp reg
1036 as used, and assignes it the same register as Rdividend. We use rscratch1
1037 instead.
1038
1039 assert(Rdividend != Rscratch, "");
1040 assert(Rdivisor != Rscratch, "");
1041 */
1042
1043 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1044 // convert division by a power of two into some shifts and logical operations
1045 }
1046
1047 __ corrected_idivl(Rresult, Rdividend, Rdivisor, is_irem, rscratch1);
1048 }
1049
1050 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1051 switch (op->code()) {
1052 case lir_idiv:
1053 arithmetic_idiv(op, false);
1054 break;
1055 case lir_irem:
1056 arithmetic_idiv(op, true);
1057 break;
1058 case lir_fmad:
1059 __ fmaddd(op->result_opr()->as_double_reg(),
1060 op->in_opr1()->as_double_reg(),
1061 op->in_opr2()->as_double_reg(),
1062 op->in_opr3()->as_double_reg());
1063 break;
1064 case lir_fmaf:
1065 __ fmadds(op->result_opr()->as_float_reg(),
1066 op->in_opr1()->as_float_reg(),
1067 op->in_opr2()->as_float_reg(),
1068 op->in_opr3()->as_float_reg());
1069 break;
1070 default: ShouldNotReachHere(); break;
1071 }
1072 }
1073
1074 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1075 #ifdef ASSERT
1076 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1077 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1078 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1079 #endif
1080
1081 if (op->cond() == lir_cond_always) {
1082 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1083 __ b(*(op->label()));
1084 } else {
1085 Assembler::Condition acond;
1086 if (op->code() == lir_cond_float_branch) {
1087 bool is_unordered = (op->ublock() == op->block());
1088 // Assembler::EQ does not permit unordered branches, so we add
1089 // another branch here. Likewise, Assembler::NE does not permit
1090 // ordered branches.
1091 if (is_unordered && op->cond() == lir_cond_equal
1092 || !is_unordered && op->cond() == lir_cond_notEqual)
1093 __ br(Assembler::VS, *(op->ublock()->label()));
1094 switch(op->cond()) {
1095 case lir_cond_equal: acond = Assembler::EQ; break;
1096 case lir_cond_notEqual: acond = Assembler::NE; break;
1097 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1098 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1099 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1100 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1101 default: ShouldNotReachHere();
1102 acond = Assembler::EQ; // unreachable
1103 }
1104 } else {
1105 switch (op->cond()) {
1106 case lir_cond_equal: acond = Assembler::EQ; break;
1107 case lir_cond_notEqual: acond = Assembler::NE; break;
1108 case lir_cond_less: acond = Assembler::LT; break;
1109 case lir_cond_lessEqual: acond = Assembler::LE; break;
1110 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1111 case lir_cond_greater: acond = Assembler::GT; break;
1112 case lir_cond_belowEqual: acond = Assembler::LS; break;
1113 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1114 default: ShouldNotReachHere();
1115 acond = Assembler::EQ; // unreachable
1116 }
1117 }
1118 __ br(acond,*(op->label()));
1119 }
1120 }
1121
1122
1123
1124 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1125 LIR_Opr src = op->in_opr();
1126 LIR_Opr dest = op->result_opr();
1127
1128 switch (op->bytecode()) {
1129 case Bytecodes::_i2f:
1130 {
1131 __ scvtfws(dest->as_float_reg(), src->as_register());
1132 break;
1133 }
1134 case Bytecodes::_i2d:
1135 {
1136 __ scvtfwd(dest->as_double_reg(), src->as_register());
1137 break;
1138 }
1139 case Bytecodes::_l2d:
1140 {
1141 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1142 break;
1143 }
1144 case Bytecodes::_l2f:
1145 {
1146 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1147 break;
1148 }
1149 case Bytecodes::_f2d:
1150 {
1151 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1152 break;
1153 }
1154 case Bytecodes::_d2f:
1155 {
1156 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1157 break;
1158 }
1159 case Bytecodes::_i2c:
1160 {
1161 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1162 break;
1163 }
1164 case Bytecodes::_i2l:
1165 {
1166 __ sxtw(dest->as_register_lo(), src->as_register());
1167 break;
1168 }
1169 case Bytecodes::_i2s:
1170 {
1171 __ sxth(dest->as_register(), src->as_register());
1172 break;
1173 }
1174 case Bytecodes::_i2b:
1175 {
1176 __ sxtb(dest->as_register(), src->as_register());
1177 break;
1178 }
1179 case Bytecodes::_l2i:
1180 {
1181 _masm->block_comment("FIXME: This could be a no-op");
1182 __ uxtw(dest->as_register(), src->as_register_lo());
1183 break;
1184 }
1185 case Bytecodes::_d2l:
1186 {
1187 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1188 break;
1189 }
1190 case Bytecodes::_f2i:
1191 {
1192 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1193 break;
1194 }
1195 case Bytecodes::_f2l:
1196 {
1197 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1198 break;
1199 }
1200 case Bytecodes::_d2i:
1201 {
1202 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1203 break;
1204 }
1205 default: ShouldNotReachHere();
1206 }
1207 }
1208
1209 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1210 if (op->init_check()) {
1211 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1212 InstanceKlass::init_state_offset()));
1213 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1214 add_debug_info_for_null_check_here(op->stub()->info());
1215 __ br(Assembler::NE, *op->stub()->entry());
1216 }
1217 __ allocate_object(op->obj()->as_register(),
1218 op->tmp1()->as_register(),
1219 op->tmp2()->as_register(),
1220 op->header_size(),
1221 op->object_size(),
1222 op->klass()->as_register(),
1223 *op->stub()->entry());
1224 __ bind(*op->stub()->continuation());
1225 }
1226
1227 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1228 Register len = op->len()->as_register();
1229 __ uxtw(len, len);
1230
1231 if (UseSlowPath ||
1232 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1233 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1234 __ b(*op->stub()->entry());
1235 } else {
1236 Register tmp1 = op->tmp1()->as_register();
1237 Register tmp2 = op->tmp2()->as_register();
1238 Register tmp3 = op->tmp3()->as_register();
1239 if (len == tmp1) {
1240 tmp1 = tmp3;
1241 } else if (len == tmp2) {
1242 tmp2 = tmp3;
1243 } else if (len == tmp3) {
1244 // everything is ok
1245 } else {
1246 __ mov(tmp3, len);
1247 }
1248 __ allocate_array(op->obj()->as_register(),
1249 len,
1250 tmp1,
1251 tmp2,
1252 arrayOopDesc::header_size(op->type()),
1253 array_element_size(op->type()),
1254 op->klass()->as_register(),
1255 *op->stub()->entry());
1256 }
1257 __ bind(*op->stub()->continuation());
1258 }
1259
1260 void LIR_Assembler::type_profile_helper(Register mdo,
1261 ciMethodData *md, ciProfileData *data,
1262 Register recv, Label* update_done) {
1263 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1264 Label next_test;
1265 // See if the receiver is receiver[n].
1266 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1267 __ ldr(rscratch1, Address(rscratch2));
1268 __ cmp(recv, rscratch1);
1269 __ br(Assembler::NE, next_test);
1270 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1271 __ addptr(data_addr, DataLayout::counter_increment);
1272 __ b(*update_done);
1273 __ bind(next_test);
1274 }
1275
1276 // Didn't find receiver; find next empty slot and fill it in
1277 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1278 Label next_test;
1279 __ lea(rscratch2,
1280 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1281 Address recv_addr(rscratch2);
1282 __ ldr(rscratch1, recv_addr);
1283 __ cbnz(rscratch1, next_test);
1284 __ str(recv, recv_addr);
1285 __ mov(rscratch1, DataLayout::counter_increment);
1286 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1287 __ str(rscratch1, Address(rscratch2));
1288 __ b(*update_done);
1289 __ bind(next_test);
1290 }
1291 }
1292
1293 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1294 // we always need a stub for the failure case.
1295 CodeStub* stub = op->stub();
1296 Register obj = op->object()->as_register();
1297 Register k_RInfo = op->tmp1()->as_register();
1298 Register klass_RInfo = op->tmp2()->as_register();
1299 Register dst = op->result_opr()->as_register();
1300 ciKlass* k = op->klass();
1301 Register Rtmp1 = noreg;
1302
1303 // check if it needs to be profiled
1304 ciMethodData* md;
1305 ciProfileData* data;
1306
1307 const bool should_profile = op->should_profile();
1308
1309 if (should_profile) {
1310 ciMethod* method = op->profiled_method();
1311 assert(method != NULL, "Should have method");
1312 int bci = op->profiled_bci();
1313 md = method->method_data_or_null();
1314 assert(md != NULL, "Sanity");
1315 data = md->bci_to_data(bci);
1316 assert(data != NULL, "need data for type check");
1317 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1318 }
1319 Label profile_cast_success, profile_cast_failure;
1320 Label *success_target = should_profile ? &profile_cast_success : success;
1321 Label *failure_target = should_profile ? &profile_cast_failure : failure;
1322
1323 if (obj == k_RInfo) {
1324 k_RInfo = dst;
1325 } else if (obj == klass_RInfo) {
1326 klass_RInfo = dst;
1327 }
1328 if (k->is_loaded() && !UseCompressedClassPointers) {
1329 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1330 } else {
1331 Rtmp1 = op->tmp3()->as_register();
1332 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1333 }
1334
1335 assert_different_registers(obj, k_RInfo, klass_RInfo);
1336
1337 if (should_profile) {
1338 Label not_null;
1339 __ cbnz(obj, not_null);
1340 // Object is null; update MDO and exit
1341 Register mdo = klass_RInfo;
1342 __ mov_metadata(mdo, md->constant_encoding());
1343 Address data_addr
1344 = __ form_address(rscratch2, mdo,
1345 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1346 LogBytesPerWord);
1347 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1348 __ ldr(rscratch1, data_addr);
1349 __ orr(rscratch1, rscratch1, header_bits);
1350 __ str(rscratch1, data_addr);
1351 __ b(*obj_is_null);
1352 __ bind(not_null);
1353 } else {
1354 __ cbz(obj, *obj_is_null);
1355 }
1356
1357 if (!k->is_loaded()) {
1358 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1359 } else {
1360 __ mov_metadata(k_RInfo, k->constant_encoding());
1361 }
1362 __ verify_oop(obj);
1363
1364 if (op->fast_check()) {
1365 // get object class
1366 // not a safepoint as obj null check happens earlier
1367 __ load_klass(rscratch1, obj);
1368 __ cmp( rscratch1, k_RInfo);
1369
1370 __ br(Assembler::NE, *failure_target);
1371 // successful cast, fall through to profile or jump
1372 } else {
1373 // get object class
1374 // not a safepoint as obj null check happens earlier
1375 __ load_klass(klass_RInfo, obj);
1376 if (k->is_loaded()) {
1377 // See if we get an immediate positive hit
1378 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1379 __ cmp(k_RInfo, rscratch1);
1380 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1381 __ br(Assembler::NE, *failure_target);
1382 // successful cast, fall through to profile or jump
1383 } else {
1384 // See if we get an immediate positive hit
1385 __ br(Assembler::EQ, *success_target);
1386 // check for self
1387 __ cmp(klass_RInfo, k_RInfo);
1388 __ br(Assembler::EQ, *success_target);
1389
1390 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1391 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1392 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1393 // result is a boolean
1394 __ cbzw(klass_RInfo, *failure_target);
1395 // successful cast, fall through to profile or jump
1396 }
1397 } else {
1398 // perform the fast part of the checking logic
1399 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1400 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1401 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1402 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1403 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1404 // result is a boolean
1405 __ cbz(k_RInfo, *failure_target);
1406 // successful cast, fall through to profile or jump
1407 }
1408 }
1409 if (should_profile) {
1410 Register mdo = klass_RInfo, recv = k_RInfo;
1411 __ bind(profile_cast_success);
1412 __ mov_metadata(mdo, md->constant_encoding());
1413 __ load_klass(recv, obj);
1414 Label update_done;
1415 type_profile_helper(mdo, md, data, recv, success);
1416 __ b(*success);
1417
1418 __ bind(profile_cast_failure);
1419 __ mov_metadata(mdo, md->constant_encoding());
1420 Address counter_addr
1421 = __ form_address(rscratch2, mdo,
1422 md->byte_offset_of_slot(data, CounterData::count_offset()),
1423 LogBytesPerWord);
1424 __ ldr(rscratch1, counter_addr);
1425 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1426 __ str(rscratch1, counter_addr);
1427 __ b(*failure);
1428 }
1429 __ b(*success);
1430 }
1431
1432
1433 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1434 const bool should_profile = op->should_profile();
1435
1436 LIR_Code code = op->code();
1437 if (code == lir_store_check) {
1438 Register value = op->object()->as_register();
1439 Register array = op->array()->as_register();
1440 Register k_RInfo = op->tmp1()->as_register();
1441 Register klass_RInfo = op->tmp2()->as_register();
1442 Register Rtmp1 = op->tmp3()->as_register();
1443
1444 CodeStub* stub = op->stub();
1445
1446 // check if it needs to be profiled
1447 ciMethodData* md;
1448 ciProfileData* data;
1449
1450 if (should_profile) {
1451 ciMethod* method = op->profiled_method();
1452 assert(method != NULL, "Should have method");
1453 int bci = op->profiled_bci();
1454 md = method->method_data_or_null();
1455 assert(md != NULL, "Sanity");
1456 data = md->bci_to_data(bci);
1457 assert(data != NULL, "need data for type check");
1458 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1459 }
1460 Label profile_cast_success, profile_cast_failure, done;
1461 Label *success_target = should_profile ? &profile_cast_success : &done;
1462 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1463
1464 if (should_profile) {
1465 Label not_null;
1466 __ cbnz(value, not_null);
1467 // Object is null; update MDO and exit
1468 Register mdo = klass_RInfo;
1469 __ mov_metadata(mdo, md->constant_encoding());
1470 Address data_addr
1471 = __ form_address(rscratch2, mdo,
1472 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1473 LogBytesPerInt);
1474 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1475 __ ldrw(rscratch1, data_addr);
1476 __ orrw(rscratch1, rscratch1, header_bits);
1477 __ strw(rscratch1, data_addr);
1478 __ b(done);
1479 __ bind(not_null);
1480 } else {
1481 __ cbz(value, done);
1482 }
1483
1484 add_debug_info_for_null_check_here(op->info_for_exception());
1485 __ load_klass(k_RInfo, array);
1486 __ load_klass(klass_RInfo, value);
1487
1488 // get instance klass (it's already uncompressed)
1489 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1490 // perform the fast part of the checking logic
1491 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1492 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1493 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1494 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1495 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1496 // result is a boolean
1497 __ cbzw(k_RInfo, *failure_target);
1498 // fall through to the success case
1499
1500 if (should_profile) {
1501 Register mdo = klass_RInfo, recv = k_RInfo;
1502 __ bind(profile_cast_success);
1503 __ mov_metadata(mdo, md->constant_encoding());
1504 __ load_klass(recv, value);
1505 Label update_done;
1506 type_profile_helper(mdo, md, data, recv, &done);
1507 __ b(done);
1508
1509 __ bind(profile_cast_failure);
1510 __ mov_metadata(mdo, md->constant_encoding());
1511 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1512 __ lea(rscratch2, counter_addr);
1513 __ ldr(rscratch1, Address(rscratch2));
1514 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1515 __ str(rscratch1, Address(rscratch2));
1516 __ b(*stub->entry());
1517 }
1518
1519 __ bind(done);
1520 } else if (code == lir_checkcast) {
1521 Register obj = op->object()->as_register();
1522 Register dst = op->result_opr()->as_register();
1523 Label success;
1524 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1525 __ bind(success);
1526 if (dst != obj) {
1527 __ mov(dst, obj);
1528 }
1529 } else if (code == lir_instanceof) {
1530 Register obj = op->object()->as_register();
1531 Register dst = op->result_opr()->as_register();
1532 Label success, failure, done;
1533 emit_typecheck_helper(op, &success, &failure, &failure);
1534 __ bind(failure);
1535 __ mov(dst, zr);
1536 __ b(done);
1537 __ bind(success);
1538 __ mov(dst, 1);
1539 __ bind(done);
1540 } else {
1541 ShouldNotReachHere();
1542 }
1543 }
1544
1545 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1546 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1547 __ cset(rscratch1, Assembler::NE);
1548 __ membar(__ AnyAny);
1549 }
1550
1551 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1552 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1553 __ cset(rscratch1, Assembler::NE);
1554 __ membar(__ AnyAny);
1555 }
1556
1557
1558 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1559 assert(VM_Version::supports_cx8(), "wrong machine");
1560 Register addr = as_reg(op->addr());
1561 Register newval = as_reg(op->new_value());
1562 Register cmpval = as_reg(op->cmp_value());
1563 Label succeed, fail, around;
1564
1565 if (op->code() == lir_cas_obj) {
1566 if (UseCompressedOops) {
1567 Register t1 = op->tmp1()->as_register();
1568 assert(op->tmp1()->is_valid(), "must be");
1569 __ encode_heap_oop(t1, cmpval);
1570 cmpval = t1;
1571 __ encode_heap_oop(rscratch2, newval);
1572 newval = rscratch2;
1573 casw(addr, newval, cmpval);
1574 } else {
1575 casl(addr, newval, cmpval);
1576 }
1577 } else if (op->code() == lir_cas_int) {
1578 casw(addr, newval, cmpval);
1579 } else {
1580 casl(addr, newval, cmpval);
1581 }
1582 }
1583
1584
1585 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1586
1587 Assembler::Condition acond, ncond;
1588 switch (condition) {
1589 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1590 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1591 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1592 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1593 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1594 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1595 case lir_cond_belowEqual:
1596 case lir_cond_aboveEqual:
1597 default: ShouldNotReachHere();
1598 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable
1599 }
1600
1601 assert(result->is_single_cpu() || result->is_double_cpu(),
1602 "expect single register for result");
1603 if (opr1->is_constant() && opr2->is_constant()
1604 && opr1->type() == T_INT && opr2->type() == T_INT) {
1605 jint val1 = opr1->as_jint();
1606 jint val2 = opr2->as_jint();
1607 if (val1 == 0 && val2 == 1) {
1608 __ cset(result->as_register(), ncond);
1609 return;
1610 } else if (val1 == 1 && val2 == 0) {
1611 __ cset(result->as_register(), acond);
1612 return;
1613 }
1614 }
1615
1616 if (opr1->is_constant() && opr2->is_constant()
1617 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1618 jlong val1 = opr1->as_jlong();
1619 jlong val2 = opr2->as_jlong();
1620 if (val1 == 0 && val2 == 1) {
1621 __ cset(result->as_register_lo(), ncond);
1622 return;
1623 } else if (val1 == 1 && val2 == 0) {
1624 __ cset(result->as_register_lo(), acond);
1625 return;
1626 }
1627 }
1628
1629 if (opr1->is_stack()) {
1630 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1631 opr1 = FrameMap::rscratch1_opr;
1632 } else if (opr1->is_constant()) {
1633 LIR_Opr tmp
1634 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1635 const2reg(opr1, tmp, lir_patch_none, NULL);
1636 opr1 = tmp;
1637 }
1638
1639 if (opr2->is_stack()) {
1640 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1641 opr2 = FrameMap::rscratch2_opr;
1642 } else if (opr2->is_constant()) {
1643 LIR_Opr tmp
1644 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1645 const2reg(opr2, tmp, lir_patch_none, NULL);
1646 opr2 = tmp;
1647 }
1648
1649 if (result->type() == T_LONG)
1650 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1651 else
1652 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1653 }
1654
1655 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1656 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1657
1658 if (left->is_single_cpu()) {
1659 Register lreg = left->as_register();
1660 Register dreg = as_reg(dest);
1661
1662 if (right->is_single_cpu()) {
1663 // cpu register - cpu register
1664
1665 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1666 "should be");
1667 Register rreg = right->as_register();
1668 switch (code) {
1669 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1670 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1671 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1672 default: ShouldNotReachHere();
1673 }
1674
1675 } else if (right->is_double_cpu()) {
1676 Register rreg = right->as_register_lo();
1677 // single_cpu + double_cpu: can happen with obj+long
1678 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1679 switch (code) {
1680 case lir_add: __ add(dreg, lreg, rreg); break;
1681 case lir_sub: __ sub(dreg, lreg, rreg); break;
1682 default: ShouldNotReachHere();
1683 }
1684 } else if (right->is_constant()) {
1685 // cpu register - constant
1686 jlong c;
1687
1688 // FIXME. This is fugly: we really need to factor all this logic.
1689 switch(right->type()) {
1690 case T_LONG:
1691 c = right->as_constant_ptr()->as_jlong();
1692 break;
1693 case T_INT:
1694 case T_ADDRESS:
1695 c = right->as_constant_ptr()->as_jint();
1696 break;
1697 default:
1698 ShouldNotReachHere();
1699 c = 0; // unreachable
1700 break;
1701 }
1702
1703 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1704 if (c == 0 && dreg == lreg) {
1705 COMMENT("effective nop elided");
1706 return;
1707 }
1708 switch(left->type()) {
1709 case T_INT:
1710 switch (code) {
1711 case lir_add: __ addw(dreg, lreg, c); break;
1712 case lir_sub: __ subw(dreg, lreg, c); break;
1713 default: ShouldNotReachHere();
1714 }
1715 break;
1716 case T_OBJECT:
1717 case T_ADDRESS:
1718 switch (code) {
1719 case lir_add: __ add(dreg, lreg, c); break;
1720 case lir_sub: __ sub(dreg, lreg, c); break;
1721 default: ShouldNotReachHere();
1722 }
1723 break;
1724 ShouldNotReachHere();
1725 }
1726 } else {
1727 ShouldNotReachHere();
1728 }
1729
1730 } else if (left->is_double_cpu()) {
1731 Register lreg_lo = left->as_register_lo();
1732
1733 if (right->is_double_cpu()) {
1734 // cpu register - cpu register
1735 Register rreg_lo = right->as_register_lo();
1736 switch (code) {
1737 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1738 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1739 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1740 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1741 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1742 default:
1743 ShouldNotReachHere();
1744 }
1745
1746 } else if (right->is_constant()) {
1747 jlong c = right->as_constant_ptr()->as_jlong_bits();
1748 Register dreg = as_reg(dest);
1749 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1750 if (c == 0 && dreg == lreg_lo) {
1751 COMMENT("effective nop elided");
1752 return;
1753 }
1754 switch (code) {
1755 case lir_add: __ add(dreg, lreg_lo, c); break;
1756 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1757 default:
1758 ShouldNotReachHere();
1759 }
1760 } else {
1761 ShouldNotReachHere();
1762 }
1763 } else if (left->is_single_fpu()) {
1764 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1765 switch (code) {
1766 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1767 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1768 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1769 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1770 default:
1771 ShouldNotReachHere();
1772 }
1773 } else if (left->is_double_fpu()) {
1774 if (right->is_double_fpu()) {
1775 // cpu register - cpu register
1776 switch (code) {
1777 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1778 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1779 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1780 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1781 default:
1782 ShouldNotReachHere();
1783 }
1784 } else {
1785 if (right->is_constant()) {
1786 ShouldNotReachHere();
1787 }
1788 ShouldNotReachHere();
1789 }
1790 } else if (left->is_single_stack() || left->is_address()) {
1791 assert(left == dest, "left and dest must be equal");
1792 ShouldNotReachHere();
1793 } else {
1794 ShouldNotReachHere();
1795 }
1796 }
1797
1798 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1799
1800
1801 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1802 switch(code) {
1803 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1804 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1805 default : ShouldNotReachHere();
1806 }
1807 }
1808
1809 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1810
1811 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1812 Register Rleft = left->is_single_cpu() ? left->as_register() :
1813 left->as_register_lo();
1814 if (dst->is_single_cpu()) {
1815 Register Rdst = dst->as_register();
1816 if (right->is_constant()) {
1817 switch (code) {
1818 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1819 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1820 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1821 default: ShouldNotReachHere(); break;
1822 }
1823 } else {
1824 Register Rright = right->is_single_cpu() ? right->as_register() :
1825 right->as_register_lo();
1826 switch (code) {
1827 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1828 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1829 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1830 default: ShouldNotReachHere(); break;
1831 }
1832 }
1833 } else {
1834 Register Rdst = dst->as_register_lo();
1835 if (right->is_constant()) {
1836 switch (code) {
1837 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1838 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1839 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1840 default: ShouldNotReachHere(); break;
1841 }
1842 } else {
1843 Register Rright = right->is_single_cpu() ? right->as_register() :
1844 right->as_register_lo();
1845 switch (code) {
1846 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1847 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1848 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1849 default: ShouldNotReachHere(); break;
1850 }
1851 }
1852 }
1853 }
1854
1855
1856
1857 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1858
1859
1860 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1861 if (opr1->is_constant() && opr2->is_single_cpu()) {
1862 // tableswitch
1863 Register reg = as_reg(opr2);
1864 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1865 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1866 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1867 Register reg1 = as_reg(opr1);
1868 if (opr2->is_single_cpu()) {
1869 // cpu register - cpu register
1870 Register reg2 = opr2->as_register();
1871 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1872 __ cmp(reg1, reg2);
1873 } else {
1874 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1875 __ cmpw(reg1, reg2);
1876 }
1877 return;
1878 }
1879 if (opr2->is_double_cpu()) {
1880 // cpu register - cpu register
1881 Register reg2 = opr2->as_register_lo();
1882 __ cmp(reg1, reg2);
1883 return;
1884 }
1885
1886 if (opr2->is_constant()) {
1887 bool is_32bit = false; // width of register operand
1888 jlong imm;
1889
1890 switch(opr2->type()) {
1891 case T_INT:
1892 imm = opr2->as_constant_ptr()->as_jint();
1893 is_32bit = true;
1894 break;
1895 case T_LONG:
1896 imm = opr2->as_constant_ptr()->as_jlong();
1897 break;
1898 case T_ADDRESS:
1899 imm = opr2->as_constant_ptr()->as_jint();
1900 break;
1901 case T_OBJECT:
1902 case T_ARRAY:
1903 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1904 break;
1905 default:
1906 ShouldNotReachHere();
1907 imm = 0; // unreachable
1908 break;
1909 }
1910
1911 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1912 if (is_32bit)
1913 __ cmpw(reg1, imm);
1914 else
1915 __ cmp(reg1, imm);
1916 return;
1917 } else {
1918 __ mov(rscratch1, imm);
1919 if (is_32bit)
1920 __ cmpw(reg1, rscratch1);
1921 else
1922 __ cmp(reg1, rscratch1);
1923 return;
1924 }
1925 } else
1926 ShouldNotReachHere();
1927 } else if (opr1->is_single_fpu()) {
1928 FloatRegister reg1 = opr1->as_float_reg();
1929 assert(opr2->is_single_fpu(), "expect single float register");
1930 FloatRegister reg2 = opr2->as_float_reg();
1931 __ fcmps(reg1, reg2);
1932 } else if (opr1->is_double_fpu()) {
1933 FloatRegister reg1 = opr1->as_double_reg();
1934 assert(opr2->is_double_fpu(), "expect double float register");
1935 FloatRegister reg2 = opr2->as_double_reg();
1936 __ fcmpd(reg1, reg2);
1937 } else {
1938 ShouldNotReachHere();
1939 }
1940 }
1941
1942 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1943 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1944 bool is_unordered_less = (code == lir_ucmp_fd2i);
1945 if (left->is_single_fpu()) {
1946 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1947 } else if (left->is_double_fpu()) {
1948 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1949 } else {
1950 ShouldNotReachHere();
1951 }
1952 } else if (code == lir_cmp_l2i) {
1953 Label done;
1954 __ cmp(left->as_register_lo(), right->as_register_lo());
1955 __ mov(dst->as_register(), (u_int64_t)-1L);
1956 __ br(Assembler::LT, done);
1957 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1958 __ bind(done);
1959 } else {
1960 ShouldNotReachHere();
1961 }
1962 }
1963
1964
1965 void LIR_Assembler::align_call(LIR_Code code) { }
1966
1967
1968 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1969 address call = __ trampoline_call(Address(op->addr(), rtype));
1970 if (call == NULL) {
1971 bailout("trampoline stub overflow");
1972 return;
1973 }
1974 add_call_info(code_offset(), op->info());
1975 }
1976
1977
1978 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1979 address call = __ ic_call(op->addr());
1980 if (call == NULL) {
1981 bailout("trampoline stub overflow");
1982 return;
1983 }
1984 add_call_info(code_offset(), op->info());
1985 }
1986
1987
1988 /* Currently, vtable-dispatch is only enabled for sparc platforms */
1989 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
1990 ShouldNotReachHere();
1991 }
1992
1993
1994 void LIR_Assembler::emit_static_call_stub() {
1995 address call_pc = __ pc();
1996 address stub = __ start_a_stub(call_stub_size());
1997 if (stub == NULL) {
1998 bailout("static call stub overflow");
1999 return;
2000 }
2001
2002 int start = __ offset();
2003
2004 __ relocate(static_stub_Relocation::spec(call_pc));
2005 __ mov_metadata(rmethod, (Metadata*)NULL);
2006 __ movptr(rscratch1, 0);
2007 __ br(rscratch1);
2008
2009 assert(__ offset() - start <= call_stub_size(), "stub too big");
2010 __ end_a_stub();
2011 }
2012
2013
2014 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2015 assert(exceptionOop->as_register() == r0, "must match");
2016 assert(exceptionPC->as_register() == r3, "must match");
2017
2018 // exception object is not added to oop map by LinearScan
2019 // (LinearScan assumes that no oops are in fixed registers)
2020 info->add_register_oop(exceptionOop);
2021 Runtime1::StubID unwind_id;
2022
2023 // get current pc information
2024 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2025 int pc_for_athrow_offset = __ offset();
2026 InternalAddress pc_for_athrow(__ pc());
2027 __ adr(exceptionPC->as_register(), pc_for_athrow);
2028 add_call_info(pc_for_athrow_offset, info); // for exception handler
2029
2030 __ verify_not_null_oop(r0);
2031 // search an exception handler (r0: exception oop, r3: throwing pc)
2032 if (compilation()->has_fpu_code()) {
2033 unwind_id = Runtime1::handle_exception_id;
2034 } else {
2035 unwind_id = Runtime1::handle_exception_nofpu_id;
2036 }
2037 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2038
2039 // FIXME: enough room for two byte trap ????
2040 __ nop();
2041 }
2042
2043
2044 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2045 assert(exceptionOop->as_register() == r0, "must match");
2046
2047 __ b(_unwind_handler_entry);
2048 }
2049
2050
2051 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2052 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2053 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2054
2055 switch (left->type()) {
2056 case T_INT: {
2057 switch (code) {
2058 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2059 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2060 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2061 default:
2062 ShouldNotReachHere();
2063 break;
2064 }
2065 break;
2066 case T_LONG:
2067 case T_ADDRESS:
2068 case T_OBJECT:
2069 switch (code) {
2070 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2071 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2072 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2073 default:
2074 ShouldNotReachHere();
2075 break;
2076 }
2077 break;
2078 default:
2079 ShouldNotReachHere();
2080 break;
2081 }
2082 }
2083 }
2084
2085
2086 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2087 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2088 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2089
2090 switch (left->type()) {
2091 case T_INT: {
2092 switch (code) {
2093 case lir_shl: __ lslw (dreg, lreg, count); break;
2094 case lir_shr: __ asrw (dreg, lreg, count); break;
2095 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2096 default:
2097 ShouldNotReachHere();
2098 break;
2099 }
2100 break;
2101 case T_LONG:
2102 case T_ADDRESS:
2103 case T_OBJECT:
2104 switch (code) {
2105 case lir_shl: __ lsl (dreg, lreg, count); break;
2106 case lir_shr: __ asr (dreg, lreg, count); break;
2107 case lir_ushr: __ lsr (dreg, lreg, count); break;
2108 default:
2109 ShouldNotReachHere();
2110 break;
2111 }
2112 break;
2113 default:
2114 ShouldNotReachHere();
2115 break;
2116 }
2117 }
2118 }
2119
2120
2121 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2122 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2123 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2124 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2125 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2126 }
2127
2128
2129 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2130 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2131 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2132 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2133 __ mov (rscratch1, c);
2134 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2135 }
2136
2137
2138 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2139 ShouldNotReachHere();
2140 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2141 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2142 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2143 __ lea(rscratch1, __ constant_oop_address(o));
2144 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2145 }
2146
2147
2148 // This code replaces a call to arraycopy; no exception may
2149 // be thrown in this code, they must be thrown in the System.arraycopy
2150 // activation frame; we could save some checks if this would not be the case
2151 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2152 ciArrayKlass* default_type = op->expected_type();
2153 Register src = op->src()->as_register();
2154 Register dst = op->dst()->as_register();
2155 Register src_pos = op->src_pos()->as_register();
2156 Register dst_pos = op->dst_pos()->as_register();
2157 Register length = op->length()->as_register();
2158 Register tmp = op->tmp()->as_register();
2159
2160 CodeStub* stub = op->stub();
2161 int flags = op->flags();
2162 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2163 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2164
2165 // if we don't know anything, just go through the generic arraycopy
2166 if (default_type == NULL // || basic_type == T_OBJECT
2167 ) {
2168 Label done;
2169 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2170
2171 // Save the arguments in case the generic arraycopy fails and we
2172 // have to fall back to the JNI stub
2173 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2174 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2175 __ str(src, Address(sp, 4*BytesPerWord));
2176
2177 address copyfunc_addr = StubRoutines::generic_arraycopy();
2178 assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2179
2180 // The arguments are in java calling convention so we shift them
2181 // to C convention
2182 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2183 __ mov(c_rarg0, j_rarg0);
2184 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2185 __ mov(c_rarg1, j_rarg1);
2186 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2187 __ mov(c_rarg2, j_rarg2);
2188 assert_different_registers(c_rarg3, j_rarg4);
2189 __ mov(c_rarg3, j_rarg3);
2190 __ mov(c_rarg4, j_rarg4);
2191 #ifndef PRODUCT
2192 if (PrintC1Statistics) {
2193 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2194 }
2195 #endif
2196 __ far_call(RuntimeAddress(copyfunc_addr));
2197
2198 __ cbz(r0, *stub->continuation());
2199
2200 // Reload values from the stack so they are where the stub
2201 // expects them.
2202 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2203 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2204 __ ldr(src, Address(sp, 4*BytesPerWord));
2205
2206 // r0 is -1^K where K == partial copied count
2207 __ eonw(rscratch1, r0, 0);
2208 // adjust length down and src/end pos up by partial copied count
2209 __ subw(length, length, rscratch1);
2210 __ addw(src_pos, src_pos, rscratch1);
2211 __ addw(dst_pos, dst_pos, rscratch1);
2212 __ b(*stub->entry());
2213
2214 __ bind(*stub->continuation());
2215 return;
2216 }
2217
2218 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2219
2220 int elem_size = type2aelembytes(basic_type);
2221 int shift_amount;
2222 int scale = exact_log2(elem_size);
2223
2224 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2225 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2226 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2227 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2228
2229 // test for NULL
2230 if (flags & LIR_OpArrayCopy::src_null_check) {
2231 __ cbz(src, *stub->entry());
2232 }
2233 if (flags & LIR_OpArrayCopy::dst_null_check) {
2234 __ cbz(dst, *stub->entry());
2235 }
2236
2237 // If the compiler was not able to prove that exact type of the source or the destination
2238 // of the arraycopy is an array type, check at runtime if the source or the destination is
2239 // an instance type.
2240 if (flags & LIR_OpArrayCopy::type_check) {
2241 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2242 __ load_klass(tmp, dst);
2243 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2244 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2245 __ br(Assembler::GE, *stub->entry());
2246 }
2247
2248 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2249 __ load_klass(tmp, src);
2250 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2251 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2252 __ br(Assembler::GE, *stub->entry());
2253 }
2254 }
2255
2256 // check if negative
2257 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2258 __ cmpw(src_pos, 0);
2259 __ br(Assembler::LT, *stub->entry());
2260 }
2261 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2262 __ cmpw(dst_pos, 0);
2263 __ br(Assembler::LT, *stub->entry());
2264 }
2265
2266 if (flags & LIR_OpArrayCopy::length_positive_check) {
2267 __ cmpw(length, 0);
2268 __ br(Assembler::LT, *stub->entry());
2269 }
2270
2271 if (flags & LIR_OpArrayCopy::src_range_check) {
2272 __ addw(tmp, src_pos, length);
2273 __ ldrw(rscratch1, src_length_addr);
2274 __ cmpw(tmp, rscratch1);
2275 __ br(Assembler::HI, *stub->entry());
2276 }
2277 if (flags & LIR_OpArrayCopy::dst_range_check) {
2278 __ addw(tmp, dst_pos, length);
2279 __ ldrw(rscratch1, dst_length_addr);
2280 __ cmpw(tmp, rscratch1);
2281 __ br(Assembler::HI, *stub->entry());
2282 }
2283
2284 if (flags & LIR_OpArrayCopy::type_check) {
2285 // We don't know the array types are compatible
2286 if (basic_type != T_OBJECT) {
2287 // Simple test for basic type arrays
2288 if (UseCompressedClassPointers) {
2289 __ ldrw(tmp, src_klass_addr);
2290 __ ldrw(rscratch1, dst_klass_addr);
2291 __ cmpw(tmp, rscratch1);
2292 } else {
2293 __ ldr(tmp, src_klass_addr);
2294 __ ldr(rscratch1, dst_klass_addr);
2295 __ cmp(tmp, rscratch1);
2296 }
2297 __ br(Assembler::NE, *stub->entry());
2298 } else {
2299 // For object arrays, if src is a sub class of dst then we can
2300 // safely do the copy.
2301 Label cont, slow;
2302
2303 #define PUSH(r1, r2) \
2304 stp(r1, r2, __ pre(sp, -2 * wordSize));
2305
2306 #define POP(r1, r2) \
2307 ldp(r1, r2, __ post(sp, 2 * wordSize));
2308
2309 __ PUSH(src, dst);
2310
2311 __ load_klass(src, src);
2312 __ load_klass(dst, dst);
2313
2314 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2315
2316 __ PUSH(src, dst);
2317 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2318 __ POP(src, dst);
2319
2320 __ cbnz(src, cont);
2321
2322 __ bind(slow);
2323 __ POP(src, dst);
2324
2325 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2326 if (copyfunc_addr != NULL) { // use stub if available
2327 // src is not a sub class of dst so we have to do a
2328 // per-element check.
2329
2330 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2331 if ((flags & mask) != mask) {
2332 // Check that at least both of them object arrays.
2333 assert(flags & mask, "one of the two should be known to be an object array");
2334
2335 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2336 __ load_klass(tmp, src);
2337 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2338 __ load_klass(tmp, dst);
2339 }
2340 int lh_offset = in_bytes(Klass::layout_helper_offset());
2341 Address klass_lh_addr(tmp, lh_offset);
2342 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2343 __ ldrw(rscratch1, klass_lh_addr);
2344 __ mov(rscratch2, objArray_lh);
2345 __ eorw(rscratch1, rscratch1, rscratch2);
2346 __ cbnzw(rscratch1, *stub->entry());
2347 }
2348
2349 // Spill because stubs can use any register they like and it's
2350 // easier to restore just those that we care about.
2351 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2352 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2353 __ str(src, Address(sp, 4*BytesPerWord));
2354
2355 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2356 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2357 assert_different_registers(c_rarg0, dst, dst_pos, length);
2358 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2359 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2360 assert_different_registers(c_rarg1, dst, length);
2361 __ uxtw(c_rarg2, length);
2362 assert_different_registers(c_rarg2, dst);
2363
2364 __ load_klass(c_rarg4, dst);
2365 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2366 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2367 __ far_call(RuntimeAddress(copyfunc_addr));
2368
2369 #ifndef PRODUCT
2370 if (PrintC1Statistics) {
2371 Label failed;
2372 __ cbnz(r0, failed);
2373 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2374 __ bind(failed);
2375 }
2376 #endif
2377
2378 __ cbz(r0, *stub->continuation());
2379
2380 #ifndef PRODUCT
2381 if (PrintC1Statistics) {
2382 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2383 }
2384 #endif
2385 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2386
2387 // Restore previously spilled arguments
2388 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2389 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2390 __ ldr(src, Address(sp, 4*BytesPerWord));
2391
2392 // return value is -1^K where K is partial copied count
2393 __ eonw(rscratch1, r0, zr);
2394 // adjust length down and src/end pos up by partial copied count
2395 __ subw(length, length, rscratch1);
2396 __ addw(src_pos, src_pos, rscratch1);
2397 __ addw(dst_pos, dst_pos, rscratch1);
2398 }
2399
2400 __ b(*stub->entry());
2401
2402 __ bind(cont);
2403 __ POP(src, dst);
2404 }
2405 }
2406
2407 #ifdef ASSERT
2408 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2409 // Sanity check the known type with the incoming class. For the
2410 // primitive case the types must match exactly with src.klass and
2411 // dst.klass each exactly matching the default type. For the
2412 // object array case, if no type check is needed then either the
2413 // dst type is exactly the expected type and the src type is a
2414 // subtype which we can't check or src is the same array as dst
2415 // but not necessarily exactly of type default_type.
2416 Label known_ok, halt;
2417 __ mov_metadata(tmp, default_type->constant_encoding());
2418 if (UseCompressedClassPointers) {
2419 __ encode_klass_not_null(tmp);
2420 }
2421
2422 if (basic_type != T_OBJECT) {
2423
2424 if (UseCompressedClassPointers) {
2425 __ ldrw(rscratch1, dst_klass_addr);
2426 __ cmpw(tmp, rscratch1);
2427 } else {
2428 __ ldr(rscratch1, dst_klass_addr);
2429 __ cmp(tmp, rscratch1);
2430 }
2431 __ br(Assembler::NE, halt);
2432 if (UseCompressedClassPointers) {
2433 __ ldrw(rscratch1, src_klass_addr);
2434 __ cmpw(tmp, rscratch1);
2435 } else {
2436 __ ldr(rscratch1, src_klass_addr);
2437 __ cmp(tmp, rscratch1);
2438 }
2439 __ br(Assembler::EQ, known_ok);
2440 } else {
2441 if (UseCompressedClassPointers) {
2442 __ ldrw(rscratch1, dst_klass_addr);
2443 __ cmpw(tmp, rscratch1);
2444 } else {
2445 __ ldr(rscratch1, dst_klass_addr);
2446 __ cmp(tmp, rscratch1);
2447 }
2448 __ br(Assembler::EQ, known_ok);
2449 __ cmp(src, dst);
2450 __ br(Assembler::EQ, known_ok);
2451 }
2452 __ bind(halt);
2453 __ stop("incorrect type information in arraycopy");
2454 __ bind(known_ok);
2455 }
2456 #endif
2457
2458 #ifndef PRODUCT
2459 if (PrintC1Statistics) {
2460 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2461 }
2462 #endif
2463
2464 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2465 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2466 assert_different_registers(c_rarg0, dst, dst_pos, length);
2467 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2468 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2469 assert_different_registers(c_rarg1, dst, length);
2470 __ uxtw(c_rarg2, length);
2471 assert_different_registers(c_rarg2, dst);
2472
2473 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2474 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2475 const char *name;
2476 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2477
2478 CodeBlob *cb = CodeCache::find_blob(entry);
2479 if (cb) {
2480 __ far_call(RuntimeAddress(entry));
2481 } else {
2482 __ call_VM_leaf(entry, 3);
2483 }
2484
2485 __ bind(*stub->continuation());
2486 }
2487
2488
2489
2490
2491 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2492 Register obj = op->obj_opr()->as_register(); // may not be an oop
2493 Register hdr = op->hdr_opr()->as_register();
2494 Register lock = op->lock_opr()->as_register();
2495 if (!UseFastLocking) {
2496 __ b(*op->stub()->entry());
2497 } else if (op->code() == lir_lock) {
2498 Register scratch = noreg;
2499 if (UseBiasedLocking) {
2500 scratch = op->scratch_opr()->as_register();
2501 }
2502 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2503 // add debug info for NullPointerException only if one is possible
2504 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2505 if (op->info() != NULL) {
2506 add_debug_info_for_null_check(null_check_offset, op->info());
2507 }
2508 // done
2509 } else if (op->code() == lir_unlock) {
2510 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2511 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2512 } else {
2513 Unimplemented();
2514 }
2515 __ bind(*op->stub()->continuation());
2516 }
2517
2518
2519 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2520 ciMethod* method = op->profiled_method();
2521 int bci = op->profiled_bci();
2522 ciMethod* callee = op->profiled_callee();
2523
2524 // Update counter for all call types
2525 ciMethodData* md = method->method_data_or_null();
2526 assert(md != NULL, "Sanity");
2527 ciProfileData* data = md->bci_to_data(bci);
2528 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2529 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2530 Register mdo = op->mdo()->as_register();
2531 __ mov_metadata(mdo, md->constant_encoding());
2532 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2533 // Perform additional virtual call profiling for invokevirtual and
2534 // invokeinterface bytecodes
2535 if (op->should_profile_receiver_type()) {
2536 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2537 Register recv = op->recv()->as_register();
2538 assert_different_registers(mdo, recv);
2539 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2540 ciKlass* known_klass = op->known_holder();
2541 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2542 // We know the type that will be seen at this call site; we can
2543 // statically update the MethodData* rather than needing to do
2544 // dynamic tests on the receiver type
2545
2546 // NOTE: we should probably put a lock around this search to
2547 // avoid collisions by concurrent compilations
2548 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2549 uint i;
2550 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2551 ciKlass* receiver = vc_data->receiver(i);
2552 if (known_klass->equals(receiver)) {
2553 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2554 __ addptr(data_addr, DataLayout::counter_increment);
2555 return;
2556 }
2557 }
2558
2559 // Receiver type not found in profile data; select an empty slot
2560
2561 // Note that this is less efficient than it should be because it
2562 // always does a write to the receiver part of the
2563 // VirtualCallData rather than just the first time
2564 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2565 ciKlass* receiver = vc_data->receiver(i);
2566 if (receiver == NULL) {
2567 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2568 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2569 __ lea(rscratch2, recv_addr);
2570 __ str(rscratch1, Address(rscratch2));
2571 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2572 __ addptr(data_addr, DataLayout::counter_increment);
2573 return;
2574 }
2575 }
2576 } else {
2577 __ load_klass(recv, recv);
2578 Label update_done;
2579 type_profile_helper(mdo, md, data, recv, &update_done);
2580 // Receiver did not match any saved receiver and there is no empty row for it.
2581 // Increment total counter to indicate polymorphic case.
2582 __ addptr(counter_addr, DataLayout::counter_increment);
2583
2584 __ bind(update_done);
2585 }
2586 } else {
2587 // Static call
2588 __ addptr(counter_addr, DataLayout::counter_increment);
2589 }
2590 }
2591
2592
2593 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2594 Unimplemented();
2595 }
2596
2597
2598 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2599 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2600 }
2601
2602 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2603 assert(op->crc()->is_single_cpu(), "crc must be register");
2604 assert(op->val()->is_single_cpu(), "byte value must be register");
2605 assert(op->result_opr()->is_single_cpu(), "result must be register");
2606 Register crc = op->crc()->as_register();
2607 Register val = op->val()->as_register();
2608 Register res = op->result_opr()->as_register();
2609
2610 assert_different_registers(val, crc, res);
2611 unsigned long offset;
2612 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2613 if (offset) __ add(res, res, offset);
2614
2615 __ mvnw(crc, crc); // ~crc
2616 __ update_byte_crc32(crc, val, res);
2617 __ mvnw(res, crc); // ~crc
2618 }
2619
2620 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2621 COMMENT("emit_profile_type {");
2622 Register obj = op->obj()->as_register();
2623 Register tmp = op->tmp()->as_pointer_register();
2624 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2625 ciKlass* exact_klass = op->exact_klass();
2626 intptr_t current_klass = op->current_klass();
2627 bool not_null = op->not_null();
2628 bool no_conflict = op->no_conflict();
2629
2630 Label update, next, none;
2631
2632 bool do_null = !not_null;
2633 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2634 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2635
2636 assert(do_null || do_update, "why are we here?");
2637 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2638 assert(mdo_addr.base() != rscratch1, "wrong register");
2639
2640 __ verify_oop(obj);
2641
2642 if (tmp != obj) {
2643 __ mov(tmp, obj);
2644 }
2645 if (do_null) {
2646 __ cbnz(tmp, update);
2647 if (!TypeEntries::was_null_seen(current_klass)) {
2648 __ ldr(rscratch2, mdo_addr);
2649 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2650 __ str(rscratch2, mdo_addr);
2651 }
2652 if (do_update) {
2653 #ifndef ASSERT
2654 __ b(next);
2655 }
2656 #else
2657 __ b(next);
2658 }
2659 } else {
2660 __ cbnz(tmp, update);
2661 __ stop("unexpected null obj");
2662 #endif
2663 }
2664
2665 __ bind(update);
2666
2667 if (do_update) {
2668 #ifdef ASSERT
2669 if (exact_klass != NULL) {
2670 Label ok;
2671 __ load_klass(tmp, tmp);
2672 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2673 __ eor(rscratch1, tmp, rscratch1);
2674 __ cbz(rscratch1, ok);
2675 __ stop("exact klass and actual klass differ");
2676 __ bind(ok);
2677 }
2678 #endif
2679 if (!no_conflict) {
2680 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2681 if (exact_klass != NULL) {
2682 __ mov_metadata(tmp, exact_klass->constant_encoding());
2683 } else {
2684 __ load_klass(tmp, tmp);
2685 }
2686
2687 __ ldr(rscratch2, mdo_addr);
2688 __ eor(tmp, tmp, rscratch2);
2689 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2690 // klass seen before, nothing to do. The unknown bit may have been
2691 // set already but no need to check.
2692 __ cbz(rscratch1, next);
2693
2694 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2695
2696 if (TypeEntries::is_type_none(current_klass)) {
2697 __ cbz(rscratch2, none);
2698 __ cmp(rscratch2, TypeEntries::null_seen);
2699 __ br(Assembler::EQ, none);
2700 // There is a chance that the checks above (re-reading profiling
2701 // data from memory) fail if another thread has just set the
2702 // profiling to this obj's klass
2703 __ dmb(Assembler::ISHLD);
2704 __ ldr(rscratch2, mdo_addr);
2705 __ eor(tmp, tmp, rscratch2);
2706 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2707 __ cbz(rscratch1, next);
2708 }
2709 } else {
2710 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2711 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2712
2713 __ ldr(tmp, mdo_addr);
2714 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2715 }
2716
2717 // different than before. Cannot keep accurate profile.
2718 __ ldr(rscratch2, mdo_addr);
2719 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2720 __ str(rscratch2, mdo_addr);
2721
2722 if (TypeEntries::is_type_none(current_klass)) {
2723 __ b(next);
2724
2725 __ bind(none);
2726 // first time here. Set profile type.
2727 __ str(tmp, mdo_addr);
2728 }
2729 } else {
2730 // There's a single possible klass at this profile point
2731 assert(exact_klass != NULL, "should be");
2732 if (TypeEntries::is_type_none(current_klass)) {
2733 __ mov_metadata(tmp, exact_klass->constant_encoding());
2734 __ ldr(rscratch2, mdo_addr);
2735 __ eor(tmp, tmp, rscratch2);
2736 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2737 __ cbz(rscratch1, next);
2738 #ifdef ASSERT
2739 {
2740 Label ok;
2741 __ ldr(rscratch1, mdo_addr);
2742 __ cbz(rscratch1, ok);
2743 __ cmp(rscratch1, TypeEntries::null_seen);
2744 __ br(Assembler::EQ, ok);
2745 // may have been set by another thread
2746 __ dmb(Assembler::ISHLD);
2747 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2748 __ ldr(rscratch2, mdo_addr);
2749 __ eor(rscratch2, rscratch1, rscratch2);
2750 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2751 __ cbz(rscratch2, ok);
2752
2753 __ stop("unexpected profiling mismatch");
2754 __ bind(ok);
2755 }
2756 #endif
2757 // first time here. Set profile type.
2758 __ ldr(tmp, mdo_addr);
2759 } else {
2760 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2761 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2762
2763 __ ldr(tmp, mdo_addr);
2764 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2765
2766 __ orr(tmp, tmp, TypeEntries::type_unknown);
2767 __ str(tmp, mdo_addr);
2768 // FIXME: Write barrier needed here?
2769 }
2770 }
2771
2772 __ bind(next);
2773 }
2774 COMMENT("} emit_profile_type");
2775 }
2776
2777
2778 void LIR_Assembler::align_backward_branch_target() {
2779 }
2780
2781
2782 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2783 if (left->is_single_cpu()) {
2784 assert(dest->is_single_cpu(), "expect single result reg");
2785 __ negw(dest->as_register(), left->as_register());
2786 } else if (left->is_double_cpu()) {
2787 assert(dest->is_double_cpu(), "expect double result reg");
2788 __ neg(dest->as_register_lo(), left->as_register_lo());
2789 } else if (left->is_single_fpu()) {
2790 assert(dest->is_single_fpu(), "expect single float result reg");
2791 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2792 } else {
2793 assert(left->is_double_fpu(), "expect double float operand reg");
2794 assert(dest->is_double_fpu(), "expect double float result reg");
2795 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2796 }
2797 }
2798
2799
2800 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2801 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2802 }
2803
2804
2805 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2806 assert(!tmp->is_valid(), "don't need temporary");
2807
2808 CodeBlob *cb = CodeCache::find_blob(dest);
2809 if (cb) {
2810 __ far_call(RuntimeAddress(dest));
2811 } else {
2812 __ mov(rscratch1, RuntimeAddress(dest));
2813 int len = args->length();
2814 int type = 0;
2815 if (! result->is_illegal()) {
2816 switch (result->type()) {
2817 case T_VOID:
2818 type = 0;
2819 break;
2820 case T_INT:
2821 case T_LONG:
2822 case T_OBJECT:
2823 type = 1;
2824 break;
2825 case T_FLOAT:
2826 type = 2;
2827 break;
2828 case T_DOUBLE:
2829 type = 3;
2830 break;
2831 default:
2832 ShouldNotReachHere();
2833 break;
2834 }
2835 }
2836 int num_gpargs = 0;
2837 int num_fpargs = 0;
2838 for (int i = 0; i < args->length(); i++) {
2839 LIR_Opr arg = args->at(i);
2840 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2841 num_fpargs++;
2842 } else {
2843 num_gpargs++;
2844 }
2845 }
2846 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2847 }
2848
2849 if (info != NULL) {
2850 add_call_info_here(info);
2851 }
2852 __ maybe_isb();
2853 }
2854
2855 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2856 if (dest->is_address() || src->is_address()) {
2857 move_op(src, dest, type, lir_patch_none, info,
2858 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2859 } else {
2860 ShouldNotReachHere();
2861 }
2862 }
2863
2864 #ifdef ASSERT
2865 // emit run-time assertion
2866 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2867 assert(op->code() == lir_assert, "must be");
2868
2869 if (op->in_opr1()->is_valid()) {
2870 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2871 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2872 } else {
2873 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2874 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2875 }
2876
2877 Label ok;
2878 if (op->condition() != lir_cond_always) {
2879 Assembler::Condition acond = Assembler::AL;
2880 switch (op->condition()) {
2881 case lir_cond_equal: acond = Assembler::EQ; break;
2882 case lir_cond_notEqual: acond = Assembler::NE; break;
2883 case lir_cond_less: acond = Assembler::LT; break;
2884 case lir_cond_lessEqual: acond = Assembler::LE; break;
2885 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2886 case lir_cond_greater: acond = Assembler::GT; break;
2887 case lir_cond_belowEqual: acond = Assembler::LS; break;
2888 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2889 default: ShouldNotReachHere();
2890 }
2891 __ br(acond, ok);
2892 }
2893 if (op->halt()) {
2894 const char* str = __ code_string(op->msg());
2895 __ stop(str);
2896 } else {
2897 breakpoint();
2898 }
2899 __ bind(ok);
2900 }
2901 #endif
2902
2903 #ifndef PRODUCT
2904 #define COMMENT(x) do { __ block_comment(x); } while (0)
2905 #else
2906 #define COMMENT(x)
2907 #endif
2908
2909 void LIR_Assembler::membar() {
2910 COMMENT("membar");
2911 __ membar(MacroAssembler::AnyAny);
2912 }
2913
2914 void LIR_Assembler::membar_acquire() {
2915 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2916 }
2917
2918 void LIR_Assembler::membar_release() {
2919 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2920 }
2921
2922 void LIR_Assembler::membar_loadload() {
2923 __ membar(Assembler::LoadLoad);
2924 }
2925
2926 void LIR_Assembler::membar_storestore() {
2927 __ membar(MacroAssembler::StoreStore);
2928 }
2929
2930 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2931
2932 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2933
2934 void LIR_Assembler::on_spin_wait() {
2935 Unimplemented();
2936 }
2937
2938 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2939 __ mov(result_reg->as_register(), rthread);
2940 }
2941
2942
2943 void LIR_Assembler::peephole(LIR_List *lir) {
2944 #if 0
2945 if (tableswitch_count >= max_tableswitches)
2946 return;
2947
2948 /*
2949 This finite-state automaton recognizes sequences of compare-and-
2950 branch instructions. We will turn them into a tableswitch. You
2951 could argue that C1 really shouldn't be doing this sort of
2952 optimization, but without it the code is really horrible.
2953 */
2954
2955 enum { start_s, cmp1_s, beq_s, cmp_s } state;
2956 int first_key, last_key = -2147483648;
2957 int next_key = 0;
2958 int start_insn = -1;
2959 int last_insn = -1;
2960 Register reg = noreg;
2961 LIR_Opr reg_opr;
2962 state = start_s;
2963
2964 LIR_OpList* inst = lir->instructions_list();
2965 for (int i = 0; i < inst->length(); i++) {
2966 LIR_Op* op = inst->at(i);
2967 switch (state) {
2968 case start_s:
2969 first_key = -1;
2970 start_insn = i;
2971 switch (op->code()) {
2972 case lir_cmp:
2973 LIR_Opr opr1 = op->as_Op2()->in_opr1();
2974 LIR_Opr opr2 = op->as_Op2()->in_opr2();
2975 if (opr1->is_cpu_register() && opr1->is_single_cpu()
2976 && opr2->is_constant()
2977 && opr2->type() == T_INT) {
2978 reg_opr = opr1;
2979 reg = opr1->as_register();
2980 first_key = opr2->as_constant_ptr()->as_jint();
2981 next_key = first_key + 1;
2982 state = cmp_s;
2983 goto next_state;
2984 }
2985 break;
2986 }
2987 break;
2988 case cmp_s:
2989 switch (op->code()) {
2990 case lir_branch:
2991 if (op->as_OpBranch()->cond() == lir_cond_equal) {
2992 state = beq_s;
2993 last_insn = i;
2994 goto next_state;
2995 }
2996 }
2997 state = start_s;
2998 break;
2999 case beq_s:
3000 switch (op->code()) {
3001 case lir_cmp: {
3002 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3003 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3004 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3005 && opr1->as_register() == reg
3006 && opr2->is_constant()
3007 && opr2->type() == T_INT
3008 && opr2->as_constant_ptr()->as_jint() == next_key) {
3009 last_key = next_key;
3010 next_key++;
3011 state = cmp_s;
3012 goto next_state;
3013 }
3014 }
3015 }
3016 last_key = next_key;
3017 state = start_s;
3018 break;
3019 default:
3020 assert(false, "impossible state");
3021 }
3022 if (state == start_s) {
3023 if (first_key < last_key - 5L && reg != noreg) {
3024 {
3025 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3026 // reg->encoding(),
3027 // start_insn, first_key, last_key);
3028 // for (int i = 0; i < inst->length(); i++) {
3029 // inst->at(i)->print();
3030 // tty->print("\n");
3031 // }
3032 // tty->print("\n");
3033 }
3034
3035 struct tableswitch *sw = &switches[tableswitch_count];
3036 sw->_insn_index = start_insn, sw->_first_key = first_key,
3037 sw->_last_key = last_key, sw->_reg = reg;
3038 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3039 {
3040 // Insert the new table of branches
3041 int offset = last_insn;
3042 for (int n = first_key; n < last_key; n++) {
3043 inst->insert_before
3044 (last_insn + 1,
3045 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3046 inst->at(offset)->as_OpBranch()->label()));
3047 offset -= 2, i++;
3048 }
3049 }
3050 // Delete all the old compare-and-branch instructions
3051 for (int n = first_key; n < last_key; n++) {
3052 inst->remove_at(start_insn);
3053 inst->remove_at(start_insn);
3054 }
3055 // Insert the tableswitch instruction
3056 inst->insert_before(start_insn,
3057 new LIR_Op2(lir_cmp, lir_cond_always,
3058 LIR_OprFact::intConst(tableswitch_count),
3059 reg_opr));
3060 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3061 tableswitch_count++;
3062 }
3063 reg = noreg;
3064 last_key = -2147483648;
3065 }
3066 next_state:
3067 ;
3068 }
3069 #endif
3070 }
3071
3072 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3073 Address addr = as_Address(src->as_address_ptr());
3074 BasicType type = src->type();
3075 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3076
3077 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3078 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3079
3080 switch(type) {
3081 case T_INT:
3082 xchg = &MacroAssembler::atomic_xchgalw;
3083 add = &MacroAssembler::atomic_addalw;
3084 break;
3085 case T_LONG:
3086 xchg = &MacroAssembler::atomic_xchgal;
3087 add = &MacroAssembler::atomic_addal;
3088 break;
3089 case T_OBJECT:
3090 case T_ARRAY:
3091 if (UseCompressedOops) {
3092 xchg = &MacroAssembler::atomic_xchgalw;
3093 add = &MacroAssembler::atomic_addalw;
3094 } else {
3095 xchg = &MacroAssembler::atomic_xchgal;
3096 add = &MacroAssembler::atomic_addal;
3097 }
3098 break;
3099 default:
3100 ShouldNotReachHere();
3101 xchg = &MacroAssembler::atomic_xchgal;
3102 add = &MacroAssembler::atomic_addal; // unreachable
3103 }
3104
3105 switch (code) {
3106 case lir_xadd:
3107 {
3108 RegisterOrConstant inc;
3109 Register tmp = as_reg(tmp_op);
3110 Register dst = as_reg(dest);
3111 if (data->is_constant()) {
3112 inc = RegisterOrConstant(as_long(data));
3113 assert_different_registers(dst, addr.base(), tmp,
3114 rscratch1, rscratch2);
3115 } else {
3116 inc = RegisterOrConstant(as_reg(data));
3117 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3118 rscratch1, rscratch2);
3119 }
3120 __ lea(tmp, addr);
3121 (_masm->*add)(dst, inc, tmp);
3122 break;
3123 }
3124 case lir_xchg:
3125 {
3126 Register tmp = tmp_op->as_register();
3127 Register obj = as_reg(data);
3128 Register dst = as_reg(dest);
3129 if (is_oop && UseCompressedOops) {
3130 __ encode_heap_oop(rscratch2, obj);
3131 obj = rscratch2;
3132 }
3133 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3134 __ lea(tmp, addr);
3135 (_masm->*xchg)(dst, obj, tmp);
3136 if (is_oop && UseCompressedOops) {
3137 __ decode_heap_oop(dst);
3138 }
3139 }
3140 break;
3141 default:
3142 ShouldNotReachHere();
3143 }
3144 __ membar(__ AnyAny);
3145 }
3146
3147 #undef __