1 /*
2 * Copyright (c) 2013, Red Hat Inc.
3 * Copyright (c) 2000, 2019, Oracle and/or its affiliates.
4 * All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "precompiled.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "gc_interface/collectedHeap.hpp"
38 #include "memory/barrierSet.hpp"
39 #include "memory/cardTableModRefBS.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "vmreg_aarch64.inline.hpp"
44
45
46
47 #ifndef PRODUCT
48 #define COMMENT(x) do { __ block_comment(x); } while (0)
49 #else
50 #define COMMENT(x)
51 #endif
52
53 NEEDS_CLEANUP // remove this definitions ?
54 const Register IC_Klass = rscratch2; // where the IC klass is cached
55 const Register SYNC_header = r0; // synchronization header
56 const Register SHIFT_count = r0; // where count for shift operations must be
57
58 #define __ _masm->
59
60
61 static void select_different_registers(Register preserve,
62 Register extra,
63 Register &tmp1,
64 Register &tmp2) {
65 if (tmp1 == preserve) {
66 assert_different_registers(tmp1, tmp2, extra);
67 tmp1 = extra;
68 } else if (tmp2 == preserve) {
69 assert_different_registers(tmp1, tmp2, extra);
70 tmp2 = extra;
71 }
72 assert_different_registers(preserve, tmp1, tmp2);
73 }
74
75
76
77 static void select_different_registers(Register preserve,
78 Register extra,
79 Register &tmp1,
80 Register &tmp2,
81 Register &tmp3) {
82 if (tmp1 == preserve) {
83 assert_different_registers(tmp1, tmp2, tmp3, extra);
84 tmp1 = extra;
85 } else if (tmp2 == preserve) {
86 assert_different_registers(tmp1, tmp2, tmp3, extra);
87 tmp2 = extra;
88 } else if (tmp3 == preserve) {
89 assert_different_registers(tmp1, tmp2, tmp3, extra);
90 tmp3 = extra;
91 }
92 assert_different_registers(preserve, tmp1, tmp2, tmp3);
93 }
94
95
96 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
97
98
99 LIR_Opr LIR_Assembler::receiverOpr() {
100 return FrameMap::receiver_opr;
101 }
102
103 LIR_Opr LIR_Assembler::osrBufferPointer() {
104 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
105 }
106
107 //--------------fpu register translations-----------------------
108
109
110 address LIR_Assembler::float_constant(float f) {
111 address const_addr = __ float_constant(f);
112 if (const_addr == NULL) {
113 bailout("const section overflow");
114 return __ code()->consts()->start();
115 } else {
116 return const_addr;
117 }
118 }
119
120
121 address LIR_Assembler::double_constant(double d) {
122 address const_addr = __ double_constant(d);
123 if (const_addr == NULL) {
124 bailout("const section overflow");
125 return __ code()->consts()->start();
126 } else {
127 return const_addr;
128 }
129 }
130
131 address LIR_Assembler::int_constant(jlong n) {
132 address const_addr = __ long_constant(n);
133 if (const_addr == NULL) {
134 bailout("const section overflow");
135 return __ code()->consts()->start();
136 } else {
137 return const_addr;
138 }
139 }
140
141 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
142
143 void LIR_Assembler::reset_FPU() { Unimplemented(); }
144
145 void LIR_Assembler::fpop() { Unimplemented(); }
146
147 void LIR_Assembler::fxch(int i) { Unimplemented(); }
148
149 void LIR_Assembler::fld(int i) { Unimplemented(); }
150
151 void LIR_Assembler::ffree(int i) { Unimplemented(); }
152
153 void LIR_Assembler::breakpoint() { Unimplemented(); }
154
155 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
156
157 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
158
159 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
160 //-------------------------------------------
161
162 static Register as_reg(LIR_Opr op) {
163 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
164 }
165
166 static jlong as_long(LIR_Opr data) {
167 jlong result;
168 switch (data->type()) {
169 case T_INT:
170 result = (data->as_jint());
171 break;
172 case T_LONG:
173 result = (data->as_jlong());
174 break;
175 default:
176 ShouldNotReachHere();
177 }
178 return result;
179 }
180
181 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
182 Register base = addr->base()->as_pointer_register();
183 LIR_Opr opr = addr->index();
184 if (opr->is_cpu_register()) {
185 Register index;
186 if (opr->is_single_cpu())
187 index = opr->as_register();
188 else
189 index = opr->as_register_lo();
190 assert(addr->disp() == 0, "must be");
191 switch(opr->type()) {
192 case T_INT:
193 return Address(base, index, Address::sxtw(addr->scale()));
194 case T_LONG:
195 return Address(base, index, Address::lsl(addr->scale()));
196 default:
197 ShouldNotReachHere();
198 }
199 } else {
200 intptr_t addr_offset = intptr_t(addr->disp());
201 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
202 return Address(base, addr_offset, Address::lsl(addr->scale()));
203 else {
204 __ mov(tmp, addr_offset);
205 return Address(base, tmp, Address::lsl(addr->scale()));
206 }
207 }
208 return Address();
209 }
210
211 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
212 ShouldNotReachHere();
213 return Address();
214 }
215
216 Address LIR_Assembler::as_Address(LIR_Address* addr) {
217 return as_Address(addr, rscratch1);
218 }
219
220 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
221 return as_Address(addr, rscratch1); // Ouch
222 // FIXME: This needs to be much more clever. See x86.
223 }
224
225
226 void LIR_Assembler::osr_entry() {
227 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
228 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
229 ValueStack* entry_state = osr_entry->state();
230 int number_of_locks = entry_state->locks_size();
231
232 // we jump here if osr happens with the interpreter
233 // state set up to continue at the beginning of the
234 // loop that triggered osr - in particular, we have
235 // the following registers setup:
236 //
237 // r2: osr buffer
238 //
239
240 // build frame
241 ciMethod* m = compilation()->method();
242 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
243
244 // OSR buffer is
245 //
246 // locals[nlocals-1..0]
247 // monitors[0..number_of_locks]
248 //
249 // locals is a direct copy of the interpreter frame so in the osr buffer
250 // so first slot in the local array is the last local from the interpreter
251 // and last slot is local[0] (receiver) from the interpreter
252 //
253 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
254 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
255 // in the interpreter frame (the method lock if a sync method)
256
257 // Initialize monitors in the compiled activation.
258 // r2: pointer to osr buffer
259 //
260 // All other registers are dead at this point and the locals will be
261 // copied into place by code emitted in the IR.
262
263 Register OSR_buf = osrBufferPointer()->as_pointer_register();
264 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
265 int monitor_offset = BytesPerWord * method()->max_locals() +
266 (2 * BytesPerWord) * (number_of_locks - 1);
267 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
268 // the OSR buffer using 2 word entries: first the lock and then
269 // the oop.
270 for (int i = 0; i < number_of_locks; i++) {
271 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
272 #ifdef ASSERT
273 // verify the interpreter's monitor has a non-null object
274 {
275 Label L;
276 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
277 __ cbnz(rscratch1, L);
278 __ stop("locked object is NULL");
279 __ bind(L);
280 }
281 #endif
282 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
283 __ str(r19, frame_map()->address_for_monitor_lock(i));
284 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
285 __ str(r19, frame_map()->address_for_monitor_object(i));
286 }
287 }
288 }
289
290
291 // inline cache check; done before the frame is built.
292 int LIR_Assembler::check_icache() {
293 Register receiver = FrameMap::receiver_opr->as_register();
294 Register ic_klass = IC_Klass;
295 int start_offset = __ offset();
296 __ inline_cache_check(receiver, ic_klass);
297
298 // if icache check fails, then jump to runtime routine
299 // Note: RECEIVER must still contain the receiver!
300 Label dont;
301 __ br(Assembler::EQ, dont);
302 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
303
304 // We align the verified entry point unless the method body
305 // (including its inline cache check) will fit in a single 64-byte
306 // icache line.
307 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
308 // force alignment after the cache check.
309 __ align(CodeEntryAlignment);
310 }
311
312 __ bind(dont);
313 return start_offset;
314 }
315
316
317 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
318 if (o == NULL) {
319 __ mov(reg, zr);
320 } else {
321 __ movoop(reg, o, /*immediate*/true);
322 }
323 }
324
325 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
326 address target = NULL;
327 relocInfo::relocType reloc_type = relocInfo::none;
328
329 switch (patching_id(info)) {
330 case PatchingStub::access_field_id:
331 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
332 reloc_type = relocInfo::section_word_type;
333 break;
334 case PatchingStub::load_klass_id:
335 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
336 reloc_type = relocInfo::metadata_type;
337 break;
338 case PatchingStub::load_mirror_id:
339 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
340 reloc_type = relocInfo::oop_type;
341 break;
342 case PatchingStub::load_appendix_id:
343 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
344 reloc_type = relocInfo::oop_type;
345 break;
346 default: ShouldNotReachHere();
347 }
348
349 __ far_call(RuntimeAddress(target));
350 add_call_info_here(info);
351 }
352
353 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
354 deoptimize_trap(info);
355 }
356
357
358 // This specifies the rsp decrement needed to build the frame
359 int LIR_Assembler::initial_frame_size_in_bytes() const {
360 // if rounding, must let FrameMap know!
361
362 // The frame_map records size in slots (32bit word)
363
364 // subtract two words to account for return address and link
365 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
366 }
367
368
369 int LIR_Assembler::emit_exception_handler() {
370 // if the last instruction is a call (typically to do a throw which
371 // is coming at the end after block reordering) the return address
372 // must still point into the code area in order to avoid assertion
373 // failures when searching for the corresponding bci => add a nop
374 // (was bug 5/14/1999 - gri)
375 __ nop();
376
377 // generate code for exception handler
378 address handler_base = __ start_a_stub(exception_handler_size);
379 if (handler_base == NULL) {
380 // not enough space left for the handler
381 bailout("exception handler overflow");
382 return -1;
383 }
384
385 int offset = code_offset();
386
387 // the exception oop and pc are in r0, and r3
388 // no other registers need to be preserved, so invalidate them
389 __ invalidate_registers(false, true, true, false, true, true);
390
391 // check that there is really an exception
392 __ verify_not_null_oop(r0);
393
394 // search an exception handler (r0: exception oop, r3: throwing pc)
395 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
396 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
397 __ end_a_stub();
398
399 return offset;
400 }
401
402
403 // Emit the code to remove the frame from the stack in the exception
404 // unwind path.
405 int LIR_Assembler::emit_unwind_handler() {
406 #ifndef PRODUCT
407 if (CommentedAssembly) {
408 _masm->block_comment("Unwind handler");
409 }
410 #endif
411
412 int offset = code_offset();
413
414 // Fetch the exception from TLS and clear out exception related thread state
415 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
417 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
418
419 __ bind(_unwind_handler_entry);
420 __ verify_not_null_oop(r0);
421 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
422 __ mov(r19, r0); // Preserve the exception
423 }
424
425 // Preform needed unlocking
426 MonitorExitStub* stub = NULL;
427 if (method()->is_synchronized()) {
428 monitor_address(0, FrameMap::r0_opr);
429 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
430 __ unlock_object(r5, r4, r0, *stub->entry());
431 __ bind(*stub->continuation());
432 }
433
434 if (compilation()->env()->dtrace_method_probes()) {
435 __ call_Unimplemented();
436 #if 0
437 __ movptr(Address(rsp, 0), rax);
438 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
439 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
440 #endif
441 }
442
443 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
444 __ mov(r0, r19); // Restore the exception
445 }
446
447 // remove the activation and dispatch to the unwind handler
448 __ block_comment("remove_frame and dispatch to the unwind handler");
449 __ remove_frame(initial_frame_size_in_bytes());
450 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
451
452 // Emit the slow path assembly
453 if (stub != NULL) {
454 stub->emit_code(this);
455 }
456
457 return offset;
458 }
459
460
461 int LIR_Assembler::emit_deopt_handler() {
462 // if the last instruction is a call (typically to do a throw which
463 // is coming at the end after block reordering) the return address
464 // must still point into the code area in order to avoid assertion
465 // failures when searching for the corresponding bci => add a nop
466 // (was bug 5/14/1999 - gri)
467 __ nop();
468
469 // generate code for exception handler
470 address handler_base = __ start_a_stub(deopt_handler_size);
471 if (handler_base == NULL) {
472 // not enough space left for the handler
473 bailout("deopt handler overflow");
474 return -1;
475 }
476
477 int offset = code_offset();
478
479 __ adr(lr, pc());
480 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
481 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
482 __ end_a_stub();
483
484 return offset;
485 }
486
487
488 // This is the fast version of java.lang.String.compare; it has not
489 // OSR-entry and therefore, we generate a slow version for OSR's
490 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info) {
491 __ mov(r2, (address)__FUNCTION__);
492 __ call_Unimplemented();
493 }
494
495
496 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
497 _masm->code_section()->relocate(adr, relocInfo::poll_type);
498 int pc_offset = code_offset();
499 flush_debug_info(pc_offset);
500 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
501 if (info->exception_handlers() != NULL) {
502 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
503 }
504 }
505
506 // Rather than take a segfault when the polling page is protected,
507 // explicitly check for a safepoint in progress and if there is one,
508 // fake a call to the handler as if a segfault had been caught.
509 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
510 __ mov(rscratch1, SafepointSynchronize::address_of_state());
511 __ ldrb(rscratch1, Address(rscratch1));
512 Label nope, poll;
513 __ cbz(rscratch1, nope);
514 __ block_comment("safepoint");
515 __ enter();
516 __ push(0x3, sp); // r0 & r1
517 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
518 __ adr(r0, poll);
519 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
520 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
521 __ blr(rscratch1);
522 __ maybe_isb();
523 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
524 __ mov(rscratch1, r0);
525 __ pop(0x3, sp); // r0 & r1
526 __ leave();
527 __ br(rscratch1);
528 address polling_page(os::get_polling_page());
529 assert(os::is_poll_address(polling_page), "should be");
530 unsigned long off;
531 __ adrp(rscratch1, Address(polling_page, rtype), off);
532 __ bind(poll);
533 if (info)
534 add_debug_info_for_branch(info); // This isn't just debug info:
535 // it's the oop map
536 else
537 __ code_section()->relocate(pc(), rtype);
538 __ ldrw(zr, Address(rscratch1, off));
539 __ bind(nope);
540 }
541
542 void LIR_Assembler::return_op(LIR_Opr result) {
543 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
544 // Pop the stack before the safepoint code
545 __ remove_frame(initial_frame_size_in_bytes());
546 if (UseCompilerSafepoints) {
547 address polling_page(os::get_polling_page());
548 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
549 } else {
550 poll_for_safepoint(relocInfo::poll_return_type);
551 }
552 __ ret(lr);
553 }
554
555 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
556 address polling_page(os::get_polling_page());
557 if (UseCompilerSafepoints) {
558 guarantee(info != NULL, "Shouldn't be NULL");
559 assert(os::is_poll_address(polling_page), "should be");
560 unsigned long off;
561 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
562 assert(off == 0, "must be");
563 add_debug_info_for_branch(info); // This isn't just debug info:
564 // it's the oop map
565 __ read_polling_page(rscratch1, relocInfo::poll_type);
566 } else {
567 poll_for_safepoint(relocInfo::poll_type, info);
568 }
569
570 return __ offset();
571 }
572
573
574 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
575 if (from_reg == r31_sp)
576 from_reg = sp;
577 if (to_reg == r31_sp)
578 to_reg = sp;
579 __ mov(to_reg, from_reg);
580 }
581
582 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
583
584
585 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
586 assert(src->is_constant(), "should not call otherwise");
587 assert(dest->is_register(), "should not call otherwise");
588 LIR_Const* c = src->as_constant_ptr();
589
590 switch (c->type()) {
591 case T_INT: {
592 assert(patch_code == lir_patch_none, "no patching handled here");
593 __ movw(dest->as_register(), c->as_jint());
594 break;
595 }
596
597 case T_ADDRESS: {
598 assert(patch_code == lir_patch_none, "no patching handled here");
599 __ mov(dest->as_register(), c->as_jint());
600 break;
601 }
602
603 case T_LONG: {
604 assert(patch_code == lir_patch_none, "no patching handled here");
605 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
606 break;
607 }
608
609 case T_OBJECT: {
610 if (patch_code == lir_patch_none) {
611 jobject2reg(c->as_jobject(), dest->as_register());
612 } else {
613 jobject2reg_with_patching(dest->as_register(), info);
614 }
615 break;
616 }
617
618 case T_METADATA: {
619 if (patch_code != lir_patch_none) {
620 klass2reg_with_patching(dest->as_register(), info);
621 } else {
622 __ mov_metadata(dest->as_register(), c->as_metadata());
623 }
624 break;
625 }
626
627 case T_FLOAT: {
628 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
629 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
630 } else {
631 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
632 __ ldrs(dest->as_float_reg(), Address(rscratch1));
633 }
634 break;
635 }
636
637 case T_DOUBLE: {
638 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
639 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
640 } else {
641 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
642 __ ldrd(dest->as_double_reg(), Address(rscratch1));
643 }
644 break;
645 }
646
647 default:
648 ShouldNotReachHere();
649 }
650 }
651
652 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
653 LIR_Const* c = src->as_constant_ptr();
654 switch (c->type()) {
655 case T_OBJECT:
656 {
657 if (! c->as_jobject())
658 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
659 else {
660 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
661 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
662 }
663 }
664 break;
665 case T_ADDRESS:
666 {
667 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
668 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
669 }
670 case T_INT:
671 case T_FLOAT:
672 {
673 Register reg = zr;
674 if (c->as_jint_bits() == 0)
675 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
676 else {
677 __ movw(rscratch1, c->as_jint_bits());
678 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
679 }
680 }
681 break;
682 case T_LONG:
683 case T_DOUBLE:
684 {
685 Register reg = zr;
686 if (c->as_jlong_bits() == 0)
687 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
688 lo_word_offset_in_bytes));
689 else {
690 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
691 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
692 lo_word_offset_in_bytes));
693 }
694 }
695 break;
696 default:
697 ShouldNotReachHere();
698 }
699 }
700
701 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
702 assert(src->is_constant(), "should not call otherwise");
703 LIR_Const* c = src->as_constant_ptr();
704 LIR_Address* to_addr = dest->as_address_ptr();
705
706 void (Assembler::* insn)(Register Rt, const Address &adr);
707
708 switch (type) {
709 case T_ADDRESS:
710 assert(c->as_jint() == 0, "should be");
711 insn = &Assembler::str;
712 break;
713 case T_LONG:
714 assert(c->as_jlong() == 0, "should be");
715 insn = &Assembler::str;
716 break;
717 case T_INT:
718 assert(c->as_jint() == 0, "should be");
719 insn = &Assembler::strw;
720 break;
721 case T_OBJECT:
722 case T_ARRAY:
723 assert(c->as_jobject() == 0, "should be");
724 if (UseCompressedOops && !wide) {
725 insn = &Assembler::strw;
726 } else {
727 insn = &Assembler::str;
728 }
729 break;
730 case T_CHAR:
731 case T_SHORT:
732 assert(c->as_jint() == 0, "should be");
733 insn = &Assembler::strh;
734 break;
735 case T_BOOLEAN:
736 case T_BYTE:
737 assert(c->as_jint() == 0, "should be");
738 insn = &Assembler::strb;
739 break;
740 default:
741 ShouldNotReachHere();
742 }
743
744 if (info) add_debug_info_for_null_check_here(info);
745 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
746 }
747
748 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
749 assert(src->is_register(), "should not call otherwise");
750 assert(dest->is_register(), "should not call otherwise");
751
752 // move between cpu-registers
753 if (dest->is_single_cpu()) {
754 if (src->type() == T_LONG) {
755 // Can do LONG -> OBJECT
756 move_regs(src->as_register_lo(), dest->as_register());
757 return;
758 }
759 assert(src->is_single_cpu(), "must match");
760 if (src->type() == T_OBJECT) {
761 __ verify_oop(src->as_register());
762 }
763 move_regs(src->as_register(), dest->as_register());
764
765 } else if (dest->is_double_cpu()) {
766 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
767 // Surprising to me but we can see move of a long to t_object
768 __ verify_oop(src->as_register());
769 move_regs(src->as_register(), dest->as_register_lo());
770 return;
771 }
772 assert(src->is_double_cpu(), "must match");
773 Register f_lo = src->as_register_lo();
774 Register f_hi = src->as_register_hi();
775 Register t_lo = dest->as_register_lo();
776 Register t_hi = dest->as_register_hi();
777 assert(f_hi == f_lo, "must be same");
778 assert(t_hi == t_lo, "must be same");
779 move_regs(f_lo, t_lo);
780
781 } else if (dest->is_single_fpu()) {
782 __ fmovs(dest->as_float_reg(), src->as_float_reg());
783
784 } else if (dest->is_double_fpu()) {
785 __ fmovd(dest->as_double_reg(), src->as_double_reg());
786
787 } else {
788 ShouldNotReachHere();
789 }
790 }
791
792 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
793 if (src->is_single_cpu()) {
794 if (type == T_ARRAY || type == T_OBJECT) {
795 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
796 __ verify_oop(src->as_register());
797 } else if (type == T_METADATA || type == T_DOUBLE) {
798 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
799 } else {
800 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
801 }
802
803 } else if (src->is_double_cpu()) {
804 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
805 __ str(src->as_register_lo(), dest_addr_LO);
806
807 } else if (src->is_single_fpu()) {
808 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
809 __ strs(src->as_float_reg(), dest_addr);
810
811 } else if (src->is_double_fpu()) {
812 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
813 __ strd(src->as_double_reg(), dest_addr);
814
815 } else {
816 ShouldNotReachHere();
817 }
818
819 }
820
821
822 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
823 LIR_Address* to_addr = dest->as_address_ptr();
824 PatchingStub* patch = NULL;
825 Register compressed_src = rscratch1;
826
827 if (patch_code != lir_patch_none) {
828 deoptimize_trap(info);
829 return;
830 }
831
832 if (type == T_ARRAY || type == T_OBJECT) {
833 __ verify_oop(src->as_register());
834
835 if (UseCompressedOops && !wide) {
836 __ encode_heap_oop(compressed_src, src->as_register());
837 } else {
838 compressed_src = src->as_register();
839 }
840 }
841
842 int null_check_here = code_offset();
843 switch (type) {
844 case T_FLOAT: {
845 __ strs(src->as_float_reg(), as_Address(to_addr));
846 break;
847 }
848
849 case T_DOUBLE: {
850 __ strd(src->as_double_reg(), as_Address(to_addr));
851 break;
852 }
853
854 case T_ARRAY: // fall through
855 case T_OBJECT: // fall through
856 if (UseCompressedOops && !wide) {
857 __ strw(compressed_src, as_Address(to_addr, rscratch2));
858 } else {
859 __ str(compressed_src, as_Address(to_addr));
860 }
861 break;
862 case T_METADATA:
863 // We get here to store a method pointer to the stack to pass to
864 // a dtrace runtime call. This can't work on 64 bit with
865 // compressed klass ptrs: T_METADATA can be a compressed klass
866 // ptr or a 64 bit method pointer.
867 LP64_ONLY(ShouldNotReachHere());
868 __ str(src->as_register(), as_Address(to_addr));
869 break;
870 case T_ADDRESS:
871 __ str(src->as_register(), as_Address(to_addr));
872 break;
873 case T_INT:
874 __ strw(src->as_register(), as_Address(to_addr));
875 break;
876
877 case T_LONG: {
878 __ str(src->as_register_lo(), as_Address_lo(to_addr));
879 break;
880 }
881
882 case T_BYTE: // fall through
883 case T_BOOLEAN: {
884 __ strb(src->as_register(), as_Address(to_addr));
885 break;
886 }
887
888 case T_CHAR: // fall through
889 case T_SHORT:
890 __ strh(src->as_register(), as_Address(to_addr));
891 break;
892
893 default:
894 ShouldNotReachHere();
895 }
896 if (info != NULL) {
897 add_debug_info_for_null_check(null_check_here, info);
898 }
899 }
900
901
902 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
903 assert(src->is_stack(), "should not call otherwise");
904 assert(dest->is_register(), "should not call otherwise");
905
906 if (dest->is_single_cpu()) {
907 if (type == T_ARRAY || type == T_OBJECT) {
908 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
909 __ verify_oop(dest->as_register());
910 } else if (type == T_METADATA) {
911 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
912 } else {
913 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
914 }
915
916 } else if (dest->is_double_cpu()) {
917 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
918 __ ldr(dest->as_register_lo(), src_addr_LO);
919
920 } else if (dest->is_single_fpu()) {
921 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
922 __ ldrs(dest->as_float_reg(), src_addr);
923
924 } else if (dest->is_double_fpu()) {
925 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
926 __ ldrd(dest->as_double_reg(), src_addr);
927
928 } else {
929 ShouldNotReachHere();
930 }
931 }
932
933
934 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
935 address target = NULL;
936 relocInfo::relocType reloc_type = relocInfo::none;
937
938 switch (patching_id(info)) {
939 case PatchingStub::access_field_id:
940 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
941 reloc_type = relocInfo::section_word_type;
942 break;
943 case PatchingStub::load_klass_id:
944 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
945 reloc_type = relocInfo::metadata_type;
946 break;
947 case PatchingStub::load_mirror_id:
948 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
949 reloc_type = relocInfo::oop_type;
950 break;
951 case PatchingStub::load_appendix_id:
952 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
953 reloc_type = relocInfo::oop_type;
954 break;
955 default: ShouldNotReachHere();
956 }
957
958 __ far_call(RuntimeAddress(target));
959 add_call_info_here(info);
960 }
961
962 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
963
964 LIR_Opr temp;
965 if (type == T_LONG || type == T_DOUBLE)
966 temp = FrameMap::rscratch1_long_opr;
967 else
968 temp = FrameMap::rscratch1_opr;
969
970 stack2reg(src, temp, src->type());
971 reg2stack(temp, dest, dest->type(), false);
972 }
973
974
975 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
976 LIR_Address* addr = src->as_address_ptr();
977 LIR_Address* from_addr = src->as_address_ptr();
978
979 if (addr->base()->type() == T_OBJECT) {
980 __ verify_oop(addr->base()->as_pointer_register());
981 }
982
983 if (patch_code != lir_patch_none) {
984 deoptimize_trap(info);
985 return;
986 }
987
988 if (info != NULL) {
989 add_debug_info_for_null_check_here(info);
990 }
991 int null_check_here = code_offset();
992 switch (type) {
993 case T_FLOAT: {
994 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
995 break;
996 }
997
998 case T_DOUBLE: {
999 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
1000 break;
1001 }
1002
1003 case T_ARRAY: // fall through
1004 case T_OBJECT: // fall through
1005 if (UseCompressedOops && !wide) {
1006 __ ldrw(dest->as_register(), as_Address(from_addr));
1007 } else {
1008 __ ldr(dest->as_register(), as_Address(from_addr));
1009 }
1010 break;
1011 case T_METADATA:
1012 // We get here to store a method pointer to the stack to pass to
1013 // a dtrace runtime call. This can't work on 64 bit with
1014 // compressed klass ptrs: T_METADATA can be a compressed klass
1015 // ptr or a 64 bit method pointer.
1016 LP64_ONLY(ShouldNotReachHere());
1017 __ ldr(dest->as_register(), as_Address(from_addr));
1018 break;
1019 case T_ADDRESS:
1020 // FIXME: OMG this is a horrible kludge. Any offset from an
1021 // address that matches klass_offset_in_bytes() will be loaded
1022 // as a word, not a long.
1023 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1024 __ ldrw(dest->as_register(), as_Address(from_addr));
1025 } else {
1026 __ ldr(dest->as_register(), as_Address(from_addr));
1027 }
1028 break;
1029 case T_INT:
1030 __ ldrw(dest->as_register(), as_Address(from_addr));
1031 break;
1032
1033 case T_LONG: {
1034 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1035 break;
1036 }
1037
1038 case T_BYTE:
1039 __ ldrsb(dest->as_register(), as_Address(from_addr));
1040 break;
1041 case T_BOOLEAN: {
1042 __ ldrb(dest->as_register(), as_Address(from_addr));
1043 break;
1044 }
1045
1046 case T_CHAR:
1047 __ ldrh(dest->as_register(), as_Address(from_addr));
1048 break;
1049 case T_SHORT:
1050 __ ldrsh(dest->as_register(), as_Address(from_addr));
1051 break;
1052
1053 default:
1054 ShouldNotReachHere();
1055 }
1056
1057 if (type == T_ARRAY || type == T_OBJECT) {
1058 #ifdef _LP64
1059 if (UseCompressedOops && !wide) {
1060 __ decode_heap_oop(dest->as_register());
1061 }
1062 #endif
1063 __ verify_oop(dest->as_register());
1064 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1065 #ifdef _LP64
1066 if (UseCompressedClassPointers) {
1067 __ decode_klass_not_null(dest->as_register());
1068 }
1069 #endif
1070 }
1071 }
1072
1073
1074 void LIR_Assembler::prefetchr(LIR_Opr src) { Unimplemented(); }
1075
1076
1077 void LIR_Assembler::prefetchw(LIR_Opr src) { Unimplemented(); }
1078
1079
1080 int LIR_Assembler::array_element_size(BasicType type) const {
1081 int elem_size = type2aelembytes(type);
1082 return exact_log2(elem_size);
1083 }
1084
1085 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1086 Register Rdividend = op->in_opr1()->as_register();
1087 Register Rdivisor = op->in_opr2()->as_register();
1088 Register Rscratch = op->in_opr3()->as_register();
1089 Register Rresult = op->result_opr()->as_register();
1090 int divisor = -1;
1091
1092 /*
1093 TODO: For some reason, using the Rscratch that gets passed in is
1094 not possible because the register allocator does not see the tmp reg
1095 as used, and assignes it the same register as Rdividend. We use rscratch1
1096 instead.
1097
1098 assert(Rdividend != Rscratch, "");
1099 assert(Rdivisor != Rscratch, "");
1100 */
1101
1102 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1103 // convert division by a power of two into some shifts and logical operations
1104 }
1105
1106 if (op->code() == lir_irem) {
1107 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1108 } else if (op->code() == lir_idiv) {
1109 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1110 } else
1111 ShouldNotReachHere();
1112 }
1113
1114 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1115 #ifdef ASSERT
1116 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1117 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1118 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1119 #endif
1120
1121 if (op->cond() == lir_cond_always) {
1122 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1123 __ b(*(op->label()));
1124 } else {
1125 Assembler::Condition acond;
1126 if (op->code() == lir_cond_float_branch) {
1127 bool is_unordered = (op->ublock() == op->block());
1128 // Assembler::EQ does not permit unordered branches, so we add
1129 // another branch here. Likewise, Assembler::NE does not permit
1130 // ordered branches.
1131 if ((is_unordered && op->cond() == lir_cond_equal)
1132 || (!is_unordered && op->cond() == lir_cond_notEqual))
1133 __ br(Assembler::VS, *(op->ublock()->label()));
1134 switch(op->cond()) {
1135 case lir_cond_equal: acond = Assembler::EQ; break;
1136 case lir_cond_notEqual: acond = Assembler::NE; break;
1137 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1138 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1139 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1140 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1141 default: ShouldNotReachHere();
1142 }
1143 } else {
1144 switch (op->cond()) {
1145 case lir_cond_equal: acond = Assembler::EQ; break;
1146 case lir_cond_notEqual: acond = Assembler::NE; break;
1147 case lir_cond_less: acond = Assembler::LT; break;
1148 case lir_cond_lessEqual: acond = Assembler::LE; break;
1149 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1150 case lir_cond_greater: acond = Assembler::GT; break;
1151 case lir_cond_belowEqual: acond = Assembler::LS; break;
1152 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1153 default: ShouldNotReachHere();
1154 }
1155 }
1156 __ br(acond,*(op->label()));
1157 }
1158 }
1159
1160
1161
1162 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1163 LIR_Opr src = op->in_opr();
1164 LIR_Opr dest = op->result_opr();
1165
1166 switch (op->bytecode()) {
1167 case Bytecodes::_i2f:
1168 {
1169 __ scvtfws(dest->as_float_reg(), src->as_register());
1170 break;
1171 }
1172 case Bytecodes::_i2d:
1173 {
1174 __ scvtfwd(dest->as_double_reg(), src->as_register());
1175 break;
1176 }
1177 case Bytecodes::_l2d:
1178 {
1179 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1180 break;
1181 }
1182 case Bytecodes::_l2f:
1183 {
1184 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1185 break;
1186 }
1187 case Bytecodes::_f2d:
1188 {
1189 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1190 break;
1191 }
1192 case Bytecodes::_d2f:
1193 {
1194 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1195 break;
1196 }
1197 case Bytecodes::_i2c:
1198 {
1199 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1200 break;
1201 }
1202 case Bytecodes::_i2l:
1203 {
1204 __ sxtw(dest->as_register_lo(), src->as_register());
1205 break;
1206 }
1207 case Bytecodes::_i2s:
1208 {
1209 __ sxth(dest->as_register(), src->as_register());
1210 break;
1211 }
1212 case Bytecodes::_i2b:
1213 {
1214 __ sxtb(dest->as_register(), src->as_register());
1215 break;
1216 }
1217 case Bytecodes::_l2i:
1218 {
1219 _masm->block_comment("FIXME: This could be a no-op");
1220 __ uxtw(dest->as_register(), src->as_register_lo());
1221 break;
1222 }
1223 case Bytecodes::_d2l:
1224 {
1225 Register tmp = op->tmp1()->as_register();
1226 __ clear_fpsr();
1227 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1228 __ get_fpsr(tmp);
1229 __ tst(tmp, 1); // FPSCR.IOC
1230 __ br(Assembler::NE, *(op->stub()->entry()));
1231 __ bind(*op->stub()->continuation());
1232 break;
1233 }
1234 case Bytecodes::_f2i:
1235 {
1236 Register tmp = op->tmp1()->as_register();
1237 __ clear_fpsr();
1238 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1239 __ get_fpsr(tmp);
1240 __ tst(tmp, 1); // FPSCR.IOC
1241 __ br(Assembler::NE, *(op->stub()->entry()));
1242 __ bind(*op->stub()->continuation());
1243 break;
1244 }
1245 case Bytecodes::_f2l:
1246 {
1247 Register tmp = op->tmp1()->as_register();
1248 __ clear_fpsr();
1249 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1250 __ get_fpsr(tmp);
1251 __ tst(tmp, 1); // FPSCR.IOC
1252 __ br(Assembler::NE, *(op->stub()->entry()));
1253 __ bind(*op->stub()->continuation());
1254 break;
1255 }
1256 case Bytecodes::_d2i:
1257 {
1258 Register tmp = op->tmp1()->as_register();
1259 __ clear_fpsr();
1260 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1261 __ get_fpsr(tmp);
1262 __ tst(tmp, 1); // FPSCR.IOC
1263 __ br(Assembler::NE, *(op->stub()->entry()));
1264 __ bind(*op->stub()->continuation());
1265 break;
1266 }
1267 default: ShouldNotReachHere();
1268 }
1269 }
1270
1271 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1272 if (op->init_check()) {
1273 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1274 InstanceKlass::init_state_offset()));
1275 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1276 add_debug_info_for_null_check_here(op->stub()->info());
1277 __ br(Assembler::NE, *op->stub()->entry());
1278 }
1279 __ allocate_object(op->obj()->as_register(),
1280 op->tmp1()->as_register(),
1281 op->tmp2()->as_register(),
1282 op->header_size(),
1283 op->object_size(),
1284 op->klass()->as_register(),
1285 *op->stub()->entry());
1286 __ bind(*op->stub()->continuation());
1287 }
1288
1289 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1290 Register len = op->len()->as_register();
1291 __ uxtw(len, len);
1292
1293 if (UseSlowPath ||
1294 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1295 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1296 __ b(*op->stub()->entry());
1297 } else {
1298 Register tmp1 = op->tmp1()->as_register();
1299 Register tmp2 = op->tmp2()->as_register();
1300 Register tmp3 = op->tmp3()->as_register();
1301 if (len == tmp1) {
1302 tmp1 = tmp3;
1303 } else if (len == tmp2) {
1304 tmp2 = tmp3;
1305 } else if (len == tmp3) {
1306 // everything is ok
1307 } else {
1308 __ mov(tmp3, len);
1309 }
1310 __ allocate_array(op->obj()->as_register(),
1311 len,
1312 tmp1,
1313 tmp2,
1314 arrayOopDesc::header_size(op->type()),
1315 array_element_size(op->type()),
1316 op->klass()->as_register(),
1317 *op->stub()->entry());
1318 }
1319 __ bind(*op->stub()->continuation());
1320 }
1321
1322 void LIR_Assembler::type_profile_helper(Register mdo,
1323 ciMethodData *md, ciProfileData *data,
1324 Register recv, Label* update_done) {
1325 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1326 Label next_test;
1327 // See if the receiver is receiver[n].
1328 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1329 __ ldr(rscratch1, Address(rscratch2));
1330 __ cmp(recv, rscratch1);
1331 __ br(Assembler::NE, next_test);
1332 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1333 __ addptr(data_addr, DataLayout::counter_increment);
1334 __ b(*update_done);
1335 __ bind(next_test);
1336 }
1337
1338 // Didn't find receiver; find next empty slot and fill it in
1339 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1340 Label next_test;
1341 __ lea(rscratch2,
1342 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1343 Address recv_addr(rscratch2);
1344 __ ldr(rscratch1, recv_addr);
1345 __ cbnz(rscratch1, next_test);
1346 __ str(recv, recv_addr);
1347 __ mov(rscratch1, DataLayout::counter_increment);
1348 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1349 __ str(rscratch1, Address(rscratch2));
1350 __ b(*update_done);
1351 __ bind(next_test);
1352 }
1353 }
1354
1355 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1356 // we always need a stub for the failure case.
1357 CodeStub* stub = op->stub();
1358 Register obj = op->object()->as_register();
1359 Register k_RInfo = op->tmp1()->as_register();
1360 Register klass_RInfo = op->tmp2()->as_register();
1361 Register dst = op->result_opr()->as_register();
1362 ciKlass* k = op->klass();
1363 Register Rtmp1 = noreg;
1364
1365 // check if it needs to be profiled
1366 ciMethodData* md;
1367 ciProfileData* data;
1368
1369 if (op->should_profile()) {
1370 ciMethod* method = op->profiled_method();
1371 assert(method != NULL, "Should have method");
1372 int bci = op->profiled_bci();
1373 md = method->method_data_or_null();
1374 assert(md != NULL, "Sanity");
1375 data = md->bci_to_data(bci);
1376 assert(data != NULL, "need data for type check");
1377 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1378 }
1379 Label profile_cast_success, profile_cast_failure;
1380 Label *success_target = op->should_profile() ? &profile_cast_success : success;
1381 Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1382
1383 if (obj == k_RInfo) {
1384 k_RInfo = dst;
1385 } else if (obj == klass_RInfo) {
1386 klass_RInfo = dst;
1387 }
1388 if (k->is_loaded() && !UseCompressedClassPointers) {
1389 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1390 } else {
1391 Rtmp1 = op->tmp3()->as_register();
1392 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1393 }
1394
1395 assert_different_registers(obj, k_RInfo, klass_RInfo);
1396
1397 if (op->should_profile()) {
1398 Label not_null;
1399 __ cbnz(obj, not_null);
1400 // Object is null; update MDO and exit
1401 Register mdo = klass_RInfo;
1402 __ mov_metadata(mdo, md->constant_encoding());
1403 Address data_addr
1404 = __ form_address(rscratch2, mdo,
1405 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1406 LogBytesPerWord);
1407 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1408 __ ldr(rscratch1, data_addr);
1409 __ orr(rscratch1, rscratch1, header_bits);
1410 __ str(rscratch1, data_addr);
1411 __ b(*obj_is_null);
1412 __ bind(not_null);
1413 } else {
1414 __ cbz(obj, *obj_is_null);
1415 }
1416
1417 if (!k->is_loaded()) {
1418 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1419 } else {
1420 #ifdef _LP64
1421 __ mov_metadata(k_RInfo, k->constant_encoding());
1422 #endif // _LP64
1423 }
1424 __ verify_oop(obj);
1425
1426 if (op->fast_check()) {
1427 // get object class
1428 // not a safepoint as obj null check happens earlier
1429 __ load_klass(rscratch1, obj);
1430 __ cmp( rscratch1, k_RInfo);
1431
1432 __ br(Assembler::NE, *failure_target);
1433 // successful cast, fall through to profile or jump
1434 } else {
1435 // get object class
1436 // not a safepoint as obj null check happens earlier
1437 __ load_klass(klass_RInfo, obj);
1438 if (k->is_loaded()) {
1439 // See if we get an immediate positive hit
1440 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1441 __ cmp(k_RInfo, rscratch1);
1442 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1443 __ br(Assembler::NE, *failure_target);
1444 // successful cast, fall through to profile or jump
1445 } else {
1446 // See if we get an immediate positive hit
1447 __ br(Assembler::EQ, *success_target);
1448 // check for self
1449 __ cmp(klass_RInfo, k_RInfo);
1450 __ br(Assembler::EQ, *success_target);
1451
1452 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1453 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1454 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1455 // result is a boolean
1456 __ cbzw(klass_RInfo, *failure_target);
1457 // successful cast, fall through to profile or jump
1458 }
1459 } else {
1460 // perform the fast part of the checking logic
1461 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1462 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1463 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1464 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1465 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1466 // result is a boolean
1467 __ cbz(k_RInfo, *failure_target);
1468 // successful cast, fall through to profile or jump
1469 }
1470 }
1471 if (op->should_profile()) {
1472 Register mdo = klass_RInfo, recv = k_RInfo;
1473 __ bind(profile_cast_success);
1474 __ mov_metadata(mdo, md->constant_encoding());
1475 __ load_klass(recv, obj);
1476 Label update_done;
1477 type_profile_helper(mdo, md, data, recv, success);
1478 __ b(*success);
1479
1480 __ bind(profile_cast_failure);
1481 __ mov_metadata(mdo, md->constant_encoding());
1482 Address counter_addr
1483 = __ form_address(rscratch2, mdo,
1484 md->byte_offset_of_slot(data, CounterData::count_offset()),
1485 LogBytesPerWord);
1486 __ ldr(rscratch1, counter_addr);
1487 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1488 __ str(rscratch1, counter_addr);
1489 __ b(*failure);
1490 }
1491 __ b(*success);
1492 }
1493
1494
1495 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1496 LIR_Code code = op->code();
1497 if (code == lir_store_check) {
1498 Register value = op->object()->as_register();
1499 Register array = op->array()->as_register();
1500 Register k_RInfo = op->tmp1()->as_register();
1501 Register klass_RInfo = op->tmp2()->as_register();
1502 Register Rtmp1 = op->tmp3()->as_register();
1503
1504 CodeStub* stub = op->stub();
1505
1506 // check if it needs to be profiled
1507 ciMethodData* md;
1508 ciProfileData* data;
1509
1510 if (op->should_profile()) {
1511 ciMethod* method = op->profiled_method();
1512 assert(method != NULL, "Should have method");
1513 int bci = op->profiled_bci();
1514 md = method->method_data_or_null();
1515 assert(md != NULL, "Sanity");
1516 data = md->bci_to_data(bci);
1517 assert(data != NULL, "need data for type check");
1518 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1519 }
1520 Label profile_cast_success, profile_cast_failure, done;
1521 Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1522 Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1523
1524 if (op->should_profile()) {
1525 Label not_null;
1526 __ cbnz(value, not_null);
1527 // Object is null; update MDO and exit
1528 Register mdo = klass_RInfo;
1529 __ mov_metadata(mdo, md->constant_encoding());
1530 Address data_addr
1531 = __ form_address(rscratch2, mdo,
1532 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1533 LogBytesPerInt);
1534 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1535 __ ldrw(rscratch1, data_addr);
1536 __ orrw(rscratch1, rscratch1, header_bits);
1537 __ strw(rscratch1, data_addr);
1538 __ b(done);
1539 __ bind(not_null);
1540 } else {
1541 __ cbz(value, done);
1542 }
1543
1544 add_debug_info_for_null_check_here(op->info_for_exception());
1545 __ load_klass(k_RInfo, array);
1546 __ load_klass(klass_RInfo, value);
1547
1548 // get instance klass (it's already uncompressed)
1549 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1550 // perform the fast part of the checking logic
1551 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1552 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1553 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1554 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1555 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1556 // result is a boolean
1557 __ cbzw(k_RInfo, *failure_target);
1558 // fall through to the success case
1559
1560 if (op->should_profile()) {
1561 Register mdo = klass_RInfo, recv = k_RInfo;
1562 __ bind(profile_cast_success);
1563 __ mov_metadata(mdo, md->constant_encoding());
1564 __ load_klass(recv, value);
1565 Label update_done;
1566 type_profile_helper(mdo, md, data, recv, &done);
1567 __ b(done);
1568
1569 __ bind(profile_cast_failure);
1570 __ mov_metadata(mdo, md->constant_encoding());
1571 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1572 __ lea(rscratch2, counter_addr);
1573 __ ldr(rscratch1, Address(rscratch2));
1574 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1575 __ str(rscratch1, Address(rscratch2));
1576 __ b(*stub->entry());
1577 }
1578
1579 __ bind(done);
1580 } else if (code == lir_checkcast) {
1581 Register obj = op->object()->as_register();
1582 Register dst = op->result_opr()->as_register();
1583 Label success;
1584 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1585 __ bind(success);
1586 if (dst != obj) {
1587 __ mov(dst, obj);
1588 }
1589 } else if (code == lir_instanceof) {
1590 Register obj = op->object()->as_register();
1591 Register dst = op->result_opr()->as_register();
1592 Label success, failure, done;
1593 emit_typecheck_helper(op, &success, &failure, &failure);
1594 __ bind(failure);
1595 __ mov(dst, zr);
1596 __ b(done);
1597 __ bind(success);
1598 __ mov(dst, 1);
1599 __ bind(done);
1600 } else {
1601 ShouldNotReachHere();
1602 }
1603 }
1604
1605 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1606 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, rscratch1);
1607 __ cset(rscratch1, Assembler::NE);
1608 __ membar(__ AnyAny);
1609 }
1610
1611 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1612 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, rscratch1);
1613 __ cset(rscratch1, Assembler::NE);
1614 __ membar(__ AnyAny);
1615 }
1616
1617
1618 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1619 assert(VM_Version::supports_cx8(), "wrong machine");
1620 Register addr = as_reg(op->addr());
1621 Register newval = as_reg(op->new_value());
1622 Register cmpval = as_reg(op->cmp_value());
1623 Label succeed, fail, around;
1624
1625 if (op->code() == lir_cas_obj) {
1626 if (UseCompressedOops) {
1627 Register t1 = op->tmp1()->as_register();
1628 assert(op->tmp1()->is_valid(), "must be");
1629 __ encode_heap_oop(t1, cmpval);
1630 cmpval = t1;
1631 __ encode_heap_oop(rscratch2, newval);
1632 newval = rscratch2;
1633 casw(addr, newval, cmpval);
1634 } else {
1635 casl(addr, newval, cmpval);
1636 }
1637 } else if (op->code() == lir_cas_int) {
1638 casw(addr, newval, cmpval);
1639 } else {
1640 casl(addr, newval, cmpval);
1641 }
1642 }
1643
1644
1645 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1646
1647 Assembler::Condition acond, ncond;
1648 switch (condition) {
1649 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1650 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1651 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1652 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1653 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1654 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1655 case lir_cond_belowEqual: Unimplemented(); break;
1656 case lir_cond_aboveEqual: Unimplemented(); break;
1657 default: ShouldNotReachHere();
1658 }
1659
1660 assert(result->is_single_cpu() || result->is_double_cpu(),
1661 "expect single register for result");
1662 if (opr1->is_constant() && opr2->is_constant()
1663 && opr1->type() == T_INT && opr2->type() == T_INT) {
1664 jint val1 = opr1->as_jint();
1665 jint val2 = opr2->as_jint();
1666 if (val1 == 0 && val2 == 1) {
1667 __ cset(result->as_register(), ncond);
1668 return;
1669 } else if (val1 == 1 && val2 == 0) {
1670 __ cset(result->as_register(), acond);
1671 return;
1672 }
1673 }
1674
1675 if (opr1->is_constant() && opr2->is_constant()
1676 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1677 jlong val1 = opr1->as_jlong();
1678 jlong val2 = opr2->as_jlong();
1679 if (val1 == 0 && val2 == 1) {
1680 __ cset(result->as_register_lo(), ncond);
1681 return;
1682 } else if (val1 == 1 && val2 == 0) {
1683 __ cset(result->as_register_lo(), acond);
1684 return;
1685 }
1686 }
1687
1688 if (opr1->is_stack()) {
1689 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1690 opr1 = FrameMap::rscratch1_opr;
1691 } else if (opr1->is_constant()) {
1692 LIR_Opr tmp
1693 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1694 const2reg(opr1, tmp, lir_patch_none, NULL);
1695 opr1 = tmp;
1696 }
1697
1698 if (opr2->is_stack()) {
1699 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1700 opr2 = FrameMap::rscratch2_opr;
1701 } else if (opr2->is_constant()) {
1702 LIR_Opr tmp
1703 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1704 const2reg(opr2, tmp, lir_patch_none, NULL);
1705 opr2 = tmp;
1706 }
1707
1708 if (result->type() == T_LONG)
1709 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1710 else
1711 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1712 }
1713
1714 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1715 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1716
1717 if (left->is_single_cpu()) {
1718 Register lreg = left->as_register();
1719 Register dreg = as_reg(dest);
1720
1721 if (right->is_single_cpu()) {
1722 // cpu register - cpu register
1723
1724 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1725 "should be");
1726 Register rreg = right->as_register();
1727 switch (code) {
1728 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1729 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1730 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1731 default: ShouldNotReachHere();
1732 }
1733
1734 } else if (right->is_double_cpu()) {
1735 Register rreg = right->as_register_lo();
1736 // single_cpu + double_cpu: can happen with obj+long
1737 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1738 switch (code) {
1739 case lir_add: __ add(dreg, lreg, rreg); break;
1740 case lir_sub: __ sub(dreg, lreg, rreg); break;
1741 default: ShouldNotReachHere();
1742 }
1743 } else if (right->is_constant()) {
1744 // cpu register - constant
1745 jlong c;
1746
1747 // FIXME. This is fugly: we really need to factor all this logic.
1748 switch(right->type()) {
1749 case T_LONG:
1750 c = right->as_constant_ptr()->as_jlong();
1751 break;
1752 case T_INT:
1753 case T_ADDRESS:
1754 c = right->as_constant_ptr()->as_jint();
1755 break;
1756 default:
1757 ShouldNotReachHere();
1758 break;
1759 }
1760
1761 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1762 if (c == 0 && dreg == lreg) {
1763 COMMENT("effective nop elided");
1764 return;
1765 }
1766 switch(left->type()) {
1767 case T_INT:
1768 switch (code) {
1769 case lir_add: __ addw(dreg, lreg, c); break;
1770 case lir_sub: __ subw(dreg, lreg, c); break;
1771 default: ShouldNotReachHere();
1772 }
1773 break;
1774 case T_OBJECT:
1775 case T_ADDRESS:
1776 switch (code) {
1777 case lir_add: __ add(dreg, lreg, c); break;
1778 case lir_sub: __ sub(dreg, lreg, c); break;
1779 default: ShouldNotReachHere();
1780 }
1781 break;
1782 ShouldNotReachHere();
1783 }
1784 } else {
1785 ShouldNotReachHere();
1786 }
1787
1788 } else if (left->is_double_cpu()) {
1789 Register lreg_lo = left->as_register_lo();
1790
1791 if (right->is_double_cpu()) {
1792 // cpu register - cpu register
1793 Register rreg_lo = right->as_register_lo();
1794 switch (code) {
1795 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1796 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1797 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1798 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1799 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1800 default:
1801 ShouldNotReachHere();
1802 }
1803
1804 } else if (right->is_constant()) {
1805 jlong c = right->as_constant_ptr()->as_jlong_bits();
1806 Register dreg = as_reg(dest);
1807 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1808 if (c == 0 && dreg == lreg_lo) {
1809 COMMENT("effective nop elided");
1810 return;
1811 }
1812 switch (code) {
1813 case lir_add: __ add(dreg, lreg_lo, c); break;
1814 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1815 default:
1816 ShouldNotReachHere();
1817 }
1818 } else {
1819 ShouldNotReachHere();
1820 }
1821 } else if (left->is_single_fpu()) {
1822 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1823 switch (code) {
1824 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1825 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1826 case lir_mul_strictfp: // fall through
1827 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1828 case lir_div_strictfp: // fall through
1829 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1830 default:
1831 ShouldNotReachHere();
1832 }
1833 } else if (left->is_double_fpu()) {
1834 if (right->is_double_fpu()) {
1835 // fpu register - fpu register
1836 switch (code) {
1837 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1838 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1839 case lir_mul_strictfp: // fall through
1840 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1841 case lir_div_strictfp: // fall through
1842 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1843 default:
1844 ShouldNotReachHere();
1845 }
1846 } else {
1847 if (right->is_constant()) {
1848 ShouldNotReachHere();
1849 }
1850 ShouldNotReachHere();
1851 }
1852 } else if (left->is_single_stack() || left->is_address()) {
1853 assert(left == dest, "left and dest must be equal");
1854 ShouldNotReachHere();
1855 } else {
1856 ShouldNotReachHere();
1857 }
1858 }
1859
1860 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1861
1862
1863 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1864 switch(code) {
1865 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1866 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1867 default : ShouldNotReachHere();
1868 }
1869 }
1870
1871 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1872
1873 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1874 Register Rleft = left->is_single_cpu() ? left->as_register() :
1875 left->as_register_lo();
1876 if (dst->is_single_cpu()) {
1877 Register Rdst = dst->as_register();
1878 if (right->is_constant()) {
1879 switch (code) {
1880 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1881 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1882 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1883 default: ShouldNotReachHere(); break;
1884 }
1885 } else {
1886 Register Rright = right->is_single_cpu() ? right->as_register() :
1887 right->as_register_lo();
1888 switch (code) {
1889 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1890 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1891 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1892 default: ShouldNotReachHere(); break;
1893 }
1894 }
1895 } else {
1896 Register Rdst = dst->as_register_lo();
1897 if (right->is_constant()) {
1898 switch (code) {
1899 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1900 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1901 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1902 default: ShouldNotReachHere(); break;
1903 }
1904 } else {
1905 Register Rright = right->is_single_cpu() ? right->as_register() :
1906 right->as_register_lo();
1907 switch (code) {
1908 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1909 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1910 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1911 default: ShouldNotReachHere(); break;
1912 }
1913 }
1914 }
1915 }
1916
1917
1918
1919 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1920
1921
1922 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1923 if (opr1->is_constant() && opr2->is_single_cpu()) {
1924 // tableswitch
1925 Register reg = as_reg(opr2);
1926 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1927 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1928 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1929 Register reg1 = as_reg(opr1);
1930 if (opr2->is_single_cpu()) {
1931 // cpu register - cpu register
1932 Register reg2 = opr2->as_register();
1933 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1934 __ cmp(reg1, reg2);
1935 } else {
1936 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1937 __ cmpw(reg1, reg2);
1938 }
1939 return;
1940 }
1941 if (opr2->is_double_cpu()) {
1942 // cpu register - cpu register
1943 Register reg2 = opr2->as_register_lo();
1944 __ cmp(reg1, reg2);
1945 return;
1946 }
1947
1948 if (opr2->is_constant()) {
1949 bool is_32bit = false; // width of register operand
1950 jlong imm;
1951
1952 switch(opr2->type()) {
1953 case T_INT:
1954 imm = opr2->as_constant_ptr()->as_jint();
1955 is_32bit = true;
1956 break;
1957 case T_LONG:
1958 imm = opr2->as_constant_ptr()->as_jlong();
1959 break;
1960 case T_ADDRESS:
1961 imm = opr2->as_constant_ptr()->as_jint();
1962 break;
1963 case T_OBJECT:
1964 case T_ARRAY:
1965 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1966 break;
1967 default:
1968 ShouldNotReachHere();
1969 break;
1970 }
1971
1972 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1973 if (is_32bit)
1974 __ cmpw(reg1, imm);
1975 else
1976 __ cmp(reg1, imm);
1977 return;
1978 } else {
1979 __ mov(rscratch1, imm);
1980 if (is_32bit)
1981 __ cmpw(reg1, rscratch1);
1982 else
1983 __ cmp(reg1, rscratch1);
1984 return;
1985 }
1986 } else
1987 ShouldNotReachHere();
1988 } else if (opr1->is_single_fpu()) {
1989 FloatRegister reg1 = opr1->as_float_reg();
1990 assert(opr2->is_single_fpu(), "expect single float register");
1991 FloatRegister reg2 = opr2->as_float_reg();
1992 __ fcmps(reg1, reg2);
1993 } else if (opr1->is_double_fpu()) {
1994 FloatRegister reg1 = opr1->as_double_reg();
1995 assert(opr2->is_double_fpu(), "expect double float register");
1996 FloatRegister reg2 = opr2->as_double_reg();
1997 __ fcmpd(reg1, reg2);
1998 } else {
1999 ShouldNotReachHere();
2000 }
2001 }
2002
2003 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2004 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2005 bool is_unordered_less = (code == lir_ucmp_fd2i);
2006 if (left->is_single_fpu()) {
2007 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2008 } else if (left->is_double_fpu()) {
2009 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2010 } else {
2011 ShouldNotReachHere();
2012 }
2013 } else if (code == lir_cmp_l2i) {
2014 Label done;
2015 __ cmp(left->as_register_lo(), right->as_register_lo());
2016 __ mov(dst->as_register(), (u_int64_t)-1L);
2017 __ br(Assembler::LT, done);
2018 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2019 __ bind(done);
2020 } else {
2021 ShouldNotReachHere();
2022 }
2023 }
2024
2025
2026 void LIR_Assembler::align_call(LIR_Code code) { }
2027
2028
2029 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2030 address call = __ trampoline_call(Address(op->addr(), rtype));
2031 if (call == NULL) {
2032 bailout("trampoline stub overflow");
2033 return;
2034 }
2035 add_call_info(code_offset(), op->info());
2036 }
2037
2038
2039 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2040 address call = __ ic_call(op->addr());
2041 if (call == NULL) {
2042 bailout("trampoline stub overflow");
2043 return;
2044 }
2045 add_call_info(code_offset(), op->info());
2046 }
2047
2048
2049 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2050 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2051 ShouldNotReachHere();
2052 }
2053
2054
2055 void LIR_Assembler::emit_static_call_stub() {
2056 address call_pc = __ pc();
2057 address stub = __ start_a_stub(call_stub_size);
2058 if (stub == NULL) {
2059 bailout("static call stub overflow");
2060 return;
2061 }
2062
2063 int start = __ offset();
2064
2065 __ relocate(static_stub_Relocation::spec(call_pc));
2066 __ mov_metadata(rmethod, (Metadata*)NULL);
2067 __ movptr(rscratch1, 0);
2068 __ br(rscratch1);
2069
2070 assert(__ offset() - start <= call_stub_size, "stub too big");
2071 __ end_a_stub();
2072 }
2073
2074
2075 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2076 assert(exceptionOop->as_register() == r0, "must match");
2077 assert(exceptionPC->as_register() == r3, "must match");
2078
2079 // exception object is not added to oop map by LinearScan
2080 // (LinearScan assumes that no oops are in fixed registers)
2081 info->add_register_oop(exceptionOop);
2082 Runtime1::StubID unwind_id;
2083
2084 // get current pc information
2085 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2086 int pc_for_athrow_offset = __ offset();
2087 InternalAddress pc_for_athrow(__ pc());
2088 __ adr(exceptionPC->as_register(), pc_for_athrow);
2089 add_call_info(pc_for_athrow_offset, info); // for exception handler
2090
2091 __ verify_not_null_oop(r0);
2092 // search an exception handler (r0: exception oop, r3: throwing pc)
2093 if (compilation()->has_fpu_code()) {
2094 unwind_id = Runtime1::handle_exception_id;
2095 } else {
2096 unwind_id = Runtime1::handle_exception_nofpu_id;
2097 }
2098 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2099
2100 // FIXME: enough room for two byte trap ????
2101 __ nop();
2102 }
2103
2104
2105 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2106 assert(exceptionOop->as_register() == r0, "must match");
2107
2108 __ b(_unwind_handler_entry);
2109 }
2110
2111
2112 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2113 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2114 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2115
2116 switch (left->type()) {
2117 case T_INT: {
2118 switch (code) {
2119 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2120 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2121 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2122 default:
2123 ShouldNotReachHere();
2124 break;
2125 }
2126 break;
2127 case T_LONG:
2128 case T_ADDRESS:
2129 case T_OBJECT:
2130 switch (code) {
2131 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2132 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2133 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2134 default:
2135 ShouldNotReachHere();
2136 break;
2137 }
2138 break;
2139 default:
2140 ShouldNotReachHere();
2141 break;
2142 }
2143 }
2144 }
2145
2146
2147 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2148 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2149 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2150
2151 switch (left->type()) {
2152 case T_INT: {
2153 switch (code) {
2154 case lir_shl: __ lslw (dreg, lreg, count); break;
2155 case lir_shr: __ asrw (dreg, lreg, count); break;
2156 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2157 default:
2158 ShouldNotReachHere();
2159 break;
2160 }
2161 break;
2162 case T_LONG:
2163 case T_ADDRESS:
2164 case T_OBJECT:
2165 switch (code) {
2166 case lir_shl: __ lsl (dreg, lreg, count); break;
2167 case lir_shr: __ asr (dreg, lreg, count); break;
2168 case lir_ushr: __ lsr (dreg, lreg, count); break;
2169 default:
2170 ShouldNotReachHere();
2171 break;
2172 }
2173 break;
2174 default:
2175 ShouldNotReachHere();
2176 break;
2177 }
2178 }
2179 }
2180
2181
2182 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2183 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2184 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2185 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2186 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2187 }
2188
2189
2190 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2191 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2192 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2193 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2194 __ mov (rscratch1, c);
2195 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2196 }
2197
2198
2199 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2200 ShouldNotReachHere();
2201 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2202 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2203 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2204 __ lea(rscratch1, __ constant_oop_address(o));
2205 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2206 }
2207
2208
2209 // This code replaces a call to arraycopy; no exception may
2210 // be thrown in this code, they must be thrown in the System.arraycopy
2211 // activation frame; we could save some checks if this would not be the case
2212 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2213 ciArrayKlass* default_type = op->expected_type();
2214 Register src = op->src()->as_register();
2215 Register dst = op->dst()->as_register();
2216 Register src_pos = op->src_pos()->as_register();
2217 Register dst_pos = op->dst_pos()->as_register();
2218 Register length = op->length()->as_register();
2219 Register tmp = op->tmp()->as_register();
2220
2221 CodeStub* stub = op->stub();
2222 int flags = op->flags();
2223 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2224 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2225
2226 // if we don't know anything, just go through the generic arraycopy
2227 if (default_type == NULL // || basic_type == T_OBJECT
2228 ) {
2229 Label done;
2230 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2231
2232 // Save the arguments in case the generic arraycopy fails and we
2233 // have to fall back to the JNI stub
2234 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2235 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2236 __ str(src, Address(sp, 4*BytesPerWord));
2237
2238 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2239 address copyfunc_addr = StubRoutines::generic_arraycopy();
2240
2241 // The arguments are in java calling convention so we shift them
2242 // to C convention
2243 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2244 __ mov(c_rarg0, j_rarg0);
2245 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2246 __ mov(c_rarg1, j_rarg1);
2247 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2248 __ mov(c_rarg2, j_rarg2);
2249 assert_different_registers(c_rarg3, j_rarg4);
2250 __ mov(c_rarg3, j_rarg3);
2251 __ mov(c_rarg4, j_rarg4);
2252 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2253 __ mov(rscratch1, RuntimeAddress(C_entry));
2254 __ blr(rscratch1);
2255 } else {
2256 #ifndef PRODUCT
2257 if (PrintC1Statistics) {
2258 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2259 }
2260 #endif
2261 __ far_call(RuntimeAddress(copyfunc_addr));
2262 }
2263
2264 __ cbz(r0, *stub->continuation());
2265
2266 // Reload values from the stack so they are where the stub
2267 // expects them.
2268 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2269 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2270 __ ldr(src, Address(sp, 4*BytesPerWord));
2271
2272 if (copyfunc_addr != NULL) {
2273 // r0 is -1^K where K == partial copied count
2274 __ eonw(rscratch1, r0, zr);
2275 // adjust length down and src/end pos up by partial copied count
2276 __ subw(length, length, rscratch1);
2277 __ addw(src_pos, src_pos, rscratch1);
2278 __ addw(dst_pos, dst_pos, rscratch1);
2279 }
2280 __ b(*stub->entry());
2281
2282 __ bind(*stub->continuation());
2283 return;
2284 }
2285
2286 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2287
2288 int elem_size = type2aelembytes(basic_type);
2289 int shift_amount;
2290 int scale = exact_log2(elem_size);
2291
2292 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2293 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2294 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2295 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2296
2297 // test for NULL
2298 if (flags & LIR_OpArrayCopy::src_null_check) {
2299 __ cbz(src, *stub->entry());
2300 }
2301 if (flags & LIR_OpArrayCopy::dst_null_check) {
2302 __ cbz(dst, *stub->entry());
2303 }
2304
2305 // If the compiler was not able to prove that exact type of the source or the destination
2306 // of the arraycopy is an array type, check at runtime if the source or the destination is
2307 // an instance type.
2308 if (flags & LIR_OpArrayCopy::type_check) {
2309 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2310 __ load_klass(tmp, dst);
2311 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2312 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2313 __ br(Assembler::GE, *stub->entry());
2314 }
2315
2316 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2317 __ load_klass(tmp, src);
2318 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2319 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2320 __ br(Assembler::GE, *stub->entry());
2321 }
2322 }
2323
2324 // check if negative
2325 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2326 __ cmpw(src_pos, 0);
2327 __ br(Assembler::LT, *stub->entry());
2328 }
2329 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2330 __ cmpw(dst_pos, 0);
2331 __ br(Assembler::LT, *stub->entry());
2332 }
2333
2334 if (flags & LIR_OpArrayCopy::length_positive_check) {
2335 __ cmpw(length, 0);
2336 __ br(Assembler::LT, *stub->entry());
2337 }
2338
2339 if (flags & LIR_OpArrayCopy::src_range_check) {
2340 __ addw(tmp, src_pos, length);
2341 __ ldrw(rscratch1, src_length_addr);
2342 __ cmpw(tmp, rscratch1);
2343 __ br(Assembler::HI, *stub->entry());
2344 }
2345 if (flags & LIR_OpArrayCopy::dst_range_check) {
2346 __ addw(tmp, dst_pos, length);
2347 __ ldrw(rscratch1, dst_length_addr);
2348 __ cmpw(tmp, rscratch1);
2349 __ br(Assembler::HI, *stub->entry());
2350 }
2351
2352 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2353 // length_positive_check if the source of our length operand is an
2354 // arraylength. However, that arraylength might be zero, and the
2355 // stub that we're about to call contains an assertion that count !=
2356 // 0 . So we make this check purely in order not to trigger an
2357 // assertion failure.
2358 __ cbzw(length, *stub->continuation());
2359
2360 if (flags & LIR_OpArrayCopy::type_check) {
2361 // We don't know the array types are compatible
2362 if (basic_type != T_OBJECT) {
2363 // Simple test for basic type arrays
2364 if (UseCompressedClassPointers) {
2365 __ ldrw(tmp, src_klass_addr);
2366 __ ldrw(rscratch1, dst_klass_addr);
2367 __ cmpw(tmp, rscratch1);
2368 } else {
2369 __ ldr(tmp, src_klass_addr);
2370 __ ldr(rscratch1, dst_klass_addr);
2371 __ cmp(tmp, rscratch1);
2372 }
2373 __ br(Assembler::NE, *stub->entry());
2374 } else {
2375 // For object arrays, if src is a sub class of dst then we can
2376 // safely do the copy.
2377 Label cont, slow;
2378
2379 #define PUSH(r1, r2) \
2380 stp(r1, r2, __ pre(sp, -2 * wordSize));
2381
2382 #define POP(r1, r2) \
2383 ldp(r1, r2, __ post(sp, 2 * wordSize));
2384
2385 __ PUSH(src, dst);
2386
2387 __ load_klass(src, src);
2388 __ load_klass(dst, dst);
2389
2390 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2391
2392 __ PUSH(src, dst);
2393 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2394 __ POP(src, dst);
2395
2396 __ cbnz(src, cont);
2397
2398 __ bind(slow);
2399 __ POP(src, dst);
2400
2401 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2402 if (copyfunc_addr != NULL) { // use stub if available
2403 // src is not a sub class of dst so we have to do a
2404 // per-element check.
2405
2406 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2407 if ((flags & mask) != mask) {
2408 // Check that at least both of them object arrays.
2409 assert(flags & mask, "one of the two should be known to be an object array");
2410
2411 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2412 __ load_klass(tmp, src);
2413 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2414 __ load_klass(tmp, dst);
2415 }
2416 int lh_offset = in_bytes(Klass::layout_helper_offset());
2417 Address klass_lh_addr(tmp, lh_offset);
2418 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2419 __ ldrw(rscratch1, klass_lh_addr);
2420 __ mov(rscratch2, objArray_lh);
2421 __ eorw(rscratch1, rscratch1, rscratch2);
2422 __ cbnzw(rscratch1, *stub->entry());
2423 }
2424
2425 // Spill because stubs can use any register they like and it's
2426 // easier to restore just those that we care about.
2427 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2428 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2429 __ str(src, Address(sp, 4*BytesPerWord));
2430
2431 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2432 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2433 assert_different_registers(c_rarg0, dst, dst_pos, length);
2434 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2435 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2436 assert_different_registers(c_rarg1, dst, length);
2437 __ uxtw(c_rarg2, length);
2438 assert_different_registers(c_rarg2, dst);
2439
2440 __ load_klass(c_rarg4, dst);
2441 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2442 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2443 __ far_call(RuntimeAddress(copyfunc_addr));
2444
2445 #ifndef PRODUCT
2446 if (PrintC1Statistics) {
2447 Label failed;
2448 __ cbnz(r0, failed);
2449 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2450 __ bind(failed);
2451 }
2452 #endif
2453
2454 __ cbz(r0, *stub->continuation());
2455
2456 #ifndef PRODUCT
2457 if (PrintC1Statistics) {
2458 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2459 }
2460 #endif
2461 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2462
2463 // Restore previously spilled arguments
2464 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2465 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2466 __ ldr(src, Address(sp, 4*BytesPerWord));
2467
2468 // return value is -1^K where K is partial copied count
2469 __ eonw(rscratch1, r0, zr);
2470 // adjust length down and src/end pos up by partial copied count
2471 __ subw(length, length, rscratch1);
2472 __ addw(src_pos, src_pos, rscratch1);
2473 __ addw(dst_pos, dst_pos, rscratch1);
2474 }
2475
2476 __ b(*stub->entry());
2477
2478 __ bind(cont);
2479 __ POP(src, dst);
2480 }
2481 }
2482
2483 #ifdef ASSERT
2484 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2485 // Sanity check the known type with the incoming class. For the
2486 // primitive case the types must match exactly with src.klass and
2487 // dst.klass each exactly matching the default type. For the
2488 // object array case, if no type check is needed then either the
2489 // dst type is exactly the expected type and the src type is a
2490 // subtype which we can't check or src is the same array as dst
2491 // but not necessarily exactly of type default_type.
2492 Label known_ok, halt;
2493 __ mov_metadata(tmp, default_type->constant_encoding());
2494 #ifdef _LP64
2495 if (UseCompressedClassPointers) {
2496 __ encode_klass_not_null(tmp);
2497 }
2498 #endif
2499
2500 if (basic_type != T_OBJECT) {
2501
2502 if (UseCompressedClassPointers) {
2503 __ ldrw(rscratch1, dst_klass_addr);
2504 __ cmpw(tmp, rscratch1);
2505 } else {
2506 __ ldr(rscratch1, dst_klass_addr);
2507 __ cmp(tmp, rscratch1);
2508 }
2509 __ br(Assembler::NE, halt);
2510 if (UseCompressedClassPointers) {
2511 __ ldrw(rscratch1, src_klass_addr);
2512 __ cmpw(tmp, rscratch1);
2513 } else {
2514 __ ldr(rscratch1, src_klass_addr);
2515 __ cmp(tmp, rscratch1);
2516 }
2517 __ br(Assembler::EQ, known_ok);
2518 } else {
2519 if (UseCompressedClassPointers) {
2520 __ ldrw(rscratch1, dst_klass_addr);
2521 __ cmpw(tmp, rscratch1);
2522 } else {
2523 __ ldr(rscratch1, dst_klass_addr);
2524 __ cmp(tmp, rscratch1);
2525 }
2526 __ br(Assembler::EQ, known_ok);
2527 __ cmp(src, dst);
2528 __ br(Assembler::EQ, known_ok);
2529 }
2530 __ bind(halt);
2531 __ stop("incorrect type information in arraycopy");
2532 __ bind(known_ok);
2533 }
2534 #endif
2535
2536 #ifndef PRODUCT
2537 if (PrintC1Statistics) {
2538 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2539 }
2540 #endif
2541
2542 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2543 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2544 assert_different_registers(c_rarg0, dst, dst_pos, length);
2545 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2546 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2547 assert_different_registers(c_rarg1, dst, length);
2548 __ uxtw(c_rarg2, length);
2549 assert_different_registers(c_rarg2, dst);
2550
2551 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2552 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2553 const char *name;
2554 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2555
2556 CodeBlob *cb = CodeCache::find_blob(entry);
2557 if (cb) {
2558 __ far_call(RuntimeAddress(entry));
2559 } else {
2560 __ call_VM_leaf(entry, 3);
2561 }
2562
2563 __ bind(*stub->continuation());
2564 }
2565
2566
2567
2568
2569 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2570 Register obj = op->obj_opr()->as_register(); // may not be an oop
2571 Register hdr = op->hdr_opr()->as_register();
2572 Register lock = op->lock_opr()->as_register();
2573 if (!UseFastLocking) {
2574 __ b(*op->stub()->entry());
2575 } else if (op->code() == lir_lock) {
2576 Register scratch = noreg;
2577 if (UseBiasedLocking) {
2578 scratch = op->scratch_opr()->as_register();
2579 }
2580 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2581 // add debug info for NullPointerException only if one is possible
2582 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2583 if (op->info() != NULL) {
2584 add_debug_info_for_null_check(null_check_offset, op->info());
2585 }
2586 // done
2587 } else if (op->code() == lir_unlock) {
2588 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2589 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2590 } else {
2591 Unimplemented();
2592 }
2593 __ bind(*op->stub()->continuation());
2594 }
2595
2596
2597 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2598 ciMethod* method = op->profiled_method();
2599 int bci = op->profiled_bci();
2600 ciMethod* callee = op->profiled_callee();
2601
2602 // Update counter for all call types
2603 ciMethodData* md = method->method_data_or_null();
2604 assert(md != NULL, "Sanity");
2605 ciProfileData* data = md->bci_to_data(bci);
2606 assert(data->is_CounterData(), "need CounterData for calls");
2607 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2608 Register mdo = op->mdo()->as_register();
2609 __ mov_metadata(mdo, md->constant_encoding());
2610 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2611 Bytecodes::Code bc = method->java_code_at_bci(bci);
2612 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2613 // Perform additional virtual call profiling for invokevirtual and
2614 // invokeinterface bytecodes
2615 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2616 !callee_is_static && // required for optimized MH invokes
2617 C1ProfileVirtualCalls) {
2618 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2619 Register recv = op->recv()->as_register();
2620 assert_different_registers(mdo, recv);
2621 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2622 ciKlass* known_klass = op->known_holder();
2623 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2624 // We know the type that will be seen at this call site; we can
2625 // statically update the MethodData* rather than needing to do
2626 // dynamic tests on the receiver type
2627
2628 // NOTE: we should probably put a lock around this search to
2629 // avoid collisions by concurrent compilations
2630 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2631 uint i;
2632 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2633 ciKlass* receiver = vc_data->receiver(i);
2634 if (known_klass->equals(receiver)) {
2635 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2636 __ addptr(data_addr, DataLayout::counter_increment);
2637 return;
2638 }
2639 }
2640
2641 // Receiver type not found in profile data; select an empty slot
2642
2643 // Note that this is less efficient than it should be because it
2644 // always does a write to the receiver part of the
2645 // VirtualCallData rather than just the first time
2646 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2647 ciKlass* receiver = vc_data->receiver(i);
2648 if (receiver == NULL) {
2649 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2650 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2651 __ lea(rscratch2, recv_addr);
2652 __ str(rscratch1, Address(rscratch2));
2653 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2654 __ addptr(data_addr, DataLayout::counter_increment);
2655 return;
2656 }
2657 }
2658 } else {
2659 __ load_klass(recv, recv);
2660 Label update_done;
2661 type_profile_helper(mdo, md, data, recv, &update_done);
2662 // Receiver did not match any saved receiver and there is no empty row for it.
2663 // Increment total counter to indicate polymorphic case.
2664 __ addptr(counter_addr, DataLayout::counter_increment);
2665
2666 __ bind(update_done);
2667 }
2668 } else {
2669 // Static call
2670 __ addptr(counter_addr, DataLayout::counter_increment);
2671 }
2672 }
2673
2674
2675 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2676 Unimplemented();
2677 }
2678
2679
2680 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2681 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2682 }
2683
2684 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2685 assert(op->crc()->is_single_cpu(), "crc must be register");
2686 assert(op->val()->is_single_cpu(), "byte value must be register");
2687 assert(op->result_opr()->is_single_cpu(), "result must be register");
2688 Register crc = op->crc()->as_register();
2689 Register val = op->val()->as_register();
2690 Register res = op->result_opr()->as_register();
2691
2692 assert_different_registers(val, crc, res);
2693 unsigned long offset;
2694 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2695 if (offset) __ add(res, res, offset);
2696
2697 __ ornw(crc, zr, crc); // ~crc
2698 __ update_byte_crc32(crc, val, res);
2699 __ ornw(res, zr, crc); // ~crc
2700 }
2701
2702 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2703 COMMENT("emit_profile_type {");
2704 Register obj = op->obj()->as_register();
2705 Register tmp = op->tmp()->as_pointer_register();
2706 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2707 ciKlass* exact_klass = op->exact_klass();
2708 intptr_t current_klass = op->current_klass();
2709 bool not_null = op->not_null();
2710 bool no_conflict = op->no_conflict();
2711
2712 Label update, next, none;
2713
2714 bool do_null = !not_null;
2715 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2716 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2717
2718 assert(do_null || do_update, "why are we here?");
2719 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2720 assert(mdo_addr.base() != rscratch1, "wrong register");
2721
2722 __ verify_oop(obj);
2723
2724 if (tmp != obj) {
2725 __ mov(tmp, obj);
2726 }
2727 if (do_null) {
2728 __ cbnz(tmp, update);
2729 if (!TypeEntries::was_null_seen(current_klass)) {
2730 __ ldr(rscratch2, mdo_addr);
2731 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2732 __ str(rscratch2, mdo_addr);
2733 }
2734 if (do_update) {
2735 #ifndef ASSERT
2736 __ b(next);
2737 }
2738 #else
2739 __ b(next);
2740 }
2741 } else {
2742 __ cbnz(tmp, update);
2743 __ stop("unexpected null obj");
2744 #endif
2745 }
2746
2747 __ bind(update);
2748
2749 if (do_update) {
2750 #ifdef ASSERT
2751 if (exact_klass != NULL) {
2752 Label ok;
2753 __ load_klass(tmp, tmp);
2754 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2755 __ eor(rscratch1, tmp, rscratch1);
2756 __ cbz(rscratch1, ok);
2757 __ stop("exact klass and actual klass differ");
2758 __ bind(ok);
2759 }
2760 #endif
2761 if (!no_conflict) {
2762 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2763 if (exact_klass != NULL) {
2764 __ mov_metadata(tmp, exact_klass->constant_encoding());
2765 } else {
2766 __ load_klass(tmp, tmp);
2767 }
2768
2769 __ ldr(rscratch2, mdo_addr);
2770 __ eor(tmp, tmp, rscratch2);
2771 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2772 // klass seen before, nothing to do. The unknown bit may have been
2773 // set already but no need to check.
2774 __ cbz(rscratch1, next);
2775
2776 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2777 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2778
2779 if (TypeEntries::is_type_none(current_klass)) {
2780 __ cbz(rscratch2, none);
2781 __ cmp(rscratch2, TypeEntries::null_seen);
2782 __ br(Assembler::EQ, none);
2783 // There is a chance that the checks above (re-reading profiling
2784 // data from memory) fail if another thread has just set the
2785 // profiling to this obj's klass
2786 __ dmb(Assembler::ISHLD);
2787 __ ldr(rscratch2, mdo_addr);
2788 __ eor(tmp, tmp, rscratch2);
2789 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2790 __ cbz(rscratch1, next);
2791 }
2792 } else {
2793 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2794 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2795
2796 __ ldr(tmp, mdo_addr);
2797 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2798 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2799 }
2800
2801 // different than before. Cannot keep accurate profile.
2802 __ ldr(rscratch2, mdo_addr);
2803 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2804 __ str(rscratch2, mdo_addr);
2805
2806 if (TypeEntries::is_type_none(current_klass)) {
2807 __ b(next);
2808
2809 __ bind(none);
2810 // first time here. Set profile type.
2811 __ str(tmp, mdo_addr);
2812 }
2813 } else {
2814 // There's a single possible klass at this profile point
2815 assert(exact_klass != NULL, "should be");
2816 if (TypeEntries::is_type_none(current_klass)) {
2817 __ mov_metadata(tmp, exact_klass->constant_encoding());
2818 __ ldr(rscratch2, mdo_addr);
2819 __ eor(tmp, tmp, rscratch2);
2820 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2821 __ cbz(rscratch1, next);
2822 #ifdef ASSERT
2823 {
2824 Label ok;
2825 __ ldr(rscratch1, mdo_addr);
2826 __ cbz(rscratch1, ok);
2827 __ cmp(rscratch1, TypeEntries::null_seen);
2828 __ br(Assembler::EQ, ok);
2829 // may have been set by another thread
2830 __ dmb(Assembler::ISHLD);
2831 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2832 __ ldr(rscratch2, mdo_addr);
2833 __ eor(rscratch2, rscratch1, rscratch2);
2834 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2835 __ cbz(rscratch2, ok);
2836
2837 __ stop("unexpected profiling mismatch");
2838 __ bind(ok);
2839 }
2840 #endif
2841 // first time here. Set profile type.
2842 __ ldr(tmp, mdo_addr);
2843 } else {
2844 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2845 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2846
2847 __ ldr(tmp, mdo_addr);
2848 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2849 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2850
2851 __ orr(tmp, tmp, TypeEntries::type_unknown);
2852 __ str(tmp, mdo_addr);
2853 // FIXME: Write barrier needed here?
2854 }
2855 }
2856
2857 __ bind(next);
2858 }
2859 COMMENT("} emit_profile_type");
2860 }
2861
2862
2863 void LIR_Assembler::align_backward_branch_target() {
2864 }
2865
2866
2867 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2868 if (left->is_single_cpu()) {
2869 assert(dest->is_single_cpu(), "expect single result reg");
2870 __ negw(dest->as_register(), left->as_register());
2871 } else if (left->is_double_cpu()) {
2872 assert(dest->is_double_cpu(), "expect double result reg");
2873 __ neg(dest->as_register_lo(), left->as_register_lo());
2874 } else if (left->is_single_fpu()) {
2875 assert(dest->is_single_fpu(), "expect single float result reg");
2876 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2877 } else {
2878 assert(left->is_double_fpu(), "expect double float operand reg");
2879 assert(dest->is_double_fpu(), "expect double float result reg");
2880 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2881 }
2882 }
2883
2884
2885 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2886 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2887 }
2888
2889
2890 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2891 assert(!tmp->is_valid(), "don't need temporary");
2892
2893 CodeBlob *cb = CodeCache::find_blob(dest);
2894 if (cb) {
2895 __ far_call(RuntimeAddress(dest));
2896 } else {
2897 __ mov(rscratch1, RuntimeAddress(dest));
2898 __ blr(rscratch1);
2899 }
2900
2901 if (info != NULL) {
2902 add_call_info_here(info);
2903 }
2904 __ maybe_isb();
2905 }
2906
2907 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2908 if (dest->is_address() || src->is_address()) {
2909 move_op(src, dest, type, lir_patch_none, info,
2910 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2911 } else {
2912 ShouldNotReachHere();
2913 }
2914 }
2915
2916 #ifdef ASSERT
2917 // emit run-time assertion
2918 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2919 assert(op->code() == lir_assert, "must be");
2920
2921 if (op->in_opr1()->is_valid()) {
2922 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2923 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2924 } else {
2925 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2926 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2927 }
2928
2929 Label ok;
2930 if (op->condition() != lir_cond_always) {
2931 Assembler::Condition acond = Assembler::AL;
2932 switch (op->condition()) {
2933 case lir_cond_equal: acond = Assembler::EQ; break;
2934 case lir_cond_notEqual: acond = Assembler::NE; break;
2935 case lir_cond_less: acond = Assembler::LT; break;
2936 case lir_cond_lessEqual: acond = Assembler::LE; break;
2937 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2938 case lir_cond_greater: acond = Assembler::GT; break;
2939 case lir_cond_belowEqual: acond = Assembler::LS; break;
2940 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2941 default: ShouldNotReachHere();
2942 }
2943 __ br(acond, ok);
2944 }
2945 if (op->halt()) {
2946 const char* str = __ code_string(op->msg());
2947 __ stop(str);
2948 } else {
2949 breakpoint();
2950 }
2951 __ bind(ok);
2952 }
2953 #endif
2954
2955 #ifndef PRODUCT
2956 #define COMMENT(x) do { __ block_comment(x); } while (0)
2957 #else
2958 #define COMMENT(x)
2959 #endif
2960
2961 void LIR_Assembler::membar() {
2962 COMMENT("membar");
2963 __ membar(MacroAssembler::AnyAny);
2964 }
2965
2966 void LIR_Assembler::membar_acquire() {
2967 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2968 }
2969
2970 void LIR_Assembler::membar_release() {
2971 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2972 }
2973
2974 void LIR_Assembler::membar_loadload() {
2975 __ membar(Assembler::LoadLoad);
2976 }
2977
2978 void LIR_Assembler::membar_storestore() {
2979 __ membar(MacroAssembler::StoreStore);
2980 }
2981
2982 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2983
2984 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2985
2986 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2987 __ mov(result_reg->as_register(), rthread);
2988 }
2989
2990
2991 void LIR_Assembler::peephole(LIR_List *lir) {
2992 #if 0
2993 if (tableswitch_count >= max_tableswitches)
2994 return;
2995
2996 /*
2997 This finite-state automaton recognizes sequences of compare-and-
2998 branch instructions. We will turn them into a tableswitch. You
2999 could argue that C1 really shouldn't be doing this sort of
3000 optimization, but without it the code is really horrible.
3001 */
3002
3003 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3004 int first_key, last_key = -2147483648;
3005 int next_key = 0;
3006 int start_insn = -1;
3007 int last_insn = -1;
3008 Register reg = noreg;
3009 LIR_Opr reg_opr;
3010 state = start_s;
3011
3012 LIR_OpList* inst = lir->instructions_list();
3013 for (int i = 0; i < inst->length(); i++) {
3014 LIR_Op* op = inst->at(i);
3015 switch (state) {
3016 case start_s:
3017 first_key = -1;
3018 start_insn = i;
3019 switch (op->code()) {
3020 case lir_cmp:
3021 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3022 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3023 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3024 && opr2->is_constant()
3025 && opr2->type() == T_INT) {
3026 reg_opr = opr1;
3027 reg = opr1->as_register();
3028 first_key = opr2->as_constant_ptr()->as_jint();
3029 next_key = first_key + 1;
3030 state = cmp_s;
3031 goto next_state;
3032 }
3033 break;
3034 }
3035 break;
3036 case cmp_s:
3037 switch (op->code()) {
3038 case lir_branch:
3039 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3040 state = beq_s;
3041 last_insn = i;
3042 goto next_state;
3043 }
3044 }
3045 state = start_s;
3046 break;
3047 case beq_s:
3048 switch (op->code()) {
3049 case lir_cmp: {
3050 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3051 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3052 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3053 && opr1->as_register() == reg
3054 && opr2->is_constant()
3055 && opr2->type() == T_INT
3056 && opr2->as_constant_ptr()->as_jint() == next_key) {
3057 last_key = next_key;
3058 next_key++;
3059 state = cmp_s;
3060 goto next_state;
3061 }
3062 }
3063 }
3064 last_key = next_key;
3065 state = start_s;
3066 break;
3067 default:
3068 assert(false, "impossible state");
3069 }
3070 if (state == start_s) {
3071 if (first_key < last_key - 5L && reg != noreg) {
3072 {
3073 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3074 // reg->encoding(),
3075 // start_insn, first_key, last_key);
3076 // for (int i = 0; i < inst->length(); i++) {
3077 // inst->at(i)->print();
3078 // tty->print("\n");
3079 // }
3080 // tty->print("\n");
3081 }
3082
3083 struct tableswitch *sw = &switches[tableswitch_count];
3084 sw->_insn_index = start_insn, sw->_first_key = first_key,
3085 sw->_last_key = last_key, sw->_reg = reg;
3086 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3087 {
3088 // Insert the new table of branches
3089 int offset = last_insn;
3090 for (int n = first_key; n < last_key; n++) {
3091 inst->insert_before
3092 (last_insn + 1,
3093 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3094 inst->at(offset)->as_OpBranch()->label()));
3095 offset -= 2, i++;
3096 }
3097 }
3098 // Delete all the old compare-and-branch instructions
3099 for (int n = first_key; n < last_key; n++) {
3100 inst->remove_at(start_insn);
3101 inst->remove_at(start_insn);
3102 }
3103 // Insert the tableswitch instruction
3104 inst->insert_before(start_insn,
3105 new LIR_Op2(lir_cmp, lir_cond_always,
3106 LIR_OprFact::intConst(tableswitch_count),
3107 reg_opr));
3108 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3109 tableswitch_count++;
3110 }
3111 reg = noreg;
3112 last_key = -2147483648;
3113 }
3114 next_state:
3115 ;
3116 }
3117 #endif
3118 }
3119
3120 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3121 Address addr = as_Address(src->as_address_ptr());
3122 BasicType type = src->type();
3123 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3124
3125 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3126 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3127
3128 switch(type) {
3129 case T_INT:
3130 xchg = &MacroAssembler::atomic_xchgalw;
3131 add = &MacroAssembler::atomic_addalw;
3132 break;
3133 case T_LONG:
3134 xchg = &MacroAssembler::atomic_xchgal;
3135 add = &MacroAssembler::atomic_addal;
3136 break;
3137 case T_OBJECT:
3138 case T_ARRAY:
3139 if (UseCompressedOops) {
3140 xchg = &MacroAssembler::atomic_xchgalw;
3141 add = &MacroAssembler::atomic_addalw;
3142 } else {
3143 xchg = &MacroAssembler::atomic_xchgal;
3144 add = &MacroAssembler::atomic_addal;
3145 }
3146 break;
3147 default:
3148 ShouldNotReachHere();
3149 xchg = &MacroAssembler::atomic_xchgal;
3150 add = &MacroAssembler::atomic_addal; // unreachable
3151 }
3152
3153 switch (code) {
3154 case lir_xadd:
3155 {
3156 RegisterOrConstant inc;
3157 Register tmp = as_reg(tmp_op);
3158 Register dst = as_reg(dest);
3159 if (data->is_constant()) {
3160 inc = RegisterOrConstant(as_long(data));
3161 assert_different_registers(dst, addr.base(), tmp,
3162 rscratch1, rscratch2);
3163 } else {
3164 inc = RegisterOrConstant(as_reg(data));
3165 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3166 rscratch1, rscratch2);
3167 }
3168 __ lea(tmp, addr);
3169 (_masm->*add)(dst, inc, tmp);
3170 break;
3171 }
3172 case lir_xchg:
3173 {
3174 Register tmp = tmp_op->as_register();
3175 Register obj = as_reg(data);
3176 Register dst = as_reg(dest);
3177 if (is_oop && UseCompressedOops) {
3178 __ encode_heap_oop(rscratch2, obj);
3179 obj = rscratch2;
3180 }
3181 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3182 __ lea(tmp, addr);
3183 (_masm->*xchg)(dst, obj, tmp);
3184 if (is_oop && UseCompressedOops) {
3185 __ decode_heap_oop(dst);
3186 }
3187 }
3188 break;
3189 default:
3190 ShouldNotReachHere();
3191 }
3192 __ membar(__ AnyAny);
3193 }
3194
3195 #undef __