1 /*
2 * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2014, Red Hat Inc. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/assembler.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "gc/shenandoah/brooksPointer.hpp"
37 #include "gc/shenandoah/shenandoahHeap.hpp"
38 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
39 #include "gc/shared/barrierSet.hpp"
40 #include "gc/shared/cardTableModRefBS.hpp"
41 #include "gc/shared/collectedHeap.hpp"
42 #include "nativeInst_aarch64.hpp"
43 #include "oops/objArrayKlass.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "vmreg_aarch64.inline.hpp"
46
47
48
49 #ifndef PRODUCT
50 #define COMMENT(x) do { __ block_comment(x); } while (0)
51 #else
52 #define COMMENT(x)
53 #endif
54
55 NEEDS_CLEANUP // remove this definitions ?
56 const Register IC_Klass = rscratch2; // where the IC klass is cached
57 const Register SYNC_header = r0; // synchronization header
58 const Register SHIFT_count = r0; // where count for shift operations must be
59
60 #define __ _masm->
61
62
63 static void select_different_registers(Register preserve,
64 Register extra,
65 Register &tmp1,
66 Register &tmp2) {
67 if (tmp1 == preserve) {
68 assert_different_registers(tmp1, tmp2, extra);
69 tmp1 = extra;
70 } else if (tmp2 == preserve) {
71 assert_different_registers(tmp1, tmp2, extra);
72 tmp2 = extra;
73 }
74 assert_different_registers(preserve, tmp1, tmp2);
75 }
76
77
78
79 static void select_different_registers(Register preserve,
80 Register extra,
81 Register &tmp1,
82 Register &tmp2,
83 Register &tmp3) {
84 if (tmp1 == preserve) {
85 assert_different_registers(tmp1, tmp2, tmp3, extra);
86 tmp1 = extra;
87 } else if (tmp2 == preserve) {
88 assert_different_registers(tmp1, tmp2, tmp3, extra);
89 tmp2 = extra;
90 } else if (tmp3 == preserve) {
91 assert_different_registers(tmp1, tmp2, tmp3, extra);
92 tmp3 = extra;
93 }
94 assert_different_registers(preserve, tmp1, tmp2, tmp3);
95 }
96
97
98 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
99
100
101 LIR_Opr LIR_Assembler::receiverOpr() {
102 return FrameMap::receiver_opr;
103 }
104
105 LIR_Opr LIR_Assembler::osrBufferPointer() {
106 return FrameMap::as_pointer_opr(receiverOpr()->as_register());
107 }
108
109 //--------------fpu register translations-----------------------
110
111
112 address LIR_Assembler::float_constant(float f) {
113 address const_addr = __ float_constant(f);
114 if (const_addr == NULL) {
115 bailout("const section overflow");
116 return __ code()->consts()->start();
117 } else {
118 return const_addr;
119 }
120 }
121
122
123 address LIR_Assembler::double_constant(double d) {
124 address const_addr = __ double_constant(d);
125 if (const_addr == NULL) {
126 bailout("const section overflow");
127 return __ code()->consts()->start();
128 } else {
129 return const_addr;
130 }
131 }
132
133 address LIR_Assembler::int_constant(jlong n) {
134 address const_addr = __ long_constant(n);
135 if (const_addr == NULL) {
136 bailout("const section overflow");
137 return __ code()->consts()->start();
138 } else {
139 return const_addr;
140 }
141 }
142
143 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
144
145 void LIR_Assembler::reset_FPU() { Unimplemented(); }
146
147 void LIR_Assembler::fpop() { Unimplemented(); }
148
149 void LIR_Assembler::fxch(int i) { Unimplemented(); }
150
151 void LIR_Assembler::fld(int i) { Unimplemented(); }
152
153 void LIR_Assembler::ffree(int i) { Unimplemented(); }
154
155 void LIR_Assembler::breakpoint() { Unimplemented(); }
156
157 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
158
159 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
160
161 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
162 //-------------------------------------------
163
164 static Register as_reg(LIR_Opr op) {
165 return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
166 }
167
168 static jlong as_long(LIR_Opr data) {
169 jlong result;
170 switch (data->type()) {
171 case T_INT:
172 result = (data->as_jint());
173 break;
174 case T_LONG:
175 result = (data->as_jlong());
176 break;
177 default:
178 ShouldNotReachHere();
179 result = 0; // unreachable
180 }
181 return result;
182 }
183
184 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
185 Register base = addr->base()->as_pointer_register();
186 LIR_Opr opr = addr->index();
187 if (opr->is_cpu_register()) {
188 Register index;
189 if (opr->is_single_cpu())
190 index = opr->as_register();
191 else
192 index = opr->as_register_lo();
193 assert(addr->disp() == 0, "must be");
194 switch(opr->type()) {
195 case T_INT:
196 return Address(base, index, Address::sxtw(addr->scale()));
197 case T_LONG:
198 return Address(base, index, Address::lsl(addr->scale()));
199 default:
200 ShouldNotReachHere();
201 }
202 } else {
203 intptr_t addr_offset = intptr_t(addr->disp());
204 if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
205 return Address(base, addr_offset, Address::lsl(addr->scale()));
206 else {
207 __ mov(tmp, addr_offset);
208 return Address(base, tmp, Address::lsl(addr->scale()));
209 }
210 }
211 return Address();
212 }
213
214 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
215 ShouldNotReachHere();
216 return Address();
217 }
218
219 Address LIR_Assembler::as_Address(LIR_Address* addr) {
220 return as_Address(addr, rscratch1);
221 }
222
223 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
224 return as_Address(addr, rscratch1); // Ouch
225 // FIXME: This needs to be much more clever. See x86.
226 }
227
228
229 void LIR_Assembler::osr_entry() {
230 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
231 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
232 ValueStack* entry_state = osr_entry->state();
233 int number_of_locks = entry_state->locks_size();
234
235 // we jump here if osr happens with the interpreter
236 // state set up to continue at the beginning of the
237 // loop that triggered osr - in particular, we have
238 // the following registers setup:
239 //
240 // r2: osr buffer
241 //
242
243 // build frame
244 ciMethod* m = compilation()->method();
245 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
246
247 // OSR buffer is
248 //
249 // locals[nlocals-1..0]
250 // monitors[0..number_of_locks]
251 //
252 // locals is a direct copy of the interpreter frame so in the osr buffer
253 // so first slot in the local array is the last local from the interpreter
254 // and last slot is local[0] (receiver) from the interpreter
255 //
256 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
257 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
258 // in the interpreter frame (the method lock if a sync method)
259
260 // Initialize monitors in the compiled activation.
261 // r2: pointer to osr buffer
262 //
263 // All other registers are dead at this point and the locals will be
264 // copied into place by code emitted in the IR.
265
266 Register OSR_buf = osrBufferPointer()->as_pointer_register();
267 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
268 int monitor_offset = BytesPerWord * method()->max_locals() +
269 (2 * BytesPerWord) * (number_of_locks - 1);
270 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
271 // the OSR buffer using 2 word entries: first the lock and then
272 // the oop.
273 for (int i = 0; i < number_of_locks; i++) {
274 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
275 #ifdef ASSERT
276 // verify the interpreter's monitor has a non-null object
277 {
278 Label L;
279 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
280 __ cbnz(rscratch1, L);
281 __ stop("locked object is NULL");
282 __ bind(L);
283 }
284 #endif
285 __ ldr(r19, Address(OSR_buf, slot_offset + 0));
286 __ str(r19, frame_map()->address_for_monitor_lock(i));
287 __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
288 __ str(r19, frame_map()->address_for_monitor_object(i));
289 }
290 }
291 }
292
293
294 // inline cache check; done before the frame is built.
295 int LIR_Assembler::check_icache() {
296 Register receiver = FrameMap::receiver_opr->as_register();
297 Register ic_klass = IC_Klass;
298 int start_offset = __ offset();
299 __ inline_cache_check(receiver, ic_klass);
300
301 // if icache check fails, then jump to runtime routine
302 // Note: RECEIVER must still contain the receiver!
303 Label dont;
304 __ br(Assembler::EQ, dont);
305 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
306
307 // We align the verified entry point unless the method body
308 // (including its inline cache check) will fit in a single 64-byte
309 // icache line.
310 if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
311 // force alignment after the cache check.
312 __ align(CodeEntryAlignment);
313 }
314
315 __ bind(dont);
316 return start_offset;
317 }
318
319
320 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
321 if (o == NULL) {
322 __ mov(reg, zr);
323 } else {
324 __ movoop(reg, o, /*immediate*/true);
325 }
326 }
327
328 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
329 address target = NULL;
330 relocInfo::relocType reloc_type = relocInfo::none;
331
332 switch (patching_id(info)) {
333 case PatchingStub::access_field_id:
334 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
335 reloc_type = relocInfo::section_word_type;
336 break;
337 case PatchingStub::load_klass_id:
338 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
339 reloc_type = relocInfo::metadata_type;
340 break;
341 case PatchingStub::load_mirror_id:
342 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
343 reloc_type = relocInfo::oop_type;
344 break;
345 case PatchingStub::load_appendix_id:
346 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
347 reloc_type = relocInfo::oop_type;
348 break;
349 default: ShouldNotReachHere();
350 }
351
352 __ far_call(RuntimeAddress(target));
353 add_call_info_here(info);
354 }
355
356 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
357 deoptimize_trap(info);
358 }
359
360
361 // This specifies the rsp decrement needed to build the frame
362 int LIR_Assembler::initial_frame_size_in_bytes() const {
363 // if rounding, must let FrameMap know!
364
365 // The frame_map records size in slots (32bit word)
366
367 // subtract two words to account for return address and link
368 return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word)) * VMRegImpl::stack_slot_size;
369 }
370
371
372 int LIR_Assembler::emit_exception_handler() {
373 // if the last instruction is a call (typically to do a throw which
374 // is coming at the end after block reordering) the return address
375 // must still point into the code area in order to avoid assertion
376 // failures when searching for the corresponding bci => add a nop
377 // (was bug 5/14/1999 - gri)
378 __ nop();
379
380 // generate code for exception handler
381 address handler_base = __ start_a_stub(exception_handler_size);
382 if (handler_base == NULL) {
383 // not enough space left for the handler
384 bailout("exception handler overflow");
385 return -1;
386 }
387
388 int offset = code_offset();
389
390 // the exception oop and pc are in r0, and r3
391 // no other registers need to be preserved, so invalidate them
392 __ invalidate_registers(false, true, true, false, true, true);
393
394 // check that there is really an exception
395 __ verify_not_null_oop(r0);
396
397 // search an exception handler (r0: exception oop, r3: throwing pc)
398 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); __ should_not_reach_here();
399 guarantee(code_offset() - offset <= exception_handler_size, "overflow");
400 __ end_a_stub();
401
402 return offset;
403 }
404
405
406 // Emit the code to remove the frame from the stack in the exception
407 // unwind path.
408 int LIR_Assembler::emit_unwind_handler() {
409 #ifndef PRODUCT
410 if (CommentedAssembly) {
411 _masm->block_comment("Unwind handler");
412 }
413 #endif
414
415 int offset = code_offset();
416
417 // Fetch the exception from TLS and clear out exception related thread state
418 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
419 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
420 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
421
422 __ bind(_unwind_handler_entry);
423 __ verify_not_null_oop(r0);
424 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
425 __ mov(r19, r0); // Preserve the exception
426 }
427
428 // Preform needed unlocking
429 MonitorExitStub* stub = NULL;
430 if (method()->is_synchronized()) {
431 monitor_address(0, FrameMap::r0_opr);
432 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
433 __ unlock_object(r5, r4, r0, *stub->entry());
434 __ bind(*stub->continuation());
435 }
436
437 if (compilation()->env()->dtrace_method_probes()) {
438 __ call_Unimplemented();
439 #if 0
440 __ movptr(Address(rsp, 0), rax);
441 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
442 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
443 #endif
444 }
445
446 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
447 __ mov(r0, r19); // Restore the exception
448 }
449
450 // remove the activation and dispatch to the unwind handler
451 __ block_comment("remove_frame and dispatch to the unwind handler");
452 __ remove_frame(initial_frame_size_in_bytes());
453 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
454
455 // Emit the slow path assembly
456 if (stub != NULL) {
457 stub->emit_code(this);
458 }
459
460 return offset;
461 }
462
463
464 int LIR_Assembler::emit_deopt_handler() {
465 // if the last instruction is a call (typically to do a throw which
466 // is coming at the end after block reordering) the return address
467 // must still point into the code area in order to avoid assertion
468 // failures when searching for the corresponding bci => add a nop
469 // (was bug 5/14/1999 - gri)
470 __ nop();
471
472 // generate code for exception handler
473 address handler_base = __ start_a_stub(deopt_handler_size);
474 if (handler_base == NULL) {
475 // not enough space left for the handler
476 bailout("deopt handler overflow");
477 return -1;
478 }
479
480 int offset = code_offset();
481
482 __ adr(lr, pc());
483 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
484 guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
485 __ end_a_stub();
486
487 return offset;
488 }
489
490 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
491 _masm->code_section()->relocate(adr, relocInfo::poll_type);
492 int pc_offset = code_offset();
493 flush_debug_info(pc_offset);
494 info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
495 if (info->exception_handlers() != NULL) {
496 compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
497 }
498 }
499
500 // Rather than take a segfault when the polling page is protected,
501 // explicitly check for a safepoint in progress and if there is one,
502 // fake a call to the handler as if a segfault had been caught.
503 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
504 __ mov(rscratch1, SafepointSynchronize::address_of_state());
505 __ ldrb(rscratch1, Address(rscratch1));
506 Label nope, poll;
507 __ cbz(rscratch1, nope);
508 __ block_comment("safepoint");
509 __ enter();
510 __ push(0x3, sp); // r0 & r1
511 __ push(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
512 __ adr(r0, poll);
513 __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
514 __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
515 __ blrt(rscratch1, 1, 0, 1);
516 __ maybe_isb();
517 __ pop(0x3ffffffc, sp); // integer registers except lr & sp & r0 & r1
518 __ mov(rscratch1, r0);
519 __ pop(0x3, sp); // r0 & r1
520 __ leave();
521 __ br(rscratch1);
522 address polling_page(os::get_polling_page());
523 assert(os::is_poll_address(polling_page), "should be");
524 unsigned long off;
525 __ adrp(rscratch1, Address(polling_page, rtype), off);
526 __ bind(poll);
527 if (info)
528 add_debug_info_for_branch(info); // This isn't just debug info:
529 // it's the oop map
530 else
531 __ code_section()->relocate(pc(), rtype);
532 __ ldrw(zr, Address(rscratch1, off));
533 __ bind(nope);
534 }
535
536 void LIR_Assembler::return_op(LIR_Opr result) {
537 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
538 // Pop the stack before the safepoint code
539 __ remove_frame(initial_frame_size_in_bytes());
540 address polling_page(os::get_polling_page());
541 __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
542 __ ret(lr);
543 }
544
545 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
546 address polling_page(os::get_polling_page());
547 guarantee(info != NULL, "Shouldn't be NULL");
548 assert(os::is_poll_address(polling_page), "should be");
549 unsigned long off;
550 __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
551 assert(off == 0, "must be");
552 add_debug_info_for_branch(info); // This isn't just debug info:
553 // it's the oop map
554 __ read_polling_page(rscratch1, relocInfo::poll_type);
555 return __ offset();
556 }
557
558
559 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
560 if (from_reg == r31_sp)
561 from_reg = sp;
562 if (to_reg == r31_sp)
563 to_reg = sp;
564 __ mov(to_reg, from_reg);
565 }
566
567 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
568
569
570 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
571 assert(src->is_constant(), "should not call otherwise");
572 assert(dest->is_register(), "should not call otherwise");
573 LIR_Const* c = src->as_constant_ptr();
574
575 switch (c->type()) {
576 case T_INT: {
577 assert(patch_code == lir_patch_none, "no patching handled here");
578 __ movw(dest->as_register(), c->as_jint());
579 break;
580 }
581
582 case T_ADDRESS: {
583 assert(patch_code == lir_patch_none, "no patching handled here");
584 __ mov(dest->as_register(), c->as_jint());
585 break;
586 }
587
588 case T_LONG: {
589 assert(patch_code == lir_patch_none, "no patching handled here");
590 __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
591 break;
592 }
593
594 case T_OBJECT: {
595 if (patch_code == lir_patch_none) {
596 jobject2reg(c->as_jobject(), dest->as_register());
597 } else {
598 jobject2reg_with_patching(dest->as_register(), info);
599 }
600 break;
601 }
602
603 case T_METADATA: {
604 if (patch_code != lir_patch_none) {
605 klass2reg_with_patching(dest->as_register(), info);
606 } else {
607 __ mov_metadata(dest->as_register(), c->as_metadata());
608 }
609 break;
610 }
611
612 case T_FLOAT: {
613 if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
614 __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
615 } else {
616 __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
617 __ ldrs(dest->as_float_reg(), Address(rscratch1));
618 }
619 break;
620 }
621
622 case T_DOUBLE: {
623 if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
624 __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
625 } else {
626 __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
627 __ ldrd(dest->as_double_reg(), Address(rscratch1));
628 }
629 break;
630 }
631
632 default:
633 ShouldNotReachHere();
634 }
635 }
636
637 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
638 LIR_Const* c = src->as_constant_ptr();
639 switch (c->type()) {
640 case T_OBJECT:
641 {
642 if (! c->as_jobject())
643 __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
644 else {
645 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
646 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
647 }
648 }
649 break;
650 case T_ADDRESS:
651 {
652 const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
653 reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
654 }
655 case T_INT:
656 case T_FLOAT:
657 {
658 Register reg = zr;
659 if (c->as_jint_bits() == 0)
660 __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
661 else {
662 __ movw(rscratch1, c->as_jint_bits());
663 __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
664 }
665 }
666 break;
667 case T_LONG:
668 case T_DOUBLE:
669 {
670 Register reg = zr;
671 if (c->as_jlong_bits() == 0)
672 __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
673 lo_word_offset_in_bytes));
674 else {
675 __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
676 __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
677 lo_word_offset_in_bytes));
678 }
679 }
680 break;
681 default:
682 ShouldNotReachHere();
683 }
684 }
685
686 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
687 assert(src->is_constant(), "should not call otherwise");
688 LIR_Const* c = src->as_constant_ptr();
689 LIR_Address* to_addr = dest->as_address_ptr();
690
691 void (Assembler::* insn)(Register Rt, const Address &adr);
692
693 switch (type) {
694 case T_ADDRESS:
695 assert(c->as_jint() == 0, "should be");
696 insn = &Assembler::str;
697 break;
698 case T_LONG:
699 assert(c->as_jlong() == 0, "should be");
700 insn = &Assembler::str;
701 break;
702 case T_INT:
703 assert(c->as_jint() == 0, "should be");
704 insn = &Assembler::strw;
705 break;
706 case T_OBJECT:
707 case T_ARRAY:
708 assert(c->as_jobject() == 0, "should be");
709 if (UseCompressedOops && !wide) {
710 insn = &Assembler::strw;
711 } else {
712 insn = &Assembler::str;
713 }
714 break;
715 case T_CHAR:
716 case T_SHORT:
717 assert(c->as_jint() == 0, "should be");
718 insn = &Assembler::strh;
719 break;
720 case T_BOOLEAN:
721 case T_BYTE:
722 assert(c->as_jint() == 0, "should be");
723 insn = &Assembler::strb;
724 break;
725 default:
726 ShouldNotReachHere();
727 insn = &Assembler::str; // unreachable
728 }
729
730 if (info) add_debug_info_for_null_check_here(info);
731 (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
732 }
733
734 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
735 assert(src->is_register(), "should not call otherwise");
736 assert(dest->is_register(), "should not call otherwise");
737
738 // move between cpu-registers
739 if (dest->is_single_cpu()) {
740 if (src->type() == T_LONG) {
741 // Can do LONG -> OBJECT
742 move_regs(src->as_register_lo(), dest->as_register());
743 return;
744 }
745 assert(src->is_single_cpu(), "must match");
746 if (src->type() == T_OBJECT) {
747 __ verify_oop(src->as_register());
748 }
749 move_regs(src->as_register(), dest->as_register());
750
751 } else if (dest->is_double_cpu()) {
752 if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
753 // Surprising to me but we can see move of a long to t_object
754 __ verify_oop(src->as_register());
755 move_regs(src->as_register(), dest->as_register_lo());
756 return;
757 }
758 assert(src->is_double_cpu(), "must match");
759 Register f_lo = src->as_register_lo();
760 Register f_hi = src->as_register_hi();
761 Register t_lo = dest->as_register_lo();
762 Register t_hi = dest->as_register_hi();
763 assert(f_hi == f_lo, "must be same");
764 assert(t_hi == t_lo, "must be same");
765 move_regs(f_lo, t_lo);
766
767 } else if (dest->is_single_fpu()) {
768 __ fmovs(dest->as_float_reg(), src->as_float_reg());
769
770 } else if (dest->is_double_fpu()) {
771 __ fmovd(dest->as_double_reg(), src->as_double_reg());
772
773 } else {
774 ShouldNotReachHere();
775 }
776 }
777
778 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
779 if (src->is_single_cpu()) {
780 if (type == T_ARRAY || type == T_OBJECT) {
781 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
782 __ verify_oop(src->as_register());
783 } else if (type == T_METADATA || type == T_DOUBLE) {
784 __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
785 } else {
786 __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
787 }
788
789 } else if (src->is_double_cpu()) {
790 Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
791 __ str(src->as_register_lo(), dest_addr_LO);
792
793 } else if (src->is_single_fpu()) {
794 Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
795 __ strs(src->as_float_reg(), dest_addr);
796
797 } else if (src->is_double_fpu()) {
798 Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
799 __ strd(src->as_double_reg(), dest_addr);
800
801 } else {
802 ShouldNotReachHere();
803 }
804
805 }
806
807
808 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
809 LIR_Address* to_addr = dest->as_address_ptr();
810 PatchingStub* patch = NULL;
811 Register compressed_src = rscratch1;
812
813 if (patch_code != lir_patch_none) {
814 deoptimize_trap(info);
815 return;
816 }
817
818 if (type == T_ARRAY || type == T_OBJECT) {
819 __ verify_oop(src->as_register());
820
821 __ shenandoah_store_check(as_Address(to_addr), src->as_register());
822
823 if (UseCompressedOops && !wide) {
824 __ encode_heap_oop(compressed_src, src->as_register());
825 } else {
826 compressed_src = src->as_register();
827 }
828 } else {
829 __ shenandoah_store_addr_check(to_addr->base()->as_pointer_register());
830 }
831
832 int null_check_here = code_offset();
833 switch (type) {
834 case T_FLOAT: {
835 __ strs(src->as_float_reg(), as_Address(to_addr));
836 break;
837 }
838
839 case T_DOUBLE: {
840 __ strd(src->as_double_reg(), as_Address(to_addr));
841 break;
842 }
843
844 case T_ARRAY: // fall through
845 case T_OBJECT: // fall through
846 if (UseCompressedOops && !wide) {
847 __ strw(compressed_src, as_Address(to_addr, rscratch2));
848 } else {
849 __ str(compressed_src, as_Address(to_addr));
850 }
851 break;
852 case T_METADATA:
853 // We get here to store a method pointer to the stack to pass to
854 // a dtrace runtime call. This can't work on 64 bit with
855 // compressed klass ptrs: T_METADATA can be a compressed klass
856 // ptr or a 64 bit method pointer.
857 ShouldNotReachHere();
858 __ str(src->as_register(), as_Address(to_addr));
859 break;
860 case T_ADDRESS:
861 __ str(src->as_register(), as_Address(to_addr));
862 break;
863 case T_INT:
864 __ strw(src->as_register(), as_Address(to_addr));
865 break;
866
867 case T_LONG: {
868 __ str(src->as_register_lo(), as_Address_lo(to_addr));
869 break;
870 }
871
872 case T_BYTE: // fall through
873 case T_BOOLEAN: {
874 __ strb(src->as_register(), as_Address(to_addr));
875 break;
876 }
877
878 case T_CHAR: // fall through
879 case T_SHORT:
880 __ strh(src->as_register(), as_Address(to_addr));
881 break;
882
883 default:
884 ShouldNotReachHere();
885 }
886 if (info != NULL) {
887 add_debug_info_for_null_check(null_check_here, info);
888 }
889 }
890
891
892 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
893 assert(src->is_stack(), "should not call otherwise");
894 assert(dest->is_register(), "should not call otherwise");
895
896 if (dest->is_single_cpu()) {
897 if (type == T_ARRAY || type == T_OBJECT) {
898 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
899 __ verify_oop(dest->as_register());
900 } else if (type == T_METADATA) {
901 __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
902 } else {
903 __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
904 }
905
906 } else if (dest->is_double_cpu()) {
907 Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
908 __ ldr(dest->as_register_lo(), src_addr_LO);
909
910 } else if (dest->is_single_fpu()) {
911 Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
912 __ ldrs(dest->as_float_reg(), src_addr);
913
914 } else if (dest->is_double_fpu()) {
915 Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
916 __ ldrd(dest->as_double_reg(), src_addr);
917
918 } else {
919 ShouldNotReachHere();
920 }
921 }
922
923
924 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
925 address target = NULL;
926 relocInfo::relocType reloc_type = relocInfo::none;
927
928 switch (patching_id(info)) {
929 case PatchingStub::access_field_id:
930 target = Runtime1::entry_for(Runtime1::access_field_patching_id);
931 reloc_type = relocInfo::section_word_type;
932 break;
933 case PatchingStub::load_klass_id:
934 target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
935 reloc_type = relocInfo::metadata_type;
936 break;
937 case PatchingStub::load_mirror_id:
938 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
939 reloc_type = relocInfo::oop_type;
940 break;
941 case PatchingStub::load_appendix_id:
942 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
943 reloc_type = relocInfo::oop_type;
944 break;
945 default: ShouldNotReachHere();
946 }
947
948 __ far_call(RuntimeAddress(target));
949 add_call_info_here(info);
950 }
951
952 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
953
954 LIR_Opr temp;
955 if (type == T_LONG || type == T_DOUBLE)
956 temp = FrameMap::rscratch1_long_opr;
957 else
958 temp = FrameMap::rscratch1_opr;
959
960 stack2reg(src, temp, src->type());
961 reg2stack(temp, dest, dest->type(), false);
962 }
963
964
965 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
966 LIR_Address* addr = src->as_address_ptr();
967 LIR_Address* from_addr = src->as_address_ptr();
968
969 if (addr->base()->type() == T_OBJECT) {
970 __ verify_oop(addr->base()->as_pointer_register());
971 }
972
973 if (patch_code != lir_patch_none) {
974 deoptimize_trap(info);
975 return;
976 }
977
978 if (info != NULL) {
979 add_debug_info_for_null_check_here(info);
980 }
981 int null_check_here = code_offset();
982 switch (type) {
983 case T_FLOAT: {
984 __ ldrs(dest->as_float_reg(), as_Address(from_addr));
985 break;
986 }
987
988 case T_DOUBLE: {
989 __ ldrd(dest->as_double_reg(), as_Address(from_addr));
990 break;
991 }
992
993 case T_ARRAY: // fall through
994 case T_OBJECT: // fall through
995 if (UseCompressedOops && !wide) {
996 __ ldrw(dest->as_register(), as_Address(from_addr));
997 } else {
998 __ ldr(dest->as_register(), as_Address(from_addr));
999 }
1000 break;
1001 case T_METADATA:
1002 // We get here to store a method pointer to the stack to pass to
1003 // a dtrace runtime call. This can't work on 64 bit with
1004 // compressed klass ptrs: T_METADATA can be a compressed klass
1005 // ptr or a 64 bit method pointer.
1006 ShouldNotReachHere();
1007 __ ldr(dest->as_register(), as_Address(from_addr));
1008 break;
1009 case T_ADDRESS:
1010 // FIXME: OMG this is a horrible kludge. Any offset from an
1011 // address that matches klass_offset_in_bytes() will be loaded
1012 // as a word, not a long.
1013 if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1014 __ ldrw(dest->as_register(), as_Address(from_addr));
1015 } else {
1016 __ ldr(dest->as_register(), as_Address(from_addr));
1017 }
1018 break;
1019 case T_INT:
1020 __ ldrw(dest->as_register(), as_Address(from_addr));
1021 break;
1022
1023 case T_LONG: {
1024 __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1025 break;
1026 }
1027
1028 case T_BYTE:
1029 __ ldrsb(dest->as_register(), as_Address(from_addr));
1030 break;
1031 case T_BOOLEAN: {
1032 __ ldrb(dest->as_register(), as_Address(from_addr));
1033 break;
1034 }
1035
1036 case T_CHAR:
1037 __ ldrh(dest->as_register(), as_Address(from_addr));
1038 break;
1039 case T_SHORT:
1040 __ ldrsh(dest->as_register(), as_Address(from_addr));
1041 break;
1042
1043 default:
1044 ShouldNotReachHere();
1045 }
1046
1047 if (type == T_ARRAY || type == T_OBJECT) {
1048 if (UseCompressedOops && !wide) {
1049 __ decode_heap_oop(dest->as_register());
1050 }
1051 __ verify_oop(dest->as_register());
1052 } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1053 if (UseCompressedClassPointers) {
1054 __ decode_klass_not_null(dest->as_register());
1055 }
1056 }
1057 }
1058
1059
1060 int LIR_Assembler::array_element_size(BasicType type) const {
1061 int elem_size = type2aelembytes(type);
1062 return exact_log2(elem_size);
1063 }
1064
1065 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1066 Register Rdividend = op->in_opr1()->as_register();
1067 Register Rdivisor = op->in_opr2()->as_register();
1068 Register Rscratch = op->in_opr3()->as_register();
1069 Register Rresult = op->result_opr()->as_register();
1070 int divisor = -1;
1071
1072 /*
1073 TODO: For some reason, using the Rscratch that gets passed in is
1074 not possible because the register allocator does not see the tmp reg
1075 as used, and assignes it the same register as Rdividend. We use rscratch1
1076 instead.
1077
1078 assert(Rdividend != Rscratch, "");
1079 assert(Rdivisor != Rscratch, "");
1080 */
1081
1082 if (Rdivisor == noreg && is_power_of_2(divisor)) {
1083 // convert division by a power of two into some shifts and logical operations
1084 }
1085
1086 if (op->code() == lir_irem) {
1087 __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1088 } else if (op->code() == lir_idiv) {
1089 __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1090 } else
1091 ShouldNotReachHere();
1092 }
1093
1094 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1095 #ifdef ASSERT
1096 assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1097 if (op->block() != NULL) _branch_target_blocks.append(op->block());
1098 if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1099 #endif
1100
1101 if (op->cond() == lir_cond_always) {
1102 if (op->info() != NULL) add_debug_info_for_branch(op->info());
1103 __ b(*(op->label()));
1104 } else {
1105 Assembler::Condition acond;
1106 if (op->code() == lir_cond_float_branch) {
1107 bool is_unordered = (op->ublock() == op->block());
1108 // Assembler::EQ does not permit unordered branches, so we add
1109 // another branch here. Likewise, Assembler::NE does not permit
1110 // ordered branches.
1111 if (is_unordered && op->cond() == lir_cond_equal
1112 || !is_unordered && op->cond() == lir_cond_notEqual)
1113 __ br(Assembler::VS, *(op->ublock()->label()));
1114 switch(op->cond()) {
1115 case lir_cond_equal: acond = Assembler::EQ; break;
1116 case lir_cond_notEqual: acond = Assembler::NE; break;
1117 case lir_cond_less: acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1118 case lir_cond_lessEqual: acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1119 case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1120 case lir_cond_greater: acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1121 default: ShouldNotReachHere();
1122 acond = Assembler::EQ; // unreachable
1123 }
1124 } else {
1125 switch (op->cond()) {
1126 case lir_cond_equal: acond = Assembler::EQ; break;
1127 case lir_cond_notEqual: acond = Assembler::NE; break;
1128 case lir_cond_less: acond = Assembler::LT; break;
1129 case lir_cond_lessEqual: acond = Assembler::LE; break;
1130 case lir_cond_greaterEqual: acond = Assembler::GE; break;
1131 case lir_cond_greater: acond = Assembler::GT; break;
1132 case lir_cond_belowEqual: acond = Assembler::LS; break;
1133 case lir_cond_aboveEqual: acond = Assembler::HS; break;
1134 default: ShouldNotReachHere();
1135 acond = Assembler::EQ; // unreachable
1136 }
1137 }
1138 __ br(acond,*(op->label()));
1139 }
1140 }
1141
1142
1143 void LIR_Assembler::emit_opShenandoahWriteBarrier(LIR_OpShenandoahWriteBarrier* op) {
1144
1145 Register obj = op->in_opr()->as_register();
1146 Register res = op->result_opr()->as_register();
1147
1148 Label done;
1149
1150 __ block_comment("Shenandoah write barrier {");
1151
1152 if (res != obj) {
1153 __ mov(res, obj);
1154 }
1155 // Check for null.
1156 if (op->need_null_check()) {
1157 __ cbz(res, done);
1158 }
1159
1160 __ shenandoah_write_barrier(res);
1161
1162 __ bind(done);
1163
1164 __ block_comment("} Shenandoah write barrier");
1165
1166 }
1167
1168
1169 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1170 LIR_Opr src = op->in_opr();
1171 LIR_Opr dest = op->result_opr();
1172
1173 switch (op->bytecode()) {
1174 case Bytecodes::_i2f:
1175 {
1176 __ scvtfws(dest->as_float_reg(), src->as_register());
1177 break;
1178 }
1179 case Bytecodes::_i2d:
1180 {
1181 __ scvtfwd(dest->as_double_reg(), src->as_register());
1182 break;
1183 }
1184 case Bytecodes::_l2d:
1185 {
1186 __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1187 break;
1188 }
1189 case Bytecodes::_l2f:
1190 {
1191 __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1192 break;
1193 }
1194 case Bytecodes::_f2d:
1195 {
1196 __ fcvts(dest->as_double_reg(), src->as_float_reg());
1197 break;
1198 }
1199 case Bytecodes::_d2f:
1200 {
1201 __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1202 break;
1203 }
1204 case Bytecodes::_i2c:
1205 {
1206 __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1207 break;
1208 }
1209 case Bytecodes::_i2l:
1210 {
1211 __ sxtw(dest->as_register_lo(), src->as_register());
1212 break;
1213 }
1214 case Bytecodes::_i2s:
1215 {
1216 __ sxth(dest->as_register(), src->as_register());
1217 break;
1218 }
1219 case Bytecodes::_i2b:
1220 {
1221 __ sxtb(dest->as_register(), src->as_register());
1222 break;
1223 }
1224 case Bytecodes::_l2i:
1225 {
1226 _masm->block_comment("FIXME: This could be a no-op");
1227 __ uxtw(dest->as_register(), src->as_register_lo());
1228 break;
1229 }
1230 case Bytecodes::_d2l:
1231 {
1232 __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1233 break;
1234 }
1235 case Bytecodes::_f2i:
1236 {
1237 __ fcvtzsw(dest->as_register(), src->as_float_reg());
1238 break;
1239 }
1240 case Bytecodes::_f2l:
1241 {
1242 __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1243 break;
1244 }
1245 case Bytecodes::_d2i:
1246 {
1247 __ fcvtzdw(dest->as_register(), src->as_double_reg());
1248 break;
1249 }
1250 default: ShouldNotReachHere();
1251 }
1252 }
1253
1254 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1255 if (op->init_check()) {
1256 __ ldrb(rscratch1, Address(op->klass()->as_register(),
1257 InstanceKlass::init_state_offset()));
1258 __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1259 add_debug_info_for_null_check_here(op->stub()->info());
1260 __ br(Assembler::NE, *op->stub()->entry());
1261 }
1262 __ allocate_object(op->obj()->as_register(),
1263 op->tmp1()->as_register(),
1264 op->tmp2()->as_register(),
1265 op->header_size(),
1266 op->object_size(),
1267 op->klass()->as_register(),
1268 *op->stub()->entry());
1269 __ bind(*op->stub()->continuation());
1270 }
1271
1272 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1273 Register len = op->len()->as_register();
1274 __ uxtw(len, len);
1275
1276 if (UseSlowPath ||
1277 (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1278 (!UseFastNewTypeArray && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1279 __ b(*op->stub()->entry());
1280 } else {
1281 Register tmp1 = op->tmp1()->as_register();
1282 Register tmp2 = op->tmp2()->as_register();
1283 Register tmp3 = op->tmp3()->as_register();
1284 if (len == tmp1) {
1285 tmp1 = tmp3;
1286 } else if (len == tmp2) {
1287 tmp2 = tmp3;
1288 } else if (len == tmp3) {
1289 // everything is ok
1290 } else {
1291 __ mov(tmp3, len);
1292 }
1293 __ allocate_array(op->obj()->as_register(),
1294 len,
1295 tmp1,
1296 tmp2,
1297 arrayOopDesc::header_size(op->type()),
1298 array_element_size(op->type()),
1299 op->klass()->as_register(),
1300 *op->stub()->entry());
1301 }
1302 __ bind(*op->stub()->continuation());
1303 }
1304
1305 void LIR_Assembler::type_profile_helper(Register mdo,
1306 ciMethodData *md, ciProfileData *data,
1307 Register recv, Label* update_done) {
1308 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1309 Label next_test;
1310 // See if the receiver is receiver[n].
1311 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1312 __ ldr(rscratch1, Address(rscratch2));
1313 __ cmp(recv, rscratch1);
1314 __ br(Assembler::NE, next_test);
1315 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1316 __ addptr(data_addr, DataLayout::counter_increment);
1317 __ b(*update_done);
1318 __ bind(next_test);
1319 }
1320
1321 // Didn't find receiver; find next empty slot and fill it in
1322 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1323 Label next_test;
1324 __ lea(rscratch2,
1325 Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1326 Address recv_addr(rscratch2);
1327 __ ldr(rscratch1, recv_addr);
1328 __ cbnz(rscratch1, next_test);
1329 __ str(recv, recv_addr);
1330 __ mov(rscratch1, DataLayout::counter_increment);
1331 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1332 __ str(rscratch1, Address(rscratch2));
1333 __ b(*update_done);
1334 __ bind(next_test);
1335 }
1336 }
1337
1338 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1339 // we always need a stub for the failure case.
1340 CodeStub* stub = op->stub();
1341 Register obj = op->object()->as_register();
1342 Register k_RInfo = op->tmp1()->as_register();
1343 Register klass_RInfo = op->tmp2()->as_register();
1344 Register dst = op->result_opr()->as_register();
1345 ciKlass* k = op->klass();
1346 Register Rtmp1 = noreg;
1347
1348 // check if it needs to be profiled
1349 ciMethodData* md;
1350 ciProfileData* data;
1351
1352 const bool should_profile = op->should_profile();
1353
1354 if (should_profile) {
1355 ciMethod* method = op->profiled_method();
1356 assert(method != NULL, "Should have method");
1357 int bci = op->profiled_bci();
1358 md = method->method_data_or_null();
1359 assert(md != NULL, "Sanity");
1360 data = md->bci_to_data(bci);
1361 assert(data != NULL, "need data for type check");
1362 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1363 }
1364 Label profile_cast_success, profile_cast_failure;
1365 Label *success_target = should_profile ? &profile_cast_success : success;
1366 Label *failure_target = should_profile ? &profile_cast_failure : failure;
1367
1368 if (obj == k_RInfo) {
1369 k_RInfo = dst;
1370 } else if (obj == klass_RInfo) {
1371 klass_RInfo = dst;
1372 }
1373 if (k->is_loaded() && !UseCompressedClassPointers) {
1374 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1375 } else {
1376 Rtmp1 = op->tmp3()->as_register();
1377 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1378 }
1379
1380 assert_different_registers(obj, k_RInfo, klass_RInfo);
1381
1382 if (should_profile) {
1383 Label not_null;
1384 __ cbnz(obj, not_null);
1385 // Object is null; update MDO and exit
1386 Register mdo = klass_RInfo;
1387 __ mov_metadata(mdo, md->constant_encoding());
1388 Address data_addr
1389 = __ form_address(rscratch2, mdo,
1390 md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1391 LogBytesPerWord);
1392 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1393 __ ldr(rscratch1, data_addr);
1394 __ orr(rscratch1, rscratch1, header_bits);
1395 __ str(rscratch1, data_addr);
1396 __ b(*obj_is_null);
1397 __ bind(not_null);
1398 } else {
1399 __ cbz(obj, *obj_is_null);
1400 }
1401
1402 if (!k->is_loaded()) {
1403 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1404 } else {
1405 __ mov_metadata(k_RInfo, k->constant_encoding());
1406 }
1407 __ verify_oop(obj);
1408
1409 if (op->fast_check()) {
1410 // get object class
1411 // not a safepoint as obj null check happens earlier
1412 __ load_klass(rscratch1, obj);
1413 __ cmp( rscratch1, k_RInfo);
1414
1415 __ br(Assembler::NE, *failure_target);
1416 // successful cast, fall through to profile or jump
1417 } else {
1418 // get object class
1419 // not a safepoint as obj null check happens earlier
1420 __ load_klass(klass_RInfo, obj);
1421 if (k->is_loaded()) {
1422 // See if we get an immediate positive hit
1423 __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1424 __ cmp(k_RInfo, rscratch1);
1425 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1426 __ br(Assembler::NE, *failure_target);
1427 // successful cast, fall through to profile or jump
1428 } else {
1429 // See if we get an immediate positive hit
1430 __ br(Assembler::EQ, *success_target);
1431 // check for self
1432 __ cmp(klass_RInfo, k_RInfo);
1433 __ br(Assembler::EQ, *success_target);
1434
1435 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1436 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1437 __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1438 // result is a boolean
1439 __ cbzw(klass_RInfo, *failure_target);
1440 // successful cast, fall through to profile or jump
1441 }
1442 } else {
1443 // perform the fast part of the checking logic
1444 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1445 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1446 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1447 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1448 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1449 // result is a boolean
1450 __ cbz(k_RInfo, *failure_target);
1451 // successful cast, fall through to profile or jump
1452 }
1453 }
1454 if (should_profile) {
1455 Register mdo = klass_RInfo, recv = k_RInfo;
1456 __ bind(profile_cast_success);
1457 __ mov_metadata(mdo, md->constant_encoding());
1458 __ load_klass(recv, obj);
1459 Label update_done;
1460 type_profile_helper(mdo, md, data, recv, success);
1461 __ b(*success);
1462
1463 __ bind(profile_cast_failure);
1464 __ mov_metadata(mdo, md->constant_encoding());
1465 Address counter_addr
1466 = __ form_address(rscratch2, mdo,
1467 md->byte_offset_of_slot(data, CounterData::count_offset()),
1468 LogBytesPerWord);
1469 __ ldr(rscratch1, counter_addr);
1470 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1471 __ str(rscratch1, counter_addr);
1472 __ b(*failure);
1473 }
1474 __ b(*success);
1475 }
1476
1477
1478 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1479 const bool should_profile = op->should_profile();
1480
1481 LIR_Code code = op->code();
1482 if (code == lir_store_check) {
1483 Register value = op->object()->as_register();
1484 Register array = op->array()->as_register();
1485 Register k_RInfo = op->tmp1()->as_register();
1486 Register klass_RInfo = op->tmp2()->as_register();
1487 Register Rtmp1 = op->tmp3()->as_register();
1488
1489 CodeStub* stub = op->stub();
1490
1491 // check if it needs to be profiled
1492 ciMethodData* md;
1493 ciProfileData* data;
1494
1495 if (should_profile) {
1496 ciMethod* method = op->profiled_method();
1497 assert(method != NULL, "Should have method");
1498 int bci = op->profiled_bci();
1499 md = method->method_data_or_null();
1500 assert(md != NULL, "Sanity");
1501 data = md->bci_to_data(bci);
1502 assert(data != NULL, "need data for type check");
1503 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1504 }
1505 Label profile_cast_success, profile_cast_failure, done;
1506 Label *success_target = should_profile ? &profile_cast_success : &done;
1507 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1508
1509 if (should_profile) {
1510 Label not_null;
1511 __ cbnz(value, not_null);
1512 // Object is null; update MDO and exit
1513 Register mdo = klass_RInfo;
1514 __ mov_metadata(mdo, md->constant_encoding());
1515 Address data_addr
1516 = __ form_address(rscratch2, mdo,
1517 md->byte_offset_of_slot(data, DataLayout::header_offset()),
1518 LogBytesPerInt);
1519 int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1520 __ ldrw(rscratch1, data_addr);
1521 __ orrw(rscratch1, rscratch1, header_bits);
1522 __ strw(rscratch1, data_addr);
1523 __ b(done);
1524 __ bind(not_null);
1525 } else {
1526 __ cbz(value, done);
1527 }
1528
1529 add_debug_info_for_null_check_here(op->info_for_exception());
1530 __ load_klass(k_RInfo, array);
1531 __ load_klass(klass_RInfo, value);
1532
1533 // get instance klass (it's already uncompressed)
1534 __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1535 // perform the fast part of the checking logic
1536 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1537 // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1538 __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1539 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1540 __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1541 // result is a boolean
1542 __ cbzw(k_RInfo, *failure_target);
1543 // fall through to the success case
1544
1545 if (should_profile) {
1546 Register mdo = klass_RInfo, recv = k_RInfo;
1547 __ bind(profile_cast_success);
1548 __ mov_metadata(mdo, md->constant_encoding());
1549 __ load_klass(recv, value);
1550 Label update_done;
1551 type_profile_helper(mdo, md, data, recv, &done);
1552 __ b(done);
1553
1554 __ bind(profile_cast_failure);
1555 __ mov_metadata(mdo, md->constant_encoding());
1556 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1557 __ lea(rscratch2, counter_addr);
1558 __ ldr(rscratch1, Address(rscratch2));
1559 __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1560 __ str(rscratch1, Address(rscratch2));
1561 __ b(*stub->entry());
1562 }
1563
1564 __ bind(done);
1565 } else if (code == lir_checkcast) {
1566 Register obj = op->object()->as_register();
1567 Register dst = op->result_opr()->as_register();
1568 Label success;
1569 emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1570 __ bind(success);
1571 if (dst != obj) {
1572 __ mov(dst, obj);
1573 }
1574 } else if (code == lir_instanceof) {
1575 Register obj = op->object()->as_register();
1576 Register dst = op->result_opr()->as_register();
1577 Label success, failure, done;
1578 emit_typecheck_helper(op, &success, &failure, &failure);
1579 __ bind(failure);
1580 __ mov(dst, zr);
1581 __ b(done);
1582 __ bind(success);
1583 __ mov(dst, 1);
1584 __ bind(done);
1585 } else {
1586 ShouldNotReachHere();
1587 }
1588 }
1589
1590 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1591 __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1592 __ cset(rscratch1, Assembler::NE);
1593 __ membar(__ AnyAny);
1594 }
1595
1596 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1597 __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1598 __ cset(rscratch1, Assembler::NE);
1599 __ membar(__ AnyAny);
1600 }
1601
1602
1603 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1604 Register addr = as_reg(op->addr());
1605 Register newval = as_reg(op->new_value());
1606 Register cmpval = as_reg(op->cmp_value());
1607 Register res = op->result_opr()->as_register();
1608
1609 if (op->code() == lir_cas_obj) {
1610 assert(op->tmp1()->is_valid(), "must be");
1611 Register t1 = op->tmp1()->as_register();
1612 if (UseCompressedOops) {
1613 if (UseShenandoahGC) {
1614 __ encode_heap_oop(t1, cmpval);
1615 cmpval = t1;
1616 assert(op->tmp2()->is_valid(), "must be");
1617 Register t2 = op->tmp2()->as_register();
1618 __ encode_heap_oop(t2, newval);
1619 newval = t2;
1620 __ cmpxchg_oop_shenandoah(addr, cmpval, newval, Assembler::word, true, true, false);
1621 __ csetw(res, Assembler::EQ);
1622 } else {
1623 __ encode_heap_oop(t1, cmpval);
1624 cmpval = t1;
1625 __ encode_heap_oop(rscratch2, newval);
1626 newval = rscratch2;
1627 casw(addr, newval, cmpval);
1628 __ eorw (res, r8, 1);
1629 }
1630 } else {
1631 if (UseShenandoahGC) {
1632 __ cmpxchg_oop_shenandoah(addr, cmpval, newval, Assembler::xword, true, true, false);
1633 __ csetw(res, Assembler::EQ);
1634 } else {
1635 casl(addr, newval, cmpval);
1636 __ eorw (res, r8, 1);
1637 }
1638 }
1639 } else if (op->code() == lir_cas_int) {
1640 casw(addr, newval, cmpval);
1641 __ eorw (res, r8, 1);
1642 } else {
1643 casl(addr, newval, cmpval);
1644 __ eorw (res, r8, 1);
1645 }
1646 }
1647
1648
1649 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1650
1651 Assembler::Condition acond, ncond;
1652 switch (condition) {
1653 case lir_cond_equal: acond = Assembler::EQ; ncond = Assembler::NE; break;
1654 case lir_cond_notEqual: acond = Assembler::NE; ncond = Assembler::EQ; break;
1655 case lir_cond_less: acond = Assembler::LT; ncond = Assembler::GE; break;
1656 case lir_cond_lessEqual: acond = Assembler::LE; ncond = Assembler::GT; break;
1657 case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1658 case lir_cond_greater: acond = Assembler::GT; ncond = Assembler::LE; break;
1659 case lir_cond_belowEqual:
1660 case lir_cond_aboveEqual:
1661 default: ShouldNotReachHere();
1662 acond = Assembler::EQ; ncond = Assembler::NE; // unreachable
1663 }
1664
1665 assert(result->is_single_cpu() || result->is_double_cpu(),
1666 "expect single register for result");
1667 if (opr1->is_constant() && opr2->is_constant()
1668 && opr1->type() == T_INT && opr2->type() == T_INT) {
1669 jint val1 = opr1->as_jint();
1670 jint val2 = opr2->as_jint();
1671 if (val1 == 0 && val2 == 1) {
1672 __ cset(result->as_register(), ncond);
1673 return;
1674 } else if (val1 == 1 && val2 == 0) {
1675 __ cset(result->as_register(), acond);
1676 return;
1677 }
1678 }
1679
1680 if (opr1->is_constant() && opr2->is_constant()
1681 && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1682 jlong val1 = opr1->as_jlong();
1683 jlong val2 = opr2->as_jlong();
1684 if (val1 == 0 && val2 == 1) {
1685 __ cset(result->as_register_lo(), ncond);
1686 return;
1687 } else if (val1 == 1 && val2 == 0) {
1688 __ cset(result->as_register_lo(), acond);
1689 return;
1690 }
1691 }
1692
1693 if (opr1->is_stack()) {
1694 stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1695 opr1 = FrameMap::rscratch1_opr;
1696 } else if (opr1->is_constant()) {
1697 LIR_Opr tmp
1698 = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1699 const2reg(opr1, tmp, lir_patch_none, NULL);
1700 opr1 = tmp;
1701 }
1702
1703 if (opr2->is_stack()) {
1704 stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1705 opr2 = FrameMap::rscratch2_opr;
1706 } else if (opr2->is_constant()) {
1707 LIR_Opr tmp
1708 = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1709 const2reg(opr2, tmp, lir_patch_none, NULL);
1710 opr2 = tmp;
1711 }
1712
1713 if (result->type() == T_LONG)
1714 __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1715 else
1716 __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1717 }
1718
1719 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1720 assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1721
1722 if (left->is_single_cpu()) {
1723 Register lreg = left->as_register();
1724 Register dreg = as_reg(dest);
1725
1726 if (right->is_single_cpu()) {
1727 // cpu register - cpu register
1728
1729 assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1730 "should be");
1731 Register rreg = right->as_register();
1732 switch (code) {
1733 case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1734 case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1735 case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1736 default: ShouldNotReachHere();
1737 }
1738
1739 } else if (right->is_double_cpu()) {
1740 Register rreg = right->as_register_lo();
1741 // single_cpu + double_cpu: can happen with obj+long
1742 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1743 switch (code) {
1744 case lir_add: __ add(dreg, lreg, rreg); break;
1745 case lir_sub: __ sub(dreg, lreg, rreg); break;
1746 default: ShouldNotReachHere();
1747 }
1748 } else if (right->is_constant()) {
1749 // cpu register - constant
1750 jlong c;
1751
1752 // FIXME. This is fugly: we really need to factor all this logic.
1753 switch(right->type()) {
1754 case T_LONG:
1755 c = right->as_constant_ptr()->as_jlong();
1756 break;
1757 case T_INT:
1758 case T_ADDRESS:
1759 c = right->as_constant_ptr()->as_jint();
1760 break;
1761 default:
1762 ShouldNotReachHere();
1763 c = 0; // unreachable
1764 break;
1765 }
1766
1767 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1768 if (c == 0 && dreg == lreg) {
1769 COMMENT("effective nop elided");
1770 return;
1771 }
1772 switch(left->type()) {
1773 case T_INT:
1774 switch (code) {
1775 case lir_add: __ addw(dreg, lreg, c); break;
1776 case lir_sub: __ subw(dreg, lreg, c); break;
1777 default: ShouldNotReachHere();
1778 }
1779 break;
1780 case T_OBJECT:
1781 case T_ADDRESS:
1782 switch (code) {
1783 case lir_add: __ add(dreg, lreg, c); break;
1784 case lir_sub: __ sub(dreg, lreg, c); break;
1785 default: ShouldNotReachHere();
1786 }
1787 break;
1788 ShouldNotReachHere();
1789 }
1790 } else {
1791 ShouldNotReachHere();
1792 }
1793
1794 } else if (left->is_double_cpu()) {
1795 Register lreg_lo = left->as_register_lo();
1796
1797 if (right->is_double_cpu()) {
1798 // cpu register - cpu register
1799 Register rreg_lo = right->as_register_lo();
1800 switch (code) {
1801 case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1802 case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1803 case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1804 case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1805 case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1806 default:
1807 ShouldNotReachHere();
1808 }
1809
1810 } else if (right->is_constant()) {
1811 jlong c = right->as_constant_ptr()->as_jlong_bits();
1812 Register dreg = as_reg(dest);
1813 assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1814 if (c == 0 && dreg == lreg_lo) {
1815 COMMENT("effective nop elided");
1816 return;
1817 }
1818 switch (code) {
1819 case lir_add: __ add(dreg, lreg_lo, c); break;
1820 case lir_sub: __ sub(dreg, lreg_lo, c); break;
1821 default:
1822 ShouldNotReachHere();
1823 }
1824 } else {
1825 ShouldNotReachHere();
1826 }
1827 } else if (left->is_single_fpu()) {
1828 assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1829 switch (code) {
1830 case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1831 case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1832 case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1833 case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1834 default:
1835 ShouldNotReachHere();
1836 }
1837 } else if (left->is_double_fpu()) {
1838 if (right->is_double_fpu()) {
1839 // cpu register - cpu register
1840 switch (code) {
1841 case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1842 case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1843 case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1844 case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1845 default:
1846 ShouldNotReachHere();
1847 }
1848 } else {
1849 if (right->is_constant()) {
1850 ShouldNotReachHere();
1851 }
1852 ShouldNotReachHere();
1853 }
1854 } else if (left->is_single_stack() || left->is_address()) {
1855 assert(left == dest, "left and dest must be equal");
1856 ShouldNotReachHere();
1857 } else {
1858 ShouldNotReachHere();
1859 }
1860 }
1861
1862 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1863
1864
1865 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1866 switch(code) {
1867 case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1868 case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1869 default : ShouldNotReachHere();
1870 }
1871 }
1872
1873 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1874
1875 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1876 Register Rleft = left->is_single_cpu() ? left->as_register() :
1877 left->as_register_lo();
1878 if (dst->is_single_cpu()) {
1879 Register Rdst = dst->as_register();
1880 if (right->is_constant()) {
1881 switch (code) {
1882 case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1883 case lir_logic_or: __ orrw (Rdst, Rleft, right->as_jint()); break;
1884 case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1885 default: ShouldNotReachHere(); break;
1886 }
1887 } else {
1888 Register Rright = right->is_single_cpu() ? right->as_register() :
1889 right->as_register_lo();
1890 switch (code) {
1891 case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1892 case lir_logic_or: __ orrw (Rdst, Rleft, Rright); break;
1893 case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1894 default: ShouldNotReachHere(); break;
1895 }
1896 }
1897 } else {
1898 Register Rdst = dst->as_register_lo();
1899 if (right->is_constant()) {
1900 switch (code) {
1901 case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1902 case lir_logic_or: __ orr (Rdst, Rleft, right->as_jlong()); break;
1903 case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1904 default: ShouldNotReachHere(); break;
1905 }
1906 } else {
1907 Register Rright = right->is_single_cpu() ? right->as_register() :
1908 right->as_register_lo();
1909 switch (code) {
1910 case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1911 case lir_logic_or: __ orr (Rdst, Rleft, Rright); break;
1912 case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1913 default: ShouldNotReachHere(); break;
1914 }
1915 }
1916 }
1917 }
1918
1919
1920
1921 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1922
1923
1924 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1925 if (opr1->is_constant() && opr2->is_single_cpu()) {
1926 // tableswitch
1927 Register reg = as_reg(opr2);
1928 struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1929 __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1930 } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1931 Register reg1 = as_reg(opr1);
1932 if (opr2->is_single_cpu()) {
1933 // cpu register - cpu register
1934 Register reg2 = opr2->as_register();
1935 if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1936 __ cmp(reg1, reg2);
1937 if (UseShenandoahGC) {
1938 oopDesc::bs()->asm_acmp_barrier(masm(), reg1, reg2);
1939 }
1940 } else {
1941 assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1942 __ cmpw(reg1, reg2);
1943 }
1944 return;
1945 }
1946 if (opr2->is_double_cpu()) {
1947 // cpu register - cpu register
1948 Register reg2 = opr2->as_register_lo();
1949 __ cmp(reg1, reg2);
1950 return;
1951 }
1952
1953 if (opr2->is_constant()) {
1954 jlong imm;
1955 switch(opr2->type()) {
1956 case T_LONG:
1957 imm = opr2->as_constant_ptr()->as_jlong();
1958 break;
1959 case T_INT:
1960 case T_ADDRESS:
1961 imm = opr2->as_constant_ptr()->as_jint();
1962 break;
1963 case T_OBJECT:
1964 case T_ARRAY:
1965 imm = jlong(opr2->as_constant_ptr()->as_jobject());
1966 break;
1967 default:
1968 ShouldNotReachHere();
1969 imm = 0; // unreachable
1970 break;
1971 }
1972
1973 if (opr2->type() == T_OBJECT || opr2->type() == T_ARRAY) {
1974 jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
1975 __ cmp(reg1, rscratch1);
1976 if (UseShenandoahGC
1977 && (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY)) {
1978 oopDesc::bs()->asm_acmp_barrier(masm(), reg1, rscratch1);
1979 }
1980 return;
1981 }
1982
1983 if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1984 if (type2aelembytes(opr1->type()) <= 4)
1985 __ cmpw(reg1, imm);
1986 else
1987 __ cmp(reg1, imm);
1988 return;
1989 } else {
1990 __ mov(rscratch1, imm);
1991 if (type2aelembytes(opr1->type()) <= 4)
1992 __ cmpw(reg1, rscratch1);
1993 else
1994 __ cmp(reg1, rscratch1);
1995 return;
1996 }
1997 } else
1998 ShouldNotReachHere();
1999 } else if (opr1->is_single_fpu()) {
2000 FloatRegister reg1 = opr1->as_float_reg();
2001 assert(opr2->is_single_fpu(), "expect single float register");
2002 FloatRegister reg2 = opr2->as_float_reg();
2003 __ fcmps(reg1, reg2);
2004 } else if (opr1->is_double_fpu()) {
2005 FloatRegister reg1 = opr1->as_double_reg();
2006 assert(opr2->is_double_fpu(), "expect double float register");
2007 FloatRegister reg2 = opr2->as_double_reg();
2008 __ fcmpd(reg1, reg2);
2009 } else {
2010 ShouldNotReachHere();
2011 }
2012 }
2013
2014 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2015 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2016 bool is_unordered_less = (code == lir_ucmp_fd2i);
2017 if (left->is_single_fpu()) {
2018 __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2019 } else if (left->is_double_fpu()) {
2020 __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2021 } else {
2022 ShouldNotReachHere();
2023 }
2024 } else if (code == lir_cmp_l2i) {
2025 Label done;
2026 __ cmp(left->as_register_lo(), right->as_register_lo());
2027 __ mov(dst->as_register(), (u_int64_t)-1L);
2028 __ br(Assembler::LT, done);
2029 __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2030 __ bind(done);
2031 } else {
2032 ShouldNotReachHere();
2033 }
2034 }
2035
2036
2037 void LIR_Assembler::align_call(LIR_Code code) { }
2038
2039
2040 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2041 address call = __ trampoline_call(Address(op->addr(), rtype));
2042 if (call == NULL) {
2043 bailout("trampoline stub overflow");
2044 return;
2045 }
2046 add_call_info(code_offset(), op->info());
2047 }
2048
2049
2050 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2051 address call = __ ic_call(op->addr());
2052 if (call == NULL) {
2053 bailout("trampoline stub overflow");
2054 return;
2055 }
2056 add_call_info(code_offset(), op->info());
2057 }
2058
2059
2060 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2061 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2062 ShouldNotReachHere();
2063 }
2064
2065
2066 void LIR_Assembler::emit_static_call_stub() {
2067 address call_pc = __ pc();
2068 address stub = __ start_a_stub(call_stub_size);
2069 if (stub == NULL) {
2070 bailout("static call stub overflow");
2071 return;
2072 }
2073
2074 int start = __ offset();
2075
2076 __ relocate(static_stub_Relocation::spec(call_pc));
2077 __ mov_metadata(rmethod, (Metadata*)NULL);
2078 __ movptr(rscratch1, 0);
2079 __ br(rscratch1);
2080
2081 assert(__ offset() - start <= call_stub_size, "stub too big");
2082 __ end_a_stub();
2083 }
2084
2085
2086 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2087 assert(exceptionOop->as_register() == r0, "must match");
2088 assert(exceptionPC->as_register() == r3, "must match");
2089
2090 // exception object is not added to oop map by LinearScan
2091 // (LinearScan assumes that no oops are in fixed registers)
2092 info->add_register_oop(exceptionOop);
2093 Runtime1::StubID unwind_id;
2094
2095 // get current pc information
2096 // pc is only needed if the method has an exception handler, the unwind code does not need it.
2097 int pc_for_athrow_offset = __ offset();
2098 InternalAddress pc_for_athrow(__ pc());
2099 __ adr(exceptionPC->as_register(), pc_for_athrow);
2100 add_call_info(pc_for_athrow_offset, info); // for exception handler
2101
2102 __ verify_not_null_oop(r0);
2103 // search an exception handler (r0: exception oop, r3: throwing pc)
2104 if (compilation()->has_fpu_code()) {
2105 unwind_id = Runtime1::handle_exception_id;
2106 } else {
2107 unwind_id = Runtime1::handle_exception_nofpu_id;
2108 }
2109 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2110
2111 // FIXME: enough room for two byte trap ????
2112 __ nop();
2113 }
2114
2115
2116 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2117 assert(exceptionOop->as_register() == r0, "must match");
2118
2119 __ b(_unwind_handler_entry);
2120 }
2121
2122
2123 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2124 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2125 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2126
2127 switch (left->type()) {
2128 case T_INT: {
2129 switch (code) {
2130 case lir_shl: __ lslvw (dreg, lreg, count->as_register()); break;
2131 case lir_shr: __ asrvw (dreg, lreg, count->as_register()); break;
2132 case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2133 default:
2134 ShouldNotReachHere();
2135 break;
2136 }
2137 break;
2138 case T_LONG:
2139 case T_ADDRESS:
2140 case T_OBJECT:
2141 switch (code) {
2142 case lir_shl: __ lslv (dreg, lreg, count->as_register()); break;
2143 case lir_shr: __ asrv (dreg, lreg, count->as_register()); break;
2144 case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2145 default:
2146 ShouldNotReachHere();
2147 break;
2148 }
2149 break;
2150 default:
2151 ShouldNotReachHere();
2152 break;
2153 }
2154 }
2155 }
2156
2157
2158 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2159 Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2160 Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2161
2162 switch (left->type()) {
2163 case T_INT: {
2164 switch (code) {
2165 case lir_shl: __ lslw (dreg, lreg, count); break;
2166 case lir_shr: __ asrw (dreg, lreg, count); break;
2167 case lir_ushr: __ lsrw (dreg, lreg, count); break;
2168 default:
2169 ShouldNotReachHere();
2170 break;
2171 }
2172 break;
2173 case T_LONG:
2174 case T_ADDRESS:
2175 case T_OBJECT:
2176 switch (code) {
2177 case lir_shl: __ lsl (dreg, lreg, count); break;
2178 case lir_shr: __ asr (dreg, lreg, count); break;
2179 case lir_ushr: __ lsr (dreg, lreg, count); break;
2180 default:
2181 ShouldNotReachHere();
2182 break;
2183 }
2184 break;
2185 default:
2186 ShouldNotReachHere();
2187 break;
2188 }
2189 }
2190 }
2191
2192
2193 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2194 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2195 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2196 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2197 __ str (r, Address(sp, offset_from_rsp_in_bytes));
2198 }
2199
2200
2201 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) {
2202 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2203 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2204 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2205 __ mov (rscratch1, c);
2206 __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2207 }
2208
2209
2210 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2211 ShouldNotReachHere();
2212 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2213 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2214 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2215 __ lea(rscratch1, __ constant_oop_address(o));
2216 __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2217 }
2218
2219
2220 // This code replaces a call to arraycopy; no exception may
2221 // be thrown in this code, they must be thrown in the System.arraycopy
2222 // activation frame; we could save some checks if this would not be the case
2223 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2224 ciArrayKlass* default_type = op->expected_type();
2225 Register src = op->src()->as_register();
2226 Register dst = op->dst()->as_register();
2227 Register src_pos = op->src_pos()->as_register();
2228 Register dst_pos = op->dst_pos()->as_register();
2229 Register length = op->length()->as_register();
2230 Register tmp = op->tmp()->as_register();
2231
2232 CodeStub* stub = op->stub();
2233 int flags = op->flags();
2234 BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2235 if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2236
2237 // if we don't know anything, just go through the generic arraycopy
2238 if (default_type == NULL // || basic_type == T_OBJECT
2239 ) {
2240 Label done;
2241 assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2242
2243 // Save the arguments in case the generic arraycopy fails and we
2244 // have to fall back to the JNI stub
2245 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2246 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2247 __ str(src, Address(sp, 4*BytesPerWord));
2248
2249 address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2250 address copyfunc_addr = StubRoutines::generic_arraycopy();
2251
2252 // The arguments are in java calling convention so we shift them
2253 // to C convention
2254 assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2255 __ mov(c_rarg0, j_rarg0);
2256 assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2257 __ mov(c_rarg1, j_rarg1);
2258 assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2259 __ mov(c_rarg2, j_rarg2);
2260 assert_different_registers(c_rarg3, j_rarg4);
2261 __ mov(c_rarg3, j_rarg3);
2262 __ mov(c_rarg4, j_rarg4);
2263 if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2264 __ mov(rscratch1, RuntimeAddress(C_entry));
2265 __ blrt(rscratch1, 5, 0, 1);
2266 } else {
2267 #ifndef PRODUCT
2268 if (PrintC1Statistics) {
2269 __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2270 }
2271 #endif
2272 __ far_call(RuntimeAddress(copyfunc_addr));
2273 }
2274
2275 __ cbz(r0, *stub->continuation());
2276
2277 // Reload values from the stack so they are where the stub
2278 // expects them.
2279 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2280 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2281 __ ldr(src, Address(sp, 4*BytesPerWord));
2282
2283 if (copyfunc_addr != NULL) {
2284 // r0 is -1^K where K == partial copied count
2285 __ eonw(rscratch1, r0, 0);
2286 // adjust length down and src/end pos up by partial copied count
2287 __ subw(length, length, rscratch1);
2288 __ addw(src_pos, src_pos, rscratch1);
2289 __ addw(dst_pos, dst_pos, rscratch1);
2290 }
2291 __ b(*stub->entry());
2292
2293 __ bind(*stub->continuation());
2294 return;
2295 }
2296
2297 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2298
2299 int elem_size = type2aelembytes(basic_type);
2300 int shift_amount;
2301 int scale = exact_log2(elem_size);
2302
2303 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2304 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2305 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2306 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2307
2308 // test for NULL
2309 if (flags & LIR_OpArrayCopy::src_null_check) {
2310 __ cbz(src, *stub->entry());
2311 }
2312 if (flags & LIR_OpArrayCopy::dst_null_check) {
2313 __ cbz(dst, *stub->entry());
2314 }
2315
2316 // check if negative
2317 if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2318 __ cmpw(src_pos, 0);
2319 __ br(Assembler::LT, *stub->entry());
2320 }
2321 if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2322 __ cmpw(dst_pos, 0);
2323 __ br(Assembler::LT, *stub->entry());
2324 }
2325
2326 if (flags & LIR_OpArrayCopy::length_positive_check) {
2327 __ cmpw(length, 0);
2328 __ br(Assembler::LT, *stub->entry());
2329 }
2330
2331 if (flags & LIR_OpArrayCopy::src_range_check) {
2332 __ addw(tmp, src_pos, length);
2333 __ ldrw(rscratch1, src_length_addr);
2334 __ cmpw(tmp, rscratch1);
2335 __ br(Assembler::HI, *stub->entry());
2336 }
2337 if (flags & LIR_OpArrayCopy::dst_range_check) {
2338 __ addw(tmp, dst_pos, length);
2339 __ ldrw(rscratch1, dst_length_addr);
2340 __ cmpw(tmp, rscratch1);
2341 __ br(Assembler::HI, *stub->entry());
2342 }
2343
2344 // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2345 // length_positive_check if the source of our length operand is an
2346 // arraylength. However, that arraylength might be zero, and the
2347 // stub that we're about to call contains an assertion that count !=
2348 // 0 . So we make this check purely in order not to trigger an
2349 // assertion failure.
2350 __ cbzw(length, *stub->continuation());
2351
2352 if (flags & LIR_OpArrayCopy::type_check) {
2353 // We don't know the array types are compatible
2354 if (basic_type != T_OBJECT) {
2355 // Simple test for basic type arrays
2356 if (UseCompressedClassPointers) {
2357 __ ldrw(tmp, src_klass_addr);
2358 __ ldrw(rscratch1, dst_klass_addr);
2359 __ cmpw(tmp, rscratch1);
2360 } else {
2361 __ ldr(tmp, src_klass_addr);
2362 __ ldr(rscratch1, dst_klass_addr);
2363 __ cmp(tmp, rscratch1);
2364 }
2365 __ br(Assembler::NE, *stub->entry());
2366 } else {
2367 // For object arrays, if src is a sub class of dst then we can
2368 // safely do the copy.
2369 Label cont, slow;
2370
2371 #define PUSH(r1, r2) \
2372 stp(r1, r2, __ pre(sp, -2 * wordSize));
2373
2374 #define POP(r1, r2) \
2375 ldp(r1, r2, __ post(sp, 2 * wordSize));
2376
2377 __ PUSH(src, dst);
2378
2379 __ load_klass(src, src);
2380 __ load_klass(dst, dst);
2381
2382 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2383
2384 __ PUSH(src, dst);
2385 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2386 __ POP(src, dst);
2387
2388 __ cbnz(src, cont);
2389
2390 __ bind(slow);
2391 __ POP(src, dst);
2392
2393 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2394 if (copyfunc_addr != NULL) { // use stub if available
2395 // src is not a sub class of dst so we have to do a
2396 // per-element check.
2397
2398 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2399 if ((flags & mask) != mask) {
2400 // Check that at least both of them object arrays.
2401 assert(flags & mask, "one of the two should be known to be an object array");
2402
2403 if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2404 __ load_klass(tmp, src);
2405 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2406 __ load_klass(tmp, dst);
2407 }
2408 int lh_offset = in_bytes(Klass::layout_helper_offset());
2409 Address klass_lh_addr(tmp, lh_offset);
2410 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2411 __ ldrw(rscratch1, klass_lh_addr);
2412 __ mov(rscratch2, objArray_lh);
2413 __ eorw(rscratch1, rscratch1, rscratch2);
2414 __ cbnzw(rscratch1, *stub->entry());
2415 }
2416
2417 // Spill because stubs can use any register they like and it's
2418 // easier to restore just those that we care about.
2419 __ stp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2420 __ stp(length, src_pos, Address(sp, 2*BytesPerWord));
2421 __ str(src, Address(sp, 4*BytesPerWord));
2422
2423 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2424 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2425 assert_different_registers(c_rarg0, dst, dst_pos, length);
2426 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2427 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2428 assert_different_registers(c_rarg1, dst, length);
2429 __ uxtw(c_rarg2, length);
2430 assert_different_registers(c_rarg2, dst);
2431
2432 __ load_klass(c_rarg4, dst);
2433 __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2434 __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2435 __ far_call(RuntimeAddress(copyfunc_addr));
2436
2437 #ifndef PRODUCT
2438 if (PrintC1Statistics) {
2439 Label failed;
2440 __ cbnz(r0, failed);
2441 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2442 __ bind(failed);
2443 }
2444 #endif
2445
2446 __ cbz(r0, *stub->continuation());
2447
2448 #ifndef PRODUCT
2449 if (PrintC1Statistics) {
2450 __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2451 }
2452 #endif
2453 assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2454
2455 // Restore previously spilled arguments
2456 __ ldp(dst, dst_pos, Address(sp, 0*BytesPerWord));
2457 __ ldp(length, src_pos, Address(sp, 2*BytesPerWord));
2458 __ ldr(src, Address(sp, 4*BytesPerWord));
2459
2460 // return value is -1^K where K is partial copied count
2461 __ eonw(rscratch1, r0, zr);
2462 // adjust length down and src/end pos up by partial copied count
2463 __ subw(length, length, rscratch1);
2464 __ addw(src_pos, src_pos, rscratch1);
2465 __ addw(dst_pos, dst_pos, rscratch1);
2466 }
2467
2468 __ b(*stub->entry());
2469
2470 __ bind(cont);
2471 __ POP(src, dst);
2472 }
2473 }
2474
2475 #ifdef ASSERT
2476 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2477 // Sanity check the known type with the incoming class. For the
2478 // primitive case the types must match exactly with src.klass and
2479 // dst.klass each exactly matching the default type. For the
2480 // object array case, if no type check is needed then either the
2481 // dst type is exactly the expected type and the src type is a
2482 // subtype which we can't check or src is the same array as dst
2483 // but not necessarily exactly of type default_type.
2484 Label known_ok, halt;
2485 __ mov_metadata(tmp, default_type->constant_encoding());
2486 if (UseCompressedClassPointers) {
2487 __ encode_klass_not_null(tmp);
2488 }
2489
2490 if (basic_type != T_OBJECT) {
2491
2492 if (UseCompressedClassPointers) {
2493 __ ldrw(rscratch1, dst_klass_addr);
2494 __ cmpw(tmp, rscratch1);
2495 } else {
2496 __ ldr(rscratch1, dst_klass_addr);
2497 __ cmp(tmp, rscratch1);
2498 }
2499 __ br(Assembler::NE, halt);
2500 if (UseCompressedClassPointers) {
2501 __ ldrw(rscratch1, src_klass_addr);
2502 __ cmpw(tmp, rscratch1);
2503 } else {
2504 __ ldr(rscratch1, src_klass_addr);
2505 __ cmp(tmp, rscratch1);
2506 }
2507 __ br(Assembler::EQ, known_ok);
2508 } else {
2509 if (UseCompressedClassPointers) {
2510 __ ldrw(rscratch1, dst_klass_addr);
2511 __ cmpw(tmp, rscratch1);
2512 } else {
2513 __ ldr(rscratch1, dst_klass_addr);
2514 __ cmp(tmp, rscratch1);
2515 }
2516 __ br(Assembler::EQ, known_ok);
2517 __ cmp(src, dst);
2518 __ br(Assembler::EQ, known_ok);
2519 }
2520 __ bind(halt);
2521 __ stop("incorrect type information in arraycopy");
2522 __ bind(known_ok);
2523 }
2524 #endif
2525
2526 #ifndef PRODUCT
2527 if (PrintC1Statistics) {
2528 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2529 }
2530 #endif
2531
2532 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2533 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2534 assert_different_registers(c_rarg0, dst, dst_pos, length);
2535 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2536 __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2537 assert_different_registers(c_rarg1, dst, length);
2538 __ uxtw(c_rarg2, length);
2539 assert_different_registers(c_rarg2, dst);
2540
2541 bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2542 bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2543 const char *name;
2544 address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2545
2546 CodeBlob *cb = CodeCache::find_blob(entry);
2547 if (cb) {
2548 __ far_call(RuntimeAddress(entry));
2549 } else {
2550 __ call_VM_leaf(entry, 3);
2551 }
2552
2553 __ bind(*stub->continuation());
2554 }
2555
2556
2557
2558
2559 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2560 Register obj = op->obj_opr()->as_register(); // may not be an oop
2561 Register hdr = op->hdr_opr()->as_register();
2562 Register lock = op->lock_opr()->as_register();
2563 if (!UseFastLocking) {
2564 __ b(*op->stub()->entry());
2565 } else if (op->code() == lir_lock) {
2566 Register scratch = noreg;
2567 if (UseBiasedLocking) {
2568 scratch = op->scratch_opr()->as_register();
2569 }
2570 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2571 // add debug info for NullPointerException only if one is possible
2572 int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2573 if (op->info() != NULL) {
2574 add_debug_info_for_null_check(null_check_offset, op->info());
2575 }
2576 // done
2577 } else if (op->code() == lir_unlock) {
2578 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2579 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2580 } else {
2581 Unimplemented();
2582 }
2583 __ bind(*op->stub()->continuation());
2584 }
2585
2586
2587 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2588 ciMethod* method = op->profiled_method();
2589 int bci = op->profiled_bci();
2590 ciMethod* callee = op->profiled_callee();
2591
2592 // Update counter for all call types
2593 ciMethodData* md = method->method_data_or_null();
2594 assert(md != NULL, "Sanity");
2595 ciProfileData* data = md->bci_to_data(bci);
2596 assert(data->is_CounterData(), "need CounterData for calls");
2597 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2598 Register mdo = op->mdo()->as_register();
2599 __ mov_metadata(mdo, md->constant_encoding());
2600 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2601 Bytecodes::Code bc = method->java_code_at_bci(bci);
2602 const bool callee_is_static = callee->is_loaded() && callee->is_static();
2603 // Perform additional virtual call profiling for invokevirtual and
2604 // invokeinterface bytecodes
2605 if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2606 !callee_is_static && // required for optimized MH invokes
2607 C1ProfileVirtualCalls) {
2608 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2609 Register recv = op->recv()->as_register();
2610 assert_different_registers(mdo, recv);
2611 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2612 ciKlass* known_klass = op->known_holder();
2613 if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2614 // We know the type that will be seen at this call site; we can
2615 // statically update the MethodData* rather than needing to do
2616 // dynamic tests on the receiver type
2617
2618 // NOTE: we should probably put a lock around this search to
2619 // avoid collisions by concurrent compilations
2620 ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2621 uint i;
2622 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2623 ciKlass* receiver = vc_data->receiver(i);
2624 if (known_klass->equals(receiver)) {
2625 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2626 __ addptr(data_addr, DataLayout::counter_increment);
2627 return;
2628 }
2629 }
2630
2631 // Receiver type not found in profile data; select an empty slot
2632
2633 // Note that this is less efficient than it should be because it
2634 // always does a write to the receiver part of the
2635 // VirtualCallData rather than just the first time
2636 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2637 ciKlass* receiver = vc_data->receiver(i);
2638 if (receiver == NULL) {
2639 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2640 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2641 __ lea(rscratch2, recv_addr);
2642 __ str(rscratch1, Address(rscratch2));
2643 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2644 __ addptr(data_addr, DataLayout::counter_increment);
2645 return;
2646 }
2647 }
2648 } else {
2649 __ load_klass(recv, recv);
2650 Label update_done;
2651 type_profile_helper(mdo, md, data, recv, &update_done);
2652 // Receiver did not match any saved receiver and there is no empty row for it.
2653 // Increment total counter to indicate polymorphic case.
2654 __ addptr(counter_addr, DataLayout::counter_increment);
2655
2656 __ bind(update_done);
2657 }
2658 } else {
2659 // Static call
2660 __ addptr(counter_addr, DataLayout::counter_increment);
2661 }
2662 }
2663
2664
2665 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2666 Unimplemented();
2667 }
2668
2669
2670 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2671 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2672 }
2673
2674 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2675 assert(op->crc()->is_single_cpu(), "crc must be register");
2676 assert(op->val()->is_single_cpu(), "byte value must be register");
2677 assert(op->result_opr()->is_single_cpu(), "result must be register");
2678 Register crc = op->crc()->as_register();
2679 Register val = op->val()->as_register();
2680 Register res = op->result_opr()->as_register();
2681
2682 assert_different_registers(val, crc, res);
2683 unsigned long offset;
2684 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2685 if (offset) __ add(res, res, offset);
2686
2687 __ ornw(crc, zr, crc); // ~crc
2688 __ update_byte_crc32(crc, val, res);
2689 __ ornw(res, zr, crc); // ~crc
2690 }
2691
2692 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2693 COMMENT("emit_profile_type {");
2694 Register obj = op->obj()->as_register();
2695 Register tmp = op->tmp()->as_pointer_register();
2696 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2697 ciKlass* exact_klass = op->exact_klass();
2698 intptr_t current_klass = op->current_klass();
2699 bool not_null = op->not_null();
2700 bool no_conflict = op->no_conflict();
2701
2702 Label update, next, none;
2703
2704 bool do_null = !not_null;
2705 bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2706 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2707
2708 assert(do_null || do_update, "why are we here?");
2709 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2710 assert(mdo_addr.base() != rscratch1, "wrong register");
2711
2712 __ verify_oop(obj);
2713
2714 if (tmp != obj) {
2715 __ mov(tmp, obj);
2716 }
2717 if (do_null) {
2718 __ cbnz(tmp, update);
2719 if (!TypeEntries::was_null_seen(current_klass)) {
2720 __ ldr(rscratch2, mdo_addr);
2721 __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2722 __ str(rscratch2, mdo_addr);
2723 }
2724 if (do_update) {
2725 #ifndef ASSERT
2726 __ b(next);
2727 }
2728 #else
2729 __ b(next);
2730 }
2731 } else {
2732 __ cbnz(tmp, update);
2733 __ stop("unexpected null obj");
2734 #endif
2735 }
2736
2737 __ bind(update);
2738
2739 if (do_update) {
2740 #ifdef ASSERT
2741 if (exact_klass != NULL) {
2742 Label ok;
2743 __ load_klass(tmp, tmp);
2744 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2745 __ eor(rscratch1, tmp, rscratch1);
2746 __ cbz(rscratch1, ok);
2747 __ stop("exact klass and actual klass differ");
2748 __ bind(ok);
2749 }
2750 #endif
2751 if (!no_conflict) {
2752 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2753 if (exact_klass != NULL) {
2754 __ mov_metadata(tmp, exact_klass->constant_encoding());
2755 } else {
2756 __ load_klass(tmp, tmp);
2757 }
2758
2759 __ ldr(rscratch2, mdo_addr);
2760 __ eor(tmp, tmp, rscratch2);
2761 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2762 // klass seen before, nothing to do. The unknown bit may have been
2763 // set already but no need to check.
2764 __ cbz(rscratch1, next);
2765
2766 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2767 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2768
2769 if (TypeEntries::is_type_none(current_klass)) {
2770 __ cbz(rscratch2, none);
2771 __ cmp(rscratch2, TypeEntries::null_seen);
2772 __ br(Assembler::EQ, none);
2773 // There is a chance that the checks above (re-reading profiling
2774 // data from memory) fail if another thread has just set the
2775 // profiling to this obj's klass
2776 __ dmb(Assembler::ISHLD);
2777 __ ldr(rscratch2, mdo_addr);
2778 __ eor(tmp, tmp, rscratch2);
2779 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2780 __ cbz(rscratch1, next);
2781 }
2782 } else {
2783 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2784 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2785
2786 __ ldr(tmp, mdo_addr);
2787 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2788 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2789 }
2790
2791 // different than before. Cannot keep accurate profile.
2792 __ ldr(rscratch2, mdo_addr);
2793 __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2794 __ str(rscratch2, mdo_addr);
2795
2796 if (TypeEntries::is_type_none(current_klass)) {
2797 __ b(next);
2798
2799 __ bind(none);
2800 // first time here. Set profile type.
2801 __ str(tmp, mdo_addr);
2802 }
2803 } else {
2804 // There's a single possible klass at this profile point
2805 assert(exact_klass != NULL, "should be");
2806 if (TypeEntries::is_type_none(current_klass)) {
2807 __ mov_metadata(tmp, exact_klass->constant_encoding());
2808 __ ldr(rscratch2, mdo_addr);
2809 __ eor(tmp, tmp, rscratch2);
2810 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2811 __ cbz(rscratch1, next);
2812 #ifdef ASSERT
2813 {
2814 Label ok;
2815 __ ldr(rscratch1, mdo_addr);
2816 __ cbz(rscratch1, ok);
2817 __ cmp(rscratch1, TypeEntries::null_seen);
2818 __ br(Assembler::EQ, ok);
2819 // may have been set by another thread
2820 __ dmb(Assembler::ISHLD);
2821 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2822 __ ldr(rscratch2, mdo_addr);
2823 __ eor(rscratch2, rscratch1, rscratch2);
2824 __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2825 __ cbz(rscratch2, ok);
2826
2827 __ stop("unexpected profiling mismatch");
2828 __ bind(ok);
2829 }
2830 #endif
2831 // first time here. Set profile type.
2832 __ ldr(tmp, mdo_addr);
2833 } else {
2834 assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2835 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2836
2837 __ ldr(tmp, mdo_addr);
2838 __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2839 __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2840
2841 __ orr(tmp, tmp, TypeEntries::type_unknown);
2842 __ str(tmp, mdo_addr);
2843 // FIXME: Write barrier needed here?
2844 }
2845 }
2846
2847 __ bind(next);
2848 }
2849 COMMENT("} emit_profile_type");
2850 }
2851
2852
2853 void LIR_Assembler::align_backward_branch_target() {
2854 }
2855
2856
2857 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2858 if (left->is_single_cpu()) {
2859 assert(dest->is_single_cpu(), "expect single result reg");
2860 __ negw(dest->as_register(), left->as_register());
2861 } else if (left->is_double_cpu()) {
2862 assert(dest->is_double_cpu(), "expect double result reg");
2863 __ neg(dest->as_register_lo(), left->as_register_lo());
2864 } else if (left->is_single_fpu()) {
2865 assert(dest->is_single_fpu(), "expect single float result reg");
2866 __ fnegs(dest->as_float_reg(), left->as_float_reg());
2867 } else {
2868 assert(left->is_double_fpu(), "expect double float operand reg");
2869 assert(dest->is_double_fpu(), "expect double float result reg");
2870 __ fnegd(dest->as_double_reg(), left->as_double_reg());
2871 }
2872 }
2873
2874
2875 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2876 __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2877 }
2878
2879
2880 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2881 assert(!tmp->is_valid(), "don't need temporary");
2882
2883 CodeBlob *cb = CodeCache::find_blob(dest);
2884 if (cb) {
2885 __ far_call(RuntimeAddress(dest));
2886 } else {
2887 __ mov(rscratch1, RuntimeAddress(dest));
2888 int len = args->length();
2889 int type = 0;
2890 if (! result->is_illegal()) {
2891 switch (result->type()) {
2892 case T_VOID:
2893 type = 0;
2894 break;
2895 case T_INT:
2896 case T_LONG:
2897 case T_OBJECT:
2898 type = 1;
2899 break;
2900 case T_FLOAT:
2901 type = 2;
2902 break;
2903 case T_DOUBLE:
2904 type = 3;
2905 break;
2906 default:
2907 ShouldNotReachHere();
2908 break;
2909 }
2910 }
2911 int num_gpargs = 0;
2912 int num_fpargs = 0;
2913 for (int i = 0; i < args->length(); i++) {
2914 LIR_Opr arg = args->at(i);
2915 if (arg->type() == T_FLOAT || arg->type() == T_DOUBLE) {
2916 num_fpargs++;
2917 } else {
2918 num_gpargs++;
2919 }
2920 }
2921 __ blrt(rscratch1, num_gpargs, num_fpargs, type);
2922 }
2923
2924 if (info != NULL) {
2925 add_call_info_here(info);
2926 }
2927 __ maybe_isb();
2928 }
2929
2930 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2931 if (dest->is_address() || src->is_address()) {
2932 move_op(src, dest, type, lir_patch_none, info,
2933 /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2934 } else {
2935 ShouldNotReachHere();
2936 }
2937 }
2938
2939 #ifdef ASSERT
2940 // emit run-time assertion
2941 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2942 assert(op->code() == lir_assert, "must be");
2943
2944 if (op->in_opr1()->is_valid()) {
2945 assert(op->in_opr2()->is_valid(), "both operands must be valid");
2946 comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2947 } else {
2948 assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2949 assert(op->condition() == lir_cond_always, "no other conditions allowed");
2950 }
2951
2952 Label ok;
2953 if (op->condition() != lir_cond_always) {
2954 Assembler::Condition acond = Assembler::AL;
2955 switch (op->condition()) {
2956 case lir_cond_equal: acond = Assembler::EQ; break;
2957 case lir_cond_notEqual: acond = Assembler::NE; break;
2958 case lir_cond_less: acond = Assembler::LT; break;
2959 case lir_cond_lessEqual: acond = Assembler::LE; break;
2960 case lir_cond_greaterEqual: acond = Assembler::GE; break;
2961 case lir_cond_greater: acond = Assembler::GT; break;
2962 case lir_cond_belowEqual: acond = Assembler::LS; break;
2963 case lir_cond_aboveEqual: acond = Assembler::HS; break;
2964 default: ShouldNotReachHere();
2965 }
2966 __ br(acond, ok);
2967 }
2968 if (op->halt()) {
2969 const char* str = __ code_string(op->msg());
2970 __ stop(str);
2971 } else {
2972 breakpoint();
2973 }
2974 __ bind(ok);
2975 }
2976 #endif
2977
2978 #ifndef PRODUCT
2979 #define COMMENT(x) do { __ block_comment(x); } while (0)
2980 #else
2981 #define COMMENT(x)
2982 #endif
2983
2984 void LIR_Assembler::membar() {
2985 COMMENT("membar");
2986 __ membar(MacroAssembler::AnyAny);
2987 }
2988
2989 void LIR_Assembler::membar_acquire() {
2990 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2991 }
2992
2993 void LIR_Assembler::membar_release() {
2994 __ membar(Assembler::LoadStore|Assembler::StoreStore);
2995 }
2996
2997 void LIR_Assembler::membar_loadload() {
2998 __ membar(Assembler::LoadLoad);
2999 }
3000
3001 void LIR_Assembler::membar_storestore() {
3002 __ membar(MacroAssembler::StoreStore);
3003 }
3004
3005 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3006
3007 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3008
3009 void LIR_Assembler::on_spin_wait() {
3010 Unimplemented();
3011 }
3012
3013 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3014 __ mov(result_reg->as_register(), rthread);
3015 }
3016
3017
3018 void LIR_Assembler::peephole(LIR_List *lir) {
3019 #if 0
3020 if (tableswitch_count >= max_tableswitches)
3021 return;
3022
3023 /*
3024 This finite-state automaton recognizes sequences of compare-and-
3025 branch instructions. We will turn them into a tableswitch. You
3026 could argue that C1 really shouldn't be doing this sort of
3027 optimization, but without it the code is really horrible.
3028 */
3029
3030 enum { start_s, cmp1_s, beq_s, cmp_s } state;
3031 int first_key, last_key = -2147483648;
3032 int next_key = 0;
3033 int start_insn = -1;
3034 int last_insn = -1;
3035 Register reg = noreg;
3036 LIR_Opr reg_opr;
3037 state = start_s;
3038
3039 LIR_OpList* inst = lir->instructions_list();
3040 for (int i = 0; i < inst->length(); i++) {
3041 LIR_Op* op = inst->at(i);
3042 switch (state) {
3043 case start_s:
3044 first_key = -1;
3045 start_insn = i;
3046 switch (op->code()) {
3047 case lir_cmp:
3048 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3049 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3050 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3051 && opr2->is_constant()
3052 && opr2->type() == T_INT) {
3053 reg_opr = opr1;
3054 reg = opr1->as_register();
3055 first_key = opr2->as_constant_ptr()->as_jint();
3056 next_key = first_key + 1;
3057 state = cmp_s;
3058 goto next_state;
3059 }
3060 break;
3061 }
3062 break;
3063 case cmp_s:
3064 switch (op->code()) {
3065 case lir_branch:
3066 if (op->as_OpBranch()->cond() == lir_cond_equal) {
3067 state = beq_s;
3068 last_insn = i;
3069 goto next_state;
3070 }
3071 }
3072 state = start_s;
3073 break;
3074 case beq_s:
3075 switch (op->code()) {
3076 case lir_cmp: {
3077 LIR_Opr opr1 = op->as_Op2()->in_opr1();
3078 LIR_Opr opr2 = op->as_Op2()->in_opr2();
3079 if (opr1->is_cpu_register() && opr1->is_single_cpu()
3080 && opr1->as_register() == reg
3081 && opr2->is_constant()
3082 && opr2->type() == T_INT
3083 && opr2->as_constant_ptr()->as_jint() == next_key) {
3084 last_key = next_key;
3085 next_key++;
3086 state = cmp_s;
3087 goto next_state;
3088 }
3089 }
3090 }
3091 last_key = next_key;
3092 state = start_s;
3093 break;
3094 default:
3095 assert(false, "impossible state");
3096 }
3097 if (state == start_s) {
3098 if (first_key < last_key - 5L && reg != noreg) {
3099 {
3100 // printf("found run register %d starting at insn %d low value %d high value %d\n",
3101 // reg->encoding(),
3102 // start_insn, first_key, last_key);
3103 // for (int i = 0; i < inst->length(); i++) {
3104 // inst->at(i)->print();
3105 // tty->print("\n");
3106 // }
3107 // tty->print("\n");
3108 }
3109
3110 struct tableswitch *sw = &switches[tableswitch_count];
3111 sw->_insn_index = start_insn, sw->_first_key = first_key,
3112 sw->_last_key = last_key, sw->_reg = reg;
3113 inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3114 {
3115 // Insert the new table of branches
3116 int offset = last_insn;
3117 for (int n = first_key; n < last_key; n++) {
3118 inst->insert_before
3119 (last_insn + 1,
3120 new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3121 inst->at(offset)->as_OpBranch()->label()));
3122 offset -= 2, i++;
3123 }
3124 }
3125 // Delete all the old compare-and-branch instructions
3126 for (int n = first_key; n < last_key; n++) {
3127 inst->remove_at(start_insn);
3128 inst->remove_at(start_insn);
3129 }
3130 // Insert the tableswitch instruction
3131 inst->insert_before(start_insn,
3132 new LIR_Op2(lir_cmp, lir_cond_always,
3133 LIR_OprFact::intConst(tableswitch_count),
3134 reg_opr));
3135 inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3136 tableswitch_count++;
3137 }
3138 reg = noreg;
3139 last_key = -2147483648;
3140 }
3141 next_state:
3142 ;
3143 }
3144 #endif
3145 }
3146
3147 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3148 Address addr = as_Address(src->as_address_ptr(), noreg);
3149 BasicType type = src->type();
3150 bool is_oop = type == T_OBJECT || type == T_ARRAY;
3151
3152 void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3153 void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3154
3155 switch(type) {
3156 case T_INT:
3157 xchg = &MacroAssembler::atomic_xchgalw;
3158 add = &MacroAssembler::atomic_addalw;
3159 break;
3160 case T_LONG:
3161 xchg = &MacroAssembler::atomic_xchgal;
3162 add = &MacroAssembler::atomic_addal;
3163 break;
3164 case T_OBJECT:
3165 case T_ARRAY:
3166 if (UseCompressedOops) {
3167 xchg = &MacroAssembler::atomic_xchgalw;
3168 add = &MacroAssembler::atomic_addalw;
3169 } else {
3170 xchg = &MacroAssembler::atomic_xchgal;
3171 add = &MacroAssembler::atomic_addal;
3172 }
3173 break;
3174 default:
3175 ShouldNotReachHere();
3176 xchg = &MacroAssembler::atomic_xchgal;
3177 add = &MacroAssembler::atomic_addal; // unreachable
3178 }
3179
3180 switch (code) {
3181 case lir_xadd:
3182 {
3183 RegisterOrConstant inc;
3184 Register tmp = as_reg(tmp_op);
3185 Register dst = as_reg(dest);
3186 if (data->is_constant()) {
3187 inc = RegisterOrConstant(as_long(data));
3188 assert_different_registers(dst, addr.base(), tmp,
3189 rscratch1, rscratch2);
3190 } else {
3191 inc = RegisterOrConstant(as_reg(data));
3192 assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3193 rscratch1, rscratch2);
3194 }
3195 __ lea(tmp, addr);
3196 (_masm->*add)(dst, inc, tmp);
3197 break;
3198 }
3199 case lir_xchg:
3200 {
3201 Register tmp = tmp_op->as_register();
3202 Register obj = as_reg(data);
3203 Register dst = as_reg(dest);
3204 if (is_oop && UseCompressedOops) {
3205 __ encode_heap_oop(rscratch2, obj);
3206 obj = rscratch2;
3207 }
3208 assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3209 __ lea(tmp, addr);
3210 (_masm->*xchg)(dst, obj, tmp);
3211 if (is_oop && UseCompressedOops) {
3212 __ decode_heap_oop(dst);
3213 }
3214 }
3215 break;
3216 default:
3217 ShouldNotReachHere();
3218 }
3219 __ membar(__ AnyAny);
3220 }
3221
3222 #undef __