1 /*
2 * Copyright (c) 2003, 2017, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "code/debugInfoRec.hpp"
28 #include "code/icBuffer.hpp"
29 #include "code/vtableStubs.hpp"
30 #include "gc/shared/gcLocker.hpp"
31 #include "interpreter/interpreter.hpp"
32 #include "logging/log.hpp"
33 #include "memory/resourceArea.hpp"
34 #include "oops/compiledICHolder.hpp"
35 #include "runtime/safepointMechanism.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "runtime/vframeArray.hpp"
38 #include "utilities/align.hpp"
39 #include "vmreg_sparc.inline.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_Runtime1.hpp"
42 #endif
43 #ifdef COMPILER2
44 #include "opto/runtime.hpp"
45 #endif
46 #if INCLUDE_JVMCI
47 #include "jvmci/jvmciJavaClasses.hpp"
48 #endif
49
50 #define __ masm->
51
52
53 class RegisterSaver {
54
55 // Used for saving volatile registers. This is Gregs, Fregs, I/L/O.
56 // The Oregs are problematic. In the 32bit build the compiler can
57 // have O registers live with 64 bit quantities. A window save will
58 // cut the heads off of the registers. We have to do a very extensive
59 // stack dance to save and restore these properly.
60
61 // Note that the Oregs problem only exists if we block at either a polling
62 // page exception a compiled code safepoint that was not originally a call
63 // or deoptimize following one of these kinds of safepoints.
64
65 // Lots of registers to save. For all builds, a window save will preserve
66 // the %i and %l registers. For the 32-bit longs-in-two entries and 64-bit
67 // builds a window-save will preserve the %o registers. In the LION build
68 // we need to save the 64-bit %o registers which requires we save them
69 // before the window-save (as then they become %i registers and get their
70 // heads chopped off on interrupt). We have to save some %g registers here
71 // as well.
72 enum {
73 // This frame's save area. Includes extra space for the native call:
74 // vararg's layout space and the like. Briefly holds the caller's
75 // register save area.
76 call_args_area = frame::register_save_words_sp_offset +
77 frame::memory_parameter_word_sp_offset*wordSize,
78 // Make sure save locations are always 8 byte aligned.
79 // can't use align_up because it doesn't produce compile time constant
80 start_of_extra_save_area = ((call_args_area + 7) & ~7),
81 g1_offset = start_of_extra_save_area, // g-regs needing saving
82 g3_offset = g1_offset+8,
83 g4_offset = g3_offset+8,
84 g5_offset = g4_offset+8,
85 o0_offset = g5_offset+8,
86 o1_offset = o0_offset+8,
87 o2_offset = o1_offset+8,
88 o3_offset = o2_offset+8,
89 o4_offset = o3_offset+8,
90 o5_offset = o4_offset+8,
91 start_of_flags_save_area = o5_offset+8,
92 ccr_offset = start_of_flags_save_area,
93 fsr_offset = ccr_offset + 8,
94 d00_offset = fsr_offset+8, // Start of float save area
95 register_save_size = d00_offset+8*32
96 };
97
98
99 public:
100
101 static int Oexception_offset() { return o0_offset; };
102 static int G3_offset() { return g3_offset; };
103 static int G5_offset() { return g5_offset; };
104 static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words);
105 static void restore_live_registers(MacroAssembler* masm);
106
107 // During deoptimization only the result register need to be restored
108 // all the other values have already been extracted.
109
110 static void restore_result_registers(MacroAssembler* masm);
111 };
112
113 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words) {
114 // Record volatile registers as callee-save values in an OopMap so their save locations will be
115 // propagated to the caller frame's RegisterMap during StackFrameStream construction (needed for
116 // deoptimization; see compiledVFrame::create_stack_value). The caller's I, L and O registers
117 // are saved in register windows - I's and L's in the caller's frame and O's in the stub frame
118 // (as the stub's I's) when the runtime routine called by the stub creates its frame.
119 int i;
120 // Always make the frame size 16 byte aligned.
121 int frame_size = align_up(additional_frame_words + register_save_size, 16);
122 // OopMap frame size is in c2 stack slots (sizeof(jint)) not bytes or words
123 int frame_size_in_slots = frame_size / sizeof(jint);
124 // CodeBlob frame size is in words.
125 *total_frame_words = frame_size / wordSize;
126 // OopMap* map = new OopMap(*total_frame_words, 0);
127 OopMap* map = new OopMap(frame_size_in_slots, 0);
128
129 __ save(SP, -frame_size, SP);
130
131
132 int debug_offset = 0;
133 // Save the G's
134 __ stx(G1, SP, g1_offset+STACK_BIAS);
135 map->set_callee_saved(VMRegImpl::stack2reg((g1_offset + debug_offset)>>2), G1->as_VMReg());
136
137 __ stx(G3, SP, g3_offset+STACK_BIAS);
138 map->set_callee_saved(VMRegImpl::stack2reg((g3_offset + debug_offset)>>2), G3->as_VMReg());
139
140 __ stx(G4, SP, g4_offset+STACK_BIAS);
141 map->set_callee_saved(VMRegImpl::stack2reg((g4_offset + debug_offset)>>2), G4->as_VMReg());
142
143 __ stx(G5, SP, g5_offset+STACK_BIAS);
144 map->set_callee_saved(VMRegImpl::stack2reg((g5_offset + debug_offset)>>2), G5->as_VMReg());
145
146 // This is really a waste but we'll keep things as they were for now
147 if (true) {
148 }
149
150
151 // Save the flags
152 __ rdccr( G5 );
153 __ stx(G5, SP, ccr_offset+STACK_BIAS);
154 __ stxfsr(SP, fsr_offset+STACK_BIAS);
155
156 // Save all the FP registers: 32 doubles (32 floats correspond to the 2 halves of the first 16 doubles)
157 int offset = d00_offset;
158 for( int i=0; i<FloatRegisterImpl::number_of_registers; i+=2 ) {
159 FloatRegister f = as_FloatRegister(i);
160 __ stf(FloatRegisterImpl::D, f, SP, offset+STACK_BIAS);
161 // Record as callee saved both halves of double registers (2 float registers).
162 map->set_callee_saved(VMRegImpl::stack2reg(offset>>2), f->as_VMReg());
163 map->set_callee_saved(VMRegImpl::stack2reg((offset + sizeof(float))>>2), f->as_VMReg()->next());
164 offset += sizeof(double);
165 }
166
167 // And we're done.
168
169 return map;
170 }
171
172
173 // Pop the current frame and restore all the registers that we
174 // saved.
175 void RegisterSaver::restore_live_registers(MacroAssembler* masm) {
176
177 // Restore all the FP registers
178 for( int i=0; i<FloatRegisterImpl::number_of_registers; i+=2 ) {
179 __ ldf(FloatRegisterImpl::D, SP, d00_offset+i*sizeof(float)+STACK_BIAS, as_FloatRegister(i));
180 }
181
182 __ ldx(SP, ccr_offset+STACK_BIAS, G1);
183 __ wrccr (G1) ;
184
185 // Restore the G's
186 // Note that G2 (AKA GThread) must be saved and restored separately.
187 // TODO-FIXME: save and restore some of the other ASRs, viz., %asi and %gsr.
188
189 __ ldx(SP, g1_offset+STACK_BIAS, G1);
190 __ ldx(SP, g3_offset+STACK_BIAS, G3);
191 __ ldx(SP, g4_offset+STACK_BIAS, G4);
192 __ ldx(SP, g5_offset+STACK_BIAS, G5);
193
194 // Restore flags
195
196 __ ldxfsr(SP, fsr_offset+STACK_BIAS);
197
198 __ restore();
199
200 }
201
202 // Pop the current frame and restore the registers that might be holding
203 // a result.
204 void RegisterSaver::restore_result_registers(MacroAssembler* masm) {
205
206 __ ldf(FloatRegisterImpl::D, SP, d00_offset+STACK_BIAS, as_FloatRegister(0));
207
208 __ restore();
209
210 }
211
212 // Is vector's size (in bytes) bigger than a size saved by default?
213 // 8 bytes FP registers are saved by default on SPARC.
214 bool SharedRuntime::is_wide_vector(int size) {
215 // Note, MaxVectorSize == 8 on SPARC.
216 assert(size <= 8, "%d bytes vectors are not supported", size);
217 return size > 8;
218 }
219
220 size_t SharedRuntime::trampoline_size() {
221 return 40;
222 }
223
224 void SharedRuntime::generate_trampoline(MacroAssembler *masm, address destination) {
225 __ set((intptr_t)destination, G3_scratch);
226 __ JMP(G3_scratch, 0);
227 __ delayed()->nop();
228 }
229
230 // The java_calling_convention describes stack locations as ideal slots on
231 // a frame with no abi restrictions. Since we must observe abi restrictions
232 // (like the placement of the register window) the slots must be biased by
233 // the following value.
234 static int reg2offset(VMReg r) {
235 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
236 }
237
238 static VMRegPair reg64_to_VMRegPair(Register r) {
239 VMRegPair ret;
240 if (wordSize == 8) {
241 ret.set2(r->as_VMReg());
242 } else {
243 ret.set_pair(r->successor()->as_VMReg(), r->as_VMReg());
244 }
245 return ret;
246 }
247
248 // ---------------------------------------------------------------------------
249 // Read the array of BasicTypes from a signature, and compute where the
250 // arguments should go. Values in the VMRegPair regs array refer to 4-byte (VMRegImpl::stack_slot_size)
251 // quantities. Values less than VMRegImpl::stack0 are registers, those above
252 // refer to 4-byte stack slots. All stack slots are based off of the window
253 // top. VMRegImpl::stack0 refers to the first slot past the 16-word window,
254 // and VMRegImpl::stack0+1 refers to the memory word 4-byes higher. Register
255 // values 0-63 (up to RegisterImpl::number_of_registers) are the 64-bit
256 // integer registers. Values 64-95 are the (32-bit only) float registers.
257 // Each 32-bit quantity is given its own number, so the integer registers
258 // (in either 32- or 64-bit builds) use 2 numbers. For example, there is
259 // an O0-low and an O0-high. Essentially, all int register numbers are doubled.
260
261 // Register results are passed in O0-O5, for outgoing call arguments. To
262 // convert to incoming arguments, convert all O's to I's. The regs array
263 // refer to the low and hi 32-bit words of 64-bit registers or stack slots.
264 // If the regs[].second() field is set to VMRegImpl::Bad(), it means it's unused (a
265 // 32-bit value was passed). If both are VMRegImpl::Bad(), it means no value was
266 // passed (used as a placeholder for the other half of longs and doubles in
267 // the 64-bit build). regs[].second() is either VMRegImpl::Bad() or regs[].second() is
268 // regs[].first()+1 (regs[].first() may be misaligned in the C calling convention).
269 // Sparc never passes a value in regs[].second() but not regs[].first() (regs[].first()
270 // == VMRegImpl::Bad() && regs[].second() != VMRegImpl::Bad()) nor unrelated values in the
271 // same VMRegPair.
272
273 // Note: the INPUTS in sig_bt are in units of Java argument words, which are
274 // either 32-bit or 64-bit depending on the build. The OUTPUTS are in 32-bit
275 // units regardless of build.
276
277
278 // ---------------------------------------------------------------------------
279 // The compiled Java calling convention. The Java convention always passes
280 // 64-bit values in adjacent aligned locations (either registers or stack),
281 // floats in float registers and doubles in aligned float pairs. There is
282 // no backing varargs store for values in registers.
283 // In the 32-bit build, longs are passed on the stack (cannot be
284 // passed in I's, because longs in I's get their heads chopped off at
285 // interrupt).
286 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
287 VMRegPair *regs,
288 int total_args_passed,
289 int is_outgoing) {
290 assert(F31->as_VMReg()->is_reg(), "overlapping stack/register numbers");
291
292 const int int_reg_max = SPARC_ARGS_IN_REGS_NUM;
293 const int flt_reg_max = 8;
294
295 int int_reg = 0;
296 int flt_reg = 0;
297 int slot = 0;
298
299 for (int i = 0; i < total_args_passed; i++) {
300 switch (sig_bt[i]) {
301 case T_INT:
302 case T_SHORT:
303 case T_CHAR:
304 case T_BYTE:
305 case T_BOOLEAN:
306 if (int_reg < int_reg_max) {
307 Register r = is_outgoing ? as_oRegister(int_reg++) : as_iRegister(int_reg++);
308 regs[i].set1(r->as_VMReg());
309 } else {
310 regs[i].set1(VMRegImpl::stack2reg(slot++));
311 }
312 break;
313
314 case T_LONG:
315 assert((i + 1) < total_args_passed && sig_bt[i+1] == T_VOID, "expecting VOID in other half");
316 // fall-through
317 case T_OBJECT:
318 case T_ARRAY:
319 case T_ADDRESS: // Used, e.g., in slow-path locking for the lock's stack address
320 if (int_reg < int_reg_max) {
321 Register r = is_outgoing ? as_oRegister(int_reg++) : as_iRegister(int_reg++);
322 regs[i].set2(r->as_VMReg());
323 } else {
324 slot = align_up(slot, 2); // align
325 regs[i].set2(VMRegImpl::stack2reg(slot));
326 slot += 2;
327 }
328 break;
329
330 case T_FLOAT:
331 if (flt_reg < flt_reg_max) {
332 FloatRegister r = as_FloatRegister(flt_reg++);
333 regs[i].set1(r->as_VMReg());
334 } else {
335 regs[i].set1(VMRegImpl::stack2reg(slot++));
336 }
337 break;
338
339 case T_DOUBLE:
340 assert((i + 1) < total_args_passed && sig_bt[i+1] == T_VOID, "expecting half");
341 if (align_up(flt_reg, 2) + 1 < flt_reg_max) {
342 flt_reg = align_up(flt_reg, 2); // align
343 FloatRegister r = as_FloatRegister(flt_reg);
344 regs[i].set2(r->as_VMReg());
345 flt_reg += 2;
346 } else {
347 slot = align_up(slot, 2); // align
348 regs[i].set2(VMRegImpl::stack2reg(slot));
349 slot += 2;
350 }
351 break;
352
353 case T_VOID:
354 regs[i].set_bad(); // Halves of longs & doubles
355 break;
356
357 default:
358 fatal("unknown basic type %d", sig_bt[i]);
359 break;
360 }
361 }
362
363 // retun the amount of stack space these arguments will need.
364 return slot;
365 }
366
367 // Helper class mostly to avoid passing masm everywhere, and handle
368 // store displacement overflow logic.
369 class AdapterGenerator {
370 MacroAssembler *masm;
371 Register Rdisp;
372 void set_Rdisp(Register r) { Rdisp = r; }
373
374 void patch_callers_callsite();
375
376 // base+st_off points to top of argument
377 int arg_offset(const int st_off) { return st_off; }
378 int next_arg_offset(const int st_off) {
379 return st_off - Interpreter::stackElementSize;
380 }
381
382 // Argument slot values may be loaded first into a register because
383 // they might not fit into displacement.
384 RegisterOrConstant arg_slot(const int st_off);
385 RegisterOrConstant next_arg_slot(const int st_off);
386
387 // Stores long into offset pointed to by base
388 void store_c2i_long(Register r, Register base,
389 const int st_off, bool is_stack);
390 void store_c2i_object(Register r, Register base,
391 const int st_off);
392 void store_c2i_int(Register r, Register base,
393 const int st_off);
394 void store_c2i_double(VMReg r_2,
395 VMReg r_1, Register base, const int st_off);
396 void store_c2i_float(FloatRegister f, Register base,
397 const int st_off);
398
399 public:
400 void gen_c2i_adapter(int total_args_passed,
401 // VMReg max_arg,
402 int comp_args_on_stack, // VMRegStackSlots
403 const BasicType *sig_bt,
404 const VMRegPair *regs,
405 Label& skip_fixup);
406 void gen_i2c_adapter(int total_args_passed,
407 // VMReg max_arg,
408 int comp_args_on_stack, // VMRegStackSlots
409 const BasicType *sig_bt,
410 const VMRegPair *regs);
411
412 AdapterGenerator(MacroAssembler *_masm) : masm(_masm) {}
413 };
414
415
416 // Patch the callers callsite with entry to compiled code if it exists.
417 void AdapterGenerator::patch_callers_callsite() {
418 Label L;
419 __ ld_ptr(G5_method, in_bytes(Method::code_offset()), G3_scratch);
420 __ br_null(G3_scratch, false, Assembler::pt, L);
421 __ delayed()->nop();
422 // Call into the VM to patch the caller, then jump to compiled callee
423 __ save_frame(4); // Args in compiled layout; do not blow them
424
425 // Must save all the live Gregs the list is:
426 // G1: 1st Long arg (32bit build)
427 // G2: global allocated to TLS
428 // G3: used in inline cache check (scratch)
429 // G4: 2nd Long arg (32bit build);
430 // G5: used in inline cache check (Method*)
431
432 // The longs must go to the stack by hand since in the 32 bit build they can be trashed by window ops.
433
434 // mov(s,d)
435 __ mov(G1, L1);
436 __ mov(G4, L4);
437 __ mov(G5_method, L5);
438 __ mov(G5_method, O0); // VM needs target method
439 __ mov(I7, O1); // VM needs caller's callsite
440 // Must be a leaf call...
441 // can be very far once the blob has been relocated
442 AddressLiteral dest(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite));
443 __ relocate(relocInfo::runtime_call_type);
444 __ jumpl_to(dest, O7, O7);
445 __ delayed()->mov(G2_thread, L7_thread_cache);
446 __ mov(L7_thread_cache, G2_thread);
447 __ mov(L1, G1);
448 __ mov(L4, G4);
449 __ mov(L5, G5_method);
450
451 __ restore(); // Restore args
452 __ bind(L);
453 }
454
455
456 RegisterOrConstant AdapterGenerator::arg_slot(const int st_off) {
457 RegisterOrConstant roc(arg_offset(st_off));
458 return __ ensure_simm13_or_reg(roc, Rdisp);
459 }
460
461 RegisterOrConstant AdapterGenerator::next_arg_slot(const int st_off) {
462 RegisterOrConstant roc(next_arg_offset(st_off));
463 return __ ensure_simm13_or_reg(roc, Rdisp);
464 }
465
466
467 // Stores long into offset pointed to by base
468 void AdapterGenerator::store_c2i_long(Register r, Register base,
469 const int st_off, bool is_stack) {
470 // In V9, longs are given 2 64-bit slots in the interpreter, but the
471 // data is passed in only 1 slot.
472 __ stx(r, base, next_arg_slot(st_off));
473 }
474
475 void AdapterGenerator::store_c2i_object(Register r, Register base,
476 const int st_off) {
477 __ st_ptr (r, base, arg_slot(st_off));
478 }
479
480 void AdapterGenerator::store_c2i_int(Register r, Register base,
481 const int st_off) {
482 __ st (r, base, arg_slot(st_off));
483 }
484
485 // Stores into offset pointed to by base
486 void AdapterGenerator::store_c2i_double(VMReg r_2,
487 VMReg r_1, Register base, const int st_off) {
488 // In V9, doubles are given 2 64-bit slots in the interpreter, but the
489 // data is passed in only 1 slot.
490 __ stf(FloatRegisterImpl::D, r_1->as_FloatRegister(), base, next_arg_slot(st_off));
491 }
492
493 void AdapterGenerator::store_c2i_float(FloatRegister f, Register base,
494 const int st_off) {
495 __ stf(FloatRegisterImpl::S, f, base, arg_slot(st_off));
496 }
497
498 void AdapterGenerator::gen_c2i_adapter(
499 int total_args_passed,
500 // VMReg max_arg,
501 int comp_args_on_stack, // VMRegStackSlots
502 const BasicType *sig_bt,
503 const VMRegPair *regs,
504 Label& L_skip_fixup) {
505
506 // Before we get into the guts of the C2I adapter, see if we should be here
507 // at all. We've come from compiled code and are attempting to jump to the
508 // interpreter, which means the caller made a static call to get here
509 // (vcalls always get a compiled target if there is one). Check for a
510 // compiled target. If there is one, we need to patch the caller's call.
511 // However we will run interpreted if we come thru here. The next pass
512 // thru the call site will run compiled. If we ran compiled here then
513 // we can (theorectically) do endless i2c->c2i->i2c transitions during
514 // deopt/uncommon trap cycles. If we always go interpreted here then
515 // we can have at most one and don't need to play any tricks to keep
516 // from endlessly growing the stack.
517 //
518 // Actually if we detected that we had an i2c->c2i transition here we
519 // ought to be able to reset the world back to the state of the interpreted
520 // call and not bother building another interpreter arg area. We don't
521 // do that at this point.
522
523 patch_callers_callsite();
524
525 __ bind(L_skip_fixup);
526
527 // Since all args are passed on the stack, total_args_passed*wordSize is the
528 // space we need. Add in varargs area needed by the interpreter. Round up
529 // to stack alignment.
530 const int arg_size = total_args_passed * Interpreter::stackElementSize;
531 const int varargs_area =
532 (frame::varargs_offset - frame::register_save_words)*wordSize;
533 const int extraspace = align_up(arg_size + varargs_area, 2*wordSize);
534
535 const int bias = STACK_BIAS;
536 const int interp_arg_offset = frame::varargs_offset*wordSize +
537 (total_args_passed-1)*Interpreter::stackElementSize;
538
539 const Register base = SP;
540
541 // Make some extra space on the stack.
542 __ sub(SP, __ ensure_simm13_or_reg(extraspace, G3_scratch), SP);
543 set_Rdisp(G3_scratch);
544
545 // Write the args into the outgoing interpreter space.
546 for (int i = 0; i < total_args_passed; i++) {
547 const int st_off = interp_arg_offset - (i*Interpreter::stackElementSize) + bias;
548 VMReg r_1 = regs[i].first();
549 VMReg r_2 = regs[i].second();
550 if (!r_1->is_valid()) {
551 assert(!r_2->is_valid(), "");
552 continue;
553 }
554 if (r_1->is_stack()) { // Pretend stack targets are loaded into G1
555 RegisterOrConstant ld_off = reg2offset(r_1) + extraspace + bias;
556 ld_off = __ ensure_simm13_or_reg(ld_off, Rdisp);
557 r_1 = G1_scratch->as_VMReg();// as part of the load/store shuffle
558 if (!r_2->is_valid()) __ ld (base, ld_off, G1_scratch);
559 else __ ldx(base, ld_off, G1_scratch);
560 }
561
562 if (r_1->is_Register()) {
563 Register r = r_1->as_Register()->after_restore();
564 if (sig_bt[i] == T_OBJECT || sig_bt[i] == T_ARRAY) {
565 store_c2i_object(r, base, st_off);
566 } else if (sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
567 store_c2i_long(r, base, st_off, r_2->is_stack());
568 } else {
569 store_c2i_int(r, base, st_off);
570 }
571 } else {
572 assert(r_1->is_FloatRegister(), "");
573 if (sig_bt[i] == T_FLOAT) {
574 store_c2i_float(r_1->as_FloatRegister(), base, st_off);
575 } else {
576 assert(sig_bt[i] == T_DOUBLE, "wrong type");
577 store_c2i_double(r_2, r_1, base, st_off);
578 }
579 }
580 }
581
582 // Load the interpreter entry point.
583 __ ld_ptr(G5_method, in_bytes(Method::interpreter_entry_offset()), G3_scratch);
584
585 // Pass O5_savedSP as an argument to the interpreter.
586 // The interpreter will restore SP to this value before returning.
587 __ add(SP, __ ensure_simm13_or_reg(extraspace, G1), O5_savedSP);
588
589 __ mov((frame::varargs_offset)*wordSize -
590 1*Interpreter::stackElementSize+bias+BytesPerWord, G1);
591 // Jump to the interpreter just as if interpreter was doing it.
592 __ jmpl(G3_scratch, 0, G0);
593 // Setup Lesp for the call. Cannot actually set Lesp as the current Lesp
594 // (really L0) is in use by the compiled frame as a generic temp. However,
595 // the interpreter does not know where its args are without some kind of
596 // arg pointer being passed in. Pass it in Gargs.
597 __ delayed()->add(SP, G1, Gargs);
598 }
599
600 static void range_check(MacroAssembler* masm, Register pc_reg, Register temp_reg, Register temp2_reg,
601 address code_start, address code_end,
602 Label& L_ok) {
603 Label L_fail;
604 __ set(ExternalAddress(code_start), temp_reg);
605 __ set(pointer_delta(code_end, code_start, 1), temp2_reg);
606 __ cmp(pc_reg, temp_reg);
607 __ brx(Assembler::lessEqualUnsigned, false, Assembler::pn, L_fail);
608 __ delayed()->add(temp_reg, temp2_reg, temp_reg);
609 __ cmp(pc_reg, temp_reg);
610 __ cmp_and_brx_short(pc_reg, temp_reg, Assembler::lessUnsigned, Assembler::pt, L_ok);
611 __ bind(L_fail);
612 }
613
614 void AdapterGenerator::gen_i2c_adapter(int total_args_passed,
615 // VMReg max_arg,
616 int comp_args_on_stack, // VMRegStackSlots
617 const BasicType *sig_bt,
618 const VMRegPair *regs) {
619 // Generate an I2C adapter: adjust the I-frame to make space for the C-frame
620 // layout. Lesp was saved by the calling I-frame and will be restored on
621 // return. Meanwhile, outgoing arg space is all owned by the callee
622 // C-frame, so we can mangle it at will. After adjusting the frame size,
623 // hoist register arguments and repack other args according to the compiled
624 // code convention. Finally, end in a jump to the compiled code. The entry
625 // point address is the start of the buffer.
626
627 // We will only enter here from an interpreted frame and never from after
628 // passing thru a c2i. Azul allowed this but we do not. If we lose the
629 // race and use a c2i we will remain interpreted for the race loser(s).
630 // This removes all sorts of headaches on the x86 side and also eliminates
631 // the possibility of having c2i -> i2c -> c2i -> ... endless transitions.
632
633 // More detail:
634 // Adapters can be frameless because they do not require the caller
635 // to perform additional cleanup work, such as correcting the stack pointer.
636 // An i2c adapter is frameless because the *caller* frame, which is interpreted,
637 // routinely repairs its own stack pointer (from interpreter_frame_last_sp),
638 // even if a callee has modified the stack pointer.
639 // A c2i adapter is frameless because the *callee* frame, which is interpreted,
640 // routinely repairs its caller's stack pointer (from sender_sp, which is set
641 // up via the senderSP register).
642 // In other words, if *either* the caller or callee is interpreted, we can
643 // get the stack pointer repaired after a call.
644 // This is why c2i and i2c adapters cannot be indefinitely composed.
645 // In particular, if a c2i adapter were to somehow call an i2c adapter,
646 // both caller and callee would be compiled methods, and neither would
647 // clean up the stack pointer changes performed by the two adapters.
648 // If this happens, control eventually transfers back to the compiled
649 // caller, but with an uncorrected stack, causing delayed havoc.
650
651 if (VerifyAdapterCalls &&
652 (Interpreter::code() != NULL || StubRoutines::code1() != NULL)) {
653 // So, let's test for cascading c2i/i2c adapters right now.
654 // assert(Interpreter::contains($return_addr) ||
655 // StubRoutines::contains($return_addr),
656 // "i2c adapter must return to an interpreter frame");
657 __ block_comment("verify_i2c { ");
658 Label L_ok;
659 if (Interpreter::code() != NULL)
660 range_check(masm, O7, O0, O1,
661 Interpreter::code()->code_start(), Interpreter::code()->code_end(),
662 L_ok);
663 if (StubRoutines::code1() != NULL)
664 range_check(masm, O7, O0, O1,
665 StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
666 L_ok);
667 if (StubRoutines::code2() != NULL)
668 range_check(masm, O7, O0, O1,
669 StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
670 L_ok);
671 const char* msg = "i2c adapter must return to an interpreter frame";
672 __ block_comment(msg);
673 __ stop(msg);
674 __ bind(L_ok);
675 __ block_comment("} verify_i2ce ");
676 }
677
678 // As you can see from the list of inputs & outputs there are not a lot
679 // of temp registers to work with: mostly G1, G3 & G4.
680
681 // Inputs:
682 // G2_thread - TLS
683 // G5_method - Method oop
684 // G4 (Gargs) - Pointer to interpreter's args
685 // O0..O4 - free for scratch
686 // O5_savedSP - Caller's saved SP, to be restored if needed
687 // O6 - Current SP!
688 // O7 - Valid return address
689 // L0-L7, I0-I7 - Caller's temps (no frame pushed yet)
690
691 // Outputs:
692 // G2_thread - TLS
693 // O0-O5 - Outgoing args in compiled layout
694 // O6 - Adjusted or restored SP
695 // O7 - Valid return address
696 // L0-L7, I0-I7 - Caller's temps (no frame pushed yet)
697 // F0-F7 - more outgoing args
698
699
700 // Gargs is the incoming argument base, and also an outgoing argument.
701 __ sub(Gargs, BytesPerWord, Gargs);
702
703 // ON ENTRY TO THE CODE WE ARE MAKING, WE HAVE AN INTERPRETED FRAME
704 // WITH O7 HOLDING A VALID RETURN PC
705 //
706 // | |
707 // : java stack :
708 // | |
709 // +--------------+ <--- start of outgoing args
710 // | receiver | |
711 // : rest of args : |---size is java-arg-words
712 // | | |
713 // +--------------+ <--- O4_args (misaligned) and Lesp if prior is not C2I
714 // | | |
715 // : unused : |---Space for max Java stack, plus stack alignment
716 // | | |
717 // +--------------+ <--- SP + 16*wordsize
718 // | |
719 // : window :
720 // | |
721 // +--------------+ <--- SP
722
723 // WE REPACK THE STACK. We use the common calling convention layout as
724 // discovered by calling SharedRuntime::calling_convention. We assume it
725 // causes an arbitrary shuffle of memory, which may require some register
726 // temps to do the shuffle. We hope for (and optimize for) the case where
727 // temps are not needed. We may have to resize the stack slightly, in case
728 // we need alignment padding (32-bit interpreter can pass longs & doubles
729 // misaligned, but the compilers expect them aligned).
730 //
731 // | |
732 // : java stack :
733 // | |
734 // +--------------+ <--- start of outgoing args
735 // | pad, align | |
736 // +--------------+ |
737 // | ints, longs, | |
738 // | floats, | |---Outgoing stack args.
739 // : doubles : | First few args in registers.
740 // | | |
741 // +--------------+ <--- SP' + 16*wordsize
742 // | |
743 // : window :
744 // | |
745 // +--------------+ <--- SP'
746
747 // ON EXIT FROM THE CODE WE ARE MAKING, WE STILL HAVE AN INTERPRETED FRAME
748 // WITH O7 HOLDING A VALID RETURN PC - ITS JUST THAT THE ARGS ARE NOW SETUP
749 // FOR COMPILED CODE AND THE FRAME SLIGHTLY GROWN.
750
751 // Cut-out for having no stack args. Since up to 6 args are passed
752 // in registers, we will commonly have no stack args.
753 if (comp_args_on_stack > 0) {
754 // Convert VMReg stack slots to words.
755 int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
756 // Round up to miminum stack alignment, in wordSize
757 comp_words_on_stack = align_up(comp_words_on_stack, 2);
758 // Now compute the distance from Lesp to SP. This calculation does not
759 // include the space for total_args_passed because Lesp has not yet popped
760 // the arguments.
761 __ sub(SP, (comp_words_on_stack)*wordSize, SP);
762 }
763
764 // Now generate the shuffle code. Pick up all register args and move the
765 // rest through G1_scratch.
766 for (int i = 0; i < total_args_passed; i++) {
767 if (sig_bt[i] == T_VOID) {
768 // Longs and doubles are passed in native word order, but misaligned
769 // in the 32-bit build.
770 assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
771 continue;
772 }
773
774 // Pick up 0, 1 or 2 words from Lesp+offset. Assume mis-aligned in the
775 // 32-bit build and aligned in the 64-bit build. Look for the obvious
776 // ldx/lddf optimizations.
777
778 // Load in argument order going down.
779 const int ld_off = (total_args_passed-i)*Interpreter::stackElementSize;
780 set_Rdisp(G1_scratch);
781
782 VMReg r_1 = regs[i].first();
783 VMReg r_2 = regs[i].second();
784 if (!r_1->is_valid()) {
785 assert(!r_2->is_valid(), "");
786 continue;
787 }
788 if (r_1->is_stack()) { // Pretend stack targets are loaded into F8/F9
789 r_1 = F8->as_VMReg(); // as part of the load/store shuffle
790 if (r_2->is_valid()) r_2 = r_1->next();
791 }
792 if (r_1->is_Register()) { // Register argument
793 Register r = r_1->as_Register()->after_restore();
794 if (!r_2->is_valid()) {
795 __ ld(Gargs, arg_slot(ld_off), r);
796 } else {
797 // In V9, longs are given 2 64-bit slots in the interpreter, but the
798 // data is passed in only 1 slot.
799 RegisterOrConstant slot = (sig_bt[i] == T_LONG) ?
800 next_arg_slot(ld_off) : arg_slot(ld_off);
801 __ ldx(Gargs, slot, r);
802 }
803 } else {
804 assert(r_1->is_FloatRegister(), "");
805 if (!r_2->is_valid()) {
806 __ ldf(FloatRegisterImpl::S, Gargs, arg_slot(ld_off), r_1->as_FloatRegister());
807 } else {
808 // In V9, doubles are given 2 64-bit slots in the interpreter, but the
809 // data is passed in only 1 slot. This code also handles longs that
810 // are passed on the stack, but need a stack-to-stack move through a
811 // spare float register.
812 RegisterOrConstant slot = (sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) ?
813 next_arg_slot(ld_off) : arg_slot(ld_off);
814 __ ldf(FloatRegisterImpl::D, Gargs, slot, r_1->as_FloatRegister());
815 }
816 }
817 // Was the argument really intended to be on the stack, but was loaded
818 // into F8/F9?
819 if (regs[i].first()->is_stack()) {
820 assert(r_1->as_FloatRegister() == F8, "fix this code");
821 // Convert stack slot to an SP offset
822 int st_off = reg2offset(regs[i].first()) + STACK_BIAS;
823 // Store down the shuffled stack word. Target address _is_ aligned.
824 RegisterOrConstant slot = __ ensure_simm13_or_reg(st_off, Rdisp);
825 if (!r_2->is_valid()) __ stf(FloatRegisterImpl::S, r_1->as_FloatRegister(), SP, slot);
826 else __ stf(FloatRegisterImpl::D, r_1->as_FloatRegister(), SP, slot);
827 }
828 }
829
830 // Jump to the compiled code just as if compiled code was doing it.
831 __ ld_ptr(G5_method, in_bytes(Method::from_compiled_offset()), G3);
832 #if INCLUDE_JVMCI
833 if (EnableJVMCI) {
834 // check if this call should be routed towards a specific entry point
835 __ ld(Address(G2_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), G1);
836 __ cmp(G0, G1);
837 Label no_alternative_target;
838 __ br(Assembler::equal, false, Assembler::pn, no_alternative_target);
839 __ delayed()->nop();
840
841 __ ld_ptr(G2_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset()), G3);
842 __ st_ptr(G0, Address(G2_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
843
844 __ bind(no_alternative_target);
845 }
846 #endif // INCLUDE_JVMCI
847
848 // 6243940 We might end up in handle_wrong_method if
849 // the callee is deoptimized as we race thru here. If that
850 // happens we don't want to take a safepoint because the
851 // caller frame will look interpreted and arguments are now
852 // "compiled" so it is much better to make this transition
853 // invisible to the stack walking code. Unfortunately if
854 // we try and find the callee by normal means a safepoint
855 // is possible. So we stash the desired callee in the thread
856 // and the vm will find there should this case occur.
857 Address callee_target_addr(G2_thread, JavaThread::callee_target_offset());
858 __ st_ptr(G5_method, callee_target_addr);
859 __ jmpl(G3, 0, G0);
860 __ delayed()->nop();
861 }
862
863 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
864 int total_args_passed,
865 int comp_args_on_stack,
866 const BasicType *sig_bt,
867 const VMRegPair *regs) {
868 AdapterGenerator agen(masm);
869 agen.gen_i2c_adapter(total_args_passed, comp_args_on_stack, sig_bt, regs);
870 }
871
872 // ---------------------------------------------------------------
873 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
874 int total_args_passed,
875 // VMReg max_arg,
876 int comp_args_on_stack, // VMRegStackSlots
877 const BasicType *sig_bt,
878 const VMRegPair *regs,
879 AdapterFingerPrint* fingerprint) {
880 address i2c_entry = __ pc();
881
882 gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
883
884
885 // -------------------------------------------------------------------------
886 // Generate a C2I adapter. On entry we know G5 holds the Method*. The
887 // args start out packed in the compiled layout. They need to be unpacked
888 // into the interpreter layout. This will almost always require some stack
889 // space. We grow the current (compiled) stack, then repack the args. We
890 // finally end in a jump to the generic interpreter entry point. On exit
891 // from the interpreter, the interpreter will restore our SP (lest the
892 // compiled code, which relys solely on SP and not FP, get sick).
893
894 address c2i_unverified_entry = __ pc();
895 Label L_skip_fixup;
896 {
897 Register R_temp = G1; // another scratch register
898
899 AddressLiteral ic_miss(SharedRuntime::get_ic_miss_stub());
900
901 __ verify_oop(O0);
902 __ load_klass(O0, G3_scratch);
903
904 __ ld_ptr(G5_method, CompiledICHolder::holder_klass_offset(), R_temp);
905 __ cmp(G3_scratch, R_temp);
906
907 Label ok, ok2;
908 __ brx(Assembler::equal, false, Assembler::pt, ok);
909 __ delayed()->ld_ptr(G5_method, CompiledICHolder::holder_metadata_offset(), G5_method);
910 __ jump_to(ic_miss, G3_scratch);
911 __ delayed()->nop();
912
913 __ bind(ok);
914 // Method might have been compiled since the call site was patched to
915 // interpreted if that is the case treat it as a miss so we can get
916 // the call site corrected.
917 __ ld_ptr(G5_method, in_bytes(Method::code_offset()), G3_scratch);
918 __ bind(ok2);
919 __ br_null(G3_scratch, false, Assembler::pt, L_skip_fixup);
920 __ delayed()->nop();
921 __ jump_to(ic_miss, G3_scratch);
922 __ delayed()->nop();
923
924 }
925
926 address c2i_entry = __ pc();
927 AdapterGenerator agen(masm);
928 agen.gen_c2i_adapter(total_args_passed, comp_args_on_stack, sig_bt, regs, L_skip_fixup);
929
930 __ flush();
931 return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry);
932
933 }
934
935 // Helper function for native calling conventions
936 static VMReg int_stk_helper( int i ) {
937 // Bias any stack based VMReg we get by ignoring the window area
938 // but not the register parameter save area.
939 //
940 // This is strange for the following reasons. We'd normally expect
941 // the calling convention to return an VMReg for a stack slot
942 // completely ignoring any abi reserved area. C2 thinks of that
943 // abi area as only out_preserve_stack_slots. This does not include
944 // the area allocated by the C abi to store down integer arguments
945 // because the java calling convention does not use it. So
946 // since c2 assumes that there are only out_preserve_stack_slots
947 // to bias the optoregs (which impacts VMRegs) when actually referencing any actual stack
948 // location the c calling convention must add in this bias amount
949 // to make up for the fact that the out_preserve_stack_slots is
950 // insufficient for C calls. What a mess. I sure hope those 6
951 // stack words were worth it on every java call!
952
953 // Another way of cleaning this up would be for out_preserve_stack_slots
954 // to take a parameter to say whether it was C or java calling conventions.
955 // Then things might look a little better (but not much).
956
957 int mem_parm_offset = i - SPARC_ARGS_IN_REGS_NUM;
958 if( mem_parm_offset < 0 ) {
959 return as_oRegister(i)->as_VMReg();
960 } else {
961 int actual_offset = (mem_parm_offset + frame::memory_parameter_word_sp_offset) * VMRegImpl::slots_per_word;
962 // Now return a biased offset that will be correct when out_preserve_slots is added back in
963 return VMRegImpl::stack2reg(actual_offset - SharedRuntime::out_preserve_stack_slots());
964 }
965 }
966
967
968 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
969 VMRegPair *regs,
970 VMRegPair *regs2,
971 int total_args_passed) {
972 assert(regs2 == NULL, "not needed on sparc");
973
974 // Return the number of VMReg stack_slots needed for the args.
975 // This value does not include an abi space (like register window
976 // save area).
977
978 // The native convention is V8 if !LP64
979 // The LP64 convention is the V9 convention which is slightly more sane.
980
981 // We return the amount of VMReg stack slots we need to reserve for all
982 // the arguments NOT counting out_preserve_stack_slots. Since we always
983 // have space for storing at least 6 registers to memory we start with that.
984 // See int_stk_helper for a further discussion.
985 int max_stack_slots = (frame::varargs_offset * VMRegImpl::slots_per_word) - SharedRuntime::out_preserve_stack_slots();
986
987 // V9 convention: All things "as-if" on double-wide stack slots.
988 // Hoist any int/ptr/long's in the first 6 to int regs.
989 // Hoist any flt/dbl's in the first 16 dbl regs.
990 int j = 0; // Count of actual args, not HALVES
991 VMRegPair param_array_reg; // location of the argument in the parameter array
992 for (int i = 0; i < total_args_passed; i++, j++) {
993 param_array_reg.set_bad();
994 switch (sig_bt[i]) {
995 case T_BOOLEAN:
996 case T_BYTE:
997 case T_CHAR:
998 case T_INT:
999 case T_SHORT:
1000 regs[i].set1(int_stk_helper(j));
1001 break;
1002 case T_LONG:
1003 assert((i + 1) < total_args_passed && sig_bt[i+1] == T_VOID, "expecting half");
1004 case T_ADDRESS: // raw pointers, like current thread, for VM calls
1005 case T_ARRAY:
1006 case T_OBJECT:
1007 case T_METADATA:
1008 regs[i].set2(int_stk_helper(j));
1009 break;
1010 case T_FLOAT:
1011 // Per SPARC Compliance Definition 2.4.1, page 3P-12 available here
1012 // http://www.sparc.org/wp-content/uploads/2014/01/SCD.2.4.1.pdf.gz
1013 //
1014 // "When a callee prototype exists, and does not indicate variable arguments,
1015 // floating-point values assigned to locations %sp+BIAS+128 through %sp+BIAS+248
1016 // will be promoted to floating-point registers"
1017 //
1018 // By "promoted" it means that the argument is located in two places, an unused
1019 // spill slot in the "parameter array" (starts at %sp+BIAS+128), and a live
1020 // float register. In most cases, there are 6 or fewer arguments of any type,
1021 // and the standard parameter array slots (%sp+BIAS+128 to %sp+BIAS+176 exclusive)
1022 // serve as shadow slots. Per the spec floating point registers %d6 to %d16
1023 // require slots beyond that (up to %sp+BIAS+248).
1024 //
1025 {
1026 // V9ism: floats go in ODD registers and stack slots
1027 int float_index = 1 + (j << 1);
1028 param_array_reg.set1(VMRegImpl::stack2reg(float_index));
1029 if (j < 16) {
1030 regs[i].set1(as_FloatRegister(float_index)->as_VMReg());
1031 } else {
1032 regs[i] = param_array_reg;
1033 }
1034 }
1035 break;
1036 case T_DOUBLE:
1037 {
1038 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
1039 // V9ism: doubles go in EVEN/ODD regs and stack slots
1040 int double_index = (j << 1);
1041 param_array_reg.set2(VMRegImpl::stack2reg(double_index));
1042 if (j < 16) {
1043 regs[i].set2(as_FloatRegister(double_index)->as_VMReg());
1044 } else {
1045 // V9ism: doubles go in EVEN/ODD stack slots
1046 regs[i] = param_array_reg;
1047 }
1048 }
1049 break;
1050 case T_VOID:
1051 regs[i].set_bad();
1052 j--;
1053 break; // Do not count HALVES
1054 default:
1055 ShouldNotReachHere();
1056 }
1057 // Keep track of the deepest parameter array slot.
1058 if (!param_array_reg.first()->is_valid()) {
1059 param_array_reg = regs[i];
1060 }
1061 if (param_array_reg.first()->is_stack()) {
1062 int off = param_array_reg.first()->reg2stack();
1063 if (off > max_stack_slots) max_stack_slots = off;
1064 }
1065 if (param_array_reg.second()->is_stack()) {
1066 int off = param_array_reg.second()->reg2stack();
1067 if (off > max_stack_slots) max_stack_slots = off;
1068 }
1069 }
1070 return align_up(max_stack_slots + 1, 2);
1071
1072 }
1073
1074
1075 // ---------------------------------------------------------------------------
1076 void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1077 switch (ret_type) {
1078 case T_FLOAT:
1079 __ stf(FloatRegisterImpl::S, F0, SP, frame_slots*VMRegImpl::stack_slot_size - 4+STACK_BIAS);
1080 break;
1081 case T_DOUBLE:
1082 __ stf(FloatRegisterImpl::D, F0, SP, frame_slots*VMRegImpl::stack_slot_size - 8+STACK_BIAS);
1083 break;
1084 }
1085 }
1086
1087 void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1088 switch (ret_type) {
1089 case T_FLOAT:
1090 __ ldf(FloatRegisterImpl::S, SP, frame_slots*VMRegImpl::stack_slot_size - 4+STACK_BIAS, F0);
1091 break;
1092 case T_DOUBLE:
1093 __ ldf(FloatRegisterImpl::D, SP, frame_slots*VMRegImpl::stack_slot_size - 8+STACK_BIAS, F0);
1094 break;
1095 }
1096 }
1097
1098 // Check and forward and pending exception. Thread is stored in
1099 // L7_thread_cache and possibly NOT in G2_thread. Since this is a native call, there
1100 // is no exception handler. We merely pop this frame off and throw the
1101 // exception in the caller's frame.
1102 static void check_forward_pending_exception(MacroAssembler *masm, Register Rex_oop) {
1103 Label L;
1104 __ br_null(Rex_oop, false, Assembler::pt, L);
1105 __ delayed()->mov(L7_thread_cache, G2_thread); // restore in case we have exception
1106 // Since this is a native call, we *know* the proper exception handler
1107 // without calling into the VM: it's the empty function. Just pop this
1108 // frame and then jump to forward_exception_entry; O7 will contain the
1109 // native caller's return PC.
1110 AddressLiteral exception_entry(StubRoutines::forward_exception_entry());
1111 __ jump_to(exception_entry, G3_scratch);
1112 __ delayed()->restore(); // Pop this frame off.
1113 __ bind(L);
1114 }
1115
1116 // A simple move of integer like type
1117 static void simple_move32(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1118 if (src.first()->is_stack()) {
1119 if (dst.first()->is_stack()) {
1120 // stack to stack
1121 __ ld(FP, reg2offset(src.first()) + STACK_BIAS, L5);
1122 __ st(L5, SP, reg2offset(dst.first()) + STACK_BIAS);
1123 } else {
1124 // stack to reg
1125 __ ld(FP, reg2offset(src.first()) + STACK_BIAS, dst.first()->as_Register());
1126 }
1127 } else if (dst.first()->is_stack()) {
1128 // reg to stack
1129 __ st(src.first()->as_Register(), SP, reg2offset(dst.first()) + STACK_BIAS);
1130 } else {
1131 __ mov(src.first()->as_Register(), dst.first()->as_Register());
1132 }
1133 }
1134
1135 // On 64 bit we will store integer like items to the stack as
1136 // 64 bits items (sparc abi) even though java would only store
1137 // 32bits for a parameter. On 32bit it will simply be 32 bits
1138 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
1139 static void move32_64(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1140 if (src.first()->is_stack()) {
1141 if (dst.first()->is_stack()) {
1142 // stack to stack
1143 __ ld(FP, reg2offset(src.first()) + STACK_BIAS, L5);
1144 __ st_ptr(L5, SP, reg2offset(dst.first()) + STACK_BIAS);
1145 } else {
1146 // stack to reg
1147 __ ld(FP, reg2offset(src.first()) + STACK_BIAS, dst.first()->as_Register());
1148 }
1149 } else if (dst.first()->is_stack()) {
1150 // reg to stack
1151 // Some compilers (gcc) expect a clean 32 bit value on function entry
1152 __ signx(src.first()->as_Register(), L5);
1153 __ st_ptr(L5, SP, reg2offset(dst.first()) + STACK_BIAS);
1154 } else {
1155 // Some compilers (gcc) expect a clean 32 bit value on function entry
1156 __ signx(src.first()->as_Register(), dst.first()->as_Register());
1157 }
1158 }
1159
1160
1161 static void move_ptr(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1162 if (src.first()->is_stack()) {
1163 if (dst.first()->is_stack()) {
1164 // stack to stack
1165 __ ld_ptr(FP, reg2offset(src.first()) + STACK_BIAS, L5);
1166 __ st_ptr(L5, SP, reg2offset(dst.first()) + STACK_BIAS);
1167 } else {
1168 // stack to reg
1169 __ ld_ptr(FP, reg2offset(src.first()) + STACK_BIAS, dst.first()->as_Register());
1170 }
1171 } else if (dst.first()->is_stack()) {
1172 // reg to stack
1173 __ st_ptr(src.first()->as_Register(), SP, reg2offset(dst.first()) + STACK_BIAS);
1174 } else {
1175 __ mov(src.first()->as_Register(), dst.first()->as_Register());
1176 }
1177 }
1178
1179
1180 // An oop arg. Must pass a handle not the oop itself
1181 static void object_move(MacroAssembler* masm,
1182 OopMap* map,
1183 int oop_handle_offset,
1184 int framesize_in_slots,
1185 VMRegPair src,
1186 VMRegPair dst,
1187 bool is_receiver,
1188 int* receiver_offset) {
1189
1190 // must pass a handle. First figure out the location we use as a handle
1191
1192 if (src.first()->is_stack()) {
1193 // Oop is already on the stack
1194 Register rHandle = dst.first()->is_stack() ? L5 : dst.first()->as_Register();
1195 __ add(FP, reg2offset(src.first()) + STACK_BIAS, rHandle);
1196 __ ld_ptr(rHandle, 0, L4);
1197 __ movr( Assembler::rc_z, L4, G0, rHandle );
1198 if (dst.first()->is_stack()) {
1199 __ st_ptr(rHandle, SP, reg2offset(dst.first()) + STACK_BIAS);
1200 }
1201 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1202 if (is_receiver) {
1203 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
1204 }
1205 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
1206 } else {
1207 // Oop is in an input register pass we must flush it to the stack
1208 const Register rOop = src.first()->as_Register();
1209 const Register rHandle = L5;
1210 int oop_slot = rOop->input_number() * VMRegImpl::slots_per_word + oop_handle_offset;
1211 int offset = oop_slot * VMRegImpl::stack_slot_size;
1212 __ st_ptr(rOop, SP, offset + STACK_BIAS);
1213 if (is_receiver) {
1214 *receiver_offset = offset;
1215 }
1216 map->set_oop(VMRegImpl::stack2reg(oop_slot));
1217 __ add(SP, offset + STACK_BIAS, rHandle);
1218 __ movr( Assembler::rc_z, rOop, G0, rHandle );
1219
1220 if (dst.first()->is_stack()) {
1221 __ st_ptr(rHandle, SP, reg2offset(dst.first()) + STACK_BIAS);
1222 } else {
1223 __ mov(rHandle, dst.first()->as_Register());
1224 }
1225 }
1226 }
1227
1228 // A float arg may have to do float reg int reg conversion
1229 static void float_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1230 assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
1231
1232 if (src.first()->is_stack()) {
1233 if (dst.first()->is_stack()) {
1234 // stack to stack the easiest of the bunch
1235 __ ld(FP, reg2offset(src.first()) + STACK_BIAS, L5);
1236 __ st(L5, SP, reg2offset(dst.first()) + STACK_BIAS);
1237 } else {
1238 // stack to reg
1239 if (dst.first()->is_Register()) {
1240 __ ld(FP, reg2offset(src.first()) + STACK_BIAS, dst.first()->as_Register());
1241 } else {
1242 __ ldf(FloatRegisterImpl::S, FP, reg2offset(src.first()) + STACK_BIAS, dst.first()->as_FloatRegister());
1243 }
1244 }
1245 } else if (dst.first()->is_stack()) {
1246 // reg to stack
1247 if (src.first()->is_Register()) {
1248 __ st(src.first()->as_Register(), SP, reg2offset(dst.first()) + STACK_BIAS);
1249 } else {
1250 __ stf(FloatRegisterImpl::S, src.first()->as_FloatRegister(), SP, reg2offset(dst.first()) + STACK_BIAS);
1251 }
1252 } else {
1253 // reg to reg
1254 if (src.first()->is_Register()) {
1255 if (dst.first()->is_Register()) {
1256 // gpr -> gpr
1257 __ mov(src.first()->as_Register(), dst.first()->as_Register());
1258 } else {
1259 // gpr -> fpr
1260 __ st(src.first()->as_Register(), FP, -4 + STACK_BIAS);
1261 __ ldf(FloatRegisterImpl::S, FP, -4 + STACK_BIAS, dst.first()->as_FloatRegister());
1262 }
1263 } else if (dst.first()->is_Register()) {
1264 // fpr -> gpr
1265 __ stf(FloatRegisterImpl::S, src.first()->as_FloatRegister(), FP, -4 + STACK_BIAS);
1266 __ ld(FP, -4 + STACK_BIAS, dst.first()->as_Register());
1267 } else {
1268 // fpr -> fpr
1269 // In theory these overlap but the ordering is such that this is likely a nop
1270 if ( src.first() != dst.first()) {
1271 __ fmov(FloatRegisterImpl::S, src.first()->as_FloatRegister(), dst.first()->as_FloatRegister());
1272 }
1273 }
1274 }
1275 }
1276
1277 static void split_long_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1278 VMRegPair src_lo(src.first());
1279 VMRegPair src_hi(src.second());
1280 VMRegPair dst_lo(dst.first());
1281 VMRegPair dst_hi(dst.second());
1282 simple_move32(masm, src_lo, dst_lo);
1283 simple_move32(masm, src_hi, dst_hi);
1284 }
1285
1286 // A long move
1287 static void long_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1288
1289 // Do the simple ones here else do two int moves
1290 if (src.is_single_phys_reg() ) {
1291 if (dst.is_single_phys_reg()) {
1292 __ mov(src.first()->as_Register(), dst.first()->as_Register());
1293 } else {
1294 // split src into two separate registers
1295 // Remember hi means hi address or lsw on sparc
1296 // Move msw to lsw
1297 if (dst.second()->is_reg()) {
1298 // MSW -> MSW
1299 __ srax(src.first()->as_Register(), 32, dst.first()->as_Register());
1300 // Now LSW -> LSW
1301 // this will only move lo -> lo and ignore hi
1302 VMRegPair split(dst.second());
1303 simple_move32(masm, src, split);
1304 } else {
1305 VMRegPair split(src.first(), L4->as_VMReg());
1306 // MSW -> MSW (lo ie. first word)
1307 __ srax(src.first()->as_Register(), 32, L4);
1308 split_long_move(masm, split, dst);
1309 }
1310 }
1311 } else if (dst.is_single_phys_reg()) {
1312 if (src.is_adjacent_aligned_on_stack(2)) {
1313 __ ldx(FP, reg2offset(src.first()) + STACK_BIAS, dst.first()->as_Register());
1314 } else {
1315 // dst is a single reg.
1316 // Remember lo is low address not msb for stack slots
1317 // and lo is the "real" register for registers
1318 // src is
1319
1320 VMRegPair split;
1321
1322 if (src.first()->is_reg()) {
1323 // src.lo (msw) is a reg, src.hi is stk/reg
1324 // we will move: src.hi (LSW) -> dst.lo, src.lo (MSW) -> src.lo [the MSW is in the LSW of the reg]
1325 split.set_pair(dst.first(), src.first());
1326 } else {
1327 // msw is stack move to L5
1328 // lsw is stack move to dst.lo (real reg)
1329 // we will move: src.hi (LSW) -> dst.lo, src.lo (MSW) -> L5
1330 split.set_pair(dst.first(), L5->as_VMReg());
1331 }
1332
1333 // src.lo -> src.lo/L5, src.hi -> dst.lo (the real reg)
1334 // msw -> src.lo/L5, lsw -> dst.lo
1335 split_long_move(masm, src, split);
1336
1337 // So dst now has the low order correct position the
1338 // msw half
1339 __ sllx(split.first()->as_Register(), 32, L5);
1340
1341 const Register d = dst.first()->as_Register();
1342 __ or3(L5, d, d);
1343 }
1344 } else {
1345 // For LP64 we can probably do better.
1346 split_long_move(masm, src, dst);
1347 }
1348 }
1349
1350 // A double move
1351 static void double_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1352
1353 // The painful thing here is that like long_move a VMRegPair might be
1354 // 1: a single physical register
1355 // 2: two physical registers (v8)
1356 // 3: a physical reg [lo] and a stack slot [hi] (v8)
1357 // 4: two stack slots
1358
1359 // Since src is always a java calling convention we know that the src pair
1360 // is always either all registers or all stack (and aligned?)
1361
1362 // in a register [lo] and a stack slot [hi]
1363 if (src.first()->is_stack()) {
1364 if (dst.first()->is_stack()) {
1365 // stack to stack the easiest of the bunch
1366 // ought to be a way to do this where if alignment is ok we use ldd/std when possible
1367 __ ld(FP, reg2offset(src.first()) + STACK_BIAS, L5);
1368 __ ld(FP, reg2offset(src.second()) + STACK_BIAS, L4);
1369 __ st(L5, SP, reg2offset(dst.first()) + STACK_BIAS);
1370 __ st(L4, SP, reg2offset(dst.second()) + STACK_BIAS);
1371 } else {
1372 // stack to reg
1373 if (dst.second()->is_stack()) {
1374 // stack -> reg, stack -> stack
1375 __ ld(FP, reg2offset(src.second()) + STACK_BIAS, L4);
1376 if (dst.first()->is_Register()) {
1377 __ ld(FP, reg2offset(src.first()) + STACK_BIAS, dst.first()->as_Register());
1378 } else {
1379 __ ldf(FloatRegisterImpl::S, FP, reg2offset(src.first()) + STACK_BIAS, dst.first()->as_FloatRegister());
1380 }
1381 // This was missing. (very rare case)
1382 __ st(L4, SP, reg2offset(dst.second()) + STACK_BIAS);
1383 } else {
1384 // stack -> reg
1385 // Eventually optimize for alignment QQQ
1386 if (dst.first()->is_Register()) {
1387 __ ld(FP, reg2offset(src.first()) + STACK_BIAS, dst.first()->as_Register());
1388 __ ld(FP, reg2offset(src.second()) + STACK_BIAS, dst.second()->as_Register());
1389 } else {
1390 __ ldf(FloatRegisterImpl::S, FP, reg2offset(src.first()) + STACK_BIAS, dst.first()->as_FloatRegister());
1391 __ ldf(FloatRegisterImpl::S, FP, reg2offset(src.second()) + STACK_BIAS, dst.second()->as_FloatRegister());
1392 }
1393 }
1394 }
1395 } else if (dst.first()->is_stack()) {
1396 // reg to stack
1397 if (src.first()->is_Register()) {
1398 // Eventually optimize for alignment QQQ
1399 __ st(src.first()->as_Register(), SP, reg2offset(dst.first()) + STACK_BIAS);
1400 if (src.second()->is_stack()) {
1401 __ ld(FP, reg2offset(src.second()) + STACK_BIAS, L4);
1402 __ st(L4, SP, reg2offset(dst.second()) + STACK_BIAS);
1403 } else {
1404 __ st(src.second()->as_Register(), SP, reg2offset(dst.second()) + STACK_BIAS);
1405 }
1406 } else {
1407 // fpr to stack
1408 if (src.second()->is_stack()) {
1409 ShouldNotReachHere();
1410 } else {
1411 // Is the stack aligned?
1412 if (reg2offset(dst.first()) & 0x7) {
1413 // No do as pairs
1414 __ stf(FloatRegisterImpl::S, src.first()->as_FloatRegister(), SP, reg2offset(dst.first()) + STACK_BIAS);
1415 __ stf(FloatRegisterImpl::S, src.second()->as_FloatRegister(), SP, reg2offset(dst.second()) + STACK_BIAS);
1416 } else {
1417 __ stf(FloatRegisterImpl::D, src.first()->as_FloatRegister(), SP, reg2offset(dst.first()) + STACK_BIAS);
1418 }
1419 }
1420 }
1421 } else {
1422 // reg to reg
1423 if (src.first()->is_Register()) {
1424 if (dst.first()->is_Register()) {
1425 // gpr -> gpr
1426 __ mov(src.first()->as_Register(), dst.first()->as_Register());
1427 __ mov(src.second()->as_Register(), dst.second()->as_Register());
1428 } else {
1429 // gpr -> fpr
1430 // ought to be able to do a single store
1431 __ stx(src.first()->as_Register(), FP, -8 + STACK_BIAS);
1432 __ stx(src.second()->as_Register(), FP, -4 + STACK_BIAS);
1433 // ought to be able to do a single load
1434 __ ldf(FloatRegisterImpl::S, FP, -8 + STACK_BIAS, dst.first()->as_FloatRegister());
1435 __ ldf(FloatRegisterImpl::S, FP, -4 + STACK_BIAS, dst.second()->as_FloatRegister());
1436 }
1437 } else if (dst.first()->is_Register()) {
1438 // fpr -> gpr
1439 // ought to be able to do a single store
1440 __ stf(FloatRegisterImpl::D, src.first()->as_FloatRegister(), FP, -8 + STACK_BIAS);
1441 // ought to be able to do a single load
1442 // REMEMBER first() is low address not LSB
1443 __ ld(FP, -8 + STACK_BIAS, dst.first()->as_Register());
1444 if (dst.second()->is_Register()) {
1445 __ ld(FP, -4 + STACK_BIAS, dst.second()->as_Register());
1446 } else {
1447 __ ld(FP, -4 + STACK_BIAS, L4);
1448 __ st(L4, SP, reg2offset(dst.second()) + STACK_BIAS);
1449 }
1450 } else {
1451 // fpr -> fpr
1452 // In theory these overlap but the ordering is such that this is likely a nop
1453 if ( src.first() != dst.first()) {
1454 __ fmov(FloatRegisterImpl::D, src.first()->as_FloatRegister(), dst.first()->as_FloatRegister());
1455 }
1456 }
1457 }
1458 }
1459
1460 // Creates an inner frame if one hasn't already been created, and
1461 // saves a copy of the thread in L7_thread_cache
1462 static void create_inner_frame(MacroAssembler* masm, bool* already_created) {
1463 if (!*already_created) {
1464 __ save_frame(0);
1465 // Save thread in L7 (INNER FRAME); it crosses a bunch of VM calls below
1466 // Don't use save_thread because it smashes G2 and we merely want to save a
1467 // copy
1468 __ mov(G2_thread, L7_thread_cache);
1469 *already_created = true;
1470 }
1471 }
1472
1473
1474 static void save_or_restore_arguments(MacroAssembler* masm,
1475 const int stack_slots,
1476 const int total_in_args,
1477 const int arg_save_area,
1478 OopMap* map,
1479 VMRegPair* in_regs,
1480 BasicType* in_sig_bt) {
1481 // if map is non-NULL then the code should store the values,
1482 // otherwise it should load them.
1483 if (map != NULL) {
1484 // Fill in the map
1485 for (int i = 0; i < total_in_args; i++) {
1486 if (in_sig_bt[i] == T_ARRAY) {
1487 if (in_regs[i].first()->is_stack()) {
1488 int offset_in_older_frame = in_regs[i].first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1489 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + stack_slots));
1490 } else if (in_regs[i].first()->is_Register()) {
1491 map->set_oop(in_regs[i].first());
1492 } else {
1493 ShouldNotReachHere();
1494 }
1495 }
1496 }
1497 }
1498
1499 // Save or restore double word values
1500 int handle_index = 0;
1501 for (int i = 0; i < total_in_args; i++) {
1502 int slot = handle_index + arg_save_area;
1503 int offset = slot * VMRegImpl::stack_slot_size;
1504 if (in_sig_bt[i] == T_LONG && in_regs[i].first()->is_Register()) {
1505 const Register reg = in_regs[i].first()->as_Register();
1506 if (reg->is_global()) {
1507 handle_index += 2;
1508 assert(handle_index <= stack_slots, "overflow");
1509 if (map != NULL) {
1510 __ stx(reg, SP, offset + STACK_BIAS);
1511 } else {
1512 __ ldx(SP, offset + STACK_BIAS, reg);
1513 }
1514 }
1515 } else if (in_sig_bt[i] == T_DOUBLE && in_regs[i].first()->is_FloatRegister()) {
1516 handle_index += 2;
1517 assert(handle_index <= stack_slots, "overflow");
1518 if (map != NULL) {
1519 __ stf(FloatRegisterImpl::D, in_regs[i].first()->as_FloatRegister(), SP, offset + STACK_BIAS);
1520 } else {
1521 __ ldf(FloatRegisterImpl::D, SP, offset + STACK_BIAS, in_regs[i].first()->as_FloatRegister());
1522 }
1523 }
1524 }
1525 // Save floats
1526 for (int i = 0; i < total_in_args; i++) {
1527 int slot = handle_index + arg_save_area;
1528 int offset = slot * VMRegImpl::stack_slot_size;
1529 if (in_sig_bt[i] == T_FLOAT && in_regs[i].first()->is_FloatRegister()) {
1530 handle_index++;
1531 assert(handle_index <= stack_slots, "overflow");
1532 if (map != NULL) {
1533 __ stf(FloatRegisterImpl::S, in_regs[i].first()->as_FloatRegister(), SP, offset + STACK_BIAS);
1534 } else {
1535 __ ldf(FloatRegisterImpl::S, SP, offset + STACK_BIAS, in_regs[i].first()->as_FloatRegister());
1536 }
1537 }
1538 }
1539
1540 }
1541
1542
1543 // Check GCLocker::needs_gc and enter the runtime if it's true. This
1544 // keeps a new JNI critical region from starting until a GC has been
1545 // forced. Save down any oops in registers and describe them in an
1546 // OopMap.
1547 static void check_needs_gc_for_critical_native(MacroAssembler* masm,
1548 const int stack_slots,
1549 const int total_in_args,
1550 const int arg_save_area,
1551 OopMapSet* oop_maps,
1552 VMRegPair* in_regs,
1553 BasicType* in_sig_bt) {
1554 __ block_comment("check GCLocker::needs_gc");
1555 Label cont;
1556 AddressLiteral sync_state(GCLocker::needs_gc_address());
1557 __ load_bool_contents(sync_state, G3_scratch);
1558 __ cmp_zero_and_br(Assembler::equal, G3_scratch, cont);
1559 __ delayed()->nop();
1560
1561 // Save down any values that are live in registers and call into the
1562 // runtime to halt for a GC
1563 OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1564 save_or_restore_arguments(masm, stack_slots, total_in_args,
1565 arg_save_area, map, in_regs, in_sig_bt);
1566
1567 __ mov(G2_thread, L7_thread_cache);
1568
1569 __ set_last_Java_frame(SP, noreg);
1570
1571 __ block_comment("block_for_jni_critical");
1572 __ call(CAST_FROM_FN_PTR(address, SharedRuntime::block_for_jni_critical), relocInfo::runtime_call_type);
1573 __ delayed()->mov(L7_thread_cache, O0);
1574 oop_maps->add_gc_map( __ offset(), map);
1575
1576 __ restore_thread(L7_thread_cache); // restore G2_thread
1577 __ reset_last_Java_frame();
1578
1579 // Reload all the register arguments
1580 save_or_restore_arguments(masm, stack_slots, total_in_args,
1581 arg_save_area, NULL, in_regs, in_sig_bt);
1582
1583 __ bind(cont);
1584 #ifdef ASSERT
1585 if (StressCriticalJNINatives) {
1586 // Stress register saving
1587 OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1588 save_or_restore_arguments(masm, stack_slots, total_in_args,
1589 arg_save_area, map, in_regs, in_sig_bt);
1590 // Destroy argument registers
1591 for (int i = 0; i < total_in_args; i++) {
1592 if (in_regs[i].first()->is_Register()) {
1593 const Register reg = in_regs[i].first()->as_Register();
1594 if (reg->is_global()) {
1595 __ mov(G0, reg);
1596 }
1597 } else if (in_regs[i].first()->is_FloatRegister()) {
1598 __ fneg(FloatRegisterImpl::D, in_regs[i].first()->as_FloatRegister(), in_regs[i].first()->as_FloatRegister());
1599 }
1600 }
1601
1602 save_or_restore_arguments(masm, stack_slots, total_in_args,
1603 arg_save_area, NULL, in_regs, in_sig_bt);
1604 }
1605 #endif
1606 }
1607
1608 // Unpack an array argument into a pointer to the body and the length
1609 // if the array is non-null, otherwise pass 0 for both.
1610 static void unpack_array_argument(MacroAssembler* masm, VMRegPair reg, BasicType in_elem_type, VMRegPair body_arg, VMRegPair length_arg) {
1611 // Pass the length, ptr pair
1612 Label is_null, done;
1613 if (reg.first()->is_stack()) {
1614 VMRegPair tmp = reg64_to_VMRegPair(L2);
1615 // Load the arg up from the stack
1616 move_ptr(masm, reg, tmp);
1617 reg = tmp;
1618 }
1619 __ cmp(reg.first()->as_Register(), G0);
1620 __ brx(Assembler::equal, false, Assembler::pt, is_null);
1621 __ delayed()->add(reg.first()->as_Register(), arrayOopDesc::base_offset_in_bytes(in_elem_type), L4);
1622 move_ptr(masm, reg64_to_VMRegPair(L4), body_arg);
1623 __ ld(reg.first()->as_Register(), arrayOopDesc::length_offset_in_bytes(), L4);
1624 move32_64(masm, reg64_to_VMRegPair(L4), length_arg);
1625 __ ba_short(done);
1626 __ bind(is_null);
1627 // Pass zeros
1628 move_ptr(masm, reg64_to_VMRegPair(G0), body_arg);
1629 move32_64(masm, reg64_to_VMRegPair(G0), length_arg);
1630 __ bind(done);
1631 }
1632
1633 static void verify_oop_args(MacroAssembler* masm,
1634 const methodHandle& method,
1635 const BasicType* sig_bt,
1636 const VMRegPair* regs) {
1637 Register temp_reg = G5_method; // not part of any compiled calling seq
1638 if (VerifyOops) {
1639 for (int i = 0; i < method->size_of_parameters(); i++) {
1640 if (sig_bt[i] == T_OBJECT ||
1641 sig_bt[i] == T_ARRAY) {
1642 VMReg r = regs[i].first();
1643 assert(r->is_valid(), "bad oop arg");
1644 if (r->is_stack()) {
1645 RegisterOrConstant ld_off = reg2offset(r) + STACK_BIAS;
1646 ld_off = __ ensure_simm13_or_reg(ld_off, temp_reg);
1647 __ ld_ptr(SP, ld_off, temp_reg);
1648 __ verify_oop(temp_reg);
1649 } else {
1650 __ verify_oop(r->as_Register());
1651 }
1652 }
1653 }
1654 }
1655 }
1656
1657 static void gen_special_dispatch(MacroAssembler* masm,
1658 const methodHandle& method,
1659 const BasicType* sig_bt,
1660 const VMRegPair* regs) {
1661 verify_oop_args(masm, method, sig_bt, regs);
1662 vmIntrinsics::ID iid = method->intrinsic_id();
1663
1664 // Now write the args into the outgoing interpreter space
1665 bool has_receiver = false;
1666 Register receiver_reg = noreg;
1667 int member_arg_pos = -1;
1668 Register member_reg = noreg;
1669 int ref_kind = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
1670 if (ref_kind != 0) {
1671 member_arg_pos = method->size_of_parameters() - 1; // trailing MemberName argument
1672 member_reg = G5_method; // known to be free at this point
1673 has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
1674 } else if (iid == vmIntrinsics::_invokeBasic) {
1675 has_receiver = true;
1676 } else {
1677 fatal("unexpected intrinsic id %d", iid);
1678 }
1679
1680 if (member_reg != noreg) {
1681 // Load the member_arg into register, if necessary.
1682 SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
1683 VMReg r = regs[member_arg_pos].first();
1684 if (r->is_stack()) {
1685 RegisterOrConstant ld_off = reg2offset(r) + STACK_BIAS;
1686 ld_off = __ ensure_simm13_or_reg(ld_off, member_reg);
1687 __ ld_ptr(SP, ld_off, member_reg);
1688 } else {
1689 // no data motion is needed
1690 member_reg = r->as_Register();
1691 }
1692 }
1693
1694 if (has_receiver) {
1695 // Make sure the receiver is loaded into a register.
1696 assert(method->size_of_parameters() > 0, "oob");
1697 assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
1698 VMReg r = regs[0].first();
1699 assert(r->is_valid(), "bad receiver arg");
1700 if (r->is_stack()) {
1701 // Porting note: This assumes that compiled calling conventions always
1702 // pass the receiver oop in a register. If this is not true on some
1703 // platform, pick a temp and load the receiver from stack.
1704 fatal("receiver always in a register");
1705 receiver_reg = G3_scratch; // known to be free at this point
1706 RegisterOrConstant ld_off = reg2offset(r) + STACK_BIAS;
1707 ld_off = __ ensure_simm13_or_reg(ld_off, member_reg);
1708 __ ld_ptr(SP, ld_off, receiver_reg);
1709 } else {
1710 // no data motion is needed
1711 receiver_reg = r->as_Register();
1712 }
1713 }
1714
1715 // Figure out which address we are really jumping to:
1716 MethodHandles::generate_method_handle_dispatch(masm, iid,
1717 receiver_reg, member_reg, /*for_compiler_entry:*/ true);
1718 }
1719
1720 // ---------------------------------------------------------------------------
1721 // Generate a native wrapper for a given method. The method takes arguments
1722 // in the Java compiled code convention, marshals them to the native
1723 // convention (handlizes oops, etc), transitions to native, makes the call,
1724 // returns to java state (possibly blocking), unhandlizes any result and
1725 // returns.
1726 //
1727 // Critical native functions are a shorthand for the use of
1728 // GetPrimtiveArrayCritical and disallow the use of any other JNI
1729 // functions. The wrapper is expected to unpack the arguments before
1730 // passing them to the callee and perform checks before and after the
1731 // native call to ensure that they GCLocker
1732 // lock_critical/unlock_critical semantics are followed. Some other
1733 // parts of JNI setup are skipped like the tear down of the JNI handle
1734 // block and the check for pending exceptions it's impossible for them
1735 // to be thrown.
1736 //
1737 // They are roughly structured like this:
1738 // if (GCLocker::needs_gc())
1739 // SharedRuntime::block_for_jni_critical();
1740 // tranistion to thread_in_native
1741 // unpack arrray arguments and call native entry point
1742 // check for safepoint in progress
1743 // check if any thread suspend flags are set
1744 // call into JVM and possible unlock the JNI critical
1745 // if a GC was suppressed while in the critical native.
1746 // transition back to thread_in_Java
1747 // return to caller
1748 //
1749 nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
1750 const methodHandle& method,
1751 int compile_id,
1752 BasicType* in_sig_bt,
1753 VMRegPair* in_regs,
1754 BasicType ret_type) {
1755 if (method->is_method_handle_intrinsic()) {
1756 vmIntrinsics::ID iid = method->intrinsic_id();
1757 intptr_t start = (intptr_t)__ pc();
1758 int vep_offset = ((intptr_t)__ pc()) - start;
1759 gen_special_dispatch(masm,
1760 method,
1761 in_sig_bt,
1762 in_regs);
1763 int frame_complete = ((intptr_t)__ pc()) - start; // not complete, period
1764 __ flush();
1765 int stack_slots = SharedRuntime::out_preserve_stack_slots(); // no out slots at all, actually
1766 return nmethod::new_native_nmethod(method,
1767 compile_id,
1768 masm->code(),
1769 vep_offset,
1770 frame_complete,
1771 stack_slots / VMRegImpl::slots_per_word,
1772 in_ByteSize(-1),
1773 in_ByteSize(-1),
1774 (OopMapSet*)NULL);
1775 }
1776 bool is_critical_native = true;
1777 address native_func = method->critical_native_function();
1778 if (native_func == NULL) {
1779 native_func = method->native_function();
1780 is_critical_native = false;
1781 }
1782 assert(native_func != NULL, "must have function");
1783
1784 // Native nmethod wrappers never take possesion of the oop arguments.
1785 // So the caller will gc the arguments. The only thing we need an
1786 // oopMap for is if the call is static
1787 //
1788 // An OopMap for lock (and class if static), and one for the VM call itself
1789 OopMapSet *oop_maps = new OopMapSet();
1790 intptr_t start = (intptr_t)__ pc();
1791
1792 // First thing make an ic check to see if we should even be here
1793 {
1794 Label L;
1795 const Register temp_reg = G3_scratch;
1796 AddressLiteral ic_miss(SharedRuntime::get_ic_miss_stub());
1797 __ verify_oop(O0);
1798 __ load_klass(O0, temp_reg);
1799 __ cmp_and_brx_short(temp_reg, G5_inline_cache_reg, Assembler::equal, Assembler::pt, L);
1800
1801 __ jump_to(ic_miss, temp_reg);
1802 __ delayed()->nop();
1803 __ align(CodeEntryAlignment);
1804 __ bind(L);
1805 }
1806
1807 int vep_offset = ((intptr_t)__ pc()) - start;
1808
1809 #ifdef COMPILER1
1810 if ((InlineObjectHash && method->intrinsic_id() == vmIntrinsics::_hashCode) || (method->intrinsic_id() == vmIntrinsics::_identityHashCode)) {
1811 // Object.hashCode, System.identityHashCode can pull the hashCode from the
1812 // header word instead of doing a full VM transition once it's been computed.
1813 // Since hashCode is usually polymorphic at call sites we can't do this
1814 // optimization at the call site without a lot of work.
1815 Label slowCase;
1816 Label done;
1817 Register obj_reg = O0;
1818 Register result = O0;
1819 Register header = G3_scratch;
1820 Register hash = G3_scratch; // overwrite header value with hash value
1821 Register mask = G1; // to get hash field from header
1822
1823 // Unlike for Object.hashCode, System.identityHashCode is static method and
1824 // gets object as argument instead of the receiver.
1825 if (method->intrinsic_id() == vmIntrinsics::_identityHashCode) {
1826 assert(method->is_static(), "method should be static");
1827 // return 0 for null reference input
1828 __ br_null(obj_reg, false, Assembler::pn, done);
1829 __ delayed()->mov(obj_reg, hash);
1830 }
1831
1832 // Read the header and build a mask to get its hash field. Give up if the object is not unlocked.
1833 // We depend on hash_mask being at most 32 bits and avoid the use of
1834 // hash_mask_in_place because it could be larger than 32 bits in a 64-bit
1835 // vm: see markOop.hpp.
1836 __ ld_ptr(obj_reg, oopDesc::mark_offset_in_bytes(), header);
1837 __ sethi(markOopDesc::hash_mask, mask);
1838 __ btst(markOopDesc::unlocked_value, header);
1839 __ br(Assembler::zero, false, Assembler::pn, slowCase);
1840 if (UseBiasedLocking) {
1841 // Check if biased and fall through to runtime if so
1842 __ delayed()->nop();
1843 __ btst(markOopDesc::biased_lock_bit_in_place, header);
1844 __ br(Assembler::notZero, false, Assembler::pn, slowCase);
1845 }
1846 __ delayed()->or3(mask, markOopDesc::hash_mask & 0x3ff, mask);
1847
1848 // Check for a valid (non-zero) hash code and get its value.
1849 __ srlx(header, markOopDesc::hash_shift, hash);
1850 __ andcc(hash, mask, hash);
1851 __ br(Assembler::equal, false, Assembler::pn, slowCase);
1852 __ delayed()->nop();
1853
1854 // leaf return.
1855 __ bind(done);
1856 __ retl();
1857 __ delayed()->mov(hash, result);
1858 __ bind(slowCase);
1859 }
1860 #endif // COMPILER1
1861
1862
1863 // We have received a description of where all the java arg are located
1864 // on entry to the wrapper. We need to convert these args to where
1865 // the jni function will expect them. To figure out where they go
1866 // we convert the java signature to a C signature by inserting
1867 // the hidden arguments as arg[0] and possibly arg[1] (static method)
1868
1869 const int total_in_args = method->size_of_parameters();
1870 int total_c_args = total_in_args;
1871 int total_save_slots = 6 * VMRegImpl::slots_per_word;
1872 if (!is_critical_native) {
1873 total_c_args += 1;
1874 if (method->is_static()) {
1875 total_c_args++;
1876 }
1877 } else {
1878 for (int i = 0; i < total_in_args; i++) {
1879 if (in_sig_bt[i] == T_ARRAY) {
1880 // These have to be saved and restored across the safepoint
1881 total_c_args++;
1882 }
1883 }
1884 }
1885
1886 BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
1887 VMRegPair* out_regs = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
1888 BasicType* in_elem_bt = NULL;
1889
1890 int argc = 0;
1891 if (!is_critical_native) {
1892 out_sig_bt[argc++] = T_ADDRESS;
1893 if (method->is_static()) {
1894 out_sig_bt[argc++] = T_OBJECT;
1895 }
1896
1897 for (int i = 0; i < total_in_args ; i++ ) {
1898 out_sig_bt[argc++] = in_sig_bt[i];
1899 }
1900 } else {
1901 Thread* THREAD = Thread::current();
1902 in_elem_bt = NEW_RESOURCE_ARRAY(BasicType, total_in_args);
1903 SignatureStream ss(method->signature());
1904 for (int i = 0; i < total_in_args ; i++ ) {
1905 if (in_sig_bt[i] == T_ARRAY) {
1906 // Arrays are passed as int, elem* pair
1907 out_sig_bt[argc++] = T_INT;
1908 out_sig_bt[argc++] = T_ADDRESS;
1909 Symbol* atype = ss.as_symbol(CHECK_NULL);
1910 const char* at = atype->as_C_string();
1911 if (strlen(at) == 2) {
1912 assert(at[0] == '[', "must be");
1913 switch (at[1]) {
1914 case 'B': in_elem_bt[i] = T_BYTE; break;
1915 case 'C': in_elem_bt[i] = T_CHAR; break;
1916 case 'D': in_elem_bt[i] = T_DOUBLE; break;
1917 case 'F': in_elem_bt[i] = T_FLOAT; break;
1918 case 'I': in_elem_bt[i] = T_INT; break;
1919 case 'J': in_elem_bt[i] = T_LONG; break;
1920 case 'S': in_elem_bt[i] = T_SHORT; break;
1921 case 'Z': in_elem_bt[i] = T_BOOLEAN; break;
1922 default: ShouldNotReachHere();
1923 }
1924 }
1925 } else {
1926 out_sig_bt[argc++] = in_sig_bt[i];
1927 in_elem_bt[i] = T_VOID;
1928 }
1929 if (in_sig_bt[i] != T_VOID) {
1930 assert(in_sig_bt[i] == ss.type(), "must match");
1931 ss.next();
1932 }
1933 }
1934 }
1935
1936 // Now figure out where the args must be stored and how much stack space
1937 // they require (neglecting out_preserve_stack_slots but space for storing
1938 // the 1st six register arguments). It's weird see int_stk_helper.
1939 //
1940 int out_arg_slots;
1941 out_arg_slots = c_calling_convention(out_sig_bt, out_regs, NULL, total_c_args);
1942
1943 if (is_critical_native) {
1944 // Critical natives may have to call out so they need a save area
1945 // for register arguments.
1946 int double_slots = 0;
1947 int single_slots = 0;
1948 for ( int i = 0; i < total_in_args; i++) {
1949 if (in_regs[i].first()->is_Register()) {
1950 const Register reg = in_regs[i].first()->as_Register();
1951 switch (in_sig_bt[i]) {
1952 case T_ARRAY:
1953 case T_BOOLEAN:
1954 case T_BYTE:
1955 case T_SHORT:
1956 case T_CHAR:
1957 case T_INT: assert(reg->is_in(), "don't need to save these"); break;
1958 case T_LONG: if (reg->is_global()) double_slots++; break;
1959 default: ShouldNotReachHere();
1960 }
1961 } else if (in_regs[i].first()->is_FloatRegister()) {
1962 switch (in_sig_bt[i]) {
1963 case T_FLOAT: single_slots++; break;
1964 case T_DOUBLE: double_slots++; break;
1965 default: ShouldNotReachHere();
1966 }
1967 }
1968 }
1969 total_save_slots = double_slots * 2 + single_slots;
1970 }
1971
1972 // Compute framesize for the wrapper. We need to handlize all oops in
1973 // registers. We must create space for them here that is disjoint from
1974 // the windowed save area because we have no control over when we might
1975 // flush the window again and overwrite values that gc has since modified.
1976 // (The live window race)
1977 //
1978 // We always just allocate 6 word for storing down these object. This allow
1979 // us to simply record the base and use the Ireg number to decide which
1980 // slot to use. (Note that the reg number is the inbound number not the
1981 // outbound number).
1982 // We must shuffle args to match the native convention, and include var-args space.
1983
1984 // Calculate the total number of stack slots we will need.
1985
1986 // First count the abi requirement plus all of the outgoing args
1987 int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots;
1988
1989 // Now the space for the inbound oop handle area
1990
1991 int oop_handle_offset = align_up(stack_slots, 2);
1992 stack_slots += total_save_slots;
1993
1994 // Now any space we need for handlizing a klass if static method
1995
1996 int klass_slot_offset = 0;
1997 int klass_offset = -1;
1998 int lock_slot_offset = 0;
1999 bool is_static = false;
2000
2001 if (method->is_static()) {
2002 klass_slot_offset = stack_slots;
2003 stack_slots += VMRegImpl::slots_per_word;
2004 klass_offset = klass_slot_offset * VMRegImpl::stack_slot_size;
2005 is_static = true;
2006 }
2007
2008 // Plus a lock if needed
2009
2010 if (method->is_synchronized()) {
2011 lock_slot_offset = stack_slots;
2012 stack_slots += VMRegImpl::slots_per_word;
2013 }
2014
2015 // Now a place to save return value or as a temporary for any gpr -> fpr moves
2016 stack_slots += 2;
2017
2018 // Ok The space we have allocated will look like:
2019 //
2020 //
2021 // FP-> | |
2022 // |---------------------|
2023 // | 2 slots for moves |
2024 // |---------------------|
2025 // | lock box (if sync) |
2026 // |---------------------| <- lock_slot_offset
2027 // | klass (if static) |
2028 // |---------------------| <- klass_slot_offset
2029 // | oopHandle area |
2030 // |---------------------| <- oop_handle_offset
2031 // | outbound memory |
2032 // | based arguments |
2033 // | |
2034 // |---------------------|
2035 // | vararg area |
2036 // |---------------------|
2037 // | |
2038 // SP-> | out_preserved_slots |
2039 //
2040 //
2041
2042
2043 // Now compute actual number of stack words we need rounding to make
2044 // stack properly aligned.
2045 stack_slots = align_up(stack_slots, 2 * VMRegImpl::slots_per_word);
2046
2047 int stack_size = stack_slots * VMRegImpl::stack_slot_size;
2048
2049 // Generate stack overflow check before creating frame
2050 __ generate_stack_overflow_check(stack_size);
2051
2052 // Generate a new frame for the wrapper.
2053 __ save(SP, -stack_size, SP);
2054
2055 int frame_complete = ((intptr_t)__ pc()) - start;
2056
2057 __ verify_thread();
2058
2059 if (is_critical_native) {
2060 check_needs_gc_for_critical_native(masm, stack_slots, total_in_args,
2061 oop_handle_offset, oop_maps, in_regs, in_sig_bt);
2062 }
2063
2064 //
2065 // We immediately shuffle the arguments so that any vm call we have to
2066 // make from here on out (sync slow path, jvmti, etc.) we will have
2067 // captured the oops from our caller and have a valid oopMap for
2068 // them.
2069
2070 // -----------------
2071 // The Grand Shuffle
2072 //
2073 // Natives require 1 or 2 extra arguments over the normal ones: the JNIEnv*
2074 // (derived from JavaThread* which is in L7_thread_cache) and, if static,
2075 // the class mirror instead of a receiver. This pretty much guarantees that
2076 // register layout will not match. We ignore these extra arguments during
2077 // the shuffle. The shuffle is described by the two calling convention
2078 // vectors we have in our possession. We simply walk the java vector to
2079 // get the source locations and the c vector to get the destinations.
2080 // Because we have a new window and the argument registers are completely
2081 // disjoint ( I0 -> O1, I1 -> O2, ...) we have nothing to worry about
2082 // here.
2083
2084 // This is a trick. We double the stack slots so we can claim
2085 // the oops in the caller's frame. Since we are sure to have
2086 // more args than the caller doubling is enough to make
2087 // sure we can capture all the incoming oop args from the
2088 // caller.
2089 //
2090 OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
2091 // Record sp-based slot for receiver on stack for non-static methods
2092 int receiver_offset = -1;
2093
2094 // We move the arguments backward because the floating point registers
2095 // destination will always be to a register with a greater or equal register
2096 // number or the stack.
2097
2098 #ifdef ASSERT
2099 bool reg_destroyed[RegisterImpl::number_of_registers];
2100 bool freg_destroyed[FloatRegisterImpl::number_of_registers];
2101 for ( int r = 0 ; r < RegisterImpl::number_of_registers ; r++ ) {
2102 reg_destroyed[r] = false;
2103 }
2104 for ( int f = 0 ; f < FloatRegisterImpl::number_of_registers ; f++ ) {
2105 freg_destroyed[f] = false;
2106 }
2107
2108 #endif /* ASSERT */
2109
2110 for ( int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0 ; i--, c_arg-- ) {
2111
2112 #ifdef ASSERT
2113 if (in_regs[i].first()->is_Register()) {
2114 assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "ack!");
2115 } else if (in_regs[i].first()->is_FloatRegister()) {
2116 assert(!freg_destroyed[in_regs[i].first()->as_FloatRegister()->encoding(FloatRegisterImpl::S)], "ack!");
2117 }
2118 if (out_regs[c_arg].first()->is_Register()) {
2119 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2120 } else if (out_regs[c_arg].first()->is_FloatRegister()) {
2121 freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding(FloatRegisterImpl::S)] = true;
2122 }
2123 #endif /* ASSERT */
2124
2125 switch (in_sig_bt[i]) {
2126 case T_ARRAY:
2127 if (is_critical_native) {
2128 unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg], out_regs[c_arg - 1]);
2129 c_arg--;
2130 break;
2131 }
2132 case T_OBJECT:
2133 assert(!is_critical_native, "no oop arguments");
2134 object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
2135 ((i == 0) && (!is_static)),
2136 &receiver_offset);
2137 break;
2138 case T_VOID:
2139 break;
2140
2141 case T_FLOAT:
2142 float_move(masm, in_regs[i], out_regs[c_arg]);
2143 break;
2144
2145 case T_DOUBLE:
2146 assert( i + 1 < total_in_args &&
2147 in_sig_bt[i + 1] == T_VOID &&
2148 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
2149 double_move(masm, in_regs[i], out_regs[c_arg]);
2150 break;
2151
2152 case T_LONG :
2153 long_move(masm, in_regs[i], out_regs[c_arg]);
2154 break;
2155
2156 case T_ADDRESS: assert(false, "found T_ADDRESS in java args");
2157
2158 default:
2159 move32_64(masm, in_regs[i], out_regs[c_arg]);
2160 }
2161 }
2162
2163 // Pre-load a static method's oop into O1. Used both by locking code and
2164 // the normal JNI call code.
2165 if (method->is_static() && !is_critical_native) {
2166 __ set_oop_constant(JNIHandles::make_local(method->method_holder()->java_mirror()), O1);
2167
2168 // Now handlize the static class mirror in O1. It's known not-null.
2169 __ st_ptr(O1, SP, klass_offset + STACK_BIAS);
2170 map->set_oop(VMRegImpl::stack2reg(klass_slot_offset));
2171 __ add(SP, klass_offset + STACK_BIAS, O1);
2172 }
2173
2174
2175 const Register L6_handle = L6;
2176
2177 if (method->is_synchronized()) {
2178 assert(!is_critical_native, "unhandled");
2179 __ mov(O1, L6_handle);
2180 }
2181
2182 // We have all of the arguments setup at this point. We MUST NOT touch any Oregs
2183 // except O6/O7. So if we must call out we must push a new frame. We immediately
2184 // push a new frame and flush the windows.
2185 intptr_t thepc = (intptr_t) __ pc();
2186 {
2187 address here = __ pc();
2188 // Call the next instruction
2189 __ call(here + 8, relocInfo::none);
2190 __ delayed()->nop();
2191 }
2192
2193 // We use the same pc/oopMap repeatedly when we call out
2194 oop_maps->add_gc_map(thepc - start, map);
2195
2196 // O7 now has the pc loaded that we will use when we finally call to native.
2197
2198 // Save thread in L7; it crosses a bunch of VM calls below
2199 // Don't use save_thread because it smashes G2 and we merely
2200 // want to save a copy
2201 __ mov(G2_thread, L7_thread_cache);
2202
2203
2204 // If we create an inner frame once is plenty
2205 // when we create it we must also save G2_thread
2206 bool inner_frame_created = false;
2207
2208 // dtrace method entry support
2209 {
2210 SkipIfEqual skip_if(
2211 masm, G3_scratch, &DTraceMethodProbes, Assembler::zero);
2212 // create inner frame
2213 __ save_frame(0);
2214 __ mov(G2_thread, L7_thread_cache);
2215 __ set_metadata_constant(method(), O1);
2216 __ call_VM_leaf(L7_thread_cache,
2217 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
2218 G2_thread, O1);
2219 __ restore();
2220 }
2221
2222 // RedefineClasses() tracing support for obsolete method entry
2223 if (log_is_enabled(Trace, redefine, class, obsolete)) {
2224 // create inner frame
2225 __ save_frame(0);
2226 __ mov(G2_thread, L7_thread_cache);
2227 __ set_metadata_constant(method(), O1);
2228 __ call_VM_leaf(L7_thread_cache,
2229 CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
2230 G2_thread, O1);
2231 __ restore();
2232 }
2233
2234 // We are in the jni frame unless saved_frame is true in which case
2235 // we are in one frame deeper (the "inner" frame). If we are in the
2236 // "inner" frames the args are in the Iregs and if the jni frame then
2237 // they are in the Oregs.
2238 // If we ever need to go to the VM (for locking, jvmti) then
2239 // we will always be in the "inner" frame.
2240
2241 // Lock a synchronized method
2242 int lock_offset = -1; // Set if locked
2243 if (method->is_synchronized()) {
2244 Register Roop = O1;
2245 const Register L3_box = L3;
2246
2247 create_inner_frame(masm, &inner_frame_created);
2248
2249 __ ld_ptr(I1, 0, O1);
2250 Label done;
2251
2252 lock_offset = (lock_slot_offset * VMRegImpl::stack_slot_size);
2253 __ add(FP, lock_offset+STACK_BIAS, L3_box);
2254 #ifdef ASSERT
2255 if (UseBiasedLocking) {
2256 // making the box point to itself will make it clear it went unused
2257 // but also be obviously invalid
2258 __ st_ptr(L3_box, L3_box, 0);
2259 }
2260 #endif // ASSERT
2261 //
2262 // Compiler_lock_object (Roop, Rmark, Rbox, Rscratch) -- kills Rmark, Rbox, Rscratch
2263 //
2264 __ compiler_lock_object(Roop, L1, L3_box, L2);
2265 __ br(Assembler::equal, false, Assembler::pt, done);
2266 __ delayed() -> add(FP, lock_offset+STACK_BIAS, L3_box);
2267
2268
2269 // None of the above fast optimizations worked so we have to get into the
2270 // slow case of monitor enter. Inline a special case of call_VM that
2271 // disallows any pending_exception.
2272 __ mov(Roop, O0); // Need oop in O0
2273 __ mov(L3_box, O1);
2274
2275 // Record last_Java_sp, in case the VM code releases the JVM lock.
2276
2277 __ set_last_Java_frame(FP, I7);
2278
2279 // do the call
2280 __ call(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), relocInfo::runtime_call_type);
2281 __ delayed()->mov(L7_thread_cache, O2);
2282
2283 __ restore_thread(L7_thread_cache); // restore G2_thread
2284 __ reset_last_Java_frame();
2285
2286 #ifdef ASSERT
2287 { Label L;
2288 __ ld_ptr(G2_thread, in_bytes(Thread::pending_exception_offset()), O0);
2289 __ br_null_short(O0, Assembler::pt, L);
2290 __ stop("no pending exception allowed on exit from IR::monitorenter");
2291 __ bind(L);
2292 }
2293 #endif
2294 __ bind(done);
2295 }
2296
2297
2298 // Finally just about ready to make the JNI call
2299
2300 __ flushw();
2301 if (inner_frame_created) {
2302 __ restore();
2303 } else {
2304 // Store only what we need from this frame
2305 // QQQ I think that non-v9 (like we care) we don't need these saves
2306 // either as the flush traps and the current window goes too.
2307 __ st_ptr(FP, SP, FP->sp_offset_in_saved_window()*wordSize + STACK_BIAS);
2308 __ st_ptr(I7, SP, I7->sp_offset_in_saved_window()*wordSize + STACK_BIAS);
2309 }
2310
2311 // get JNIEnv* which is first argument to native
2312 if (!is_critical_native) {
2313 __ add(G2_thread, in_bytes(JavaThread::jni_environment_offset()), O0);
2314 }
2315
2316 // Use that pc we placed in O7 a while back as the current frame anchor
2317 __ set_last_Java_frame(SP, O7);
2318
2319 // We flushed the windows ages ago now mark them as flushed before transitioning.
2320 __ set(JavaFrameAnchor::flushed, G3_scratch);
2321 __ st(G3_scratch, G2_thread, JavaThread::frame_anchor_offset() + JavaFrameAnchor::flags_offset());
2322
2323 // Transition from _thread_in_Java to _thread_in_native.
2324 __ set(_thread_in_native, G3_scratch);
2325
2326 AddressLiteral dest(native_func);
2327 __ relocate(relocInfo::runtime_call_type);
2328 __ jumpl_to(dest, O7, O7);
2329 __ delayed()->st(G3_scratch, G2_thread, JavaThread::thread_state_offset());
2330
2331 __ restore_thread(L7_thread_cache); // restore G2_thread
2332
2333 // Unpack native results. For int-types, we do any needed sign-extension
2334 // and move things into I0. The return value there will survive any VM
2335 // calls for blocking or unlocking. An FP or OOP result (handle) is done
2336 // specially in the slow-path code.
2337 switch (ret_type) {
2338 case T_VOID: break; // Nothing to do!
2339 case T_FLOAT: break; // Got it where we want it (unless slow-path)
2340 case T_DOUBLE: break; // Got it where we want it (unless slow-path)
2341 // In 64 bits build result is in O0, in O0, O1 in 32bit build
2342 case T_LONG:
2343 // Fall thru
2344 case T_OBJECT: // Really a handle
2345 case T_ARRAY:
2346 case T_INT:
2347 __ mov(O0, I0);
2348 break;
2349 case T_BOOLEAN: __ subcc(G0, O0, G0); __ addc(G0, 0, I0); break; // !0 => true; 0 => false
2350 case T_BYTE : __ sll(O0, 24, O0); __ sra(O0, 24, I0); break;
2351 case T_CHAR : __ sll(O0, 16, O0); __ srl(O0, 16, I0); break; // cannot use and3, 0xFFFF too big as immediate value!
2352 case T_SHORT : __ sll(O0, 16, O0); __ sra(O0, 16, I0); break;
2353 break; // Cannot de-handlize until after reclaiming jvm_lock
2354 default:
2355 ShouldNotReachHere();
2356 }
2357
2358 Label after_transition;
2359 // must we block?
2360
2361 // Block, if necessary, before resuming in _thread_in_Java state.
2362 // In order for GC to work, don't clear the last_Java_sp until after blocking.
2363 { Label no_block;
2364
2365 // Switch thread to "native transition" state before reading the synchronization state.
2366 // This additional state is necessary because reading and testing the synchronization
2367 // state is not atomic w.r.t. GC, as this scenario demonstrates:
2368 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2369 // VM thread changes sync state to synchronizing and suspends threads for GC.
2370 // Thread A is resumed to finish this native method, but doesn't block here since it
2371 // didn't see any synchronization is progress, and escapes.
2372 __ set(_thread_in_native_trans, G3_scratch);
2373 __ st(G3_scratch, G2_thread, JavaThread::thread_state_offset());
2374 if(os::is_MP()) {
2375 if (UseMembar) {
2376 // Force this write out before the read below
2377 __ membar(Assembler::StoreLoad);
2378 } else {
2379 // Write serialization page so VM thread can do a pseudo remote membar.
2380 // We use the current thread pointer to calculate a thread specific
2381 // offset to write to within the page. This minimizes bus traffic
2382 // due to cache line collision.
2383 __ serialize_memory(G2_thread, G1_scratch, G3_scratch);
2384 }
2385 }
2386
2387 Label L;
2388 Address suspend_state(G2_thread, JavaThread::suspend_flags_offset());
2389 __ safepoint_poll(L, false, G2_thread, G3_scratch);
2390 __ delayed()->ld(suspend_state, G3_scratch);
2391 __ cmp_and_br_short(G3_scratch, 0, Assembler::equal, Assembler::pt, no_block);
2392 __ bind(L);
2393
2394 // Block. Save any potential method result value before the operation and
2395 // use a leaf call to leave the last_Java_frame setup undisturbed. Doing this
2396 // lets us share the oopMap we used when we went native rather the create
2397 // a distinct one for this pc
2398 //
2399 save_native_result(masm, ret_type, stack_slots);
2400 if (!is_critical_native) {
2401 __ call_VM_leaf(L7_thread_cache,
2402 CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans),
2403 G2_thread);
2404 } else {
2405 __ call_VM_leaf(L7_thread_cache,
2406 CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans_and_transition),
2407 G2_thread);
2408 }
2409
2410 // Restore any method result value
2411 restore_native_result(masm, ret_type, stack_slots);
2412
2413 if (is_critical_native) {
2414 // The call above performed the transition to thread_in_Java so
2415 // skip the transition logic below.
2416 __ ba(after_transition);
2417 __ delayed()->nop();
2418 }
2419
2420 __ bind(no_block);
2421 }
2422
2423 // thread state is thread_in_native_trans. Any safepoint blocking has already
2424 // happened so we can now change state to _thread_in_Java.
2425 __ set(_thread_in_Java, G3_scratch);
2426 __ st(G3_scratch, G2_thread, JavaThread::thread_state_offset());
2427 __ bind(after_transition);
2428
2429 Label no_reguard;
2430 __ ld(G2_thread, JavaThread::stack_guard_state_offset(), G3_scratch);
2431 __ cmp_and_br_short(G3_scratch, JavaThread::stack_guard_yellow_reserved_disabled, Assembler::notEqual, Assembler::pt, no_reguard);
2432
2433 save_native_result(masm, ret_type, stack_slots);
2434 __ call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
2435 __ delayed()->nop();
2436
2437 __ restore_thread(L7_thread_cache); // restore G2_thread
2438 restore_native_result(masm, ret_type, stack_slots);
2439
2440 __ bind(no_reguard);
2441
2442 // Handle possible exception (will unlock if necessary)
2443
2444 // native result if any is live in freg or I0 (and I1 if long and 32bit vm)
2445
2446 // Unlock
2447 if (method->is_synchronized()) {
2448 Label done;
2449 Register I2_ex_oop = I2;
2450 const Register L3_box = L3;
2451 // Get locked oop from the handle we passed to jni
2452 __ ld_ptr(L6_handle, 0, L4);
2453 __ add(SP, lock_offset+STACK_BIAS, L3_box);
2454 // Must save pending exception around the slow-path VM call. Since it's a
2455 // leaf call, the pending exception (if any) can be kept in a register.
2456 __ ld_ptr(G2_thread, in_bytes(Thread::pending_exception_offset()), I2_ex_oop);
2457 // Now unlock
2458 // (Roop, Rmark, Rbox, Rscratch)
2459 __ compiler_unlock_object(L4, L1, L3_box, L2);
2460 __ br(Assembler::equal, false, Assembler::pt, done);
2461 __ delayed()-> add(SP, lock_offset+STACK_BIAS, L3_box);
2462
2463 // save and restore any potential method result value around the unlocking
2464 // operation. Will save in I0 (or stack for FP returns).
2465 save_native_result(masm, ret_type, stack_slots);
2466
2467 // Must clear pending-exception before re-entering the VM. Since this is
2468 // a leaf call, pending-exception-oop can be safely kept in a register.
2469 __ st_ptr(G0, G2_thread, in_bytes(Thread::pending_exception_offset()));
2470
2471 // slow case of monitor enter. Inline a special case of call_VM that
2472 // disallows any pending_exception.
2473 __ mov(L3_box, O1);
2474
2475 // Pass in current thread pointer
2476 __ mov(G2_thread, O2);
2477
2478 __ call(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C), relocInfo::runtime_call_type);
2479 __ delayed()->mov(L4, O0); // Need oop in O0
2480
2481 __ restore_thread(L7_thread_cache); // restore G2_thread
2482
2483 #ifdef ASSERT
2484 { Label L;
2485 __ ld_ptr(G2_thread, in_bytes(Thread::pending_exception_offset()), O0);
2486 __ br_null_short(O0, Assembler::pt, L);
2487 __ stop("no pending exception allowed on exit from IR::monitorexit");
2488 __ bind(L);
2489 }
2490 #endif
2491 restore_native_result(masm, ret_type, stack_slots);
2492 // check_forward_pending_exception jump to forward_exception if any pending
2493 // exception is set. The forward_exception routine expects to see the
2494 // exception in pending_exception and not in a register. Kind of clumsy,
2495 // since all folks who branch to forward_exception must have tested
2496 // pending_exception first and hence have it in a register already.
2497 __ st_ptr(I2_ex_oop, G2_thread, in_bytes(Thread::pending_exception_offset()));
2498 __ bind(done);
2499 }
2500
2501 // Tell dtrace about this method exit
2502 {
2503 SkipIfEqual skip_if(
2504 masm, G3_scratch, &DTraceMethodProbes, Assembler::zero);
2505 save_native_result(masm, ret_type, stack_slots);
2506 __ set_metadata_constant(method(), O1);
2507 __ call_VM_leaf(L7_thread_cache,
2508 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2509 G2_thread, O1);
2510 restore_native_result(masm, ret_type, stack_slots);
2511 }
2512
2513 // Clear "last Java frame" SP and PC.
2514 __ verify_thread(); // G2_thread must be correct
2515 __ reset_last_Java_frame();
2516
2517 // Unbox oop result, e.g. JNIHandles::resolve value in I0.
2518 if (ret_type == T_OBJECT || ret_type == T_ARRAY) {
2519 Label done, not_weak;
2520 __ br_null(I0, false, Assembler::pn, done); // Use NULL as-is.
2521 __ delayed()->andcc(I0, JNIHandles::weak_tag_mask, G0); // Test for jweak
2522 __ brx(Assembler::zero, true, Assembler::pt, not_weak);
2523 __ delayed()->ld_ptr(I0, 0, I0); // Maybe resolve (untagged) jobject.
2524 // Resolve jweak.
2525 __ ld_ptr(I0, -JNIHandles::weak_tag_value, I0);
2526 #if INCLUDE_ALL_GCS
2527 if (UseG1GC) {
2528 // Copy to O0 because macro doesn't allow pre_val in input reg.
2529 __ mov(I0, O0);
2530 __ g1_write_barrier_pre(noreg /* obj */,
2531 noreg /* index */,
2532 0 /* offset */,
2533 O0 /* pre_val */,
2534 G3_scratch /* tmp */,
2535 true /* preserve_o_regs */);
2536 }
2537 #endif // INCLUDE_ALL_GCS
2538 __ bind(not_weak);
2539 __ verify_oop(I0);
2540 __ bind(done);
2541 }
2542
2543 if (CheckJNICalls) {
2544 // clear_pending_jni_exception_check
2545 __ st_ptr(G0, G2_thread, JavaThread::pending_jni_exception_check_fn_offset());
2546 }
2547
2548 if (!is_critical_native) {
2549 // reset handle block
2550 __ ld_ptr(G2_thread, in_bytes(JavaThread::active_handles_offset()), L5);
2551 __ st(G0, L5, JNIHandleBlock::top_offset_in_bytes());
2552
2553 __ ld_ptr(G2_thread, in_bytes(Thread::pending_exception_offset()), G3_scratch);
2554 check_forward_pending_exception(masm, G3_scratch);
2555 }
2556
2557
2558 // Return
2559
2560 __ ret();
2561 __ delayed()->restore();
2562
2563 __ flush();
2564
2565 nmethod *nm = nmethod::new_native_nmethod(method,
2566 compile_id,
2567 masm->code(),
2568 vep_offset,
2569 frame_complete,
2570 stack_slots / VMRegImpl::slots_per_word,
2571 (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)),
2572 in_ByteSize(lock_offset),
2573 oop_maps);
2574
2575 if (is_critical_native) {
2576 nm->set_lazy_critical_native(true);
2577 }
2578 return nm;
2579
2580 }
2581
2582 // this function returns the adjust size (in number of words) to a c2i adapter
2583 // activation for use during deoptimization
2584 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals) {
2585 assert(callee_locals >= callee_parameters,
2586 "test and remove; got more parms than locals");
2587 if (callee_locals < callee_parameters)
2588 return 0; // No adjustment for negative locals
2589 int diff = (callee_locals - callee_parameters) * Interpreter::stackElementWords;
2590 return align_up(diff, WordsPerLong);
2591 }
2592
2593 // "Top of Stack" slots that may be unused by the calling convention but must
2594 // otherwise be preserved.
2595 // On Intel these are not necessary and the value can be zero.
2596 // On Sparc this describes the words reserved for storing a register window
2597 // when an interrupt occurs.
2598 uint SharedRuntime::out_preserve_stack_slots() {
2599 return frame::register_save_words * VMRegImpl::slots_per_word;
2600 }
2601
2602 static void gen_new_frame(MacroAssembler* masm, bool deopt) {
2603 //
2604 // Common out the new frame generation for deopt and uncommon trap
2605 //
2606 Register G3pcs = G3_scratch; // Array of new pcs (input)
2607 Register Oreturn0 = O0;
2608 Register Oreturn1 = O1;
2609 Register O2UnrollBlock = O2;
2610 Register O3array = O3; // Array of frame sizes (input)
2611 Register O4array_size = O4; // number of frames (input)
2612 Register O7frame_size = O7; // number of frames (input)
2613
2614 __ ld_ptr(O3array, 0, O7frame_size);
2615 __ sub(G0, O7frame_size, O7frame_size);
2616 __ save(SP, O7frame_size, SP);
2617 __ ld_ptr(G3pcs, 0, I7); // load frame's new pc
2618
2619 #ifdef ASSERT
2620 // make sure that the frames are aligned properly
2621 #endif
2622
2623 // Deopt needs to pass some extra live values from frame to frame
2624
2625 if (deopt) {
2626 __ mov(Oreturn0->after_save(), Oreturn0);
2627 __ mov(Oreturn1->after_save(), Oreturn1);
2628 }
2629
2630 __ mov(O4array_size->after_save(), O4array_size);
2631 __ sub(O4array_size, 1, O4array_size);
2632 __ mov(O3array->after_save(), O3array);
2633 __ mov(O2UnrollBlock->after_save(), O2UnrollBlock);
2634 __ add(G3pcs, wordSize, G3pcs); // point to next pc value
2635
2636 #ifdef ASSERT
2637 // trash registers to show a clear pattern in backtraces
2638 __ set(0xDEAD0000, I0);
2639 __ add(I0, 2, I1);
2640 __ add(I0, 4, I2);
2641 __ add(I0, 6, I3);
2642 __ add(I0, 8, I4);
2643 // Don't touch I5 could have valuable savedSP
2644 __ set(0xDEADBEEF, L0);
2645 __ mov(L0, L1);
2646 __ mov(L0, L2);
2647 __ mov(L0, L3);
2648 __ mov(L0, L4);
2649 __ mov(L0, L5);
2650
2651 // trash the return value as there is nothing to return yet
2652 __ set(0xDEAD0001, O7);
2653 #endif
2654
2655 __ mov(SP, O5_savedSP);
2656 }
2657
2658
2659 static void make_new_frames(MacroAssembler* masm, bool deopt) {
2660 //
2661 // loop through the UnrollBlock info and create new frames
2662 //
2663 Register G3pcs = G3_scratch;
2664 Register Oreturn0 = O0;
2665 Register Oreturn1 = O1;
2666 Register O2UnrollBlock = O2;
2667 Register O3array = O3;
2668 Register O4array_size = O4;
2669 Label loop;
2670
2671 #ifdef ASSERT
2672 // Compilers generate code that bang the stack by as much as the
2673 // interpreter would need. So this stack banging should never
2674 // trigger a fault. Verify that it does not on non product builds.
2675 if (UseStackBanging) {
2676 // Get total frame size for interpreted frames
2677 __ ld(O2UnrollBlock, Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes(), O4);
2678 __ bang_stack_size(O4, O3, G3_scratch);
2679 }
2680 #endif
2681
2682 __ ld(O2UnrollBlock, Deoptimization::UnrollBlock::number_of_frames_offset_in_bytes(), O4array_size);
2683 __ ld_ptr(O2UnrollBlock, Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes(), G3pcs);
2684 __ ld_ptr(O2UnrollBlock, Deoptimization::UnrollBlock::frame_sizes_offset_in_bytes(), O3array);
2685
2686 // Adjust old interpreter frame to make space for new frame's extra java locals
2687 //
2688 // We capture the original sp for the transition frame only because it is needed in
2689 // order to properly calculate interpreter_sp_adjustment. Even though in real life
2690 // every interpreter frame captures a savedSP it is only needed at the transition
2691 // (fortunately). If we had to have it correct everywhere then we would need to
2692 // be told the sp_adjustment for each frame we create. If the frame size array
2693 // were to have twice the frame count entries then we could have pairs [sp_adjustment, frame_size]
2694 // for each frame we create and keep up the illusion every where.
2695 //
2696
2697 __ ld(O2UnrollBlock, Deoptimization::UnrollBlock::caller_adjustment_offset_in_bytes(), O7);
2698 __ mov(SP, O5_savedSP); // remember initial sender's original sp before adjustment
2699 __ sub(SP, O7, SP);
2700
2701 #ifdef ASSERT
2702 // make sure that there is at least one entry in the array
2703 __ tst(O4array_size);
2704 __ breakpoint_trap(Assembler::zero, Assembler::icc);
2705 #endif
2706
2707 // Now push the new interpreter frames
2708 __ bind(loop);
2709
2710 // allocate a new frame, filling the registers
2711
2712 gen_new_frame(masm, deopt); // allocate an interpreter frame
2713
2714 __ cmp_zero_and_br(Assembler::notZero, O4array_size, loop);
2715 __ delayed()->add(O3array, wordSize, O3array);
2716 __ ld_ptr(G3pcs, 0, O7); // load final frame new pc
2717
2718 }
2719
2720 //------------------------------generate_deopt_blob----------------------------
2721 // Ought to generate an ideal graph & compile, but here's some SPARC ASM
2722 // instead.
2723 void SharedRuntime::generate_deopt_blob() {
2724 // allocate space for the code
2725 ResourceMark rm;
2726 // setup code generation tools
2727 int pad = VerifyThread ? 512 : 0;// Extra slop space for more verify code
2728 #ifdef ASSERT
2729 if (UseStackBanging) {
2730 pad += (JavaThread::stack_shadow_zone_size() / os::vm_page_size())*16 + 32;
2731 }
2732 #endif
2733 #if INCLUDE_JVMCI
2734 if (EnableJVMCI) {
2735 pad += 1000; // Increase the buffer size when compiling for JVMCI
2736 }
2737 #endif
2738 CodeBuffer buffer("deopt_blob", 2100+pad, 512);
2739 MacroAssembler* masm = new MacroAssembler(&buffer);
2740 FloatRegister Freturn0 = F0;
2741 Register Greturn1 = G1;
2742 Register Oreturn0 = O0;
2743 Register Oreturn1 = O1;
2744 Register O2UnrollBlock = O2;
2745 Register L0deopt_mode = L0;
2746 Register G4deopt_mode = G4_scratch;
2747 int frame_size_words;
2748 Address saved_Freturn0_addr(FP, -sizeof(double) + STACK_BIAS);
2749 Label cont;
2750
2751 OopMapSet *oop_maps = new OopMapSet();
2752
2753 //
2754 // This is the entry point for code which is returning to a de-optimized
2755 // frame.
2756 // The steps taken by this frame are as follows:
2757 // - push a dummy "register_save" and save the return values (O0, O1, F0/F1, G1)
2758 // and all potentially live registers (at a pollpoint many registers can be live).
2759 //
2760 // - call the C routine: Deoptimization::fetch_unroll_info (this function
2761 // returns information about the number and size of interpreter frames
2762 // which are equivalent to the frame which is being deoptimized)
2763 // - deallocate the unpack frame, restoring only results values. Other
2764 // volatile registers will now be captured in the vframeArray as needed.
2765 // - deallocate the deoptimization frame
2766 // - in a loop using the information returned in the previous step
2767 // push new interpreter frames (take care to propagate the return
2768 // values through each new frame pushed)
2769 // - create a dummy "unpack_frame" and save the return values (O0, O1, F0)
2770 // - call the C routine: Deoptimization::unpack_frames (this function
2771 // lays out values on the interpreter frame which was just created)
2772 // - deallocate the dummy unpack_frame
2773 // - ensure that all the return values are correctly set and then do
2774 // a return to the interpreter entry point
2775 //
2776 // Refer to the following methods for more information:
2777 // - Deoptimization::fetch_unroll_info
2778 // - Deoptimization::unpack_frames
2779
2780 OopMap* map = NULL;
2781
2782 int start = __ offset();
2783
2784 // restore G2, the trampoline destroyed it
2785 __ get_thread();
2786
2787 // On entry we have been called by the deoptimized nmethod with a call that
2788 // replaced the original call (or safepoint polling location) so the deoptimizing
2789 // pc is now in O7. Return values are still in the expected places
2790
2791 map = RegisterSaver::save_live_registers(masm, 0, &frame_size_words);
2792 __ ba(cont);
2793 __ delayed()->mov(Deoptimization::Unpack_deopt, L0deopt_mode);
2794
2795
2796 #if INCLUDE_JVMCI
2797 Label after_fetch_unroll_info_call;
2798 int implicit_exception_uncommon_trap_offset = 0;
2799 int uncommon_trap_offset = 0;
2800
2801 if (EnableJVMCI) {
2802 masm->block_comment("BEGIN implicit_exception_uncommon_trap");
2803 implicit_exception_uncommon_trap_offset = __ offset() - start;
2804
2805 __ ld_ptr(G2_thread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset()), O7);
2806 __ st_ptr(G0, Address(G2_thread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())));
2807 __ add(O7, -8, O7);
2808
2809 uncommon_trap_offset = __ offset() - start;
2810
2811 // Save everything in sight.
2812 (void) RegisterSaver::save_live_registers(masm, 0, &frame_size_words);
2813 __ set_last_Java_frame(SP, NULL);
2814
2815 __ ld(G2_thread, in_bytes(JavaThread::pending_deoptimization_offset()), O1);
2816 __ sub(G0, 1, L1);
2817 __ st(L1, G2_thread, in_bytes(JavaThread::pending_deoptimization_offset()));
2818
2819 __ mov((int32_t)Deoptimization::Unpack_reexecute, L0deopt_mode);
2820 __ mov(G2_thread, O0);
2821 __ mov(L0deopt_mode, O2);
2822 __ call(CAST_FROM_FN_PTR(address, Deoptimization::uncommon_trap));
2823 __ delayed()->nop();
2824 oop_maps->add_gc_map( __ offset()-start, map->deep_copy());
2825 __ get_thread();
2826 __ add(O7, 8, O7);
2827 __ reset_last_Java_frame();
2828
2829 __ ba(after_fetch_unroll_info_call);
2830 __ delayed()->nop(); // Delay slot
2831 masm->block_comment("END implicit_exception_uncommon_trap");
2832 } // EnableJVMCI
2833 #endif // INCLUDE_JVMCI
2834
2835 int exception_offset = __ offset() - start;
2836
2837 // restore G2, the trampoline destroyed it
2838 __ get_thread();
2839
2840 // On entry we have been jumped to by the exception handler (or exception_blob
2841 // for server). O0 contains the exception oop and O7 contains the original
2842 // exception pc. So if we push a frame here it will look to the
2843 // stack walking code (fetch_unroll_info) just like a normal call so
2844 // state will be extracted normally.
2845
2846 // save exception oop in JavaThread and fall through into the
2847 // exception_in_tls case since they are handled in same way except
2848 // for where the pending exception is kept.
2849 __ st_ptr(Oexception, G2_thread, JavaThread::exception_oop_offset());
2850
2851 //
2852 // Vanilla deoptimization with an exception pending in exception_oop
2853 //
2854 int exception_in_tls_offset = __ offset() - start;
2855
2856 // No need to update oop_map as each call to save_live_registers will produce identical oopmap
2857 // Opens a new stack frame
2858 (void) RegisterSaver::save_live_registers(masm, 0, &frame_size_words);
2859
2860 // Restore G2_thread
2861 __ get_thread();
2862
2863 #ifdef ASSERT
2864 {
2865 // verify that there is really an exception oop in exception_oop
2866 Label has_exception;
2867 __ ld_ptr(G2_thread, JavaThread::exception_oop_offset(), Oexception);
2868 __ br_notnull_short(Oexception, Assembler::pt, has_exception);
2869 __ stop("no exception in thread");
2870 __ bind(has_exception);
2871
2872 // verify that there is no pending exception
2873 Label no_pending_exception;
2874 Address exception_addr(G2_thread, Thread::pending_exception_offset());
2875 __ ld_ptr(exception_addr, Oexception);
2876 __ br_null_short(Oexception, Assembler::pt, no_pending_exception);
2877 __ stop("must not have pending exception here");
2878 __ bind(no_pending_exception);
2879 }
2880 #endif
2881
2882 __ ba(cont);
2883 __ delayed()->mov(Deoptimization::Unpack_exception, L0deopt_mode);;
2884
2885 //
2886 // Reexecute entry, similar to c2 uncommon trap
2887 //
2888 int reexecute_offset = __ offset() - start;
2889 #if INCLUDE_JVMCI && !defined(COMPILER1)
2890 if (EnableJVMCI && UseJVMCICompiler) {
2891 // JVMCI does not use this kind of deoptimization
2892 __ should_not_reach_here();
2893 }
2894 #endif
2895 // No need to update oop_map as each call to save_live_registers will produce identical oopmap
2896 (void) RegisterSaver::save_live_registers(masm, 0, &frame_size_words);
2897
2898 __ mov(Deoptimization::Unpack_reexecute, L0deopt_mode);
2899
2900 __ bind(cont);
2901
2902 __ set_last_Java_frame(SP, noreg);
2903
2904 // do the call by hand so we can get the oopmap
2905
2906 __ mov(G2_thread, L7_thread_cache);
2907 __ mov(L0deopt_mode, O1);
2908 __ call(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info), relocInfo::runtime_call_type);
2909 __ delayed()->mov(G2_thread, O0);
2910
2911 // Set an oopmap for the call site this describes all our saved volatile registers
2912
2913 oop_maps->add_gc_map( __ offset()-start, map);
2914
2915 __ mov(L7_thread_cache, G2_thread);
2916
2917 __ reset_last_Java_frame();
2918
2919 #if INCLUDE_JVMCI
2920 if (EnableJVMCI) {
2921 __ bind(after_fetch_unroll_info_call);
2922 }
2923 #endif
2924 // NOTE: we know that only O0/O1 will be reloaded by restore_result_registers
2925 // so this move will survive
2926
2927 __ mov(L0deopt_mode, G4deopt_mode);
2928
2929 __ mov(O0, O2UnrollBlock->after_save());
2930
2931 RegisterSaver::restore_result_registers(masm);
2932
2933 __ ld(O2UnrollBlock, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes(), G4deopt_mode);
2934 Label noException;
2935 __ cmp_and_br_short(G4deopt_mode, Deoptimization::Unpack_exception, Assembler::notEqual, Assembler::pt, noException);
2936
2937 // Move the pending exception from exception_oop to Oexception so
2938 // the pending exception will be picked up the interpreter.
2939 __ ld_ptr(G2_thread, in_bytes(JavaThread::exception_oop_offset()), Oexception);
2940 __ st_ptr(G0, G2_thread, in_bytes(JavaThread::exception_oop_offset()));
2941 __ st_ptr(G0, G2_thread, in_bytes(JavaThread::exception_pc_offset()));
2942 __ bind(noException);
2943
2944 // deallocate the deoptimization frame taking care to preserve the return values
2945 __ mov(Oreturn0, Oreturn0->after_save());
2946 __ mov(Oreturn1, Oreturn1->after_save());
2947 __ mov(O2UnrollBlock, O2UnrollBlock->after_save());
2948 __ restore();
2949
2950 // Allocate new interpreter frame(s) and possible c2i adapter frame
2951
2952 make_new_frames(masm, true);
2953
2954 // push a dummy "unpack_frame" taking care of float return values and
2955 // call Deoptimization::unpack_frames to have the unpacker layout
2956 // information in the interpreter frames just created and then return
2957 // to the interpreter entry point
2958 __ save(SP, -frame_size_words*wordSize, SP);
2959 __ stf(FloatRegisterImpl::D, Freturn0, saved_Freturn0_addr);
2960 // LP64 uses g4 in set_last_Java_frame
2961 __ mov(G4deopt_mode, O1);
2962 __ set_last_Java_frame(SP, G0);
2963 __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames), G2_thread, O1);
2964 __ reset_last_Java_frame();
2965 __ ldf(FloatRegisterImpl::D, saved_Freturn0_addr, Freturn0);
2966
2967 __ ret();
2968 __ delayed()->restore();
2969
2970 masm->flush();
2971 _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_words);
2972 _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
2973 #if INCLUDE_JVMCI
2974 if (EnableJVMCI) {
2975 _deopt_blob->set_uncommon_trap_offset(uncommon_trap_offset);
2976 _deopt_blob->set_implicit_exception_uncommon_trap_offset(implicit_exception_uncommon_trap_offset);
2977 }
2978 #endif
2979 }
2980
2981 #ifdef COMPILER2
2982
2983 //------------------------------generate_uncommon_trap_blob--------------------
2984 // Ought to generate an ideal graph & compile, but here's some SPARC ASM
2985 // instead.
2986 void SharedRuntime::generate_uncommon_trap_blob() {
2987 // allocate space for the code
2988 ResourceMark rm;
2989 // setup code generation tools
2990 int pad = VerifyThread ? 512 : 0;
2991 #ifdef ASSERT
2992 if (UseStackBanging) {
2993 pad += (JavaThread::stack_shadow_zone_size() / os::vm_page_size())*16 + 32;
2994 }
2995 #endif
2996 CodeBuffer buffer("uncommon_trap_blob", 2700+pad, 512);
2997 MacroAssembler* masm = new MacroAssembler(&buffer);
2998 Register O2UnrollBlock = O2;
2999 Register O2klass_index = O2;
3000
3001 //
3002 // This is the entry point for all traps the compiler takes when it thinks
3003 // it cannot handle further execution of compilation code. The frame is
3004 // deoptimized in these cases and converted into interpreter frames for
3005 // execution
3006 // The steps taken by this frame are as follows:
3007 // - push a fake "unpack_frame"
3008 // - call the C routine Deoptimization::uncommon_trap (this function
3009 // packs the current compiled frame into vframe arrays and returns
3010 // information about the number and size of interpreter frames which
3011 // are equivalent to the frame which is being deoptimized)
3012 // - deallocate the "unpack_frame"
3013 // - deallocate the deoptimization frame
3014 // - in a loop using the information returned in the previous step
3015 // push interpreter frames;
3016 // - create a dummy "unpack_frame"
3017 // - call the C routine: Deoptimization::unpack_frames (this function
3018 // lays out values on the interpreter frame which was just created)
3019 // - deallocate the dummy unpack_frame
3020 // - return to the interpreter entry point
3021 //
3022 // Refer to the following methods for more information:
3023 // - Deoptimization::uncommon_trap
3024 // - Deoptimization::unpack_frame
3025
3026 // the unloaded class index is in O0 (first parameter to this blob)
3027
3028 // push a dummy "unpack_frame"
3029 // and call Deoptimization::uncommon_trap to pack the compiled frame into
3030 // vframe array and return the UnrollBlock information
3031 __ save_frame(0);
3032 __ set_last_Java_frame(SP, noreg);
3033 __ mov(I0, O2klass_index);
3034 __ mov(Deoptimization::Unpack_uncommon_trap, O3); // exec mode
3035 __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, Deoptimization::uncommon_trap), G2_thread, O2klass_index, O3);
3036 __ reset_last_Java_frame();
3037 __ mov(O0, O2UnrollBlock->after_save());
3038 __ restore();
3039
3040 // deallocate the deoptimized frame taking care to preserve the return values
3041 __ mov(O2UnrollBlock, O2UnrollBlock->after_save());
3042 __ restore();
3043
3044 #ifdef ASSERT
3045 { Label L;
3046 __ ld(O2UnrollBlock, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes(), O1);
3047 __ cmp_and_br_short(O1, Deoptimization::Unpack_uncommon_trap, Assembler::equal, Assembler::pt, L);
3048 __ stop("SharedRuntime::generate_deopt_blob: expected Unpack_uncommon_trap");
3049 __ bind(L);
3050 }
3051 #endif
3052
3053 // Allocate new interpreter frame(s) and possible c2i adapter frame
3054
3055 make_new_frames(masm, false);
3056
3057 // push a dummy "unpack_frame" taking care of float return values and
3058 // call Deoptimization::unpack_frames to have the unpacker layout
3059 // information in the interpreter frames just created and then return
3060 // to the interpreter entry point
3061 __ save_frame(0);
3062 __ set_last_Java_frame(SP, noreg);
3063 __ mov(Deoptimization::Unpack_uncommon_trap, O3); // indicate it is the uncommon trap case
3064 __ call_VM_leaf(L7_thread_cache, CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames), G2_thread, O3);
3065 __ reset_last_Java_frame();
3066 __ ret();
3067 __ delayed()->restore();
3068
3069 masm->flush();
3070 _uncommon_trap_blob = UncommonTrapBlob::create(&buffer, NULL, __ total_frame_size_in_bytes(0)/wordSize);
3071 }
3072
3073 #endif // COMPILER2
3074
3075 //------------------------------generate_handler_blob-------------------
3076 //
3077 // Generate a special Compile2Runtime blob that saves all registers, and sets
3078 // up an OopMap.
3079 //
3080 // This blob is jumped to (via a breakpoint and the signal handler) from a
3081 // safepoint in compiled code. On entry to this blob, O7 contains the
3082 // address in the original nmethod at which we should resume normal execution.
3083 // Thus, this blob looks like a subroutine which must preserve lots of
3084 // registers and return normally. Note that O7 is never register-allocated,
3085 // so it is guaranteed to be free here.
3086 //
3087
3088 // The hardest part of what this blob must do is to save the 64-bit %o
3089 // registers in the 32-bit build. A simple 'save' turn the %o's to %i's and
3090 // an interrupt will chop off their heads. Making space in the caller's frame
3091 // first will let us save the 64-bit %o's before save'ing, but we cannot hand
3092 // the adjusted FP off to the GC stack-crawler: this will modify the caller's
3093 // SP and mess up HIS OopMaps. So we first adjust the caller's SP, then save
3094 // the 64-bit %o's, then do a save, then fixup the caller's SP (our FP).
3095 // Tricky, tricky, tricky...
3096
3097 SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_type) {
3098 assert (StubRoutines::forward_exception_entry() != NULL, "must be generated before");
3099
3100 // allocate space for the code
3101 ResourceMark rm;
3102 // setup code generation tools
3103 // Measured 8/7/03 at 896 in 32bit debug build (no VerifyThread)
3104 // Measured 8/7/03 at 1080 in 32bit debug build (VerifyThread)
3105 CodeBuffer buffer("handler_blob", 1600, 512);
3106 MacroAssembler* masm = new MacroAssembler(&buffer);
3107 int frame_size_words;
3108 OopMapSet *oop_maps = new OopMapSet();
3109 OopMap* map = NULL;
3110
3111 int start = __ offset();
3112
3113 bool cause_return = (poll_type == POLL_AT_RETURN);
3114 // If this causes a return before the processing, then do a "restore"
3115 if (cause_return) {
3116 __ restore();
3117 } else {
3118 // Make it look like we were called via the poll
3119 // so that frame constructor always sees a valid return address
3120 __ ld_ptr(Address(G2_thread, JavaThread::saved_exception_pc_offset()), O7);
3121 __ sub(O7, frame::pc_return_offset, O7);
3122 }
3123
3124 map = RegisterSaver::save_live_registers(masm, 0, &frame_size_words);
3125
3126 // setup last_Java_sp (blows G4)
3127 __ set_last_Java_frame(SP, noreg);
3128
3129 Register saved_O7 = O7->after_save();
3130 if (!cause_return && SafepointMechanism::uses_thread_local_poll()) {
3131 // Keep a copy of the return pc in L0 to detect if it gets modified
3132 __ mov(saved_O7, L0);
3133 // Adjust and keep a copy of our npc saved by the signal handler
3134 __ ld_ptr(Address(G2_thread, JavaThread::saved_exception_npc_offset()), L1);
3135 __ sub(L1, frame::pc_return_offset, L1);
3136 }
3137
3138 // call into the runtime to handle illegal instructions exception
3139 // Do not use call_VM_leaf, because we need to make a GC map at this call site.
3140 __ mov(G2_thread, O0);
3141 __ save_thread(L7_thread_cache);
3142 __ call(call_ptr);
3143 __ delayed()->nop();
3144
3145 // Set an oopmap for the call site.
3146 // We need this not only for callee-saved registers, but also for volatile
3147 // registers that the compiler might be keeping live across a safepoint.
3148
3149 oop_maps->add_gc_map( __ offset() - start, map);
3150
3151 __ restore_thread(L7_thread_cache);
3152 // clear last_Java_sp
3153 __ reset_last_Java_frame();
3154
3155 // Check for exceptions
3156 Label pending;
3157
3158 __ ld_ptr(G2_thread, in_bytes(Thread::pending_exception_offset()), O1);
3159 __ br_notnull_short(O1, Assembler::pn, pending);
3160
3161 if (!cause_return && SafepointMechanism::uses_thread_local_poll()) {
3162 // If nobody modified our return pc then we must return to the npc which he saved in L1
3163 __ cmp(saved_O7, L0);
3164 __ movcc(Assembler::equal, false, Assembler::ptr_cc, L1, saved_O7);
3165 }
3166
3167 RegisterSaver::restore_live_registers(masm);
3168
3169 // We are back the the original state on entry and ready to go.
3170
3171 __ retl();
3172 __ delayed()->nop();
3173
3174 // Pending exception after the safepoint
3175
3176 __ bind(pending);
3177
3178 RegisterSaver::restore_live_registers(masm);
3179
3180 // We are back the the original state on entry.
3181
3182 // Tail-call forward_exception_entry, with the issuing PC in O7,
3183 // so it looks like the original nmethod called forward_exception_entry.
3184 __ set((intptr_t)StubRoutines::forward_exception_entry(), O0);
3185 __ JMP(O0, 0);
3186 __ delayed()->nop();
3187
3188 // -------------
3189 // make sure all code is generated
3190 masm->flush();
3191
3192 // return exception blob
3193 return SafepointBlob::create(&buffer, oop_maps, frame_size_words);
3194 }
3195
3196 //
3197 // generate_resolve_blob - call resolution (static/virtual/opt-virtual/ic-miss
3198 //
3199 // Generate a stub that calls into vm to find out the proper destination
3200 // of a java call. All the argument registers are live at this point
3201 // but since this is generic code we don't know what they are and the caller
3202 // must do any gc of the args.
3203 //
3204 RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const char* name) {
3205 assert (StubRoutines::forward_exception_entry() != NULL, "must be generated before");
3206
3207 // allocate space for the code
3208 ResourceMark rm;
3209 // setup code generation tools
3210 // Measured 8/7/03 at 896 in 32bit debug build (no VerifyThread)
3211 // Measured 8/7/03 at 1080 in 32bit debug build (VerifyThread)
3212 CodeBuffer buffer(name, 1600, 512);
3213 MacroAssembler* masm = new MacroAssembler(&buffer);
3214 int frame_size_words;
3215 OopMapSet *oop_maps = new OopMapSet();
3216 OopMap* map = NULL;
3217
3218 int start = __ offset();
3219
3220 map = RegisterSaver::save_live_registers(masm, 0, &frame_size_words);
3221
3222 int frame_complete = __ offset();
3223
3224 // setup last_Java_sp (blows G4)
3225 __ set_last_Java_frame(SP, noreg);
3226
3227 // call into the runtime to handle illegal instructions exception
3228 // Do not use call_VM_leaf, because we need to make a GC map at this call site.
3229 __ mov(G2_thread, O0);
3230 __ save_thread(L7_thread_cache);
3231 __ call(destination, relocInfo::runtime_call_type);
3232 __ delayed()->nop();
3233
3234 // O0 contains the address we are going to jump to assuming no exception got installed
3235
3236 // Set an oopmap for the call site.
3237 // We need this not only for callee-saved registers, but also for volatile
3238 // registers that the compiler might be keeping live across a safepoint.
3239
3240 oop_maps->add_gc_map( __ offset() - start, map);
3241
3242 __ restore_thread(L7_thread_cache);
3243 // clear last_Java_sp
3244 __ reset_last_Java_frame();
3245
3246 // Check for exceptions
3247 Label pending;
3248
3249 __ ld_ptr(G2_thread, in_bytes(Thread::pending_exception_offset()), O1);
3250 __ br_notnull_short(O1, Assembler::pn, pending);
3251
3252 // get the returned Method*
3253
3254 __ get_vm_result_2(G5_method);
3255 __ stx(G5_method, SP, RegisterSaver::G5_offset()+STACK_BIAS);
3256
3257 // O0 is where we want to jump, overwrite G3 which is saved and scratch
3258
3259 __ stx(O0, SP, RegisterSaver::G3_offset()+STACK_BIAS);
3260
3261 RegisterSaver::restore_live_registers(masm);
3262
3263 // We are back the the original state on entry and ready to go.
3264
3265 __ JMP(G3, 0);
3266 __ delayed()->nop();
3267
3268 // Pending exception after the safepoint
3269
3270 __ bind(pending);
3271
3272 RegisterSaver::restore_live_registers(masm);
3273
3274 // We are back the the original state on entry.
3275
3276 // Tail-call forward_exception_entry, with the issuing PC in O7,
3277 // so it looks like the original nmethod called forward_exception_entry.
3278 __ set((intptr_t)StubRoutines::forward_exception_entry(), O0);
3279 __ JMP(O0, 0);
3280 __ delayed()->nop();
3281
3282 // -------------
3283 // make sure all code is generated
3284 masm->flush();
3285
3286 // return the blob
3287 // frame_size_words or bytes??
3288 return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_words, oop_maps, true);
3289 }