1 /*
2 * Copyright (c) 2003, 2018, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #ifndef _WINDOWS
27 #include "alloca.h"
28 #endif
29 #include "asm/macroAssembler.hpp"
30 #include "asm/macroAssembler.inline.hpp"
31 #include "code/debugInfoRec.hpp"
32 #include "code/icBuffer.hpp"
33 #include "code/nativeInst.hpp"
34 #include "code/vtableStubs.hpp"
35 #include "gc/shared/collectedHeap.hpp"
36 #include "gc/shared/gcLocker.hpp"
37 #include "gc/shared/barrierSet.hpp"
38 #include "gc/shared/barrierSetAssembler.hpp"
39 #include "interpreter/interpreter.hpp"
40 #include "logging/log.hpp"
41 #include "memory/resourceArea.hpp"
42 #include "oops/compiledICHolder.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/vframeArray.hpp"
46 #include "utilities/align.hpp"
47 #include "utilities/formatBuffer.hpp"
48 #include "vm_version_x86.hpp"
49 #include "vmreg_x86.inline.hpp"
50 #ifdef COMPILER1
51 #include "c1/c1_Runtime1.hpp"
52 #endif
53 #ifdef COMPILER2
54 #include "opto/runtime.hpp"
55 #endif
56 #if INCLUDE_JVMCI
57 #include "jvmci/jvmciJavaClasses.hpp"
58 #endif
59
60 #define __ masm->
61
62 const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;
63
64 class SimpleRuntimeFrame {
65
66 public:
67
68 // Most of the runtime stubs have this simple frame layout.
69 // This class exists to make the layout shared in one place.
70 // Offsets are for compiler stack slots, which are jints.
71 enum layout {
72 // The frame sender code expects that rbp will be in the "natural" place and
73 // will override any oopMap setting for it. We must therefore force the layout
74 // so that it agrees with the frame sender code.
75 rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
76 rbp_off2,
77 return_off, return_off2,
78 framesize
79 };
80 };
81
82 class RegisterSaver {
83 // Capture info about frame layout. Layout offsets are in jint
84 // units because compiler frame slots are jints.
85 #define XSAVE_AREA_BEGIN 160
86 #define XSAVE_AREA_YMM_BEGIN 576
87 #define XSAVE_AREA_ZMM_BEGIN 1152
88 #define XSAVE_AREA_UPPERBANK 1664
89 #define DEF_XMM_OFFS(regnum) xmm ## regnum ## _off = xmm_off + (regnum)*16/BytesPerInt, xmm ## regnum ## H_off
90 #define DEF_YMM_OFFS(regnum) ymm ## regnum ## _off = ymm_off + (regnum)*16/BytesPerInt, ymm ## regnum ## H_off
91 #define DEF_ZMM_OFFS(regnum) zmm ## regnum ## _off = zmm_off + (regnum-16)*64/BytesPerInt, zmm ## regnum ## H_off
92 enum layout {
93 fpu_state_off = frame::arg_reg_save_area_bytes/BytesPerInt, // fxsave save area
94 xmm_off = fpu_state_off + XSAVE_AREA_BEGIN/BytesPerInt, // offset in fxsave save area
95 DEF_XMM_OFFS(0),
96 DEF_XMM_OFFS(1),
97 // 2..15 are implied in range usage
98 ymm_off = xmm_off + (XSAVE_AREA_YMM_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt,
99 DEF_YMM_OFFS(0),
100 DEF_YMM_OFFS(1),
101 // 2..15 are implied in range usage
102 zmm_high = xmm_off + (XSAVE_AREA_ZMM_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt,
103 zmm_off = xmm_off + (XSAVE_AREA_UPPERBANK - XSAVE_AREA_BEGIN)/BytesPerInt,
104 DEF_ZMM_OFFS(16),
105 DEF_ZMM_OFFS(17),
106 // 18..31 are implied in range usage
107 fpu_state_end = fpu_state_off + ((FPUStateSizeInWords-1)*wordSize / BytesPerInt),
108 fpu_stateH_end,
109 r15_off, r15H_off,
110 r14_off, r14H_off,
111 r13_off, r13H_off,
112 r12_off, r12H_off,
113 r11_off, r11H_off,
114 r10_off, r10H_off,
115 r9_off, r9H_off,
116 r8_off, r8H_off,
117 rdi_off, rdiH_off,
118 rsi_off, rsiH_off,
119 ignore_off, ignoreH_off, // extra copy of rbp
120 rsp_off, rspH_off,
121 rbx_off, rbxH_off,
122 rdx_off, rdxH_off,
123 rcx_off, rcxH_off,
124 rax_off, raxH_off,
125 // 16-byte stack alignment fill word: see MacroAssembler::push/pop_IU_state
126 align_off, alignH_off,
127 flags_off, flagsH_off,
128 // The frame sender code expects that rbp will be in the "natural" place and
129 // will override any oopMap setting for it. We must therefore force the layout
130 // so that it agrees with the frame sender code.
131 rbp_off, rbpH_off, // copy of rbp we will restore
132 return_off, returnH_off, // slot for return address
133 reg_save_size // size in compiler stack slots
134 };
135
136 public:
137 static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_vectors = false);
138 static void restore_live_registers(MacroAssembler* masm, bool restore_vectors = false);
139
140 // Offsets into the register save area
141 // Used by deoptimization when it is managing result register
142 // values on its own
143
144 static int rax_offset_in_bytes(void) { return BytesPerInt * rax_off; }
145 static int rdx_offset_in_bytes(void) { return BytesPerInt * rdx_off; }
146 static int rbx_offset_in_bytes(void) { return BytesPerInt * rbx_off; }
147 static int xmm0_offset_in_bytes(void) { return BytesPerInt * xmm0_off; }
148 static int return_offset_in_bytes(void) { return BytesPerInt * return_off; }
149
150 // During deoptimization only the result registers need to be restored,
151 // all the other values have already been extracted.
152 static void restore_result_registers(MacroAssembler* masm);
153 };
154
155 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_vectors) {
156 int off = 0;
157 int num_xmm_regs = XMMRegisterImpl::number_of_registers;
158 if (UseAVX < 3) {
159 num_xmm_regs = num_xmm_regs/2;
160 }
161 #if COMPILER2_OR_JVMCI
162 if (save_vectors) {
163 assert(UseAVX > 0, "Vectors larger than 16 byte long are supported only with AVX");
164 assert(MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported");
165 }
166 #else
167 assert(!save_vectors, "vectors are generated only by C2 and JVMCI");
168 #endif
169
170 // Always make the frame size 16-byte aligned, both vector and non vector stacks are always allocated
171 int frame_size_in_bytes = align_up(reg_save_size*BytesPerInt, num_xmm_regs);
172 // OopMap frame size is in compiler stack slots (jint's) not bytes or words
173 int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
174 // CodeBlob frame size is in words.
175 int frame_size_in_words = frame_size_in_bytes / wordSize;
176 *total_frame_words = frame_size_in_words;
177
178 // Save registers, fpu state, and flags.
179 // We assume caller has already pushed the return address onto the
180 // stack, so rsp is 8-byte aligned here.
181 // We push rpb twice in this sequence because we want the real rbp
182 // to be under the return like a normal enter.
183
184 __ enter(); // rsp becomes 16-byte aligned here
185 __ push_CPU_state(); // Push a multiple of 16 bytes
186
187 // push cpu state handles this on EVEX enabled targets
188 if (save_vectors) {
189 // Save upper half of YMM registers(0..15)
190 int base_addr = XSAVE_AREA_YMM_BEGIN;
191 for (int n = 0; n < 16; n++) {
192 __ vextractf128_high(Address(rsp, base_addr+n*16), as_XMMRegister(n));
193 }
194 if (VM_Version::supports_evex()) {
195 // Save upper half of ZMM registers(0..15)
196 base_addr = XSAVE_AREA_ZMM_BEGIN;
197 for (int n = 0; n < 16; n++) {
198 __ vextractf64x4_high(Address(rsp, base_addr+n*32), as_XMMRegister(n));
199 }
200 // Save full ZMM registers(16..num_xmm_regs)
201 base_addr = XSAVE_AREA_UPPERBANK;
202 off = 0;
203 int vector_len = Assembler::AVX_512bit;
204 for (int n = 16; n < num_xmm_regs; n++) {
205 __ evmovdqul(Address(rsp, base_addr+(off++*64)), as_XMMRegister(n), vector_len);
206 }
207 }
208 } else {
209 if (VM_Version::supports_evex()) {
210 // Save upper bank of ZMM registers(16..31) for double/float usage
211 int base_addr = XSAVE_AREA_UPPERBANK;
212 off = 0;
213 for (int n = 16; n < num_xmm_regs; n++) {
214 __ movsd(Address(rsp, base_addr+(off++*64)), as_XMMRegister(n));
215 }
216 }
217 }
218 __ vzeroupper();
219 if (frame::arg_reg_save_area_bytes != 0) {
220 // Allocate argument register save area
221 __ subptr(rsp, frame::arg_reg_save_area_bytes);
222 }
223
224 // Set an oopmap for the call site. This oopmap will map all
225 // oop-registers and debug-info registers as callee-saved. This
226 // will allow deoptimization at this safepoint to find all possible
227 // debug-info recordings, as well as let GC find all oops.
228
229 OopMapSet *oop_maps = new OopMapSet();
230 OopMap* map = new OopMap(frame_size_in_slots, 0);
231
232 #define STACK_OFFSET(x) VMRegImpl::stack2reg((x))
233
234 map->set_callee_saved(STACK_OFFSET( rax_off ), rax->as_VMReg());
235 map->set_callee_saved(STACK_OFFSET( rcx_off ), rcx->as_VMReg());
236 map->set_callee_saved(STACK_OFFSET( rdx_off ), rdx->as_VMReg());
237 map->set_callee_saved(STACK_OFFSET( rbx_off ), rbx->as_VMReg());
238 // rbp location is known implicitly by the frame sender code, needs no oopmap
239 // and the location where rbp was saved by is ignored
240 map->set_callee_saved(STACK_OFFSET( rsi_off ), rsi->as_VMReg());
241 map->set_callee_saved(STACK_OFFSET( rdi_off ), rdi->as_VMReg());
242 map->set_callee_saved(STACK_OFFSET( r8_off ), r8->as_VMReg());
243 map->set_callee_saved(STACK_OFFSET( r9_off ), r9->as_VMReg());
244 map->set_callee_saved(STACK_OFFSET( r10_off ), r10->as_VMReg());
245 map->set_callee_saved(STACK_OFFSET( r11_off ), r11->as_VMReg());
246 map->set_callee_saved(STACK_OFFSET( r12_off ), r12->as_VMReg());
247 map->set_callee_saved(STACK_OFFSET( r13_off ), r13->as_VMReg());
248 map->set_callee_saved(STACK_OFFSET( r14_off ), r14->as_VMReg());
249 map->set_callee_saved(STACK_OFFSET( r15_off ), r15->as_VMReg());
250 // For both AVX and EVEX we will use the legacy FXSAVE area for xmm0..xmm15,
251 // on EVEX enabled targets, we get it included in the xsave area
252 off = xmm0_off;
253 int delta = xmm1_off - off;
254 for (int n = 0; n < 16; n++) {
255 XMMRegister xmm_name = as_XMMRegister(n);
256 map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg());
257 off += delta;
258 }
259 if(UseAVX > 2) {
260 // Obtain xmm16..xmm31 from the XSAVE area on EVEX enabled targets
261 off = zmm16_off;
262 delta = zmm17_off - off;
263 for (int n = 16; n < num_xmm_regs; n++) {
264 XMMRegister zmm_name = as_XMMRegister(n);
265 map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg());
266 off += delta;
267 }
268 }
269
270 #if COMPILER2_OR_JVMCI
271 if (save_vectors) {
272 off = ymm0_off;
273 int delta = ymm1_off - off;
274 for (int n = 0; n < 16; n++) {
275 XMMRegister ymm_name = as_XMMRegister(n);
276 map->set_callee_saved(STACK_OFFSET(off), ymm_name->as_VMReg()->next(4));
277 off += delta;
278 }
279 }
280 #endif // COMPILER2_OR_JVMCI
281
282 // %%% These should all be a waste but we'll keep things as they were for now
283 if (true) {
284 map->set_callee_saved(STACK_OFFSET( raxH_off ), rax->as_VMReg()->next());
285 map->set_callee_saved(STACK_OFFSET( rcxH_off ), rcx->as_VMReg()->next());
286 map->set_callee_saved(STACK_OFFSET( rdxH_off ), rdx->as_VMReg()->next());
287 map->set_callee_saved(STACK_OFFSET( rbxH_off ), rbx->as_VMReg()->next());
288 // rbp location is known implicitly by the frame sender code, needs no oopmap
289 map->set_callee_saved(STACK_OFFSET( rsiH_off ), rsi->as_VMReg()->next());
290 map->set_callee_saved(STACK_OFFSET( rdiH_off ), rdi->as_VMReg()->next());
291 map->set_callee_saved(STACK_OFFSET( r8H_off ), r8->as_VMReg()->next());
292 map->set_callee_saved(STACK_OFFSET( r9H_off ), r9->as_VMReg()->next());
293 map->set_callee_saved(STACK_OFFSET( r10H_off ), r10->as_VMReg()->next());
294 map->set_callee_saved(STACK_OFFSET( r11H_off ), r11->as_VMReg()->next());
295 map->set_callee_saved(STACK_OFFSET( r12H_off ), r12->as_VMReg()->next());
296 map->set_callee_saved(STACK_OFFSET( r13H_off ), r13->as_VMReg()->next());
297 map->set_callee_saved(STACK_OFFSET( r14H_off ), r14->as_VMReg()->next());
298 map->set_callee_saved(STACK_OFFSET( r15H_off ), r15->as_VMReg()->next());
299 // For both AVX and EVEX we will use the legacy FXSAVE area for xmm0..xmm15,
300 // on EVEX enabled targets, we get it included in the xsave area
301 off = xmm0H_off;
302 delta = xmm1H_off - off;
303 for (int n = 0; n < 16; n++) {
304 XMMRegister xmm_name = as_XMMRegister(n);
305 map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg()->next());
306 off += delta;
307 }
308 if (UseAVX > 2) {
309 // Obtain xmm16..xmm31 from the XSAVE area on EVEX enabled targets
310 off = zmm16H_off;
311 delta = zmm17H_off - off;
312 for (int n = 16; n < num_xmm_regs; n++) {
313 XMMRegister zmm_name = as_XMMRegister(n);
314 map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg()->next());
315 off += delta;
316 }
317 }
318 }
319
320 return map;
321 }
322
323 void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_vectors) {
324 int num_xmm_regs = XMMRegisterImpl::number_of_registers;
325 if (UseAVX < 3) {
326 num_xmm_regs = num_xmm_regs/2;
327 }
328 if (frame::arg_reg_save_area_bytes != 0) {
329 // Pop arg register save area
330 __ addptr(rsp, frame::arg_reg_save_area_bytes);
331 }
332
333 #if COMPILER2_OR_JVMCI
334 if (restore_vectors) {
335 assert(UseAVX > 0, "Vectors larger than 16 byte long are supported only with AVX");
336 assert(MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported");
337 }
338 #else
339 assert(!restore_vectors, "vectors are generated only by C2");
340 #endif
341
342 __ vzeroupper();
343
344 // On EVEX enabled targets everything is handled in pop fpu state
345 if (restore_vectors) {
346 // Restore upper half of YMM registers (0..15)
347 int base_addr = XSAVE_AREA_YMM_BEGIN;
348 for (int n = 0; n < 16; n++) {
349 __ vinsertf128_high(as_XMMRegister(n), Address(rsp, base_addr+n*16));
350 }
351 if (VM_Version::supports_evex()) {
352 // Restore upper half of ZMM registers (0..15)
353 base_addr = XSAVE_AREA_ZMM_BEGIN;
354 for (int n = 0; n < 16; n++) {
355 __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, base_addr+n*32));
356 }
357 // Restore full ZMM registers(16..num_xmm_regs)
358 base_addr = XSAVE_AREA_UPPERBANK;
359 int vector_len = Assembler::AVX_512bit;
360 int off = 0;
361 for (int n = 16; n < num_xmm_regs; n++) {
362 __ evmovdqul(as_XMMRegister(n), Address(rsp, base_addr+(off++*64)), vector_len);
363 }
364 }
365 } else {
366 if (VM_Version::supports_evex()) {
367 // Restore upper bank of ZMM registers(16..31) for double/float usage
368 int base_addr = XSAVE_AREA_UPPERBANK;
369 int off = 0;
370 for (int n = 16; n < num_xmm_regs; n++) {
371 __ movsd(as_XMMRegister(n), Address(rsp, base_addr+(off++*64)));
372 }
373 }
374 }
375
376 // Recover CPU state
377 __ pop_CPU_state();
378 // Get the rbp described implicitly by the calling convention (no oopMap)
379 __ pop(rbp);
380 }
381
382 void RegisterSaver::restore_result_registers(MacroAssembler* masm) {
383
384 // Just restore result register. Only used by deoptimization. By
385 // now any callee save register that needs to be restored to a c2
386 // caller of the deoptee has been extracted into the vframeArray
387 // and will be stuffed into the c2i adapter we create for later
388 // restoration so only result registers need to be restored here.
389
390 // Restore fp result register
391 __ movdbl(xmm0, Address(rsp, xmm0_offset_in_bytes()));
392 // Restore integer result register
393 __ movptr(rax, Address(rsp, rax_offset_in_bytes()));
394 __ movptr(rdx, Address(rsp, rdx_offset_in_bytes()));
395
396 // Pop all of the register save are off the stack except the return address
397 __ addptr(rsp, return_offset_in_bytes());
398 }
399
400 // Is vector's size (in bytes) bigger than a size saved by default?
401 // 16 bytes XMM registers are saved by default using fxsave/fxrstor instructions.
402 bool SharedRuntime::is_wide_vector(int size) {
403 return size > 16;
404 }
405
406 size_t SharedRuntime::trampoline_size() {
407 return 16;
408 }
409
410 void SharedRuntime::generate_trampoline(MacroAssembler *masm, address destination) {
411 __ jump(RuntimeAddress(destination));
412 }
413
414 // The java_calling_convention describes stack locations as ideal slots on
415 // a frame with no abi restrictions. Since we must observe abi restrictions
416 // (like the placement of the register window) the slots must be biased by
417 // the following value.
418 static int reg2offset_in(VMReg r) {
419 // Account for saved rbp and return address
420 // This should really be in_preserve_stack_slots
421 return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
422 }
423
424 static int reg2offset_out(VMReg r) {
425 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
426 }
427
428 // ---------------------------------------------------------------------------
429 // Read the array of BasicTypes from a signature, and compute where the
430 // arguments should go. Values in the VMRegPair regs array refer to 4-byte
431 // quantities. Values less than VMRegImpl::stack0 are registers, those above
432 // refer to 4-byte stack slots. All stack slots are based off of the stack pointer
433 // as framesizes are fixed.
434 // VMRegImpl::stack0 refers to the first slot 0(sp).
435 // and VMRegImpl::stack0+1 refers to the memory word 4-byes higher. Register
436 // up to RegisterImpl::number_of_registers) are the 64-bit
437 // integer registers.
438
439 // Note: the INPUTS in sig_bt are in units of Java argument words, which are
440 // either 32-bit or 64-bit depending on the build. The OUTPUTS are in 32-bit
441 // units regardless of build. Of course for i486 there is no 64 bit build
442
443 // The Java calling convention is a "shifted" version of the C ABI.
444 // By skipping the first C ABI register we can call non-static jni methods
445 // with small numbers of arguments without having to shuffle the arguments
446 // at all. Since we control the java ABI we ought to at least get some
447 // advantage out of it.
448
449 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
450 VMRegPair *regs,
451 int total_args_passed,
452 int is_outgoing) {
453
454 // Create the mapping between argument positions and
455 // registers.
456 static const Register INT_ArgReg[Argument::n_int_register_parameters_j] = {
457 j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5
458 };
459 static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_j] = {
460 j_farg0, j_farg1, j_farg2, j_farg3,
461 j_farg4, j_farg5, j_farg6, j_farg7
462 };
463
464
465 uint int_args = 0;
466 uint fp_args = 0;
467 uint stk_args = 0; // inc by 2 each time
468
469 for (int i = 0; i < total_args_passed; i++) {
470 switch (sig_bt[i]) {
471 case T_BOOLEAN:
472 case T_CHAR:
473 case T_BYTE:
474 case T_SHORT:
475 case T_INT:
476 if (int_args < Argument::n_int_register_parameters_j) {
477 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
478 } else {
479 regs[i].set1(VMRegImpl::stack2reg(stk_args));
480 stk_args += 2;
481 }
482 break;
483 case T_VOID:
484 // halves of T_LONG or T_DOUBLE
485 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
486 regs[i].set_bad();
487 break;
488 case T_LONG:
489 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
490 // fall through
491 case T_OBJECT:
492 case T_ARRAY:
493 case T_ADDRESS:
494 if (int_args < Argument::n_int_register_parameters_j) {
495 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
496 } else {
497 regs[i].set2(VMRegImpl::stack2reg(stk_args));
498 stk_args += 2;
499 }
500 break;
501 case T_FLOAT:
502 if (fp_args < Argument::n_float_register_parameters_j) {
503 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
504 } else {
505 regs[i].set1(VMRegImpl::stack2reg(stk_args));
506 stk_args += 2;
507 }
508 break;
509 case T_DOUBLE:
510 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
511 if (fp_args < Argument::n_float_register_parameters_j) {
512 regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
513 } else {
514 regs[i].set2(VMRegImpl::stack2reg(stk_args));
515 stk_args += 2;
516 }
517 break;
518 default:
519 ShouldNotReachHere();
520 break;
521 }
522 }
523
524 return align_up(stk_args, 2);
525 }
526
527 // Patch the callers callsite with entry to compiled code if it exists.
528 static void patch_callers_callsite(MacroAssembler *masm) {
529 Label L;
530 __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), (int32_t)NULL_WORD);
531 __ jcc(Assembler::equal, L);
532
533 // Save the current stack pointer
534 __ mov(r13, rsp);
535 // Schedule the branch target address early.
536 // Call into the VM to patch the caller, then jump to compiled callee
537 // rax isn't live so capture return address while we easily can
538 __ movptr(rax, Address(rsp, 0));
539
540 // align stack so push_CPU_state doesn't fault
541 __ andptr(rsp, -(StackAlignmentInBytes));
542 __ push_CPU_state();
543 __ vzeroupper();
544 // VM needs caller's callsite
545 // VM needs target method
546 // This needs to be a long call since we will relocate this adapter to
547 // the codeBuffer and it may not reach
548
549 // Allocate argument register save area
550 if (frame::arg_reg_save_area_bytes != 0) {
551 __ subptr(rsp, frame::arg_reg_save_area_bytes);
552 }
553 __ mov(c_rarg0, rbx);
554 __ mov(c_rarg1, rax);
555 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
556
557 // De-allocate argument register save area
558 if (frame::arg_reg_save_area_bytes != 0) {
559 __ addptr(rsp, frame::arg_reg_save_area_bytes);
560 }
561
562 __ vzeroupper();
563 __ pop_CPU_state();
564 // restore sp
565 __ mov(rsp, r13);
566 __ bind(L);
567 }
568
569
570 static void gen_c2i_adapter(MacroAssembler *masm,
571 int total_args_passed,
572 int comp_args_on_stack,
573 const BasicType *sig_bt,
574 const VMRegPair *regs,
575 Label& skip_fixup) {
576 // Before we get into the guts of the C2I adapter, see if we should be here
577 // at all. We've come from compiled code and are attempting to jump to the
578 // interpreter, which means the caller made a static call to get here
579 // (vcalls always get a compiled target if there is one). Check for a
580 // compiled target. If there is one, we need to patch the caller's call.
581 patch_callers_callsite(masm);
582
583 __ bind(skip_fixup);
584
585 // Since all args are passed on the stack, total_args_passed *
586 // Interpreter::stackElementSize is the space we need. Plus 1 because
587 // we also account for the return address location since
588 // we store it first rather than hold it in rax across all the shuffling
589
590 int extraspace = (total_args_passed * Interpreter::stackElementSize) + wordSize;
591
592 // stack is aligned, keep it that way
593 extraspace = align_up(extraspace, 2*wordSize);
594
595 // Get return address
596 __ pop(rax);
597
598 // set senderSP value
599 __ mov(r13, rsp);
600
601 __ subptr(rsp, extraspace);
602
603 // Store the return address in the expected location
604 __ movptr(Address(rsp, 0), rax);
605
606 // Now write the args into the outgoing interpreter space
607 for (int i = 0; i < total_args_passed; i++) {
608 if (sig_bt[i] == T_VOID) {
609 assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
610 continue;
611 }
612
613 // offset to start parameters
614 int st_off = (total_args_passed - i) * Interpreter::stackElementSize;
615 int next_off = st_off - Interpreter::stackElementSize;
616
617 // Say 4 args:
618 // i st_off
619 // 0 32 T_LONG
620 // 1 24 T_VOID
621 // 2 16 T_OBJECT
622 // 3 8 T_BOOL
623 // - 0 return address
624 //
625 // However to make thing extra confusing. Because we can fit a long/double in
626 // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
627 // leaves one slot empty and only stores to a single slot. In this case the
628 // slot that is occupied is the T_VOID slot. See I said it was confusing.
629
630 VMReg r_1 = regs[i].first();
631 VMReg r_2 = regs[i].second();
632 if (!r_1->is_valid()) {
633 assert(!r_2->is_valid(), "");
634 continue;
635 }
636 if (r_1->is_stack()) {
637 // memory to memory use rax
638 int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
639 if (!r_2->is_valid()) {
640 // sign extend??
641 __ movl(rax, Address(rsp, ld_off));
642 __ movptr(Address(rsp, st_off), rax);
643
644 } else {
645
646 __ movq(rax, Address(rsp, ld_off));
647
648 // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
649 // T_DOUBLE and T_LONG use two slots in the interpreter
650 if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
651 // ld_off == LSW, ld_off+wordSize == MSW
652 // st_off == MSW, next_off == LSW
653 __ movq(Address(rsp, next_off), rax);
654 #ifdef ASSERT
655 // Overwrite the unused slot with known junk
656 __ mov64(rax, CONST64(0xdeadffffdeadaaaa));
657 __ movptr(Address(rsp, st_off), rax);
658 #endif /* ASSERT */
659 } else {
660 __ movq(Address(rsp, st_off), rax);
661 }
662 }
663 } else if (r_1->is_Register()) {
664 Register r = r_1->as_Register();
665 if (!r_2->is_valid()) {
666 // must be only an int (or less ) so move only 32bits to slot
667 // why not sign extend??
668 __ movl(Address(rsp, st_off), r);
669 } else {
670 // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
671 // T_DOUBLE and T_LONG use two slots in the interpreter
672 if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
673 // long/double in gpr
674 #ifdef ASSERT
675 // Overwrite the unused slot with known junk
676 __ mov64(rax, CONST64(0xdeadffffdeadaaab));
677 __ movptr(Address(rsp, st_off), rax);
678 #endif /* ASSERT */
679 __ movq(Address(rsp, next_off), r);
680 } else {
681 __ movptr(Address(rsp, st_off), r);
682 }
683 }
684 } else {
685 assert(r_1->is_XMMRegister(), "");
686 if (!r_2->is_valid()) {
687 // only a float use just part of the slot
688 __ movflt(Address(rsp, st_off), r_1->as_XMMRegister());
689 } else {
690 #ifdef ASSERT
691 // Overwrite the unused slot with known junk
692 __ mov64(rax, CONST64(0xdeadffffdeadaaac));
693 __ movptr(Address(rsp, st_off), rax);
694 #endif /* ASSERT */
695 __ movdbl(Address(rsp, next_off), r_1->as_XMMRegister());
696 }
697 }
698 }
699
700 // Schedule the branch target address early.
701 __ movptr(rcx, Address(rbx, in_bytes(Method::interpreter_entry_offset())));
702 __ jmp(rcx);
703 }
704
705 static void range_check(MacroAssembler* masm, Register pc_reg, Register temp_reg,
706 address code_start, address code_end,
707 Label& L_ok) {
708 Label L_fail;
709 __ lea(temp_reg, ExternalAddress(code_start));
710 __ cmpptr(pc_reg, temp_reg);
711 __ jcc(Assembler::belowEqual, L_fail);
712 __ lea(temp_reg, ExternalAddress(code_end));
713 __ cmpptr(pc_reg, temp_reg);
714 __ jcc(Assembler::below, L_ok);
715 __ bind(L_fail);
716 }
717
718 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
719 int total_args_passed,
720 int comp_args_on_stack,
721 const BasicType *sig_bt,
722 const VMRegPair *regs) {
723
724 // Note: r13 contains the senderSP on entry. We must preserve it since
725 // we may do a i2c -> c2i transition if we lose a race where compiled
726 // code goes non-entrant while we get args ready.
727 // In addition we use r13 to locate all the interpreter args as
728 // we must align the stack to 16 bytes on an i2c entry else we
729 // lose alignment we expect in all compiled code and register
730 // save code can segv when fxsave instructions find improperly
731 // aligned stack pointer.
732
733 // Adapters can be frameless because they do not require the caller
734 // to perform additional cleanup work, such as correcting the stack pointer.
735 // An i2c adapter is frameless because the *caller* frame, which is interpreted,
736 // routinely repairs its own stack pointer (from interpreter_frame_last_sp),
737 // even if a callee has modified the stack pointer.
738 // A c2i adapter is frameless because the *callee* frame, which is interpreted,
739 // routinely repairs its caller's stack pointer (from sender_sp, which is set
740 // up via the senderSP register).
741 // In other words, if *either* the caller or callee is interpreted, we can
742 // get the stack pointer repaired after a call.
743 // This is why c2i and i2c adapters cannot be indefinitely composed.
744 // In particular, if a c2i adapter were to somehow call an i2c adapter,
745 // both caller and callee would be compiled methods, and neither would
746 // clean up the stack pointer changes performed by the two adapters.
747 // If this happens, control eventually transfers back to the compiled
748 // caller, but with an uncorrected stack, causing delayed havoc.
749
750 // Pick up the return address
751 __ movptr(rax, Address(rsp, 0));
752
753 if (VerifyAdapterCalls &&
754 (Interpreter::code() != NULL || StubRoutines::code1() != NULL)) {
755 // So, let's test for cascading c2i/i2c adapters right now.
756 // assert(Interpreter::contains($return_addr) ||
757 // StubRoutines::contains($return_addr),
758 // "i2c adapter must return to an interpreter frame");
759 __ block_comment("verify_i2c { ");
760 Label L_ok;
761 if (Interpreter::code() != NULL)
762 range_check(masm, rax, r11,
763 Interpreter::code()->code_start(), Interpreter::code()->code_end(),
764 L_ok);
765 if (StubRoutines::code1() != NULL)
766 range_check(masm, rax, r11,
767 StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
768 L_ok);
769 if (StubRoutines::code2() != NULL)
770 range_check(masm, rax, r11,
771 StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
772 L_ok);
773 const char* msg = "i2c adapter must return to an interpreter frame";
774 __ block_comment(msg);
775 __ stop(msg);
776 __ bind(L_ok);
777 __ block_comment("} verify_i2ce ");
778 }
779
780 // Must preserve original SP for loading incoming arguments because
781 // we need to align the outgoing SP for compiled code.
782 __ movptr(r11, rsp);
783
784 // Cut-out for having no stack args. Since up to 2 int/oop args are passed
785 // in registers, we will occasionally have no stack args.
786 int comp_words_on_stack = 0;
787 if (comp_args_on_stack) {
788 // Sig words on the stack are greater-than VMRegImpl::stack0. Those in
789 // registers are below. By subtracting stack0, we either get a negative
790 // number (all values in registers) or the maximum stack slot accessed.
791
792 // Convert 4-byte c2 stack slots to words.
793 comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
794 // Round up to miminum stack alignment, in wordSize
795 comp_words_on_stack = align_up(comp_words_on_stack, 2);
796 __ subptr(rsp, comp_words_on_stack * wordSize);
797 }
798
799
800 // Ensure compiled code always sees stack at proper alignment
801 __ andptr(rsp, -16);
802
803 // push the return address and misalign the stack that youngest frame always sees
804 // as far as the placement of the call instruction
805 __ push(rax);
806
807 // Put saved SP in another register
808 const Register saved_sp = rax;
809 __ movptr(saved_sp, r11);
810
811 // Will jump to the compiled code just as if compiled code was doing it.
812 // Pre-load the register-jump target early, to schedule it better.
813 __ movptr(r11, Address(rbx, in_bytes(Method::from_compiled_offset())));
814
815 #if INCLUDE_JVMCI
816 if (EnableJVMCI || UseAOT) {
817 // check if this call should be routed towards a specific entry point
818 __ cmpptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);
819 Label no_alternative_target;
820 __ jcc(Assembler::equal, no_alternative_target);
821 __ movptr(r11, Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
822 __ movptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);
823 __ bind(no_alternative_target);
824 }
825 #endif // INCLUDE_JVMCI
826
827 // Now generate the shuffle code. Pick up all register args and move the
828 // rest through the floating point stack top.
829 for (int i = 0; i < total_args_passed; i++) {
830 if (sig_bt[i] == T_VOID) {
831 // Longs and doubles are passed in native word order, but misaligned
832 // in the 32-bit build.
833 assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
834 continue;
835 }
836
837 // Pick up 0, 1 or 2 words from SP+offset.
838
839 assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
840 "scrambled load targets?");
841 // Load in argument order going down.
842 int ld_off = (total_args_passed - i)*Interpreter::stackElementSize;
843 // Point to interpreter value (vs. tag)
844 int next_off = ld_off - Interpreter::stackElementSize;
845 //
846 //
847 //
848 VMReg r_1 = regs[i].first();
849 VMReg r_2 = regs[i].second();
850 if (!r_1->is_valid()) {
851 assert(!r_2->is_valid(), "");
852 continue;
853 }
854 if (r_1->is_stack()) {
855 // Convert stack slot to an SP offset (+ wordSize to account for return address )
856 int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size + wordSize;
857
858 // We can use r13 as a temp here because compiled code doesn't need r13 as an input
859 // and if we end up going thru a c2i because of a miss a reasonable value of r13
860 // will be generated.
861 if (!r_2->is_valid()) {
862 // sign extend???
863 __ movl(r13, Address(saved_sp, ld_off));
864 __ movptr(Address(rsp, st_off), r13);
865 } else {
866 //
867 // We are using two optoregs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE
868 // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case
869 // So we must adjust where to pick up the data to match the interpreter.
870 //
871 // Interpreter local[n] == MSW, local[n+1] == LSW however locals
872 // are accessed as negative so LSW is at LOW address
873
874 // ld_off is MSW so get LSW
875 const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
876 next_off : ld_off;
877 __ movq(r13, Address(saved_sp, offset));
878 // st_off is LSW (i.e. reg.first())
879 __ movq(Address(rsp, st_off), r13);
880 }
881 } else if (r_1->is_Register()) { // Register argument
882 Register r = r_1->as_Register();
883 assert(r != rax, "must be different");
884 if (r_2->is_valid()) {
885 //
886 // We are using two VMRegs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE
887 // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case
888 // So we must adjust where to pick up the data to match the interpreter.
889
890 const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
891 next_off : ld_off;
892
893 // this can be a misaligned move
894 __ movq(r, Address(saved_sp, offset));
895 } else {
896 // sign extend and use a full word?
897 __ movl(r, Address(saved_sp, ld_off));
898 }
899 } else {
900 if (!r_2->is_valid()) {
901 __ movflt(r_1->as_XMMRegister(), Address(saved_sp, ld_off));
902 } else {
903 __ movdbl(r_1->as_XMMRegister(), Address(saved_sp, next_off));
904 }
905 }
906 }
907
908 // 6243940 We might end up in handle_wrong_method if
909 // the callee is deoptimized as we race thru here. If that
910 // happens we don't want to take a safepoint because the
911 // caller frame will look interpreted and arguments are now
912 // "compiled" so it is much better to make this transition
913 // invisible to the stack walking code. Unfortunately if
914 // we try and find the callee by normal means a safepoint
915 // is possible. So we stash the desired callee in the thread
916 // and the vm will find there should this case occur.
917
918 __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx);
919
920 // put Method* where a c2i would expect should we end up there
921 // only needed becaus eof c2 resolve stubs return Method* as a result in
922 // rax
923 __ mov(rax, rbx);
924 __ jmp(r11);
925 }
926
927 // ---------------------------------------------------------------
928 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
929 int total_args_passed,
930 int comp_args_on_stack,
931 const BasicType *sig_bt,
932 const VMRegPair *regs,
933 AdapterFingerPrint* fingerprint) {
934 address i2c_entry = __ pc();
935
936 gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
937
938 // -------------------------------------------------------------------------
939 // Generate a C2I adapter. On entry we know rbx holds the Method* during calls
940 // to the interpreter. The args start out packed in the compiled layout. They
941 // need to be unpacked into the interpreter layout. This will almost always
942 // require some stack space. We grow the current (compiled) stack, then repack
943 // the args. We finally end in a jump to the generic interpreter entry point.
944 // On exit from the interpreter, the interpreter will restore our SP (lest the
945 // compiled code, which relys solely on SP and not RBP, get sick).
946
947 address c2i_unverified_entry = __ pc();
948 Label skip_fixup;
949 Label ok;
950
951 Register holder = rax;
952 Register receiver = j_rarg0;
953 Register temp = rbx;
954
955 {
956 __ load_klass(temp, receiver);
957 __ cmpptr(temp, Address(holder, CompiledICHolder::holder_klass_offset()));
958 __ movptr(rbx, Address(holder, CompiledICHolder::holder_metadata_offset()));
959 __ jcc(Assembler::equal, ok);
960 __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
961
962 __ bind(ok);
963 // Method might have been compiled since the call site was patched to
964 // interpreted if that is the case treat it as a miss so we can get
965 // the call site corrected.
966 __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), (int32_t)NULL_WORD);
967 __ jcc(Assembler::equal, skip_fixup);
968 __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
969 }
970
971 address c2i_entry = __ pc();
972
973 gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
974
975 __ flush();
976 return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry);
977 }
978
979 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
980 VMRegPair *regs,
981 VMRegPair *regs2,
982 int total_args_passed) {
983 assert(regs2 == NULL, "not needed on x86");
984 // We return the amount of VMRegImpl stack slots we need to reserve for all
985 // the arguments NOT counting out_preserve_stack_slots.
986
987 // NOTE: These arrays will have to change when c1 is ported
988 #ifdef _WIN64
989 static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
990 c_rarg0, c_rarg1, c_rarg2, c_rarg3
991 };
992 static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
993 c_farg0, c_farg1, c_farg2, c_farg3
994 };
995 #else
996 static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
997 c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5
998 };
999 static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
1000 c_farg0, c_farg1, c_farg2, c_farg3,
1001 c_farg4, c_farg5, c_farg6, c_farg7
1002 };
1003 #endif // _WIN64
1004
1005
1006 uint int_args = 0;
1007 uint fp_args = 0;
1008 uint stk_args = 0; // inc by 2 each time
1009
1010 for (int i = 0; i < total_args_passed; i++) {
1011 switch (sig_bt[i]) {
1012 case T_BOOLEAN:
1013 case T_CHAR:
1014 case T_BYTE:
1015 case T_SHORT:
1016 case T_INT:
1017 if (int_args < Argument::n_int_register_parameters_c) {
1018 regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
1019 #ifdef _WIN64
1020 fp_args++;
1021 // Allocate slots for callee to stuff register args the stack.
1022 stk_args += 2;
1023 #endif
1024 } else {
1025 regs[i].set1(VMRegImpl::stack2reg(stk_args));
1026 stk_args += 2;
1027 }
1028 break;
1029 case T_LONG:
1030 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
1031 // fall through
1032 case T_OBJECT:
1033 case T_ARRAY:
1034 case T_ADDRESS:
1035 case T_METADATA:
1036 if (int_args < Argument::n_int_register_parameters_c) {
1037 regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
1038 #ifdef _WIN64
1039 fp_args++;
1040 stk_args += 2;
1041 #endif
1042 } else {
1043 regs[i].set2(VMRegImpl::stack2reg(stk_args));
1044 stk_args += 2;
1045 }
1046 break;
1047 case T_FLOAT:
1048 if (fp_args < Argument::n_float_register_parameters_c) {
1049 regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
1050 #ifdef _WIN64
1051 int_args++;
1052 // Allocate slots for callee to stuff register args the stack.
1053 stk_args += 2;
1054 #endif
1055 } else {
1056 regs[i].set1(VMRegImpl::stack2reg(stk_args));
1057 stk_args += 2;
1058 }
1059 break;
1060 case T_DOUBLE:
1061 assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
1062 if (fp_args < Argument::n_float_register_parameters_c) {
1063 regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
1064 #ifdef _WIN64
1065 int_args++;
1066 // Allocate slots for callee to stuff register args the stack.
1067 stk_args += 2;
1068 #endif
1069 } else {
1070 regs[i].set2(VMRegImpl::stack2reg(stk_args));
1071 stk_args += 2;
1072 }
1073 break;
1074 case T_VOID: // Halves of longs and doubles
1075 assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
1076 regs[i].set_bad();
1077 break;
1078 default:
1079 ShouldNotReachHere();
1080 break;
1081 }
1082 }
1083 #ifdef _WIN64
1084 // windows abi requires that we always allocate enough stack space
1085 // for 4 64bit registers to be stored down.
1086 if (stk_args < 8) {
1087 stk_args = 8;
1088 }
1089 #endif // _WIN64
1090
1091 return stk_args;
1092 }
1093
1094 // On 64 bit we will store integer like items to the stack as
1095 // 64 bits items (sparc abi) even though java would only store
1096 // 32bits for a parameter. On 32bit it will simply be 32 bits
1097 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
1098 static void move32_64(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1099 if (src.first()->is_stack()) {
1100 if (dst.first()->is_stack()) {
1101 // stack to stack
1102 __ movslq(rax, Address(rbp, reg2offset_in(src.first())));
1103 __ movq(Address(rsp, reg2offset_out(dst.first())), rax);
1104 } else {
1105 // stack to reg
1106 __ movslq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
1107 }
1108 } else if (dst.first()->is_stack()) {
1109 // reg to stack
1110 // Do we really have to sign extend???
1111 // __ movslq(src.first()->as_Register(), src.first()->as_Register());
1112 __ movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
1113 } else {
1114 // Do we really have to sign extend???
1115 // __ movslq(dst.first()->as_Register(), src.first()->as_Register());
1116 if (dst.first() != src.first()) {
1117 __ movq(dst.first()->as_Register(), src.first()->as_Register());
1118 }
1119 }
1120 }
1121
1122 static void move_ptr(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1123 if (src.first()->is_stack()) {
1124 if (dst.first()->is_stack()) {
1125 // stack to stack
1126 __ movq(rax, Address(rbp, reg2offset_in(src.first())));
1127 __ movq(Address(rsp, reg2offset_out(dst.first())), rax);
1128 } else {
1129 // stack to reg
1130 __ movq(dst.first()->as_Register(), Address(rbp, reg2offset_in(src.first())));
1131 }
1132 } else if (dst.first()->is_stack()) {
1133 // reg to stack
1134 __ movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
1135 } else {
1136 if (dst.first() != src.first()) {
1137 __ movq(dst.first()->as_Register(), src.first()->as_Register());
1138 }
1139 }
1140 }
1141
1142 // An oop arg. Must pass a handle not the oop itself
1143 static void object_move(MacroAssembler* masm,
1144 OopMap* map,
1145 int oop_handle_offset,
1146 int framesize_in_slots,
1147 VMRegPair src,
1148 VMRegPair dst,
1149 bool is_receiver,
1150 int* receiver_offset) {
1151
1152 // must pass a handle. First figure out the location we use as a handle
1153
1154 Register rHandle = dst.first()->is_stack() ? rax : dst.first()->as_Register();
1155
1156 // See if oop is NULL if it is we need no handle
1157
1158 if (src.first()->is_stack()) {
1159
1160 // Oop is already on the stack as an argument
1161 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1162 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
1163 if (is_receiver) {
1164 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
1165 }
1166
1167 __ cmpptr(Address(rbp, reg2offset_in(src.first())), (int32_t)NULL_WORD);
1168 __ lea(rHandle, Address(rbp, reg2offset_in(src.first())));
1169 // conditionally move a NULL
1170 __ cmovptr(Assembler::equal, rHandle, Address(rbp, reg2offset_in(src.first())));
1171 } else {
1172
1173 // Oop is in an a register we must store it to the space we reserve
1174 // on the stack for oop_handles and pass a handle if oop is non-NULL
1175
1176 const Register rOop = src.first()->as_Register();
1177 int oop_slot;
1178 if (rOop == j_rarg0)
1179 oop_slot = 0;
1180 else if (rOop == j_rarg1)
1181 oop_slot = 1;
1182 else if (rOop == j_rarg2)
1183 oop_slot = 2;
1184 else if (rOop == j_rarg3)
1185 oop_slot = 3;
1186 else if (rOop == j_rarg4)
1187 oop_slot = 4;
1188 else {
1189 assert(rOop == j_rarg5, "wrong register");
1190 oop_slot = 5;
1191 }
1192
1193 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
1194 int offset = oop_slot*VMRegImpl::stack_slot_size;
1195
1196 map->set_oop(VMRegImpl::stack2reg(oop_slot));
1197 // Store oop in handle area, may be NULL
1198 __ movptr(Address(rsp, offset), rOop);
1199 if (is_receiver) {
1200 *receiver_offset = offset;
1201 }
1202
1203 __ cmpptr(rOop, (int32_t)NULL_WORD);
1204 __ lea(rHandle, Address(rsp, offset));
1205 // conditionally move a NULL from the handle area where it was just stored
1206 __ cmovptr(Assembler::equal, rHandle, Address(rsp, offset));
1207 }
1208
1209 // If arg is on the stack then place it otherwise it is already in correct reg.
1210 if (dst.first()->is_stack()) {
1211 __ movptr(Address(rsp, reg2offset_out(dst.first())), rHandle);
1212 }
1213 }
1214
1215 // A float arg may have to do float reg int reg conversion
1216 static void float_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1217 assert(!src.second()->is_valid() && !dst.second()->is_valid(), "bad float_move");
1218
1219 // The calling conventions assures us that each VMregpair is either
1220 // all really one physical register or adjacent stack slots.
1221 // This greatly simplifies the cases here compared to sparc.
1222
1223 if (src.first()->is_stack()) {
1224 if (dst.first()->is_stack()) {
1225 __ movl(rax, Address(rbp, reg2offset_in(src.first())));
1226 __ movptr(Address(rsp, reg2offset_out(dst.first())), rax);
1227 } else {
1228 // stack to reg
1229 assert(dst.first()->is_XMMRegister(), "only expect xmm registers as parameters");
1230 __ movflt(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_in(src.first())));
1231 }
1232 } else if (dst.first()->is_stack()) {
1233 // reg to stack
1234 assert(src.first()->is_XMMRegister(), "only expect xmm registers as parameters");
1235 __ movflt(Address(rsp, reg2offset_out(dst.first())), src.first()->as_XMMRegister());
1236 } else {
1237 // reg to reg
1238 // In theory these overlap but the ordering is such that this is likely a nop
1239 if ( src.first() != dst.first()) {
1240 __ movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
1241 }
1242 }
1243 }
1244
1245 // A long move
1246 static void long_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1247
1248 // The calling conventions assures us that each VMregpair is either
1249 // all really one physical register or adjacent stack slots.
1250 // This greatly simplifies the cases here compared to sparc.
1251
1252 if (src.is_single_phys_reg() ) {
1253 if (dst.is_single_phys_reg()) {
1254 if (dst.first() != src.first()) {
1255 __ mov(dst.first()->as_Register(), src.first()->as_Register());
1256 }
1257 } else {
1258 assert(dst.is_single_reg(), "not a stack pair");
1259 __ movq(Address(rsp, reg2offset_out(dst.first())), src.first()->as_Register());
1260 }
1261 } else if (dst.is_single_phys_reg()) {
1262 assert(src.is_single_reg(), "not a stack pair");
1263 __ movq(dst.first()->as_Register(), Address(rbp, reg2offset_out(src.first())));
1264 } else {
1265 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
1266 __ movq(rax, Address(rbp, reg2offset_in(src.first())));
1267 __ movq(Address(rsp, reg2offset_out(dst.first())), rax);
1268 }
1269 }
1270
1271 // A double move
1272 static void double_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1273
1274 // The calling conventions assures us that each VMregpair is either
1275 // all really one physical register or adjacent stack slots.
1276 // This greatly simplifies the cases here compared to sparc.
1277
1278 if (src.is_single_phys_reg() ) {
1279 if (dst.is_single_phys_reg()) {
1280 // In theory these overlap but the ordering is such that this is likely a nop
1281 if ( src.first() != dst.first()) {
1282 __ movdbl(dst.first()->as_XMMRegister(), src.first()->as_XMMRegister());
1283 }
1284 } else {
1285 assert(dst.is_single_reg(), "not a stack pair");
1286 __ movdbl(Address(rsp, reg2offset_out(dst.first())), src.first()->as_XMMRegister());
1287 }
1288 } else if (dst.is_single_phys_reg()) {
1289 assert(src.is_single_reg(), "not a stack pair");
1290 __ movdbl(dst.first()->as_XMMRegister(), Address(rbp, reg2offset_out(src.first())));
1291 } else {
1292 assert(src.is_single_reg() && dst.is_single_reg(), "not stack pairs");
1293 __ movq(rax, Address(rbp, reg2offset_in(src.first())));
1294 __ movq(Address(rsp, reg2offset_out(dst.first())), rax);
1295 }
1296 }
1297
1298
1299 void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1300 // We always ignore the frame_slots arg and just use the space just below frame pointer
1301 // which by this time is free to use
1302 switch (ret_type) {
1303 case T_FLOAT:
1304 __ movflt(Address(rbp, -wordSize), xmm0);
1305 break;
1306 case T_DOUBLE:
1307 __ movdbl(Address(rbp, -wordSize), xmm0);
1308 break;
1309 case T_VOID: break;
1310 default: {
1311 __ movptr(Address(rbp, -wordSize), rax);
1312 }
1313 }
1314 }
1315
1316 void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1317 // We always ignore the frame_slots arg and just use the space just below frame pointer
1318 // which by this time is free to use
1319 switch (ret_type) {
1320 case T_FLOAT:
1321 __ movflt(xmm0, Address(rbp, -wordSize));
1322 break;
1323 case T_DOUBLE:
1324 __ movdbl(xmm0, Address(rbp, -wordSize));
1325 break;
1326 case T_VOID: break;
1327 default: {
1328 __ movptr(rax, Address(rbp, -wordSize));
1329 }
1330 }
1331 }
1332
1333 static void save_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
1334 for ( int i = first_arg ; i < arg_count ; i++ ) {
1335 if (args[i].first()->is_Register()) {
1336 __ push(args[i].first()->as_Register());
1337 } else if (args[i].first()->is_XMMRegister()) {
1338 __ subptr(rsp, 2*wordSize);
1339 __ movdbl(Address(rsp, 0), args[i].first()->as_XMMRegister());
1340 }
1341 }
1342 }
1343
1344 static void restore_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
1345 for ( int i = arg_count - 1 ; i >= first_arg ; i-- ) {
1346 if (args[i].first()->is_Register()) {
1347 __ pop(args[i].first()->as_Register());
1348 } else if (args[i].first()->is_XMMRegister()) {
1349 __ movdbl(args[i].first()->as_XMMRegister(), Address(rsp, 0));
1350 __ addptr(rsp, 2*wordSize);
1351 }
1352 }
1353 }
1354
1355
1356 static void save_or_restore_arguments(MacroAssembler* masm,
1357 const int stack_slots,
1358 const int total_in_args,
1359 const int arg_save_area,
1360 OopMap* map,
1361 VMRegPair* in_regs,
1362 BasicType* in_sig_bt) {
1363 // if map is non-NULL then the code should store the values,
1364 // otherwise it should load them.
1365 int slot = arg_save_area;
1366 // Save down double word first
1367 for ( int i = 0; i < total_in_args; i++) {
1368 if (in_regs[i].first()->is_XMMRegister() && in_sig_bt[i] == T_DOUBLE) {
1369 int offset = slot * VMRegImpl::stack_slot_size;
1370 slot += VMRegImpl::slots_per_word;
1371 assert(slot <= stack_slots, "overflow");
1372 if (map != NULL) {
1373 __ movdbl(Address(rsp, offset), in_regs[i].first()->as_XMMRegister());
1374 } else {
1375 __ movdbl(in_regs[i].first()->as_XMMRegister(), Address(rsp, offset));
1376 }
1377 }
1378 if (in_regs[i].first()->is_Register() &&
1379 (in_sig_bt[i] == T_LONG || in_sig_bt[i] == T_ARRAY)) {
1380 int offset = slot * VMRegImpl::stack_slot_size;
1381 if (map != NULL) {
1382 __ movq(Address(rsp, offset), in_regs[i].first()->as_Register());
1383 if (in_sig_bt[i] == T_ARRAY) {
1384 map->set_oop(VMRegImpl::stack2reg(slot));;
1385 }
1386 } else {
1387 __ movq(in_regs[i].first()->as_Register(), Address(rsp, offset));
1388 }
1389 slot += VMRegImpl::slots_per_word;
1390 }
1391 }
1392 // Save or restore single word registers
1393 for ( int i = 0; i < total_in_args; i++) {
1394 if (in_regs[i].first()->is_Register()) {
1395 int offset = slot * VMRegImpl::stack_slot_size;
1396 slot++;
1397 assert(slot <= stack_slots, "overflow");
1398
1399 // Value is in an input register pass we must flush it to the stack
1400 const Register reg = in_regs[i].first()->as_Register();
1401 switch (in_sig_bt[i]) {
1402 case T_BOOLEAN:
1403 case T_CHAR:
1404 case T_BYTE:
1405 case T_SHORT:
1406 case T_INT:
1407 if (map != NULL) {
1408 __ movl(Address(rsp, offset), reg);
1409 } else {
1410 __ movl(reg, Address(rsp, offset));
1411 }
1412 break;
1413 case T_ARRAY:
1414 case T_LONG:
1415 // handled above
1416 break;
1417 case T_OBJECT:
1418 default: ShouldNotReachHere();
1419 }
1420 } else if (in_regs[i].first()->is_XMMRegister()) {
1421 if (in_sig_bt[i] == T_FLOAT) {
1422 int offset = slot * VMRegImpl::stack_slot_size;
1423 slot++;
1424 assert(slot <= stack_slots, "overflow");
1425 if (map != NULL) {
1426 __ movflt(Address(rsp, offset), in_regs[i].first()->as_XMMRegister());
1427 } else {
1428 __ movflt(in_regs[i].first()->as_XMMRegister(), Address(rsp, offset));
1429 }
1430 }
1431 } else if (in_regs[i].first()->is_stack()) {
1432 if (in_sig_bt[i] == T_ARRAY && map != NULL) {
1433 int offset_in_older_frame = in_regs[i].first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1434 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + stack_slots));
1435 }
1436 }
1437 }
1438 }
1439
1440 // Pin object, return pinned object or null in rax
1441 static void gen_pin_object(MacroAssembler* masm,
1442 VMRegPair reg) {
1443 __ block_comment("gen_pin_object {");
1444
1445 // rax always contains oop, either incoming or
1446 // pinned.
1447 Register tmp_reg = rax;
1448
1449 Label is_null;
1450 VMRegPair tmp;
1451 VMRegPair in_reg = reg;
1452
1453 tmp.set_ptr(tmp_reg->as_VMReg());
1454 if (reg.first()->is_stack()) {
1455 // Load the arg up from the stack
1456 move_ptr(masm, reg, tmp);
1457 reg = tmp;
1458 } else {
1459 __ movptr(rax, reg.first()->as_Register());
1460 }
1461 __ testptr(reg.first()->as_Register(), reg.first()->as_Register());
1462 __ jccb(Assembler::equal, is_null);
1463
1464 if (reg.first()->as_Register() != c_rarg1) {
1465 __ movptr(c_rarg1, reg.first()->as_Register());
1466 }
1467
1468 __ call_VM_leaf(
1469 CAST_FROM_FN_PTR(address, SharedRuntime::pin_object),
1470 r15_thread, c_rarg1);
1471
1472 __ bind(is_null);
1473 __ block_comment("} gen_pin_object");
1474 }
1475
1476 // Unpin object
1477 static void gen_unpin_object(MacroAssembler* masm,
1478 VMRegPair reg) {
1479 __ block_comment("gen_unpin_object {");
1480 Label is_null;
1481
1482 if (reg.first()->is_stack()) {
1483 __ movptr(c_rarg1, Address(rbp, reg2offset_in(reg.first())));
1484 } else if (reg.first()->as_Register() != c_rarg1) {
1485 __ movptr(c_rarg1, reg.first()->as_Register());
1486 }
1487
1488 __ testptr(c_rarg1, c_rarg1);
1489 __ jccb(Assembler::equal, is_null);
1490
1491 __ call_VM_leaf(
1492 CAST_FROM_FN_PTR(address, SharedRuntime::unpin_object),
1493 r15_thread, c_rarg1);
1494
1495 __ bind(is_null);
1496 __ block_comment("} gen_unpin_object");
1497 }
1498
1499 // Check GCLocker::needs_gc and enter the runtime if it's true. This
1500 // keeps a new JNI critical region from starting until a GC has been
1501 // forced. Save down any oops in registers and describe them in an
1502 // OopMap.
1503 static void check_needs_gc_for_critical_native(MacroAssembler* masm,
1504 int stack_slots,
1505 int total_c_args,
1506 int total_in_args,
1507 int arg_save_area,
1508 OopMapSet* oop_maps,
1509 VMRegPair* in_regs,
1510 BasicType* in_sig_bt) {
1511 __ block_comment("check GCLocker::needs_gc");
1512 Label cont;
1513 __ cmp8(ExternalAddress((address)GCLocker::needs_gc_address()), false);
1514 __ jcc(Assembler::equal, cont);
1515
1516 // Save down any incoming oops and call into the runtime to halt for a GC
1517
1518 OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1519 save_or_restore_arguments(masm, stack_slots, total_in_args,
1520 arg_save_area, map, in_regs, in_sig_bt);
1521
1522 address the_pc = __ pc();
1523 oop_maps->add_gc_map( __ offset(), map);
1524 __ set_last_Java_frame(rsp, noreg, the_pc);
1525
1526 __ block_comment("block_for_jni_critical");
1527 __ movptr(c_rarg0, r15_thread);
1528 __ mov(r12, rsp); // remember sp
1529 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
1530 __ andptr(rsp, -16); // align stack as required by ABI
1531 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::block_for_jni_critical)));
1532 __ mov(rsp, r12); // restore sp
1533 __ reinit_heapbase();
1534
1535 __ reset_last_Java_frame(false);
1536
1537 save_or_restore_arguments(masm, stack_slots, total_in_args,
1538 arg_save_area, NULL, in_regs, in_sig_bt);
1539 __ bind(cont);
1540 #ifdef ASSERT
1541 if (StressCriticalJNINatives) {
1542 // Stress register saving
1543 OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1544 save_or_restore_arguments(masm, stack_slots, total_in_args,
1545 arg_save_area, map, in_regs, in_sig_bt);
1546 // Destroy argument registers
1547 for (int i = 0; i < total_in_args - 1; i++) {
1548 if (in_regs[i].first()->is_Register()) {
1549 const Register reg = in_regs[i].first()->as_Register();
1550 __ xorptr(reg, reg);
1551 } else if (in_regs[i].first()->is_XMMRegister()) {
1552 __ xorpd(in_regs[i].first()->as_XMMRegister(), in_regs[i].first()->as_XMMRegister());
1553 } else if (in_regs[i].first()->is_FloatRegister()) {
1554 ShouldNotReachHere();
1555 } else if (in_regs[i].first()->is_stack()) {
1556 // Nothing to do
1557 } else {
1558 ShouldNotReachHere();
1559 }
1560 if (in_sig_bt[i] == T_LONG || in_sig_bt[i] == T_DOUBLE) {
1561 i++;
1562 }
1563 }
1564
1565 save_or_restore_arguments(masm, stack_slots, total_in_args,
1566 arg_save_area, NULL, in_regs, in_sig_bt);
1567 }
1568 #endif
1569 }
1570
1571 // Unpack an array argument into a pointer to the body and the length
1572 // if the array is non-null, otherwise pass 0 for both.
1573 static void unpack_array_argument(MacroAssembler* masm, VMRegPair reg, BasicType in_elem_type, VMRegPair body_arg, VMRegPair length_arg) {
1574 Register tmp_reg = rax;
1575 assert(!body_arg.first()->is_Register() || body_arg.first()->as_Register() != tmp_reg,
1576 "possible collision");
1577 assert(!length_arg.first()->is_Register() || length_arg.first()->as_Register() != tmp_reg,
1578 "possible collision");
1579
1580 __ block_comment("unpack_array_argument {");
1581
1582 // Pass the length, ptr pair
1583 Label is_null, done;
1584 VMRegPair tmp;
1585 tmp.set_ptr(tmp_reg->as_VMReg());
1586 if (reg.first()->is_stack()) {
1587 // Load the arg up from the stack
1588 move_ptr(masm, reg, tmp);
1589 reg = tmp;
1590 }
1591 __ testptr(reg.first()->as_Register(), reg.first()->as_Register());
1592 __ jccb(Assembler::equal, is_null);
1593 __ lea(tmp_reg, Address(reg.first()->as_Register(), arrayOopDesc::base_offset_in_bytes(in_elem_type)));
1594 move_ptr(masm, tmp, body_arg);
1595 // load the length relative to the body.
1596 __ movl(tmp_reg, Address(tmp_reg, arrayOopDesc::length_offset_in_bytes() -
1597 arrayOopDesc::base_offset_in_bytes(in_elem_type)));
1598 move32_64(masm, tmp, length_arg);
1599 __ jmpb(done);
1600 __ bind(is_null);
1601 // Pass zeros
1602 __ xorptr(tmp_reg, tmp_reg);
1603 move_ptr(masm, tmp, body_arg);
1604 move32_64(masm, tmp, length_arg);
1605 __ bind(done);
1606
1607 __ block_comment("} unpack_array_argument");
1608 }
1609
1610
1611 // Different signatures may require very different orders for the move
1612 // to avoid clobbering other arguments. There's no simple way to
1613 // order them safely. Compute a safe order for issuing stores and
1614 // break any cycles in those stores. This code is fairly general but
1615 // it's not necessary on the other platforms so we keep it in the
1616 // platform dependent code instead of moving it into a shared file.
1617 // (See bugs 7013347 & 7145024.)
1618 // Note that this code is specific to LP64.
1619 class ComputeMoveOrder: public StackObj {
1620 class MoveOperation: public ResourceObj {
1621 friend class ComputeMoveOrder;
1622 private:
1623 VMRegPair _src;
1624 VMRegPair _dst;
1625 int _src_index;
1626 int _dst_index;
1627 bool _processed;
1628 MoveOperation* _next;
1629 MoveOperation* _prev;
1630
1631 static int get_id(VMRegPair r) {
1632 return r.first()->value();
1633 }
1634
1635 public:
1636 MoveOperation(int src_index, VMRegPair src, int dst_index, VMRegPair dst):
1637 _src(src)
1638 , _dst(dst)
1639 , _src_index(src_index)
1640 , _dst_index(dst_index)
1641 , _processed(false)
1642 , _next(NULL)
1643 , _prev(NULL) {
1644 }
1645
1646 VMRegPair src() const { return _src; }
1647 int src_id() const { return get_id(src()); }
1648 int src_index() const { return _src_index; }
1649 VMRegPair dst() const { return _dst; }
1650 void set_dst(int i, VMRegPair dst) { _dst_index = i, _dst = dst; }
1651 int dst_index() const { return _dst_index; }
1652 int dst_id() const { return get_id(dst()); }
1653 MoveOperation* next() const { return _next; }
1654 MoveOperation* prev() const { return _prev; }
1655 void set_processed() { _processed = true; }
1656 bool is_processed() const { return _processed; }
1657
1658 // insert
1659 void break_cycle(VMRegPair temp_register) {
1660 // create a new store following the last store
1661 // to move from the temp_register to the original
1662 MoveOperation* new_store = new MoveOperation(-1, temp_register, dst_index(), dst());
1663
1664 // break the cycle of links and insert new_store at the end
1665 // break the reverse link.
1666 MoveOperation* p = prev();
1667 assert(p->next() == this, "must be");
1668 _prev = NULL;
1669 p->_next = new_store;
1670 new_store->_prev = p;
1671
1672 // change the original store to save it's value in the temp.
1673 set_dst(-1, temp_register);
1674 }
1675
1676 void link(GrowableArray<MoveOperation*>& killer) {
1677 // link this store in front the store that it depends on
1678 MoveOperation* n = killer.at_grow(src_id(), NULL);
1679 if (n != NULL) {
1680 assert(_next == NULL && n->_prev == NULL, "shouldn't have been set yet");
1681 _next = n;
1682 n->_prev = this;
1683 }
1684 }
1685 };
1686
1687 private:
1688 GrowableArray<MoveOperation*> edges;
1689
1690 public:
1691 ComputeMoveOrder(int total_in_args, VMRegPair* in_regs, int total_c_args, VMRegPair* out_regs,
1692 BasicType* in_sig_bt, GrowableArray<int>& arg_order, VMRegPair tmp_vmreg) {
1693 // Move operations where the dest is the stack can all be
1694 // scheduled first since they can't interfere with the other moves.
1695 for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
1696 if (in_sig_bt[i] == T_ARRAY) {
1697 c_arg--;
1698 if (out_regs[c_arg].first()->is_stack() &&
1699 out_regs[c_arg + 1].first()->is_stack()) {
1700 arg_order.push(i);
1701 arg_order.push(c_arg);
1702 } else {
1703 if (out_regs[c_arg].first()->is_stack() ||
1704 in_regs[i].first() == out_regs[c_arg].first()) {
1705 add_edge(i, in_regs[i].first(), c_arg, out_regs[c_arg + 1]);
1706 } else {
1707 add_edge(i, in_regs[i].first(), c_arg, out_regs[c_arg]);
1708 }
1709 }
1710 } else if (in_sig_bt[i] == T_VOID) {
1711 arg_order.push(i);
1712 arg_order.push(c_arg);
1713 } else {
1714 if (out_regs[c_arg].first()->is_stack() ||
1715 in_regs[i].first() == out_regs[c_arg].first()) {
1716 arg_order.push(i);
1717 arg_order.push(c_arg);
1718 } else {
1719 add_edge(i, in_regs[i].first(), c_arg, out_regs[c_arg]);
1720 }
1721 }
1722 }
1723 // Break any cycles in the register moves and emit the in the
1724 // proper order.
1725 GrowableArray<MoveOperation*>* stores = get_store_order(tmp_vmreg);
1726 for (int i = 0; i < stores->length(); i++) {
1727 arg_order.push(stores->at(i)->src_index());
1728 arg_order.push(stores->at(i)->dst_index());
1729 }
1730 }
1731
1732 // Collected all the move operations
1733 void add_edge(int src_index, VMRegPair src, int dst_index, VMRegPair dst) {
1734 if (src.first() == dst.first()) return;
1735 edges.append(new MoveOperation(src_index, src, dst_index, dst));
1736 }
1737
1738 // Walk the edges breaking cycles between moves. The result list
1739 // can be walked in order to produce the proper set of loads
1740 GrowableArray<MoveOperation*>* get_store_order(VMRegPair temp_register) {
1741 // Record which moves kill which values
1742 GrowableArray<MoveOperation*> killer;
1743 for (int i = 0; i < edges.length(); i++) {
1744 MoveOperation* s = edges.at(i);
1745 assert(killer.at_grow(s->dst_id(), NULL) == NULL, "only one killer");
1746 killer.at_put_grow(s->dst_id(), s, NULL);
1747 }
1748 assert(killer.at_grow(MoveOperation::get_id(temp_register), NULL) == NULL,
1749 "make sure temp isn't in the registers that are killed");
1750
1751 // create links between loads and stores
1752 for (int i = 0; i < edges.length(); i++) {
1753 edges.at(i)->link(killer);
1754 }
1755
1756 // at this point, all the move operations are chained together
1757 // in a doubly linked list. Processing it backwards finds
1758 // the beginning of the chain, forwards finds the end. If there's
1759 // a cycle it can be broken at any point, so pick an edge and walk
1760 // backward until the list ends or we end where we started.
1761 GrowableArray<MoveOperation*>* stores = new GrowableArray<MoveOperation*>();
1762 for (int e = 0; e < edges.length(); e++) {
1763 MoveOperation* s = edges.at(e);
1764 if (!s->is_processed()) {
1765 MoveOperation* start = s;
1766 // search for the beginning of the chain or cycle
1767 while (start->prev() != NULL && start->prev() != s) {
1768 start = start->prev();
1769 }
1770 if (start->prev() == s) {
1771 start->break_cycle(temp_register);
1772 }
1773 // walk the chain forward inserting to store list
1774 while (start != NULL) {
1775 stores->append(start);
1776 start->set_processed();
1777 start = start->next();
1778 }
1779 }
1780 }
1781 return stores;
1782 }
1783 };
1784
1785 static void verify_oop_args(MacroAssembler* masm,
1786 const methodHandle& method,
1787 const BasicType* sig_bt,
1788 const VMRegPair* regs) {
1789 Register temp_reg = rbx; // not part of any compiled calling seq
1790 if (VerifyOops) {
1791 for (int i = 0; i < method->size_of_parameters(); i++) {
1792 if (sig_bt[i] == T_OBJECT ||
1793 sig_bt[i] == T_ARRAY) {
1794 VMReg r = regs[i].first();
1795 assert(r->is_valid(), "bad oop arg");
1796 if (r->is_stack()) {
1797 __ movptr(temp_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1798 __ verify_oop(temp_reg);
1799 } else {
1800 __ verify_oop(r->as_Register());
1801 }
1802 }
1803 }
1804 }
1805 }
1806
1807 static void gen_special_dispatch(MacroAssembler* masm,
1808 const methodHandle& method,
1809 const BasicType* sig_bt,
1810 const VMRegPair* regs) {
1811 verify_oop_args(masm, method, sig_bt, regs);
1812 vmIntrinsics::ID iid = method->intrinsic_id();
1813
1814 // Now write the args into the outgoing interpreter space
1815 bool has_receiver = false;
1816 Register receiver_reg = noreg;
1817 int member_arg_pos = -1;
1818 Register member_reg = noreg;
1819 int ref_kind = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
1820 if (ref_kind != 0) {
1821 member_arg_pos = method->size_of_parameters() - 1; // trailing MemberName argument
1822 member_reg = rbx; // known to be free at this point
1823 has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
1824 } else if (iid == vmIntrinsics::_invokeBasic) {
1825 has_receiver = true;
1826 } else {
1827 fatal("unexpected intrinsic id %d", iid);
1828 }
1829
1830 if (member_reg != noreg) {
1831 // Load the member_arg into register, if necessary.
1832 SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
1833 VMReg r = regs[member_arg_pos].first();
1834 if (r->is_stack()) {
1835 __ movptr(member_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1836 } else {
1837 // no data motion is needed
1838 member_reg = r->as_Register();
1839 }
1840 }
1841
1842 if (has_receiver) {
1843 // Make sure the receiver is loaded into a register.
1844 assert(method->size_of_parameters() > 0, "oob");
1845 assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
1846 VMReg r = regs[0].first();
1847 assert(r->is_valid(), "bad receiver arg");
1848 if (r->is_stack()) {
1849 // Porting note: This assumes that compiled calling conventions always
1850 // pass the receiver oop in a register. If this is not true on some
1851 // platform, pick a temp and load the receiver from stack.
1852 fatal("receiver always in a register");
1853 receiver_reg = j_rarg0; // known to be free at this point
1854 __ movptr(receiver_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1855 } else {
1856 // no data motion is needed
1857 receiver_reg = r->as_Register();
1858 }
1859 }
1860
1861 // Figure out which address we are really jumping to:
1862 MethodHandles::generate_method_handle_dispatch(masm, iid,
1863 receiver_reg, member_reg, /*for_compiler_entry:*/ true);
1864 }
1865
1866 // ---------------------------------------------------------------------------
1867 // Generate a native wrapper for a given method. The method takes arguments
1868 // in the Java compiled code convention, marshals them to the native
1869 // convention (handlizes oops, etc), transitions to native, makes the call,
1870 // returns to java state (possibly blocking), unhandlizes any result and
1871 // returns.
1872 //
1873 // Critical native functions are a shorthand for the use of
1874 // GetPrimtiveArrayCritical and disallow the use of any other JNI
1875 // functions. The wrapper is expected to unpack the arguments before
1876 // passing them to the callee and perform checks before and after the
1877 // native call to ensure that they GCLocker
1878 // lock_critical/unlock_critical semantics are followed. Some other
1879 // parts of JNI setup are skipped like the tear down of the JNI handle
1880 // block and the check for pending exceptions it's impossible for them
1881 // to be thrown.
1882 //
1883 // They are roughly structured like this:
1884 // if (GCLocker::needs_gc())
1885 // SharedRuntime::block_for_jni_critical();
1886 // tranistion to thread_in_native
1887 // unpack arrray arguments and call native entry point
1888 // check for safepoint in progress
1889 // check if any thread suspend flags are set
1890 // call into JVM and possible unlock the JNI critical
1891 // if a GC was suppressed while in the critical native.
1892 // transition back to thread_in_Java
1893 // return to caller
1894 //
1895 nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
1896 const methodHandle& method,
1897 int compile_id,
1898 BasicType* in_sig_bt,
1899 VMRegPair* in_regs,
1900 BasicType ret_type) {
1901 if (method->is_method_handle_intrinsic()) {
1902 vmIntrinsics::ID iid = method->intrinsic_id();
1903 intptr_t start = (intptr_t)__ pc();
1904 int vep_offset = ((intptr_t)__ pc()) - start;
1905 gen_special_dispatch(masm,
1906 method,
1907 in_sig_bt,
1908 in_regs);
1909 int frame_complete = ((intptr_t)__ pc()) - start; // not complete, period
1910 __ flush();
1911 int stack_slots = SharedRuntime::out_preserve_stack_slots(); // no out slots at all, actually
1912 return nmethod::new_native_nmethod(method,
1913 compile_id,
1914 masm->code(),
1915 vep_offset,
1916 frame_complete,
1917 stack_slots / VMRegImpl::slots_per_word,
1918 in_ByteSize(-1),
1919 in_ByteSize(-1),
1920 (OopMapSet*)NULL);
1921 }
1922 bool is_critical_native = true;
1923 address native_func = method->critical_native_function();
1924 if (native_func == NULL) {
1925 native_func = method->native_function();
1926 is_critical_native = false;
1927 }
1928 assert(native_func != NULL, "must have function");
1929
1930 // An OopMap for lock (and class if static)
1931 OopMapSet *oop_maps = new OopMapSet();
1932 intptr_t start = (intptr_t)__ pc();
1933
1934 // We have received a description of where all the java arg are located
1935 // on entry to the wrapper. We need to convert these args to where
1936 // the jni function will expect them. To figure out where they go
1937 // we convert the java signature to a C signature by inserting
1938 // the hidden arguments as arg[0] and possibly arg[1] (static method)
1939
1940 const int total_in_args = method->size_of_parameters();
1941 int total_c_args = total_in_args;
1942 if (!is_critical_native) {
1943 total_c_args += 1;
1944 if (method->is_static()) {
1945 total_c_args++;
1946 }
1947 } else {
1948 for (int i = 0; i < total_in_args; i++) {
1949 if (in_sig_bt[i] == T_ARRAY) {
1950 total_c_args++;
1951 }
1952 }
1953 }
1954
1955 BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
1956 VMRegPair* out_regs = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
1957 BasicType* in_elem_bt = NULL;
1958
1959 int argc = 0;
1960 if (!is_critical_native) {
1961 out_sig_bt[argc++] = T_ADDRESS;
1962 if (method->is_static()) {
1963 out_sig_bt[argc++] = T_OBJECT;
1964 }
1965
1966 for (int i = 0; i < total_in_args ; i++ ) {
1967 out_sig_bt[argc++] = in_sig_bt[i];
1968 }
1969 } else {
1970 Thread* THREAD = Thread::current();
1971 in_elem_bt = NEW_RESOURCE_ARRAY(BasicType, total_in_args);
1972 SignatureStream ss(method->signature());
1973 for (int i = 0; i < total_in_args ; i++ ) {
1974 if (in_sig_bt[i] == T_ARRAY) {
1975 // Arrays are passed as int, elem* pair
1976 out_sig_bt[argc++] = T_INT;
1977 out_sig_bt[argc++] = T_ADDRESS;
1978 Symbol* atype = ss.as_symbol(CHECK_NULL);
1979 const char* at = atype->as_C_string();
1980 if (strlen(at) == 2) {
1981 assert(at[0] == '[', "must be");
1982 switch (at[1]) {
1983 case 'B': in_elem_bt[i] = T_BYTE; break;
1984 case 'C': in_elem_bt[i] = T_CHAR; break;
1985 case 'D': in_elem_bt[i] = T_DOUBLE; break;
1986 case 'F': in_elem_bt[i] = T_FLOAT; break;
1987 case 'I': in_elem_bt[i] = T_INT; break;
1988 case 'J': in_elem_bt[i] = T_LONG; break;
1989 case 'S': in_elem_bt[i] = T_SHORT; break;
1990 case 'Z': in_elem_bt[i] = T_BOOLEAN; break;
1991 default: ShouldNotReachHere();
1992 }
1993 }
1994 } else {
1995 out_sig_bt[argc++] = in_sig_bt[i];
1996 in_elem_bt[i] = T_VOID;
1997 }
1998 if (in_sig_bt[i] != T_VOID) {
1999 assert(in_sig_bt[i] == ss.type(), "must match");
2000 ss.next();
2001 }
2002 }
2003 }
2004
2005 // Now figure out where the args must be stored and how much stack space
2006 // they require.
2007 int out_arg_slots;
2008 out_arg_slots = c_calling_convention(out_sig_bt, out_regs, NULL, total_c_args);
2009
2010 // Compute framesize for the wrapper. We need to handlize all oops in
2011 // incoming registers
2012
2013 // Calculate the total number of stack slots we will need.
2014
2015 // First count the abi requirement plus all of the outgoing args
2016 int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots;
2017
2018 // Now the space for the inbound oop handle area
2019 int total_save_slots = 6 * VMRegImpl::slots_per_word; // 6 arguments passed in registers
2020 if (is_critical_native) {
2021 // Critical natives may have to call out so they need a save area
2022 // for register arguments.
2023 int double_slots = 0;
2024 int single_slots = 0;
2025 for ( int i = 0; i < total_in_args; i++) {
2026 if (in_regs[i].first()->is_Register()) {
2027 const Register reg = in_regs[i].first()->as_Register();
2028 switch (in_sig_bt[i]) {
2029 case T_BOOLEAN:
2030 case T_BYTE:
2031 case T_SHORT:
2032 case T_CHAR:
2033 case T_INT: single_slots++; break;
2034 case T_ARRAY: // specific to LP64 (7145024)
2035 case T_LONG: double_slots++; break;
2036 default: ShouldNotReachHere();
2037 }
2038 } else if (in_regs[i].first()->is_XMMRegister()) {
2039 switch (in_sig_bt[i]) {
2040 case T_FLOAT: single_slots++; break;
2041 case T_DOUBLE: double_slots++; break;
2042 default: ShouldNotReachHere();
2043 }
2044 } else if (in_regs[i].first()->is_FloatRegister()) {
2045 ShouldNotReachHere();
2046 }
2047 }
2048 total_save_slots = double_slots * 2 + single_slots;
2049 // align the save area
2050 if (double_slots != 0) {
2051 stack_slots = align_up(stack_slots, 2);
2052 }
2053 }
2054
2055 int oop_handle_offset = stack_slots;
2056 stack_slots += total_save_slots;
2057
2058 // Now any space we need for handlizing a klass if static method
2059
2060 int klass_slot_offset = 0;
2061 int klass_offset = -1;
2062 int lock_slot_offset = 0;
2063 bool is_static = false;
2064
2065 if (method->is_static()) {
2066 klass_slot_offset = stack_slots;
2067 stack_slots += VMRegImpl::slots_per_word;
2068 klass_offset = klass_slot_offset * VMRegImpl::stack_slot_size;
2069 is_static = true;
2070 }
2071
2072 // Plus a lock if needed
2073
2074 if (method->is_synchronized()) {
2075 lock_slot_offset = stack_slots;
2076 stack_slots += VMRegImpl::slots_per_word;
2077 }
2078
2079 // Now a place (+2) to save return values or temp during shuffling
2080 // + 4 for return address (which we own) and saved rbp
2081 stack_slots += 6;
2082
2083 // Ok The space we have allocated will look like:
2084 //
2085 //
2086 // FP-> | |
2087 // |---------------------|
2088 // | 2 slots for moves |
2089 // |---------------------|
2090 // | lock box (if sync) |
2091 // |---------------------| <- lock_slot_offset
2092 // | klass (if static) |
2093 // |---------------------| <- klass_slot_offset
2094 // | oopHandle area |
2095 // |---------------------| <- oop_handle_offset (6 java arg registers)
2096 // | outbound memory |
2097 // | based arguments |
2098 // | |
2099 // |---------------------|
2100 // | |
2101 // SP-> | out_preserved_slots |
2102 //
2103 //
2104
2105
2106 // Now compute actual number of stack words we need rounding to make
2107 // stack properly aligned.
2108 stack_slots = align_up(stack_slots, StackAlignmentInSlots);
2109
2110 int stack_size = stack_slots * VMRegImpl::stack_slot_size;
2111
2112 // First thing make an ic check to see if we should even be here
2113
2114 // We are free to use all registers as temps without saving them and
2115 // restoring them except rbp. rbp is the only callee save register
2116 // as far as the interpreter and the compiler(s) are concerned.
2117
2118
2119 const Register ic_reg = rax;
2120 const Register receiver = j_rarg0;
2121
2122 Label hit;
2123 Label exception_pending;
2124
2125 assert_different_registers(ic_reg, receiver, rscratch1);
2126 __ verify_oop(receiver);
2127 __ load_klass(rscratch1, receiver);
2128 __ cmpq(ic_reg, rscratch1);
2129 __ jcc(Assembler::equal, hit);
2130
2131 __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2132
2133 // Verified entry point must be aligned
2134 __ align(8);
2135
2136 __ bind(hit);
2137
2138 int vep_offset = ((intptr_t)__ pc()) - start;
2139
2140 #ifdef COMPILER1
2141 // For Object.hashCode, System.identityHashCode try to pull hashCode from object header if available.
2142 if ((InlineObjectHash && method->intrinsic_id() == vmIntrinsics::_hashCode) || (method->intrinsic_id() == vmIntrinsics::_identityHashCode)) {
2143 inline_check_hashcode_from_object_header(masm, method, j_rarg0 /*obj_reg*/, rax /*result*/);
2144 }
2145 #endif // COMPILER1
2146
2147 // The instruction at the verified entry point must be 5 bytes or longer
2148 // because it can be patched on the fly by make_non_entrant. The stack bang
2149 // instruction fits that requirement.
2150
2151 // Generate stack overflow check
2152
2153 if (UseStackBanging) {
2154 __ bang_stack_with_offset((int)JavaThread::stack_shadow_zone_size());
2155 } else {
2156 // need a 5 byte instruction to allow MT safe patching to non-entrant
2157 __ fat_nop();
2158 }
2159
2160 // Generate a new frame for the wrapper.
2161 __ enter();
2162 // -2 because return address is already present and so is saved rbp
2163 __ subptr(rsp, stack_size - 2*wordSize);
2164
2165 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2166 bs->nmethod_entry_barrier(masm);
2167
2168 // Frame is now completed as far as size and linkage.
2169 int frame_complete = ((intptr_t)__ pc()) - start;
2170
2171 if (UseRTMLocking) {
2172 // Abort RTM transaction before calling JNI
2173 // because critical section will be large and will be
2174 // aborted anyway. Also nmethod could be deoptimized.
2175 __ xabort(0);
2176 }
2177
2178 #ifdef ASSERT
2179 {
2180 Label L;
2181 __ mov(rax, rsp);
2182 __ andptr(rax, -16); // must be 16 byte boundary (see amd64 ABI)
2183 __ cmpptr(rax, rsp);
2184 __ jcc(Assembler::equal, L);
2185 __ stop("improperly aligned stack");
2186 __ bind(L);
2187 }
2188 #endif /* ASSERT */
2189
2190
2191 // We use r14 as the oop handle for the receiver/klass
2192 // It is callee save so it survives the call to native
2193
2194 const Register oop_handle_reg = r14;
2195
2196 if (is_critical_native && !Universe::heap()->supports_object_pinning()) {
2197 check_needs_gc_for_critical_native(masm, stack_slots, total_c_args, total_in_args,
2198 oop_handle_offset, oop_maps, in_regs, in_sig_bt);
2199 }
2200
2201 //
2202 // We immediately shuffle the arguments so that any vm call we have to
2203 // make from here on out (sync slow path, jvmti, etc.) we will have
2204 // captured the oops from our caller and have a valid oopMap for
2205 // them.
2206
2207 // -----------------
2208 // The Grand Shuffle
2209
2210 // The Java calling convention is either equal (linux) or denser (win64) than the
2211 // c calling convention. However the because of the jni_env argument the c calling
2212 // convention always has at least one more (and two for static) arguments than Java.
2213 // Therefore if we move the args from java -> c backwards then we will never have
2214 // a register->register conflict and we don't have to build a dependency graph
2215 // and figure out how to break any cycles.
2216 //
2217
2218 // Record esp-based slot for receiver on stack for non-static methods
2219 int receiver_offset = -1;
2220
2221 // This is a trick. We double the stack slots so we can claim
2222 // the oops in the caller's frame. Since we are sure to have
2223 // more args than the caller doubling is enough to make
2224 // sure we can capture all the incoming oop args from the
2225 // caller.
2226 //
2227 OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
2228
2229 // Mark location of rbp (someday)
2230 // map->set_callee_saved(VMRegImpl::stack2reg( stack_slots - 2), stack_slots * 2, 0, vmreg(rbp));
2231
2232 // Use eax, ebx as temporaries during any memory-memory moves we have to do
2233 // All inbound args are referenced based on rbp and all outbound args via rsp.
2234
2235
2236 #ifdef ASSERT
2237 bool reg_destroyed[RegisterImpl::number_of_registers];
2238 bool freg_destroyed[XMMRegisterImpl::number_of_registers];
2239 for ( int r = 0 ; r < RegisterImpl::number_of_registers ; r++ ) {
2240 reg_destroyed[r] = false;
2241 }
2242 for ( int f = 0 ; f < XMMRegisterImpl::number_of_registers ; f++ ) {
2243 freg_destroyed[f] = false;
2244 }
2245
2246 #endif /* ASSERT */
2247
2248 // This may iterate in two different directions depending on the
2249 // kind of native it is. The reason is that for regular JNI natives
2250 // the incoming and outgoing registers are offset upwards and for
2251 // critical natives they are offset down.
2252 GrowableArray<int> arg_order(2 * total_in_args);
2253 // Inbound arguments that need to be pinned for critical natives
2254 GrowableArray<int> pinned_args(total_in_args);
2255 // Current stack slot for storing register based array argument
2256 int pinned_slot = oop_handle_offset;
2257
2258 VMRegPair tmp_vmreg;
2259 tmp_vmreg.set2(rbx->as_VMReg());
2260
2261 if (!is_critical_native) {
2262 for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
2263 arg_order.push(i);
2264 arg_order.push(c_arg);
2265 }
2266 } else {
2267 // Compute a valid move order, using tmp_vmreg to break any cycles
2268 ComputeMoveOrder cmo(total_in_args, in_regs, total_c_args, out_regs, in_sig_bt, arg_order, tmp_vmreg);
2269 }
2270
2271 int temploc = -1;
2272 for (int ai = 0; ai < arg_order.length(); ai += 2) {
2273 int i = arg_order.at(ai);
2274 int c_arg = arg_order.at(ai + 1);
2275 __ block_comment(err_msg("move %d -> %d", i, c_arg));
2276 if (c_arg == -1) {
2277 assert(is_critical_native, "should only be required for critical natives");
2278 // This arg needs to be moved to a temporary
2279 __ mov(tmp_vmreg.first()->as_Register(), in_regs[i].first()->as_Register());
2280 in_regs[i] = tmp_vmreg;
2281 temploc = i;
2282 continue;
2283 } else if (i == -1) {
2284 assert(is_critical_native, "should only be required for critical natives");
2285 // Read from the temporary location
2286 assert(temploc != -1, "must be valid");
2287 i = temploc;
2288 temploc = -1;
2289 }
2290 #ifdef ASSERT
2291 if (in_regs[i].first()->is_Register()) {
2292 assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
2293 } else if (in_regs[i].first()->is_XMMRegister()) {
2294 assert(!freg_destroyed[in_regs[i].first()->as_XMMRegister()->encoding()], "destroyed reg!");
2295 }
2296 if (out_regs[c_arg].first()->is_Register()) {
2297 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2298 } else if (out_regs[c_arg].first()->is_XMMRegister()) {
2299 freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
2300 }
2301 #endif /* ASSERT */
2302 switch (in_sig_bt[i]) {
2303 case T_ARRAY:
2304 if (is_critical_native) {
2305 // pin before unpack
2306 if (Universe::heap()->supports_object_pinning()) {
2307 save_args(masm, total_c_args, 0, out_regs);
2308 gen_pin_object(masm, in_regs[i]);
2309 pinned_args.append(i);
2310 restore_args(masm, total_c_args, 0, out_regs);
2311
2312 // rax has pinned array
2313 VMRegPair result_reg;
2314 result_reg.set_ptr(rax->as_VMReg());
2315 move_ptr(masm, result_reg, in_regs[i]);
2316 if (!in_regs[i].first()->is_stack()) {
2317 assert(pinned_slot <= stack_slots, "overflow");
2318 move_ptr(masm, result_reg, VMRegImpl::stack2reg(pinned_slot));
2319 pinned_slot += VMRegImpl::slots_per_word;
2320 }
2321 }
2322 unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
2323 c_arg++;
2324 #ifdef ASSERT
2325 if (out_regs[c_arg].first()->is_Register()) {
2326 reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2327 } else if (out_regs[c_arg].first()->is_XMMRegister()) {
2328 freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
2329 }
2330 #endif
2331 break;
2332 }
2333 case T_OBJECT:
2334 assert(!is_critical_native, "no oop arguments");
2335 object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
2336 ((i == 0) && (!is_static)),
2337 &receiver_offset);
2338 break;
2339 case T_VOID:
2340 break;
2341
2342 case T_FLOAT:
2343 float_move(masm, in_regs[i], out_regs[c_arg]);
2344 break;
2345
2346 case T_DOUBLE:
2347 assert( i + 1 < total_in_args &&
2348 in_sig_bt[i + 1] == T_VOID &&
2349 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
2350 double_move(masm, in_regs[i], out_regs[c_arg]);
2351 break;
2352
2353 case T_LONG :
2354 long_move(masm, in_regs[i], out_regs[c_arg]);
2355 break;
2356
2357 case T_ADDRESS: assert(false, "found T_ADDRESS in java args");
2358
2359 default:
2360 move32_64(masm, in_regs[i], out_regs[c_arg]);
2361 }
2362 }
2363
2364 int c_arg;
2365
2366 // Pre-load a static method's oop into r14. Used both by locking code and
2367 // the normal JNI call code.
2368 if (!is_critical_native) {
2369 // point c_arg at the first arg that is already loaded in case we
2370 // need to spill before we call out
2371 c_arg = total_c_args - total_in_args;
2372
2373 if (method->is_static()) {
2374
2375 // load oop into a register
2376 __ movoop(oop_handle_reg, JNIHandles::make_local(method->method_holder()->java_mirror()));
2377
2378 // Now handlize the static class mirror it's known not-null.
2379 __ movptr(Address(rsp, klass_offset), oop_handle_reg);
2380 map->set_oop(VMRegImpl::stack2reg(klass_slot_offset));
2381
2382 // Now get the handle
2383 __ lea(oop_handle_reg, Address(rsp, klass_offset));
2384 // store the klass handle as second argument
2385 __ movptr(c_rarg1, oop_handle_reg);
2386 // and protect the arg if we must spill
2387 c_arg--;
2388 }
2389 } else {
2390 // For JNI critical methods we need to save all registers in save_args.
2391 c_arg = 0;
2392 }
2393
2394 // Change state to native (we save the return address in the thread, since it might not
2395 // be pushed on the stack when we do a a stack traversal). It is enough that the pc()
2396 // points into the right code segment. It does not have to be the correct return pc.
2397 // We use the same pc/oopMap repeatedly when we call out
2398
2399 intptr_t the_pc = (intptr_t) __ pc();
2400 oop_maps->add_gc_map(the_pc - start, map);
2401
2402 __ set_last_Java_frame(rsp, noreg, (address)the_pc);
2403
2404
2405 // We have all of the arguments setup at this point. We must not touch any register
2406 // argument registers at this point (what if we save/restore them there are no oop?
2407
2408 {
2409 SkipIfEqual skip(masm, &DTraceMethodProbes, false);
2410 // protect the args we've loaded
2411 save_args(masm, total_c_args, c_arg, out_regs);
2412 __ mov_metadata(c_rarg1, method());
2413 __ call_VM_leaf(
2414 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
2415 r15_thread, c_rarg1);
2416 restore_args(masm, total_c_args, c_arg, out_regs);
2417 }
2418
2419 // RedefineClasses() tracing support for obsolete method entry
2420 if (log_is_enabled(Trace, redefine, class, obsolete)) {
2421 // protect the args we've loaded
2422 save_args(masm, total_c_args, c_arg, out_regs);
2423 __ mov_metadata(c_rarg1, method());
2424 __ call_VM_leaf(
2425 CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
2426 r15_thread, c_rarg1);
2427 restore_args(masm, total_c_args, c_arg, out_regs);
2428 }
2429
2430 // Lock a synchronized method
2431
2432 // Register definitions used by locking and unlocking
2433
2434 const Register swap_reg = rax; // Must use rax for cmpxchg instruction
2435 const Register obj_reg = rbx; // Will contain the oop
2436 const Register lock_reg = r13; // Address of compiler lock object (BasicLock)
2437 const Register old_hdr = r13; // value of old header at unlock time
2438
2439 Label slow_path_lock;
2440 Label lock_done;
2441
2442 if (method->is_synchronized()) {
2443 assert(!is_critical_native, "unhandled");
2444
2445
2446 const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
2447
2448 // Get the handle (the 2nd argument)
2449 __ mov(oop_handle_reg, c_rarg1);
2450
2451 // Get address of the box
2452
2453 __ lea(lock_reg, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2454
2455 // Load the oop from the handle
2456 __ movptr(obj_reg, Address(oop_handle_reg, 0));
2457
2458 __ resolve(IS_NOT_NULL, obj_reg);
2459 if (UseBiasedLocking) {
2460 __ biased_locking_enter(lock_reg, obj_reg, swap_reg, rscratch1, false, lock_done, &slow_path_lock);
2461 }
2462
2463 // Load immediate 1 into swap_reg %rax
2464 __ movl(swap_reg, 1);
2465
2466 // Load (object->mark() | 1) into swap_reg %rax
2467 __ orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2468
2469 // Save (object->mark() | 1) into BasicLock's displaced header
2470 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2471
2472 // src -> dest iff dest == rax else rax <- dest
2473 __ lock();
2474 __ cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2475 __ jcc(Assembler::equal, lock_done);
2476
2477 // Hmm should this move to the slow path code area???
2478
2479 // Test if the oopMark is an obvious stack pointer, i.e.,
2480 // 1) (mark & 3) == 0, and
2481 // 2) rsp <= mark < mark + os::pagesize()
2482 // These 3 tests can be done by evaluating the following
2483 // expression: ((mark - rsp) & (3 - os::vm_page_size())),
2484 // assuming both stack pointer and pagesize have their
2485 // least significant 2 bits clear.
2486 // NOTE: the oopMark is in swap_reg %rax as the result of cmpxchg
2487
2488 __ subptr(swap_reg, rsp);
2489 __ andptr(swap_reg, 3 - os::vm_page_size());
2490
2491 // Save the test result, for recursive case, the result is zero
2492 __ movptr(Address(lock_reg, mark_word_offset), swap_reg);
2493 __ jcc(Assembler::notEqual, slow_path_lock);
2494
2495 // Slow path will re-enter here
2496
2497 __ bind(lock_done);
2498 }
2499
2500
2501 // Finally just about ready to make the JNI call
2502
2503
2504 // get JNIEnv* which is first argument to native
2505 if (!is_critical_native) {
2506 __ lea(c_rarg0, Address(r15_thread, in_bytes(JavaThread::jni_environment_offset())));
2507 }
2508
2509 // Now set thread in native
2510 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native);
2511
2512 __ call(RuntimeAddress(native_func));
2513
2514 // Verify or restore cpu control state after JNI call
2515 __ restore_cpu_control_state_after_jni();
2516
2517 // Unpack native results.
2518 switch (ret_type) {
2519 case T_BOOLEAN: __ c2bool(rax); break;
2520 case T_CHAR : __ movzwl(rax, rax); break;
2521 case T_BYTE : __ sign_extend_byte (rax); break;
2522 case T_SHORT : __ sign_extend_short(rax); break;
2523 case T_INT : /* nothing to do */ break;
2524 case T_DOUBLE :
2525 case T_FLOAT :
2526 // Result is in xmm0 we'll save as needed
2527 break;
2528 case T_ARRAY: // Really a handle
2529 case T_OBJECT: // Really a handle
2530 break; // can't de-handlize until after safepoint check
2531 case T_VOID: break;
2532 case T_LONG: break;
2533 default : ShouldNotReachHere();
2534 }
2535
2536 // unpin pinned arguments
2537 pinned_slot = oop_handle_offset;
2538 if (pinned_args.length() > 0) {
2539 // save return value that may be overwritten otherwise.
2540 save_native_result(masm, ret_type, stack_slots);
2541 for (int index = 0; index < pinned_args.length(); index ++) {
2542 int i = pinned_args.at(index);
2543 assert(pinned_slot <= stack_slots, "overflow");
2544 if (!in_regs[i].first()->is_stack()) {
2545 int offset = pinned_slot * VMRegImpl::stack_slot_size;
2546 __ movq(in_regs[i].first()->as_Register(), Address(rsp, offset));
2547 pinned_slot += VMRegImpl::slots_per_word;
2548 }
2549 gen_unpin_object(masm, in_regs[i]);
2550 }
2551 restore_native_result(masm, ret_type, stack_slots);
2552 }
2553
2554 // Switch thread to "native transition" state before reading the synchronization state.
2555 // This additional state is necessary because reading and testing the synchronization
2556 // state is not atomic w.r.t. GC, as this scenario demonstrates:
2557 // Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2558 // VM thread changes sync state to synchronizing and suspends threads for GC.
2559 // Thread A is resumed to finish this native method, but doesn't block here since it
2560 // didn't see any synchronization is progress, and escapes.
2561 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2562
2563 // Force this write out before the read below
2564 __ membar(Assembler::Membar_mask_bits(
2565 Assembler::LoadLoad | Assembler::LoadStore |
2566 Assembler::StoreLoad | Assembler::StoreStore));
2567
2568 Label after_transition;
2569
2570 // check for safepoint operation in progress and/or pending suspend requests
2571 {
2572 Label Continue;
2573 Label slow_path;
2574
2575 __ safepoint_poll(slow_path, r15_thread, rscratch1);
2576
2577 __ cmpl(Address(r15_thread, JavaThread::suspend_flags_offset()), 0);
2578 __ jcc(Assembler::equal, Continue);
2579 __ bind(slow_path);
2580
2581 // Don't use call_VM as it will see a possible pending exception and forward it
2582 // and never return here preventing us from clearing _last_native_pc down below.
2583 // Also can't use call_VM_leaf either as it will check to see if rsi & rdi are
2584 // preserved and correspond to the bcp/locals pointers. So we do a runtime call
2585 // by hand.
2586 //
2587 __ vzeroupper();
2588 save_native_result(masm, ret_type, stack_slots);
2589 __ mov(c_rarg0, r15_thread);
2590 __ mov(r12, rsp); // remember sp
2591 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
2592 __ andptr(rsp, -16); // align stack as required by ABI
2593 if (!is_critical_native) {
2594 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
2595 } else {
2596 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans_and_transition)));
2597 }
2598 __ mov(rsp, r12); // restore sp
2599 __ reinit_heapbase();
2600 // Restore any method result value
2601 restore_native_result(masm, ret_type, stack_slots);
2602
2603 if (is_critical_native) {
2604 // The call above performed the transition to thread_in_Java so
2605 // skip the transition logic below.
2606 __ jmpb(after_transition);
2607 }
2608
2609 __ bind(Continue);
2610 }
2611
2612 // change thread state
2613 __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
2614 __ bind(after_transition);
2615
2616 Label reguard;
2617 Label reguard_done;
2618 __ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()), JavaThread::stack_guard_yellow_reserved_disabled);
2619 __ jcc(Assembler::equal, reguard);
2620 __ bind(reguard_done);
2621
2622 // native result if any is live
2623
2624 // Unlock
2625 Label unlock_done;
2626 Label slow_path_unlock;
2627 if (method->is_synchronized()) {
2628
2629 // Get locked oop from the handle we passed to jni
2630 __ movptr(obj_reg, Address(oop_handle_reg, 0));
2631 __ resolve(IS_NOT_NULL, obj_reg);
2632
2633 Label done;
2634
2635 if (UseBiasedLocking) {
2636 __ biased_locking_exit(obj_reg, old_hdr, done);
2637 }
2638
2639 // Simple recursive lock?
2640
2641 __ cmpptr(Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size), (int32_t)NULL_WORD);
2642 __ jcc(Assembler::equal, done);
2643
2644 // Must save rax if if it is live now because cmpxchg must use it
2645 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2646 save_native_result(masm, ret_type, stack_slots);
2647 }
2648
2649
2650 // get address of the stack lock
2651 __ lea(rax, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2652 // get old displaced header
2653 __ movptr(old_hdr, Address(rax, 0));
2654
2655 // Atomic swap old header if oop still contains the stack lock
2656 __ lock();
2657 __ cmpxchgptr(old_hdr, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2658 __ jcc(Assembler::notEqual, slow_path_unlock);
2659
2660 // slow path re-enters here
2661 __ bind(unlock_done);
2662 if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2663 restore_native_result(masm, ret_type, stack_slots);
2664 }
2665
2666 __ bind(done);
2667
2668 }
2669 {
2670 SkipIfEqual skip(masm, &DTraceMethodProbes, false);
2671 save_native_result(masm, ret_type, stack_slots);
2672 __ mov_metadata(c_rarg1, method());
2673 __ call_VM_leaf(
2674 CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2675 r15_thread, c_rarg1);
2676 restore_native_result(masm, ret_type, stack_slots);
2677 }
2678
2679 __ reset_last_Java_frame(false);
2680
2681 // Unbox oop result, e.g. JNIHandles::resolve value.
2682 if (ret_type == T_OBJECT || ret_type == T_ARRAY) {
2683 __ resolve_jobject(rax /* value */,
2684 r15_thread /* thread */,
2685 rcx /* tmp */);
2686 }
2687
2688 if (CheckJNICalls) {
2689 // clear_pending_jni_exception_check
2690 __ movptr(Address(r15_thread, JavaThread::pending_jni_exception_check_fn_offset()), NULL_WORD);
2691 }
2692
2693 if (!is_critical_native) {
2694 // reset handle block
2695 __ movptr(rcx, Address(r15_thread, JavaThread::active_handles_offset()));
2696 __ movl(Address(rcx, JNIHandleBlock::top_offset_in_bytes()), (int32_t)NULL_WORD);
2697 }
2698
2699 // pop our frame
2700
2701 __ leave();
2702
2703 if (!is_critical_native) {
2704 // Any exception pending?
2705 __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), (int32_t)NULL_WORD);
2706 __ jcc(Assembler::notEqual, exception_pending);
2707 }
2708
2709 // Return
2710
2711 __ ret(0);
2712
2713 // Unexpected paths are out of line and go here
2714
2715 if (!is_critical_native) {
2716 // forward the exception
2717 __ bind(exception_pending);
2718
2719 // and forward the exception
2720 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2721 }
2722
2723 // Slow path locking & unlocking
2724 if (method->is_synchronized()) {
2725
2726 // BEGIN Slow path lock
2727 __ bind(slow_path_lock);
2728
2729 // has last_Java_frame setup. No exceptions so do vanilla call not call_VM
2730 // args are (oop obj, BasicLock* lock, JavaThread* thread)
2731
2732 // protect the args we've loaded
2733 save_args(masm, total_c_args, c_arg, out_regs);
2734
2735 __ mov(c_rarg0, obj_reg);
2736 __ mov(c_rarg1, lock_reg);
2737 __ mov(c_rarg2, r15_thread);
2738
2739 // Not a leaf but we have last_Java_frame setup as we want
2740 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3);
2741 restore_args(masm, total_c_args, c_arg, out_regs);
2742
2743 #ifdef ASSERT
2744 { Label L;
2745 __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), (int32_t)NULL_WORD);
2746 __ jcc(Assembler::equal, L);
2747 __ stop("no pending exception allowed on exit from monitorenter");
2748 __ bind(L);
2749 }
2750 #endif
2751 __ jmp(lock_done);
2752
2753 // END Slow path lock
2754
2755 // BEGIN Slow path unlock
2756 __ bind(slow_path_unlock);
2757
2758 // If we haven't already saved the native result we must save it now as xmm registers
2759 // are still exposed.
2760 __ vzeroupper();
2761 if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
2762 save_native_result(masm, ret_type, stack_slots);
2763 }
2764
2765 __ lea(c_rarg1, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2766
2767 __ mov(c_rarg0, obj_reg);
2768 __ mov(c_rarg2, r15_thread);
2769 __ mov(r12, rsp); // remember sp
2770 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
2771 __ andptr(rsp, -16); // align stack as required by ABI
2772
2773 // Save pending exception around call to VM (which contains an EXCEPTION_MARK)
2774 // NOTE that obj_reg == rbx currently
2775 __ movptr(rbx, Address(r15_thread, in_bytes(Thread::pending_exception_offset())));
2776 __ movptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), (int32_t)NULL_WORD);
2777
2778 // args are (oop obj, BasicLock* lock, JavaThread* thread)
2779 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C)));
2780 __ mov(rsp, r12); // restore sp
2781 __ reinit_heapbase();
2782 #ifdef ASSERT
2783 {
2784 Label L;
2785 __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), (int)NULL_WORD);
2786 __ jcc(Assembler::equal, L);
2787 __ stop("no pending exception allowed on exit complete_monitor_unlocking_C");
2788 __ bind(L);
2789 }
2790 #endif /* ASSERT */
2791
2792 __ movptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), rbx);
2793
2794 if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
2795 restore_native_result(masm, ret_type, stack_slots);
2796 }
2797 __ jmp(unlock_done);
2798
2799 // END Slow path unlock
2800
2801 } // synchronized
2802
2803 // SLOW PATH Reguard the stack if needed
2804
2805 __ bind(reguard);
2806 __ vzeroupper();
2807 save_native_result(masm, ret_type, stack_slots);
2808 __ mov(r12, rsp); // remember sp
2809 __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
2810 __ andptr(rsp, -16); // align stack as required by ABI
2811 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
2812 __ mov(rsp, r12); // restore sp
2813 __ reinit_heapbase();
2814 restore_native_result(masm, ret_type, stack_slots);
2815 // and continue
2816 __ jmp(reguard_done);
2817
2818
2819
2820 __ flush();
2821
2822 nmethod *nm = nmethod::new_native_nmethod(method,
2823 compile_id,
2824 masm->code(),
2825 vep_offset,
2826 frame_complete,
2827 stack_slots / VMRegImpl::slots_per_word,
2828 (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)),
2829 in_ByteSize(lock_slot_offset*VMRegImpl::stack_slot_size),
2830 oop_maps);
2831
2832 if (is_critical_native) {
2833 nm->set_lazy_critical_native(true);
2834 }
2835
2836 return nm;
2837
2838 }
2839
2840 // this function returns the adjust size (in number of words) to a c2i adapter
2841 // activation for use during deoptimization
2842 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals ) {
2843 return (callee_locals - callee_parameters) * Interpreter::stackElementWords;
2844 }
2845
2846
2847 uint SharedRuntime::out_preserve_stack_slots() {
2848 return 0;
2849 }
2850
2851 //------------------------------generate_deopt_blob----------------------------
2852 void SharedRuntime::generate_deopt_blob() {
2853 // Allocate space for the code
2854 ResourceMark rm;
2855 // Setup code generation tools
2856 int pad = 0;
2857 #if INCLUDE_JVMCI
2858 if (EnableJVMCI || UseAOT) {
2859 pad += 512; // Increase the buffer size when compiling for JVMCI
2860 }
2861 #endif
2862 CodeBuffer buffer("deopt_blob", 2048+pad, 1024);
2863 MacroAssembler* masm = new MacroAssembler(&buffer);
2864 int frame_size_in_words;
2865 OopMap* map = NULL;
2866 OopMapSet *oop_maps = new OopMapSet();
2867
2868 // -------------
2869 // This code enters when returning to a de-optimized nmethod. A return
2870 // address has been pushed on the the stack, and return values are in
2871 // registers.
2872 // If we are doing a normal deopt then we were called from the patched
2873 // nmethod from the point we returned to the nmethod. So the return
2874 // address on the stack is wrong by NativeCall::instruction_size
2875 // We will adjust the value so it looks like we have the original return
2876 // address on the stack (like when we eagerly deoptimized).
2877 // In the case of an exception pending when deoptimizing, we enter
2878 // with a return address on the stack that points after the call we patched
2879 // into the exception handler. We have the following register state from,
2880 // e.g., the forward exception stub (see stubGenerator_x86_64.cpp).
2881 // rax: exception oop
2882 // rbx: exception handler
2883 // rdx: throwing pc
2884 // So in this case we simply jam rdx into the useless return address and
2885 // the stack looks just like we want.
2886 //
2887 // At this point we need to de-opt. We save the argument return
2888 // registers. We call the first C routine, fetch_unroll_info(). This
2889 // routine captures the return values and returns a structure which
2890 // describes the current frame size and the sizes of all replacement frames.
2891 // The current frame is compiled code and may contain many inlined
2892 // functions, each with their own JVM state. We pop the current frame, then
2893 // push all the new frames. Then we call the C routine unpack_frames() to
2894 // populate these frames. Finally unpack_frames() returns us the new target
2895 // address. Notice that callee-save registers are BLOWN here; they have
2896 // already been captured in the vframeArray at the time the return PC was
2897 // patched.
2898 address start = __ pc();
2899 Label cont;
2900
2901 // Prolog for non exception case!
2902
2903 // Save everything in sight.
2904 map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words);
2905
2906 // Normal deoptimization. Save exec mode for unpack_frames.
2907 __ movl(r14, Deoptimization::Unpack_deopt); // callee-saved
2908 __ jmp(cont);
2909
2910 int reexecute_offset = __ pc() - start;
2911 #if INCLUDE_JVMCI && !defined(COMPILER1)
2912 if (EnableJVMCI && UseJVMCICompiler) {
2913 // JVMCI does not use this kind of deoptimization
2914 __ should_not_reach_here();
2915 }
2916 #endif
2917
2918 // Reexecute case
2919 // return address is the pc describes what bci to do re-execute at
2920
2921 // No need to update map as each call to save_live_registers will produce identical oopmap
2922 (void) RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words);
2923
2924 __ movl(r14, Deoptimization::Unpack_reexecute); // callee-saved
2925 __ jmp(cont);
2926
2927 #if INCLUDE_JVMCI
2928 Label after_fetch_unroll_info_call;
2929 int implicit_exception_uncommon_trap_offset = 0;
2930 int uncommon_trap_offset = 0;
2931
2932 if (EnableJVMCI || UseAOT) {
2933 implicit_exception_uncommon_trap_offset = __ pc() - start;
2934
2935 __ pushptr(Address(r15_thread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())));
2936 __ movptr(Address(r15_thread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())), (int32_t)NULL_WORD);
2937
2938 uncommon_trap_offset = __ pc() - start;
2939
2940 // Save everything in sight.
2941 RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words);
2942 // fetch_unroll_info needs to call last_java_frame()
2943 __ set_last_Java_frame(noreg, noreg, NULL);
2944
2945 __ movl(c_rarg1, Address(r15_thread, in_bytes(JavaThread::pending_deoptimization_offset())));
2946 __ movl(Address(r15_thread, in_bytes(JavaThread::pending_deoptimization_offset())), -1);
2947
2948 __ movl(r14, (int32_t)Deoptimization::Unpack_reexecute);
2949 __ mov(c_rarg0, r15_thread);
2950 __ movl(c_rarg2, r14); // exec mode
2951 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::uncommon_trap)));
2952 oop_maps->add_gc_map( __ pc()-start, map->deep_copy());
2953
2954 __ reset_last_Java_frame(false);
2955
2956 __ jmp(after_fetch_unroll_info_call);
2957 } // EnableJVMCI
2958 #endif // INCLUDE_JVMCI
2959
2960 int exception_offset = __ pc() - start;
2961
2962 // Prolog for exception case
2963
2964 // all registers are dead at this entry point, except for rax, and
2965 // rdx which contain the exception oop and exception pc
2966 // respectively. Set them in TLS and fall thru to the
2967 // unpack_with_exception_in_tls entry point.
2968
2969 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), rdx);
2970 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), rax);
2971
2972 int exception_in_tls_offset = __ pc() - start;
2973
2974 // new implementation because exception oop is now passed in JavaThread
2975
2976 // Prolog for exception case
2977 // All registers must be preserved because they might be used by LinearScan
2978 // Exceptiop oop and throwing PC are passed in JavaThread
2979 // tos: stack at point of call to method that threw the exception (i.e. only
2980 // args are on the stack, no return address)
2981
2982 // make room on stack for the return address
2983 // It will be patched later with the throwing pc. The correct value is not
2984 // available now because loading it from memory would destroy registers.
2985 __ push(0);
2986
2987 // Save everything in sight.
2988 map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words);
2989
2990 // Now it is safe to overwrite any register
2991
2992 // Deopt during an exception. Save exec mode for unpack_frames.
2993 __ movl(r14, Deoptimization::Unpack_exception); // callee-saved
2994
2995 // load throwing pc from JavaThread and patch it as the return address
2996 // of the current frame. Then clear the field in JavaThread
2997
2998 __ movptr(rdx, Address(r15_thread, JavaThread::exception_pc_offset()));
2999 __ movptr(Address(rbp, wordSize), rdx);
3000 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), (int32_t)NULL_WORD);
3001
3002 #ifdef ASSERT
3003 // verify that there is really an exception oop in JavaThread
3004 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
3005 __ verify_oop(rax);
3006
3007 // verify that there is no pending exception
3008 Label no_pending_exception;
3009 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
3010 __ testptr(rax, rax);
3011 __ jcc(Assembler::zero, no_pending_exception);
3012 __ stop("must not have pending exception here");
3013 __ bind(no_pending_exception);
3014 #endif
3015
3016 __ bind(cont);
3017
3018 // Call C code. Need thread and this frame, but NOT official VM entry
3019 // crud. We cannot block on this call, no GC can happen.
3020 //
3021 // UnrollBlock* fetch_unroll_info(JavaThread* thread)
3022
3023 // fetch_unroll_info needs to call last_java_frame().
3024
3025 __ set_last_Java_frame(noreg, noreg, NULL);
3026 #ifdef ASSERT
3027 { Label L;
3028 __ cmpptr(Address(r15_thread,
3029 JavaThread::last_Java_fp_offset()),
3030 (int32_t)0);
3031 __ jcc(Assembler::equal, L);
3032 __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared");
3033 __ bind(L);
3034 }
3035 #endif // ASSERT
3036 __ mov(c_rarg0, r15_thread);
3037 __ movl(c_rarg1, r14); // exec_mode
3038 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info)));
3039
3040 // Need to have an oopmap that tells fetch_unroll_info where to
3041 // find any register it might need.
3042 oop_maps->add_gc_map(__ pc() - start, map);
3043
3044 __ reset_last_Java_frame(false);
3045
3046 #if INCLUDE_JVMCI
3047 if (EnableJVMCI || UseAOT) {
3048 __ bind(after_fetch_unroll_info_call);
3049 }
3050 #endif
3051
3052 // Load UnrollBlock* into rdi
3053 __ mov(rdi, rax);
3054
3055 __ movl(r14, Address(rdi, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes()));
3056 Label noException;
3057 __ cmpl(r14, Deoptimization::Unpack_exception); // Was exception pending?
3058 __ jcc(Assembler::notEqual, noException);
3059 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
3060 // QQQ this is useless it was NULL above
3061 __ movptr(rdx, Address(r15_thread, JavaThread::exception_pc_offset()));
3062 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), (int32_t)NULL_WORD);
3063 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), (int32_t)NULL_WORD);
3064
3065 __ verify_oop(rax);
3066
3067 // Overwrite the result registers with the exception results.
3068 __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax);
3069 // I think this is useless
3070 __ movptr(Address(rsp, RegisterSaver::rdx_offset_in_bytes()), rdx);
3071
3072 __ bind(noException);
3073
3074 // Only register save data is on the stack.
3075 // Now restore the result registers. Everything else is either dead
3076 // or captured in the vframeArray.
3077 RegisterSaver::restore_result_registers(masm);
3078
3079 // All of the register save area has been popped of the stack. Only the
3080 // return address remains.
3081
3082 // Pop all the frames we must move/replace.
3083 //
3084 // Frame picture (youngest to oldest)
3085 // 1: self-frame (no frame link)
3086 // 2: deopting frame (no frame link)
3087 // 3: caller of deopting frame (could be compiled/interpreted).
3088 //
3089 // Note: by leaving the return address of self-frame on the stack
3090 // and using the size of frame 2 to adjust the stack
3091 // when we are done the return to frame 3 will still be on the stack.
3092
3093 // Pop deoptimized frame
3094 __ movl(rcx, Address(rdi, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset_in_bytes()));
3095 __ addptr(rsp, rcx);
3096
3097 // rsp should be pointing at the return address to the caller (3)
3098
3099 // Pick up the initial fp we should save
3100 // restore rbp before stack bang because if stack overflow is thrown it needs to be pushed (and preserved)
3101 __ movptr(rbp, Address(rdi, Deoptimization::UnrollBlock::initial_info_offset_in_bytes()));
3102
3103 #ifdef ASSERT
3104 // Compilers generate code that bang the stack by as much as the
3105 // interpreter would need. So this stack banging should never
3106 // trigger a fault. Verify that it does not on non product builds.
3107 if (UseStackBanging) {
3108 __ movl(rbx, Address(rdi, Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes()));
3109 __ bang_stack_size(rbx, rcx);
3110 }
3111 #endif
3112
3113 // Load address of array of frame pcs into rcx
3114 __ movptr(rcx, Address(rdi, Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
3115
3116 // Trash the old pc
3117 __ addptr(rsp, wordSize);
3118
3119 // Load address of array of frame sizes into rsi
3120 __ movptr(rsi, Address(rdi, Deoptimization::UnrollBlock::frame_sizes_offset_in_bytes()));
3121
3122 // Load counter into rdx
3123 __ movl(rdx, Address(rdi, Deoptimization::UnrollBlock::number_of_frames_offset_in_bytes()));
3124
3125 // Now adjust the caller's stack to make up for the extra locals
3126 // but record the original sp so that we can save it in the skeletal interpreter
3127 // frame and the stack walking of interpreter_sender will get the unextended sp
3128 // value and not the "real" sp value.
3129
3130 const Register sender_sp = r8;
3131
3132 __ mov(sender_sp, rsp);
3133 __ movl(rbx, Address(rdi,
3134 Deoptimization::UnrollBlock::
3135 caller_adjustment_offset_in_bytes()));
3136 __ subptr(rsp, rbx);
3137
3138 // Push interpreter frames in a loop
3139 Label loop;
3140 __ bind(loop);
3141 __ movptr(rbx, Address(rsi, 0)); // Load frame size
3142 __ subptr(rbx, 2*wordSize); // We'll push pc and ebp by hand
3143 __ pushptr(Address(rcx, 0)); // Save return address
3144 __ enter(); // Save old & set new ebp
3145 __ subptr(rsp, rbx); // Prolog
3146 // This value is corrected by layout_activation_impl
3147 __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD );
3148 __ movptr(Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize), sender_sp); // Make it walkable
3149 __ mov(sender_sp, rsp); // Pass sender_sp to next frame
3150 __ addptr(rsi, wordSize); // Bump array pointer (sizes)
3151 __ addptr(rcx, wordSize); // Bump array pointer (pcs)
3152 __ decrementl(rdx); // Decrement counter
3153 __ jcc(Assembler::notZero, loop);
3154 __ pushptr(Address(rcx, 0)); // Save final return address
3155
3156 // Re-push self-frame
3157 __ enter(); // Save old & set new ebp
3158
3159 // Allocate a full sized register save area.
3160 // Return address and rbp are in place, so we allocate two less words.
3161 __ subptr(rsp, (frame_size_in_words - 2) * wordSize);
3162
3163 // Restore frame locals after moving the frame
3164 __ movdbl(Address(rsp, RegisterSaver::xmm0_offset_in_bytes()), xmm0);
3165 __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax);
3166
3167 // Call C code. Need thread but NOT official VM entry
3168 // crud. We cannot block on this call, no GC can happen. Call should
3169 // restore return values to their stack-slots with the new SP.
3170 //
3171 // void Deoptimization::unpack_frames(JavaThread* thread, int exec_mode)
3172
3173 // Use rbp because the frames look interpreted now
3174 // Save "the_pc" since it cannot easily be retrieved using the last_java_SP after we aligned SP.
3175 // Don't need the precise return PC here, just precise enough to point into this code blob.
3176 address the_pc = __ pc();
3177 __ set_last_Java_frame(noreg, rbp, the_pc);
3178
3179 __ andptr(rsp, -(StackAlignmentInBytes)); // Fix stack alignment as required by ABI
3180 __ mov(c_rarg0, r15_thread);
3181 __ movl(c_rarg1, r14); // second arg: exec_mode
3182 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
3183 // Revert SP alignment after call since we're going to do some SP relative addressing below
3184 __ movptr(rsp, Address(r15_thread, JavaThread::last_Java_sp_offset()));
3185
3186 // Set an oopmap for the call site
3187 // Use the same PC we used for the last java frame
3188 oop_maps->add_gc_map(the_pc - start,
3189 new OopMap( frame_size_in_words, 0 ));
3190
3191 // Clear fp AND pc
3192 __ reset_last_Java_frame(true);
3193
3194 // Collect return values
3195 __ movdbl(xmm0, Address(rsp, RegisterSaver::xmm0_offset_in_bytes()));
3196 __ movptr(rax, Address(rsp, RegisterSaver::rax_offset_in_bytes()));
3197 // I think this is useless (throwing pc?)
3198 __ movptr(rdx, Address(rsp, RegisterSaver::rdx_offset_in_bytes()));
3199
3200 // Pop self-frame.
3201 __ leave(); // Epilog
3202
3203 // Jump to interpreter
3204 __ ret(0);
3205
3206 // Make sure all code is generated
3207 masm->flush();
3208
3209 _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
3210 _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
3211 #if INCLUDE_JVMCI
3212 if (EnableJVMCI || UseAOT) {
3213 _deopt_blob->set_uncommon_trap_offset(uncommon_trap_offset);
3214 _deopt_blob->set_implicit_exception_uncommon_trap_offset(implicit_exception_uncommon_trap_offset);
3215 }
3216 #endif
3217 }
3218
3219 #ifdef COMPILER2
3220 //------------------------------generate_uncommon_trap_blob--------------------
3221 void SharedRuntime::generate_uncommon_trap_blob() {
3222 // Allocate space for the code
3223 ResourceMark rm;
3224 // Setup code generation tools
3225 CodeBuffer buffer("uncommon_trap_blob", 2048, 1024);
3226 MacroAssembler* masm = new MacroAssembler(&buffer);
3227
3228 assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
3229
3230 address start = __ pc();
3231
3232 if (UseRTMLocking) {
3233 // Abort RTM transaction before possible nmethod deoptimization.
3234 __ xabort(0);
3235 }
3236
3237 // Push self-frame. We get here with a return address on the
3238 // stack, so rsp is 8-byte aligned until we allocate our frame.
3239 __ subptr(rsp, SimpleRuntimeFrame::return_off << LogBytesPerInt); // Epilog!
3240
3241 // No callee saved registers. rbp is assumed implicitly saved
3242 __ movptr(Address(rsp, SimpleRuntimeFrame::rbp_off << LogBytesPerInt), rbp);
3243
3244 // compiler left unloaded_class_index in j_rarg0 move to where the
3245 // runtime expects it.
3246 __ movl(c_rarg1, j_rarg0);
3247
3248 __ set_last_Java_frame(noreg, noreg, NULL);
3249
3250 // Call C code. Need thread but NOT official VM entry
3251 // crud. We cannot block on this call, no GC can happen. Call should
3252 // capture callee-saved registers as well as return values.
3253 // Thread is in rdi already.
3254 //
3255 // UnrollBlock* uncommon_trap(JavaThread* thread, jint unloaded_class_index);
3256
3257 __ mov(c_rarg0, r15_thread);
3258 __ movl(c_rarg2, Deoptimization::Unpack_uncommon_trap);
3259 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::uncommon_trap)));
3260
3261 // Set an oopmap for the call site
3262 OopMapSet* oop_maps = new OopMapSet();
3263 OopMap* map = new OopMap(SimpleRuntimeFrame::framesize, 0);
3264
3265 // location of rbp is known implicitly by the frame sender code
3266
3267 oop_maps->add_gc_map(__ pc() - start, map);
3268
3269 __ reset_last_Java_frame(false);
3270
3271 // Load UnrollBlock* into rdi
3272 __ mov(rdi, rax);
3273
3274 #ifdef ASSERT
3275 { Label L;
3276 __ cmpptr(Address(rdi, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes()),
3277 (int32_t)Deoptimization::Unpack_uncommon_trap);
3278 __ jcc(Assembler::equal, L);
3279 __ stop("SharedRuntime::generate_deopt_blob: expected Unpack_uncommon_trap");
3280 __ bind(L);
3281 }
3282 #endif
3283
3284 // Pop all the frames we must move/replace.
3285 //
3286 // Frame picture (youngest to oldest)
3287 // 1: self-frame (no frame link)
3288 // 2: deopting frame (no frame link)
3289 // 3: caller of deopting frame (could be compiled/interpreted).
3290
3291 // Pop self-frame. We have no frame, and must rely only on rax and rsp.
3292 __ addptr(rsp, (SimpleRuntimeFrame::framesize - 2) << LogBytesPerInt); // Epilog!
3293
3294 // Pop deoptimized frame (int)
3295 __ movl(rcx, Address(rdi,
3296 Deoptimization::UnrollBlock::
3297 size_of_deoptimized_frame_offset_in_bytes()));
3298 __ addptr(rsp, rcx);
3299
3300 // rsp should be pointing at the return address to the caller (3)
3301
3302 // Pick up the initial fp we should save
3303 // restore rbp before stack bang because if stack overflow is thrown it needs to be pushed (and preserved)
3304 __ movptr(rbp, Address(rdi, Deoptimization::UnrollBlock::initial_info_offset_in_bytes()));
3305
3306 #ifdef ASSERT
3307 // Compilers generate code that bang the stack by as much as the
3308 // interpreter would need. So this stack banging should never
3309 // trigger a fault. Verify that it does not on non product builds.
3310 if (UseStackBanging) {
3311 __ movl(rbx, Address(rdi ,Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes()));
3312 __ bang_stack_size(rbx, rcx);
3313 }
3314 #endif
3315
3316 // Load address of array of frame pcs into rcx (address*)
3317 __ movptr(rcx, Address(rdi, Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
3318
3319 // Trash the return pc
3320 __ addptr(rsp, wordSize);
3321
3322 // Load address of array of frame sizes into rsi (intptr_t*)
3323 __ movptr(rsi, Address(rdi, Deoptimization::UnrollBlock:: frame_sizes_offset_in_bytes()));
3324
3325 // Counter
3326 __ movl(rdx, Address(rdi, Deoptimization::UnrollBlock:: number_of_frames_offset_in_bytes())); // (int)
3327
3328 // Now adjust the caller's stack to make up for the extra locals but
3329 // record the original sp so that we can save it in the skeletal
3330 // interpreter frame and the stack walking of interpreter_sender
3331 // will get the unextended sp value and not the "real" sp value.
3332
3333 const Register sender_sp = r8;
3334
3335 __ mov(sender_sp, rsp);
3336 __ movl(rbx, Address(rdi, Deoptimization::UnrollBlock:: caller_adjustment_offset_in_bytes())); // (int)
3337 __ subptr(rsp, rbx);
3338
3339 // Push interpreter frames in a loop
3340 Label loop;
3341 __ bind(loop);
3342 __ movptr(rbx, Address(rsi, 0)); // Load frame size
3343 __ subptr(rbx, 2 * wordSize); // We'll push pc and rbp by hand
3344 __ pushptr(Address(rcx, 0)); // Save return address
3345 __ enter(); // Save old & set new rbp
3346 __ subptr(rsp, rbx); // Prolog
3347 __ movptr(Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize),
3348 sender_sp); // Make it walkable
3349 // This value is corrected by layout_activation_impl
3350 __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), (int32_t)NULL_WORD );
3351 __ mov(sender_sp, rsp); // Pass sender_sp to next frame
3352 __ addptr(rsi, wordSize); // Bump array pointer (sizes)
3353 __ addptr(rcx, wordSize); // Bump array pointer (pcs)
3354 __ decrementl(rdx); // Decrement counter
3355 __ jcc(Assembler::notZero, loop);
3356 __ pushptr(Address(rcx, 0)); // Save final return address
3357
3358 // Re-push self-frame
3359 __ enter(); // Save old & set new rbp
3360 __ subptr(rsp, (SimpleRuntimeFrame::framesize - 4) << LogBytesPerInt);
3361 // Prolog
3362
3363 // Use rbp because the frames look interpreted now
3364 // Save "the_pc" since it cannot easily be retrieved using the last_java_SP after we aligned SP.
3365 // Don't need the precise return PC here, just precise enough to point into this code blob.
3366 address the_pc = __ pc();
3367 __ set_last_Java_frame(noreg, rbp, the_pc);
3368
3369 // Call C code. Need thread but NOT official VM entry
3370 // crud. We cannot block on this call, no GC can happen. Call should
3371 // restore return values to their stack-slots with the new SP.
3372 // Thread is in rdi already.
3373 //
3374 // BasicType unpack_frames(JavaThread* thread, int exec_mode);
3375
3376 __ andptr(rsp, -(StackAlignmentInBytes)); // Align SP as required by ABI
3377 __ mov(c_rarg0, r15_thread);
3378 __ movl(c_rarg1, Deoptimization::Unpack_uncommon_trap);
3379 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
3380
3381 // Set an oopmap for the call site
3382 // Use the same PC we used for the last java frame
3383 oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0));
3384
3385 // Clear fp AND pc
3386 __ reset_last_Java_frame(true);
3387
3388 // Pop self-frame.
3389 __ leave(); // Epilog
3390
3391 // Jump to interpreter
3392 __ ret(0);
3393
3394 // Make sure all code is generated
3395 masm->flush();
3396
3397 _uncommon_trap_blob = UncommonTrapBlob::create(&buffer, oop_maps,
3398 SimpleRuntimeFrame::framesize >> 1);
3399 }
3400 #endif // COMPILER2
3401
3402
3403 //------------------------------generate_handler_blob------
3404 //
3405 // Generate a special Compile2Runtime blob that saves all registers,
3406 // and setup oopmap.
3407 //
3408 SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_type) {
3409 assert(StubRoutines::forward_exception_entry() != NULL,
3410 "must be generated before");
3411
3412 ResourceMark rm;
3413 OopMapSet *oop_maps = new OopMapSet();
3414 OopMap* map;
3415
3416 // Allocate space for the code. Setup code generation tools.
3417 CodeBuffer buffer("handler_blob", 2048, 1024);
3418 MacroAssembler* masm = new MacroAssembler(&buffer);
3419
3420 address start = __ pc();
3421 address call_pc = NULL;
3422 int frame_size_in_words;
3423 bool cause_return = (poll_type == POLL_AT_RETURN);
3424 bool save_vectors = (poll_type == POLL_AT_VECTOR_LOOP);
3425
3426 if (UseRTMLocking) {
3427 // Abort RTM transaction before calling runtime
3428 // because critical section will be large and will be
3429 // aborted anyway. Also nmethod could be deoptimized.
3430 __ xabort(0);
3431 }
3432
3433 // Make room for return address (or push it again)
3434 if (!cause_return) {
3435 __ push(rbx);
3436 }
3437
3438 // Save registers, fpu state, and flags
3439 map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, save_vectors);
3440
3441 // The following is basically a call_VM. However, we need the precise
3442 // address of the call in order to generate an oopmap. Hence, we do all the
3443 // work outselves.
3444
3445 __ set_last_Java_frame(noreg, noreg, NULL);
3446
3447 // The return address must always be correct so that frame constructor never
3448 // sees an invalid pc.
3449
3450 if (!cause_return) {
3451 // Get the return pc saved by the signal handler and stash it in its appropriate place on the stack.
3452 // Additionally, rbx is a callee saved register and we can look at it later to determine
3453 // if someone changed the return address for us!
3454 __ movptr(rbx, Address(r15_thread, JavaThread::saved_exception_pc_offset()));
3455 __ movptr(Address(rbp, wordSize), rbx);
3456 }
3457
3458 // Do the call
3459 __ mov(c_rarg0, r15_thread);
3460 __ call(RuntimeAddress(call_ptr));
3461
3462 // Set an oopmap for the call site. This oopmap will map all
3463 // oop-registers and debug-info registers as callee-saved. This
3464 // will allow deoptimization at this safepoint to find all possible
3465 // debug-info recordings, as well as let GC find all oops.
3466
3467 oop_maps->add_gc_map( __ pc() - start, map);
3468
3469 Label noException;
3470
3471 __ reset_last_Java_frame(false);
3472
3473 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3474 __ jcc(Assembler::equal, noException);
3475
3476 // Exception pending
3477
3478 RegisterSaver::restore_live_registers(masm, save_vectors);
3479
3480 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3481
3482 // No exception case
3483 __ bind(noException);
3484
3485 Label no_adjust;
3486 #ifdef ASSERT
3487 Label bail;
3488 #endif
3489 if (SafepointMechanism::uses_thread_local_poll() && !cause_return) {
3490 Label no_prefix, not_special;
3491
3492 // If our stashed return pc was modified by the runtime we avoid touching it
3493 __ cmpptr(rbx, Address(rbp, wordSize));
3494 __ jccb(Assembler::notEqual, no_adjust);
3495
3496 // Skip over the poll instruction.
3497 // See NativeInstruction::is_safepoint_poll()
3498 // Possible encodings:
3499 // 85 00 test %eax,(%rax)
3500 // 85 01 test %eax,(%rcx)
3501 // 85 02 test %eax,(%rdx)
3502 // 85 03 test %eax,(%rbx)
3503 // 85 06 test %eax,(%rsi)
3504 // 85 07 test %eax,(%rdi)
3505 //
3506 // 41 85 00 test %eax,(%r8)
3507 // 41 85 01 test %eax,(%r9)
3508 // 41 85 02 test %eax,(%r10)
3509 // 41 85 03 test %eax,(%r11)
3510 // 41 85 06 test %eax,(%r14)
3511 // 41 85 07 test %eax,(%r15)
3512 //
3513 // 85 04 24 test %eax,(%rsp)
3514 // 41 85 04 24 test %eax,(%r12)
3515 // 85 45 00 test %eax,0x0(%rbp)
3516 // 41 85 45 00 test %eax,0x0(%r13)
3517
3518 __ cmpb(Address(rbx, 0), NativeTstRegMem::instruction_rex_b_prefix);
3519 __ jcc(Assembler::notEqual, no_prefix);
3520 __ addptr(rbx, 1);
3521 __ bind(no_prefix);
3522 #ifdef ASSERT
3523 __ movptr(rax, rbx); // remember where 0x85 should be, for verification below
3524 #endif
3525 // r12/r13/rsp/rbp base encoding takes 3 bytes with the following register values:
3526 // r12/rsp 0x04
3527 // r13/rbp 0x05
3528 __ movzbq(rcx, Address(rbx, 1));
3529 __ andptr(rcx, 0x07); // looking for 0x04 .. 0x05
3530 __ subptr(rcx, 4); // looking for 0x00 .. 0x01
3531 __ cmpptr(rcx, 1);
3532 __ jcc(Assembler::above, not_special);
3533 __ addptr(rbx, 1);
3534 __ bind(not_special);
3535 #ifdef ASSERT
3536 // Verify the correct encoding of the poll we're about to skip.
3537 __ cmpb(Address(rax, 0), NativeTstRegMem::instruction_code_memXregl);
3538 __ jcc(Assembler::notEqual, bail);
3539 // Mask out the modrm bits
3540 __ testb(Address(rax, 1), NativeTstRegMem::modrm_mask);
3541 // rax encodes to 0, so if the bits are nonzero it's incorrect
3542 __ jcc(Assembler::notZero, bail);
3543 #endif
3544 // Adjust return pc forward to step over the safepoint poll instruction
3545 __ addptr(rbx, 2);
3546 __ movptr(Address(rbp, wordSize), rbx);
3547 }
3548
3549 __ bind(no_adjust);
3550 // Normal exit, restore registers and exit.
3551 RegisterSaver::restore_live_registers(masm, save_vectors);
3552 __ ret(0);
3553
3554 #ifdef ASSERT
3555 __ bind(bail);
3556 __ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected");
3557 #endif
3558
3559 // Make sure all code is generated
3560 masm->flush();
3561
3562 // Fill-out other meta info
3563 return SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
3564 }
3565
3566 //
3567 // generate_resolve_blob - call resolution (static/virtual/opt-virtual/ic-miss
3568 //
3569 // Generate a stub that calls into vm to find out the proper destination
3570 // of a java call. All the argument registers are live at this point
3571 // but since this is generic code we don't know what they are and the caller
3572 // must do any gc of the args.
3573 //
3574 RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const char* name) {
3575 assert (StubRoutines::forward_exception_entry() != NULL, "must be generated before");
3576
3577 // allocate space for the code
3578 ResourceMark rm;
3579
3580 CodeBuffer buffer(name, 1000, 512);
3581 MacroAssembler* masm = new MacroAssembler(&buffer);
3582
3583 int frame_size_in_words;
3584
3585 OopMapSet *oop_maps = new OopMapSet();
3586 OopMap* map = NULL;
3587
3588 int start = __ offset();
3589
3590 map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words);
3591
3592 int frame_complete = __ offset();
3593
3594 __ set_last_Java_frame(noreg, noreg, NULL);
3595
3596 __ mov(c_rarg0, r15_thread);
3597
3598 __ call(RuntimeAddress(destination));
3599
3600
3601 // Set an oopmap for the call site.
3602 // We need this not only for callee-saved registers, but also for volatile
3603 // registers that the compiler might be keeping live across a safepoint.
3604
3605 oop_maps->add_gc_map( __ offset() - start, map);
3606
3607 // rax contains the address we are going to jump to assuming no exception got installed
3608
3609 // clear last_Java_sp
3610 __ reset_last_Java_frame(false);
3611 // check for pending exceptions
3612 Label pending;
3613 __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), (int32_t)NULL_WORD);
3614 __ jcc(Assembler::notEqual, pending);
3615
3616 // get the returned Method*
3617 __ get_vm_result_2(rbx, r15_thread);
3618 __ movptr(Address(rsp, RegisterSaver::rbx_offset_in_bytes()), rbx);
3619
3620 __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax);
3621
3622 RegisterSaver::restore_live_registers(masm);
3623
3624 // We are back the the original state on entry and ready to go.
3625
3626 __ jmp(rax);
3627
3628 // Pending exception after the safepoint
3629
3630 __ bind(pending);
3631
3632 RegisterSaver::restore_live_registers(masm);
3633
3634 // exception pending => remove activation and forward to exception handler
3635
3636 __ movptr(Address(r15_thread, JavaThread::vm_result_offset()), (int)NULL_WORD);
3637
3638 __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
3639 __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3640
3641 // -------------
3642 // make sure all code is generated
3643 masm->flush();
3644
3645 // return the blob
3646 // frame_size_words or bytes??
3647 return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true);
3648 }
3649
3650
3651 //------------------------------Montgomery multiplication------------------------
3652 //
3653
3654 #ifndef _WINDOWS
3655
3656 #define ASM_SUBTRACT
3657
3658 #ifdef ASM_SUBTRACT
3659 // Subtract 0:b from carry:a. Return carry.
3660 static unsigned long
3661 sub(unsigned long a[], unsigned long b[], unsigned long carry, long len) {
3662 long i = 0, cnt = len;
3663 unsigned long tmp;
3664 asm volatile("clc; "
3665 "0: ; "
3666 "mov (%[b], %[i], 8), %[tmp]; "
3667 "sbb %[tmp], (%[a], %[i], 8); "
3668 "inc %[i]; dec %[cnt]; "
3669 "jne 0b; "
3670 "mov %[carry], %[tmp]; sbb $0, %[tmp]; "
3671 : [i]"+r"(i), [cnt]"+r"(cnt), [tmp]"=&r"(tmp)
3672 : [a]"r"(a), [b]"r"(b), [carry]"r"(carry)
3673 : "memory");
3674 return tmp;
3675 }
3676 #else // ASM_SUBTRACT
3677 typedef int __attribute__((mode(TI))) int128;
3678
3679 // Subtract 0:b from carry:a. Return carry.
3680 static unsigned long
3681 sub(unsigned long a[], unsigned long b[], unsigned long carry, int len) {
3682 int128 tmp = 0;
3683 int i;
3684 for (i = 0; i < len; i++) {
3685 tmp += a[i];
3686 tmp -= b[i];
3687 a[i] = tmp;
3688 tmp >>= 64;
3689 assert(-1 <= tmp && tmp <= 0, "invariant");
3690 }
3691 return tmp + carry;
3692 }
3693 #endif // ! ASM_SUBTRACT
3694
3695 // Multiply (unsigned) Long A by Long B, accumulating the double-
3696 // length result into the accumulator formed of T0, T1, and T2.
3697 #define MACC(A, B, T0, T1, T2) \
3698 do { \
3699 unsigned long hi, lo; \
3700 __asm__ ("mul %5; add %%rax, %2; adc %%rdx, %3; adc $0, %4" \
3701 : "=&d"(hi), "=a"(lo), "+r"(T0), "+r"(T1), "+g"(T2) \
3702 : "r"(A), "a"(B) : "cc"); \
3703 } while(0)
3704
3705 // As above, but add twice the double-length result into the
3706 // accumulator.
3707 #define MACC2(A, B, T0, T1, T2) \
3708 do { \
3709 unsigned long hi, lo; \
3710 __asm__ ("mul %5; add %%rax, %2; adc %%rdx, %3; adc $0, %4; " \
3711 "add %%rax, %2; adc %%rdx, %3; adc $0, %4" \
3712 : "=&d"(hi), "=a"(lo), "+r"(T0), "+r"(T1), "+g"(T2) \
3713 : "r"(A), "a"(B) : "cc"); \
3714 } while(0)
3715
3716 // Fast Montgomery multiplication. The derivation of the algorithm is
3717 // in A Cryptographic Library for the Motorola DSP56000,
3718 // Dusse and Kaliski, Proc. EUROCRYPT 90, pp. 230-237.
3719
3720 static void __attribute__((noinline))
3721 montgomery_multiply(unsigned long a[], unsigned long b[], unsigned long n[],
3722 unsigned long m[], unsigned long inv, int len) {
3723 unsigned long t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
3724 int i;
3725
3726 assert(inv * n[0] == -1UL, "broken inverse in Montgomery multiply");
3727
3728 for (i = 0; i < len; i++) {
3729 int j;
3730 for (j = 0; j < i; j++) {
3731 MACC(a[j], b[i-j], t0, t1, t2);
3732 MACC(m[j], n[i-j], t0, t1, t2);
3733 }
3734 MACC(a[i], b[0], t0, t1, t2);
3735 m[i] = t0 * inv;
3736 MACC(m[i], n[0], t0, t1, t2);
3737
3738 assert(t0 == 0, "broken Montgomery multiply");
3739
3740 t0 = t1; t1 = t2; t2 = 0;
3741 }
3742
3743 for (i = len; i < 2*len; i++) {
3744 int j;
3745 for (j = i-len+1; j < len; j++) {
3746 MACC(a[j], b[i-j], t0, t1, t2);
3747 MACC(m[j], n[i-j], t0, t1, t2);
3748 }
3749 m[i-len] = t0;
3750 t0 = t1; t1 = t2; t2 = 0;
3751 }
3752
3753 while (t0)
3754 t0 = sub(m, n, t0, len);
3755 }
3756
3757 // Fast Montgomery squaring. This uses asymptotically 25% fewer
3758 // multiplies so it should be up to 25% faster than Montgomery
3759 // multiplication. However, its loop control is more complex and it
3760 // may actually run slower on some machines.
3761
3762 static void __attribute__((noinline))
3763 montgomery_square(unsigned long a[], unsigned long n[],
3764 unsigned long m[], unsigned long inv, int len) {
3765 unsigned long t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
3766 int i;
3767
3768 assert(inv * n[0] == -1UL, "broken inverse in Montgomery multiply");
3769
3770 for (i = 0; i < len; i++) {
3771 int j;
3772 int end = (i+1)/2;
3773 for (j = 0; j < end; j++) {
3774 MACC2(a[j], a[i-j], t0, t1, t2);
3775 MACC(m[j], n[i-j], t0, t1, t2);
3776 }
3777 if ((i & 1) == 0) {
3778 MACC(a[j], a[j], t0, t1, t2);
3779 }
3780 for (; j < i; j++) {
3781 MACC(m[j], n[i-j], t0, t1, t2);
3782 }
3783 m[i] = t0 * inv;
3784 MACC(m[i], n[0], t0, t1, t2);
3785
3786 assert(t0 == 0, "broken Montgomery square");
3787
3788 t0 = t1; t1 = t2; t2 = 0;
3789 }
3790
3791 for (i = len; i < 2*len; i++) {
3792 int start = i-len+1;
3793 int end = start + (len - start)/2;
3794 int j;
3795 for (j = start; j < end; j++) {
3796 MACC2(a[j], a[i-j], t0, t1, t2);
3797 MACC(m[j], n[i-j], t0, t1, t2);
3798 }
3799 if ((i & 1) == 0) {
3800 MACC(a[j], a[j], t0, t1, t2);
3801 }
3802 for (; j < len; j++) {
3803 MACC(m[j], n[i-j], t0, t1, t2);
3804 }
3805 m[i-len] = t0;
3806 t0 = t1; t1 = t2; t2 = 0;
3807 }
3808
3809 while (t0)
3810 t0 = sub(m, n, t0, len);
3811 }
3812
3813 // Swap words in a longword.
3814 static unsigned long swap(unsigned long x) {
3815 return (x << 32) | (x >> 32);
3816 }
3817
3818 // Copy len longwords from s to d, word-swapping as we go. The
3819 // destination array is reversed.
3820 static void reverse_words(unsigned long *s, unsigned long *d, int len) {
3821 d += len;
3822 while(len-- > 0) {
3823 d--;
3824 *d = swap(*s);
3825 s++;
3826 }
3827 }
3828
3829 // The threshold at which squaring is advantageous was determined
3830 // experimentally on an i7-3930K (Ivy Bridge) CPU @ 3.5GHz.
3831 #define MONTGOMERY_SQUARING_THRESHOLD 64
3832
3833 void SharedRuntime::montgomery_multiply(jint *a_ints, jint *b_ints, jint *n_ints,
3834 jint len, jlong inv,
3835 jint *m_ints) {
3836 assert(len % 2 == 0, "array length in montgomery_multiply must be even");
3837 int longwords = len/2;
3838
3839 // Make very sure we don't use so much space that the stack might
3840 // overflow. 512 jints corresponds to an 16384-bit integer and
3841 // will use here a total of 8k bytes of stack space.
3842 int total_allocation = longwords * sizeof (unsigned long) * 4;
3843 guarantee(total_allocation <= 8192, "must be");
3844 unsigned long *scratch = (unsigned long *)alloca(total_allocation);
3845
3846 // Local scratch arrays
3847 unsigned long
3848 *a = scratch + 0 * longwords,
3849 *b = scratch + 1 * longwords,
3850 *n = scratch + 2 * longwords,
3851 *m = scratch + 3 * longwords;
3852
3853 reverse_words((unsigned long *)a_ints, a, longwords);
3854 reverse_words((unsigned long *)b_ints, b, longwords);
3855 reverse_words((unsigned long *)n_ints, n, longwords);
3856
3857 ::montgomery_multiply(a, b, n, m, (unsigned long)inv, longwords);
3858
3859 reverse_words(m, (unsigned long *)m_ints, longwords);
3860 }
3861
3862 void SharedRuntime::montgomery_square(jint *a_ints, jint *n_ints,
3863 jint len, jlong inv,
3864 jint *m_ints) {
3865 assert(len % 2 == 0, "array length in montgomery_square must be even");
3866 int longwords = len/2;
3867
3868 // Make very sure we don't use so much space that the stack might
3869 // overflow. 512 jints corresponds to an 16384-bit integer and
3870 // will use here a total of 6k bytes of stack space.
3871 int total_allocation = longwords * sizeof (unsigned long) * 3;
3872 guarantee(total_allocation <= 8192, "must be");
3873 unsigned long *scratch = (unsigned long *)alloca(total_allocation);
3874
3875 // Local scratch arrays
3876 unsigned long
3877 *a = scratch + 0 * longwords,
3878 *n = scratch + 1 * longwords,
3879 *m = scratch + 2 * longwords;
3880
3881 reverse_words((unsigned long *)a_ints, a, longwords);
3882 reverse_words((unsigned long *)n_ints, n, longwords);
3883
3884 if (len >= MONTGOMERY_SQUARING_THRESHOLD) {
3885 ::montgomery_square(a, n, m, (unsigned long)inv, longwords);
3886 } else {
3887 ::montgomery_multiply(a, a, n, m, (unsigned long)inv, longwords);
3888 }
3889
3890 reverse_words(m, (unsigned long *)m_ints, longwords);
3891 }
3892
3893 #endif // WINDOWS
3894
3895 #ifdef COMPILER2
3896 // This is here instead of runtime_x86_64.cpp because it uses SimpleRuntimeFrame
3897 //
3898 //------------------------------generate_exception_blob---------------------------
3899 // creates exception blob at the end
3900 // Using exception blob, this code is jumped from a compiled method.
3901 // (see emit_exception_handler in x86_64.ad file)
3902 //
3903 // Given an exception pc at a call we call into the runtime for the
3904 // handler in this method. This handler might merely restore state
3905 // (i.e. callee save registers) unwind the frame and jump to the
3906 // exception handler for the nmethod if there is no Java level handler
3907 // for the nmethod.
3908 //
3909 // This code is entered with a jmp.
3910 //
3911 // Arguments:
3912 // rax: exception oop
3913 // rdx: exception pc
3914 //
3915 // Results:
3916 // rax: exception oop
3917 // rdx: exception pc in caller or ???
3918 // destination: exception handler of caller
3919 //
3920 // Note: the exception pc MUST be at a call (precise debug information)
3921 // Registers rax, rdx, rcx, rsi, rdi, r8-r11 are not callee saved.
3922 //
3923
3924 void OptoRuntime::generate_exception_blob() {
3925 assert(!OptoRuntime::is_callee_saved_register(RDX_num), "");
3926 assert(!OptoRuntime::is_callee_saved_register(RAX_num), "");
3927 assert(!OptoRuntime::is_callee_saved_register(RCX_num), "");
3928
3929 assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
3930
3931 // Allocate space for the code
3932 ResourceMark rm;
3933 // Setup code generation tools
3934 CodeBuffer buffer("exception_blob", 2048, 1024);
3935 MacroAssembler* masm = new MacroAssembler(&buffer);
3936
3937
3938 address start = __ pc();
3939
3940 // Exception pc is 'return address' for stack walker
3941 __ push(rdx);
3942 __ subptr(rsp, SimpleRuntimeFrame::return_off << LogBytesPerInt); // Prolog
3943
3944 // Save callee-saved registers. See x86_64.ad.
3945
3946 // rbp is an implicitly saved callee saved register (i.e., the calling
3947 // convention will save/restore it in the prolog/epilog). Other than that
3948 // there are no callee save registers now that adapter frames are gone.
3949
3950 __ movptr(Address(rsp, SimpleRuntimeFrame::rbp_off << LogBytesPerInt), rbp);
3951
3952 // Store exception in Thread object. We cannot pass any arguments to the
3953 // handle_exception call, since we do not want to make any assumption
3954 // about the size of the frame where the exception happened in.
3955 // c_rarg0 is either rdi (Linux) or rcx (Windows).
3956 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()),rax);
3957 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), rdx);
3958
3959 // This call does all the hard work. It checks if an exception handler
3960 // exists in the method.
3961 // If so, it returns the handler address.
3962 // If not, it prepares for stack-unwinding, restoring the callee-save
3963 // registers of the frame being removed.
3964 //
3965 // address OptoRuntime::handle_exception_C(JavaThread* thread)
3966
3967 // At a method handle call, the stack may not be properly aligned
3968 // when returning with an exception.
3969 address the_pc = __ pc();
3970 __ set_last_Java_frame(noreg, noreg, the_pc);
3971 __ mov(c_rarg0, r15_thread);
3972 __ andptr(rsp, -(StackAlignmentInBytes)); // Align stack
3973 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, OptoRuntime::handle_exception_C)));
3974
3975 // Set an oopmap for the call site. This oopmap will only be used if we
3976 // are unwinding the stack. Hence, all locations will be dead.
3977 // Callee-saved registers will be the same as the frame above (i.e.,
3978 // handle_exception_stub), since they were restored when we got the
3979 // exception.
3980
3981 OopMapSet* oop_maps = new OopMapSet();
3982
3983 oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0));
3984
3985 __ reset_last_Java_frame(false);
3986
3987 // Restore callee-saved registers
3988
3989 // rbp is an implicitly saved callee-saved register (i.e., the calling
3990 // convention will save restore it in prolog/epilog) Other than that
3991 // there are no callee save registers now that adapter frames are gone.
3992
3993 __ movptr(rbp, Address(rsp, SimpleRuntimeFrame::rbp_off << LogBytesPerInt));
3994
3995 __ addptr(rsp, SimpleRuntimeFrame::return_off << LogBytesPerInt); // Epilog
3996 __ pop(rdx); // No need for exception pc anymore
3997
3998 // rax: exception handler
3999
4000 // We have a handler in rax (could be deopt blob).
4001 __ mov(r8, rax);
4002
4003 // Get the exception oop
4004 __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
4005 // Get the exception pc in case we are deoptimized
4006 __ movptr(rdx, Address(r15_thread, JavaThread::exception_pc_offset()));
4007 #ifdef ASSERT
4008 __ movptr(Address(r15_thread, JavaThread::exception_handler_pc_offset()), (int)NULL_WORD);
4009 __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), (int)NULL_WORD);
4010 #endif
4011 // Clear the exception oop so GC no longer processes it as a root.
4012 __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), (int)NULL_WORD);
4013
4014 // rax: exception oop
4015 // r8: exception handler
4016 // rdx: exception pc
4017 // Jump to handler
4018
4019 __ jmp(r8);
4020
4021 // Make sure all code is generated
4022 masm->flush();
4023
4024 // Set exception blob
4025 _exception_blob = ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
4026 }
4027 #endif // COMPILER2