23
24 #include "precompiled.hpp"
25 #include "asm/assembler.hpp"
26 #include "c1/c1_Defs.hpp"
27 #include "c1/c1_MacroAssembler.hpp"
28 #include "c1/c1_Runtime1.hpp"
29 #include "ci/ciUtilities.hpp"
30 #include "gc/shared/cardTable.hpp"
31 #include "gc/shared/cardTableBarrierSet.hpp"
32 #include "interpreter/interpreter.hpp"
33 #include "nativeInst_x86.hpp"
34 #include "oops/compiledICHolder.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "prims/jvmtiExport.hpp"
37 #include "register_x86.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "runtime/signature.hpp"
40 #include "runtime/vframeArray.hpp"
41 #include "utilities/macros.hpp"
42 #include "vmreg_x86.inline.hpp"
43 #if INCLUDE_ALL_GCS
44 #include "gc/g1/g1BarrierSet.hpp"
45 #include "gc/g1/g1CardTable.hpp"
46 #include "gc/g1/g1ThreadLocalData.hpp"
47 #endif
48
49
50 // Implementation of StubAssembler
51
52 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
53 // setup registers
54 const Register thread = NOT_LP64(rdi) LP64_ONLY(r15_thread); // is callee-saved register (Visual C++ calling conventions)
55 assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
56 assert(oop_result1 != thread && metadata_result != thread, "registers must be different");
57 assert(args_size >= 0, "illegal args_size");
58 bool align_stack = false;
59 #ifdef _LP64
60 // At a method handle call, the stack may not be properly aligned
61 // when returning with an exception.
62 align_stack = (stub_id() == Runtime1::handle_exception_from_callee_id);
63 #endif
64
65 #ifdef _LP64
66 mov(c_rarg0, thread);
67 set_num_rt_args(0); // Nothing on stack
|
23
24 #include "precompiled.hpp"
25 #include "asm/assembler.hpp"
26 #include "c1/c1_Defs.hpp"
27 #include "c1/c1_MacroAssembler.hpp"
28 #include "c1/c1_Runtime1.hpp"
29 #include "ci/ciUtilities.hpp"
30 #include "gc/shared/cardTable.hpp"
31 #include "gc/shared/cardTableBarrierSet.hpp"
32 #include "interpreter/interpreter.hpp"
33 #include "nativeInst_x86.hpp"
34 #include "oops/compiledICHolder.hpp"
35 #include "oops/oop.inline.hpp"
36 #include "prims/jvmtiExport.hpp"
37 #include "register_x86.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "runtime/signature.hpp"
40 #include "runtime/vframeArray.hpp"
41 #include "utilities/macros.hpp"
42 #include "vmreg_x86.inline.hpp"
43
44 // Implementation of StubAssembler
45
46 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
47 // setup registers
48 const Register thread = NOT_LP64(rdi) LP64_ONLY(r15_thread); // is callee-saved register (Visual C++ calling conventions)
49 assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
50 assert(oop_result1 != thread && metadata_result != thread, "registers must be different");
51 assert(args_size >= 0, "illegal args_size");
52 bool align_stack = false;
53 #ifdef _LP64
54 // At a method handle call, the stack may not be properly aligned
55 // when returning with an exception.
56 align_stack = (stub_id() == Runtime1::handle_exception_from_callee_id);
57 #endif
58
59 #ifdef _LP64
60 mov(c_rarg0, thread);
61 set_num_rt_args(0); // Nothing on stack
|
194 push(arg2);
195 push(arg1);
196 #endif // _LP64
197 return call_RT(oop_result1, metadata_result, entry, 3);
198 }
199
200
201 // Implementation of StubFrame
202
203 class StubFrame: public StackObj {
204 private:
205 StubAssembler* _sasm;
206
207 public:
208 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments);
209 void load_argument(int offset_in_words, Register reg);
210
211 ~StubFrame();
212 };
213
214
215 #define __ _sasm->
216
217 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) {
218 _sasm = sasm;
219 __ set_info(name, must_gc_arguments);
220 __ enter();
221 }
222
223 // load parameters that were stored with LIR_Assembler::store_parameter
224 // Note: offsets for store_parameter and load_argument must match
225 void StubFrame::load_argument(int offset_in_words, Register reg) {
226 // rbp, + 0: link
227 // + 1: return address
228 // + 2: argument with offset 0
229 // + 3: argument with offset 1
230 // + 4: ...
231
232 __ movptr(reg, Address(rbp, (offset_in_words + 2) * BytesPerWord));
233 }
234
235
236 StubFrame::~StubFrame() {
237 __ leave();
238 __ ret(0);
239 }
240
241 #undef __
242
243
244 // Implementation of Runtime1
245
246 #define __ sasm->
247
248 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
249 const int xmm_regs_as_doubles_size_in_slots = FrameMap::nof_xmm_regs * 2;
250
251 // Stack layout for saving/restoring all the registers needed during a runtime
252 // call (this includes deoptimization)
253 // Note: note that users of this frame may well have arguments to some runtime
254 // while these values are on the stack. These positions neglect those arguments
255 // but the code in save_live_registers will take the argument count into
256 // account.
257 //
258 #ifdef _LP64
259 #define SLOT2(x) x,
260 #define SLOT_PER_WORD 2
261 #else
262 #define SLOT2(x)
263 #define SLOT_PER_WORD 1
264 #endif // _LP64
265
266 enum reg_save_layout {
|
188 push(arg2);
189 push(arg1);
190 #endif // _LP64
191 return call_RT(oop_result1, metadata_result, entry, 3);
192 }
193
194
195 // Implementation of StubFrame
196
197 class StubFrame: public StackObj {
198 private:
199 StubAssembler* _sasm;
200
201 public:
202 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments);
203 void load_argument(int offset_in_words, Register reg);
204
205 ~StubFrame();
206 };
207
208 void StubAssembler::prologue(const char* name, bool must_gc_arguments) {
209 set_info(name, must_gc_arguments);
210 enter();
211 }
212
213 void StubAssembler::epilogue() {
214 leave();
215 ret(0);
216 }
217
218 #define __ _sasm->
219
220 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) {
221 _sasm = sasm;
222 __ prologue(name, must_gc_arguments);
223 }
224
225 // load parameters that were stored with LIR_Assembler::store_parameter
226 // Note: offsets for store_parameter and load_argument must match
227 void StubFrame::load_argument(int offset_in_words, Register reg) {
228 __ load_parameter(offset_in_words, reg);
229 }
230
231
232 StubFrame::~StubFrame() {
233 __ epilogue();
234 }
235
236 #undef __
237
238
239 // Implementation of Runtime1
240
241 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
242 const int xmm_regs_as_doubles_size_in_slots = FrameMap::nof_xmm_regs * 2;
243
244 // Stack layout for saving/restoring all the registers needed during a runtime
245 // call (this includes deoptimization)
246 // Note: note that users of this frame may well have arguments to some runtime
247 // while these values are on the stack. These positions neglect those arguments
248 // but the code in save_live_registers will take the argument count into
249 // account.
250 //
251 #ifdef _LP64
252 #define SLOT2(x) x,
253 #define SLOT_PER_WORD 2
254 #else
255 #define SLOT2(x)
256 #define SLOT_PER_WORD 1
257 #endif // _LP64
258
259 enum reg_save_layout {
|
292 r11_off, r11H_off, // 392, 396
293 r10_off, r10H_off, // 400, 404
294 r9_off, r9H_off, // 408, 412
295 r8_off, r8H_off, // 416, 420
296 rdi_off, rdiH_off, // 424, 428
297 #else
298 rdi_off = extra_space_offset,
299 #endif // _LP64
300 rsi_off, SLOT2(rsiH_off) // 432, 436
301 rbp_off, SLOT2(rbpH_off) // 440, 444
302 rsp_off, SLOT2(rspH_off) // 448, 452
303 rbx_off, SLOT2(rbxH_off) // 456, 460
304 rdx_off, SLOT2(rdxH_off) // 464, 468
305 rcx_off, SLOT2(rcxH_off) // 472, 476
306 rax_off, SLOT2(raxH_off) // 480, 484
307 saved_rbp_off, SLOT2(saved_rbpH_off) // 488, 492
308 return_off, SLOT2(returnH_off) // 496, 500
309 reg_save_frame_size // As noted: neglects any parameters to runtime // 504
310 };
311
312
313
314 // Save off registers which might be killed by calls into the runtime.
315 // Tries to smart of about FP registers. In particular we separate
316 // saving and describing the FPU registers for deoptimization since we
317 // have to save the FPU registers twice if we describe them and on P4
318 // saving FPU registers which don't contain anything appears
319 // expensive. The deopt blob is the only thing which needs to
320 // describe FPU registers. In all other cases it should be sufficient
321 // to simply save their current value.
322
323 static OopMap* generate_oop_map(StubAssembler* sasm, int num_rt_args,
324 bool save_fpu_registers = true) {
325
326 // In 64bit all the args are in regs so there are no additional stack slots
327 LP64_ONLY(num_rt_args = 0);
328 LP64_ONLY(assert((reg_save_frame_size * VMRegImpl::stack_slot_size) % 16 == 0, "must be 16 byte aligned");)
329 int frame_size_in_slots = reg_save_frame_size + num_rt_args; // args + thread
330 sasm->set_frame_size(frame_size_in_slots / VMRegImpl::slots_per_word);
331
332 // record saved value locations in an OopMap
|
285 r11_off, r11H_off, // 392, 396
286 r10_off, r10H_off, // 400, 404
287 r9_off, r9H_off, // 408, 412
288 r8_off, r8H_off, // 416, 420
289 rdi_off, rdiH_off, // 424, 428
290 #else
291 rdi_off = extra_space_offset,
292 #endif // _LP64
293 rsi_off, SLOT2(rsiH_off) // 432, 436
294 rbp_off, SLOT2(rbpH_off) // 440, 444
295 rsp_off, SLOT2(rspH_off) // 448, 452
296 rbx_off, SLOT2(rbxH_off) // 456, 460
297 rdx_off, SLOT2(rdxH_off) // 464, 468
298 rcx_off, SLOT2(rcxH_off) // 472, 476
299 rax_off, SLOT2(raxH_off) // 480, 484
300 saved_rbp_off, SLOT2(saved_rbpH_off) // 488, 492
301 return_off, SLOT2(returnH_off) // 496, 500
302 reg_save_frame_size // As noted: neglects any parameters to runtime // 504
303 };
304
305 // Save off registers which might be killed by calls into the runtime.
306 // Tries to smart of about FP registers. In particular we separate
307 // saving and describing the FPU registers for deoptimization since we
308 // have to save the FPU registers twice if we describe them and on P4
309 // saving FPU registers which don't contain anything appears
310 // expensive. The deopt blob is the only thing which needs to
311 // describe FPU registers. In all other cases it should be sufficient
312 // to simply save their current value.
313
314 static OopMap* generate_oop_map(StubAssembler* sasm, int num_rt_args,
315 bool save_fpu_registers = true) {
316
317 // In 64bit all the args are in regs so there are no additional stack slots
318 LP64_ONLY(num_rt_args = 0);
319 LP64_ONLY(assert((reg_save_frame_size * VMRegImpl::stack_slot_size) % 16 == 0, "must be 16 byte aligned");)
320 int frame_size_in_slots = reg_save_frame_size + num_rt_args; // args + thread
321 sasm->set_frame_size(frame_size_in_slots / VMRegImpl::slots_per_word);
322
323 // record saved value locations in an OopMap
|
400 }
401 }
402 xmm_off += 2;
403 }
404 assert(xmm_off == float_regs_as_doubles_off, "incorrect number of xmm registers");
405
406 } else if (UseSSE == 1) {
407 int xmm_off = xmm_regs_as_doubles_off;
408 for (int n = 0; n < FrameMap::nof_fpu_regs; n++) {
409 VMReg xmm_name_0 = as_XMMRegister(n)->as_VMReg();
410 map->set_callee_saved(VMRegImpl::stack2reg(xmm_off + num_rt_args), xmm_name_0);
411 xmm_off += 2;
412 }
413 assert(xmm_off == float_regs_as_doubles_off, "incorrect number of xmm registers");
414 }
415 }
416
417 return map;
418 }
419
420 static OopMap* save_live_registers(StubAssembler* sasm, int num_rt_args,
421 bool save_fpu_registers = true) {
422 __ block_comment("save_live_registers");
423
424 __ pusha(); // integer registers
425
426 // assert(float_regs_as_doubles_off % 2 == 0, "misaligned offset");
427 // assert(xmm_regs_as_doubles_off % 2 == 0, "misaligned offset");
428
429 __ subptr(rsp, extra_space_offset * VMRegImpl::stack_slot_size);
430
431 #ifdef ASSERT
432 __ movptr(Address(rsp, marker * VMRegImpl::stack_slot_size), (int32_t)0xfeedbeef);
433 #endif
434
435 if (save_fpu_registers) {
436 if (UseSSE < 2) {
437 // save FPU stack
438 __ fnsave(Address(rsp, fpu_state_off * VMRegImpl::stack_slot_size));
439 __ fwait();
440
|
391 }
392 }
393 xmm_off += 2;
394 }
395 assert(xmm_off == float_regs_as_doubles_off, "incorrect number of xmm registers");
396
397 } else if (UseSSE == 1) {
398 int xmm_off = xmm_regs_as_doubles_off;
399 for (int n = 0; n < FrameMap::nof_fpu_regs; n++) {
400 VMReg xmm_name_0 = as_XMMRegister(n)->as_VMReg();
401 map->set_callee_saved(VMRegImpl::stack2reg(xmm_off + num_rt_args), xmm_name_0);
402 xmm_off += 2;
403 }
404 assert(xmm_off == float_regs_as_doubles_off, "incorrect number of xmm registers");
405 }
406 }
407
408 return map;
409 }
410
411 #define __ this->
412
413 void C1_MacroAssembler::save_live_registers_no_oop_map(int num_rt_args, bool save_fpu_registers) {
414 __ block_comment("save_live_registers");
415
416 __ pusha(); // integer registers
417
418 // assert(float_regs_as_doubles_off % 2 == 0, "misaligned offset");
419 // assert(xmm_regs_as_doubles_off % 2 == 0, "misaligned offset");
420
421 __ subptr(rsp, extra_space_offset * VMRegImpl::stack_slot_size);
422
423 #ifdef ASSERT
424 __ movptr(Address(rsp, marker * VMRegImpl::stack_slot_size), (int32_t)0xfeedbeef);
425 #endif
426
427 if (save_fpu_registers) {
428 if (UseSSE < 2) {
429 // save FPU stack
430 __ fnsave(Address(rsp, fpu_state_off * VMRegImpl::stack_slot_size));
431 __ fwait();
432
|
475 }
476 #endif
477 for (int n = 0; n < xmm_bypass_limit; n++) {
478 XMMRegister xmm_name = as_XMMRegister(n);
479 __ movdbl(Address(rsp, xmm_regs_as_doubles_off * VMRegImpl::stack_slot_size + offset), xmm_name);
480 offset += 8;
481 }
482 } else if (UseSSE == 1) {
483 // save XMM registers as float because double not supported without SSE2(num MMX == num fpu)
484 int offset = 0;
485 for (int n = 0; n < FrameMap::nof_fpu_regs; n++) {
486 XMMRegister xmm_name = as_XMMRegister(n);
487 __ movflt(Address(rsp, xmm_regs_as_doubles_off * VMRegImpl::stack_slot_size + offset), xmm_name);
488 offset += 8;
489 }
490 }
491 }
492
493 // FPU stack must be empty now
494 __ verify_FPU(0, "save_live_registers");
495
496 return generate_oop_map(sasm, num_rt_args, save_fpu_registers);
497 }
498
499
500 static void restore_fpu(StubAssembler* sasm, bool restore_fpu_registers = true) {
501 if (restore_fpu_registers) {
502 if (UseSSE >= 2) {
503 // restore XMM registers
504 int xmm_bypass_limit = FrameMap::nof_xmm_regs;
505 #ifdef _LP64
506 if (UseAVX < 3) {
507 xmm_bypass_limit = xmm_bypass_limit / 2;
508 }
509 #endif
510 int offset = 0;
511 for (int n = 0; n < xmm_bypass_limit; n++) {
512 XMMRegister xmm_name = as_XMMRegister(n);
513 __ movdbl(xmm_name, Address(rsp, xmm_regs_as_doubles_off * VMRegImpl::stack_slot_size + offset));
514 offset += 8;
515 }
516 } else if (UseSSE == 1) {
517 // restore XMM registers(num MMX == num fpu)
518 int offset = 0;
519 for (int n = 0; n < FrameMap::nof_fpu_regs; n++) {
|
467 }
468 #endif
469 for (int n = 0; n < xmm_bypass_limit; n++) {
470 XMMRegister xmm_name = as_XMMRegister(n);
471 __ movdbl(Address(rsp, xmm_regs_as_doubles_off * VMRegImpl::stack_slot_size + offset), xmm_name);
472 offset += 8;
473 }
474 } else if (UseSSE == 1) {
475 // save XMM registers as float because double not supported without SSE2(num MMX == num fpu)
476 int offset = 0;
477 for (int n = 0; n < FrameMap::nof_fpu_regs; n++) {
478 XMMRegister xmm_name = as_XMMRegister(n);
479 __ movflt(Address(rsp, xmm_regs_as_doubles_off * VMRegImpl::stack_slot_size + offset), xmm_name);
480 offset += 8;
481 }
482 }
483 }
484
485 // FPU stack must be empty now
486 __ verify_FPU(0, "save_live_registers");
487 }
488
489 #undef __
490 #define __ sasm->
491
492 static void restore_fpu(C1_MacroAssembler* sasm, bool restore_fpu_registers) {
493 if (restore_fpu_registers) {
494 if (UseSSE >= 2) {
495 // restore XMM registers
496 int xmm_bypass_limit = FrameMap::nof_xmm_regs;
497 #ifdef _LP64
498 if (UseAVX < 3) {
499 xmm_bypass_limit = xmm_bypass_limit / 2;
500 }
501 #endif
502 int offset = 0;
503 for (int n = 0; n < xmm_bypass_limit; n++) {
504 XMMRegister xmm_name = as_XMMRegister(n);
505 __ movdbl(xmm_name, Address(rsp, xmm_regs_as_doubles_off * VMRegImpl::stack_slot_size + offset));
506 offset += 8;
507 }
508 } else if (UseSSE == 1) {
509 // restore XMM registers(num MMX == num fpu)
510 int offset = 0;
511 for (int n = 0; n < FrameMap::nof_fpu_regs; n++) {
|
531 }
532
533 } else {
534 // check that FPU stack is really empty
535 __ verify_FPU(0, "restore_live_registers");
536 }
537
538 #ifdef ASSERT
539 {
540 Label ok;
541 __ cmpptr(Address(rsp, marker * VMRegImpl::stack_slot_size), (int32_t)0xfeedbeef);
542 __ jcc(Assembler::equal, ok);
543 __ stop("bad offsets in frame");
544 __ bind(ok);
545 }
546 #endif // ASSERT
547
548 __ addptr(rsp, extra_space_offset * VMRegImpl::stack_slot_size);
549 }
550
551
552 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
553 __ block_comment("restore_live_registers");
554
555 restore_fpu(sasm, restore_fpu_registers);
556 __ popa();
557 }
558
559
560 static void restore_live_registers_except_rax(StubAssembler* sasm, bool restore_fpu_registers = true) {
561 __ block_comment("restore_live_registers_except_rax");
562
563 restore_fpu(sasm, restore_fpu_registers);
564
565 #ifdef _LP64
566 __ movptr(r15, Address(rsp, 0));
567 __ movptr(r14, Address(rsp, wordSize));
568 __ movptr(r13, Address(rsp, 2 * wordSize));
569 __ movptr(r12, Address(rsp, 3 * wordSize));
570 __ movptr(r11, Address(rsp, 4 * wordSize));
571 __ movptr(r10, Address(rsp, 5 * wordSize));
572 __ movptr(r9, Address(rsp, 6 * wordSize));
573 __ movptr(r8, Address(rsp, 7 * wordSize));
574 __ movptr(rdi, Address(rsp, 8 * wordSize));
575 __ movptr(rsi, Address(rsp, 9 * wordSize));
576 __ movptr(rbp, Address(rsp, 10 * wordSize));
577 // skip rsp
|
523 }
524
525 } else {
526 // check that FPU stack is really empty
527 __ verify_FPU(0, "restore_live_registers");
528 }
529
530 #ifdef ASSERT
531 {
532 Label ok;
533 __ cmpptr(Address(rsp, marker * VMRegImpl::stack_slot_size), (int32_t)0xfeedbeef);
534 __ jcc(Assembler::equal, ok);
535 __ stop("bad offsets in frame");
536 __ bind(ok);
537 }
538 #endif // ASSERT
539
540 __ addptr(rsp, extra_space_offset * VMRegImpl::stack_slot_size);
541 }
542
543 #undef __
544 #define __ this->
545
546 void C1_MacroAssembler::restore_live_registers(bool restore_fpu_registers) {
547 __ block_comment("restore_live_registers");
548
549 restore_fpu(this, restore_fpu_registers);
550 __ popa();
551 }
552
553 #undef __
554 #define __ sasm->
555
556 static OopMap* save_live_registers(StubAssembler* sasm, int num_rt_args,
557 bool save_fpu_registers = true) {
558 sasm->save_live_registers_no_oop_map(num_rt_args, save_fpu_registers);
559 return generate_oop_map(sasm, num_rt_args, save_fpu_registers);
560 }
561
562 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
563 sasm->restore_live_registers(restore_fpu_registers);
564 }
565
566 static void restore_live_registers_except_rax(StubAssembler* sasm, bool restore_fpu_registers = true) {
567 __ block_comment("restore_live_registers_except_rax");
568
569 restore_fpu(sasm, restore_fpu_registers);
570
571 #ifdef _LP64
572 __ movptr(r15, Address(rsp, 0));
573 __ movptr(r14, Address(rsp, wordSize));
574 __ movptr(r13, Address(rsp, 2 * wordSize));
575 __ movptr(r12, Address(rsp, 3 * wordSize));
576 __ movptr(r11, Address(rsp, 4 * wordSize));
577 __ movptr(r10, Address(rsp, 5 * wordSize));
578 __ movptr(r9, Address(rsp, 6 * wordSize));
579 __ movptr(r8, Address(rsp, 7 * wordSize));
580 __ movptr(rdi, Address(rsp, 8 * wordSize));
581 __ movptr(rsi, Address(rsp, 9 * wordSize));
582 __ movptr(rbp, Address(rsp, 10 * wordSize));
583 // skip rsp
|
1538 #endif // _LP64
1539 __ jmp(do_return);
1540
1541 __ bind(return0);
1542 __ fpop();
1543 #ifndef _LP64
1544 __ xorptr(rdx,rdx);
1545 __ xorptr(rax,rax);
1546 #else
1547 __ xorptr(rax, rax);
1548 #endif // _LP64
1549
1550 __ bind(do_return);
1551 __ addptr(rsp, 32);
1552 LP64_ONLY(__ pop(rdx);)
1553 __ pop(rcx);
1554 __ pop(rsi);
1555 __ ret(0);
1556 }
1557 break;
1558
1559 #if INCLUDE_ALL_GCS
1560 case g1_pre_barrier_slow_id:
1561 {
1562 StubFrame f(sasm, "g1_pre_barrier", dont_gc_arguments);
1563 // arg0 : previous value of memory
1564
1565 BarrierSet* bs = BarrierSet::barrier_set();
1566 if (bs->kind() != BarrierSet::G1BarrierSet) {
1567 __ movptr(rax, (int)id);
1568 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), rax);
1569 __ should_not_reach_here();
1570 break;
1571 }
1572 __ push(rax);
1573 __ push(rdx);
1574
1575 const Register pre_val = rax;
1576 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
1577 const Register tmp = rdx;
1578
1579 NOT_LP64(__ get_thread(thread);)
1580
1581 Address queue_active(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset()));
1582 Address queue_index(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset()));
1583 Address buffer(thread, in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset()));
1584
1585 Label done;
1586 Label runtime;
1587
1588 // Is marking still active?
1589 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
1590 __ cmpl(queue_active, 0);
1591 } else {
1592 assert(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
1593 __ cmpb(queue_active, 0);
1594 }
1595 __ jcc(Assembler::equal, done);
1596
1597 // Can we store original value in the thread's buffer?
1598
1599 __ movptr(tmp, queue_index);
1600 __ testptr(tmp, tmp);
1601 __ jcc(Assembler::zero, runtime);
1602 __ subptr(tmp, wordSize);
1603 __ movptr(queue_index, tmp);
1604 __ addptr(tmp, buffer);
1605
1606 // prev_val (rax)
1607 f.load_argument(0, pre_val);
1608 __ movptr(Address(tmp, 0), pre_val);
1609 __ jmp(done);
1610
1611 __ bind(runtime);
1612
1613 save_live_registers(sasm, 3);
1614
1615 // load the pre-value
1616 f.load_argument(0, rcx);
1617 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), rcx, thread);
1618
1619 restore_live_registers(sasm);
1620
1621 __ bind(done);
1622
1623 __ pop(rdx);
1624 __ pop(rax);
1625 }
1626 break;
1627
1628 case g1_post_barrier_slow_id:
1629 {
1630 StubFrame f(sasm, "g1_post_barrier", dont_gc_arguments);
1631
1632 BarrierSet* bs = BarrierSet::barrier_set();
1633 if (bs->kind() != BarrierSet::G1BarrierSet) {
1634 __ movptr(rax, (int)id);
1635 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), rax);
1636 __ should_not_reach_here();
1637 break;
1638 }
1639
1640 // arg0: store_address
1641 Address store_addr(rbp, 2*BytesPerWord);
1642
1643 Label done;
1644 Label enqueued;
1645 Label runtime;
1646
1647 // At this point we know new_value is non-NULL and the new_value crosses regions.
1648 // Must check to see if card is already dirty
1649
1650 const Register thread = NOT_LP64(rax) LP64_ONLY(r15_thread);
1651
1652 Address queue_index(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset()));
1653 Address buffer(thread, in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset()));
1654
1655 __ push(rax);
1656 __ push(rcx);
1657
1658 const Register cardtable = rax;
1659 const Register card_addr = rcx;
1660
1661 f.load_argument(0, card_addr);
1662 __ shrptr(card_addr, CardTable::card_shift);
1663 // Do not use ExternalAddress to load 'byte_map_base', since 'byte_map_base' is NOT
1664 // a valid address and therefore is not properly handled by the relocation code.
1665 __ movptr(cardtable, ci_card_table_address_as<intptr_t>());
1666 __ addptr(card_addr, cardtable);
1667
1668 NOT_LP64(__ get_thread(thread);)
1669
1670 __ cmpb(Address(card_addr, 0), (int)G1CardTable::g1_young_card_val());
1671 __ jcc(Assembler::equal, done);
1672
1673 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
1674 __ cmpb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
1675 __ jcc(Assembler::equal, done);
1676
1677 // storing region crossing non-NULL, card is clean.
1678 // dirty card and log.
1679
1680 __ movb(Address(card_addr, 0), (int)CardTable::dirty_card_val());
1681
1682 const Register tmp = rdx;
1683 __ push(rdx);
1684
1685 __ movptr(tmp, queue_index);
1686 __ testptr(tmp, tmp);
1687 __ jcc(Assembler::zero, runtime);
1688 __ subptr(tmp, wordSize);
1689 __ movptr(queue_index, tmp);
1690 __ addptr(tmp, buffer);
1691 __ movptr(Address(tmp, 0), card_addr);
1692 __ jmp(enqueued);
1693
1694 __ bind(runtime);
1695
1696 save_live_registers(sasm, 3);
1697
1698 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
1699
1700 restore_live_registers(sasm);
1701
1702 __ bind(enqueued);
1703 __ pop(rdx);
1704
1705 __ bind(done);
1706 __ pop(rcx);
1707 __ pop(rax);
1708 }
1709 break;
1710 #endif // INCLUDE_ALL_GCS
1711
1712 case predicate_failed_trap_id:
1713 {
1714 StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments);
1715
1716 OopMap* map = save_live_registers(sasm, 1);
1717
1718 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1719 oop_maps = new OopMapSet();
1720 oop_maps->add_gc_map(call_offset, map);
1721 restore_live_registers(sasm);
1722 __ leave();
1723 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1724 assert(deopt_blob != NULL, "deoptimization blob must have been created");
1725
1726 __ jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1727 }
1728 break;
1729
|
1544 #endif // _LP64
1545 __ jmp(do_return);
1546
1547 __ bind(return0);
1548 __ fpop();
1549 #ifndef _LP64
1550 __ xorptr(rdx,rdx);
1551 __ xorptr(rax,rax);
1552 #else
1553 __ xorptr(rax, rax);
1554 #endif // _LP64
1555
1556 __ bind(do_return);
1557 __ addptr(rsp, 32);
1558 LP64_ONLY(__ pop(rdx);)
1559 __ pop(rcx);
1560 __ pop(rsi);
1561 __ ret(0);
1562 }
1563 break;
1564
1565 case predicate_failed_trap_id:
1566 {
1567 StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments);
1568
1569 OopMap* map = save_live_registers(sasm, 1);
1570
1571 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1572 oop_maps = new OopMapSet();
1573 oop_maps->add_gc_map(call_offset, map);
1574 restore_live_registers(sasm);
1575 __ leave();
1576 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1577 assert(deopt_blob != NULL, "deoptimization blob must have been created");
1578
1579 __ jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1580 }
1581 break;
1582
|