29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "runtime/thread.hpp"
38 #include "utilities/macros.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
48
49 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
50 Register src, Register dst, Register count) {
51
52 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
53
54 if (is_reference_type(type)) {
55
56 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahStoreValEnqueueBarrier || ShenandoahLoadRefBarrier) {
57 #ifdef _LP64
58 Register thread = r15_thread;
59 #else
60 Register thread = rax;
61 if (thread == src || thread == dst || thread == count) {
62 thread = rbx;
63 }
64 if (thread == src || thread == dst || thread == count) {
65 thread = rcx;
66 }
67 if (thread == src || thread == dst || thread == count) {
68 thread = rdx;
273 bool need_addr_setup = (src_addr != dst);
274
275 if (need_addr_setup) {
276 __ push(src_addr);
277 __ lea(src_addr, src);
278
279 if (dst != rax) {
280 // Move obj into rax and save rax
281 __ push(rax);
282 __ movptr(rax, dst);
283 }
284 } else {
285 // dst == rsi
286 __ push(rax);
287 __ movptr(rax, dst);
288
289 // we can clobber it, since it is outgoing register
290 __ lea(src_addr, src);
291 }
292
293 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
294
295 if (need_addr_setup) {
296 if (dst != rax) {
297 __ movptr(dst, rax);
298 __ pop(rax);
299 }
300 __ pop(src_addr);
301 } else {
302 __ movptr(dst, rax);
303 __ pop(rax);
304 }
305
306 __ bind(done);
307
308 #ifndef _LP64
309 __ pop(thread);
310 #endif
311 }
312
313 void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst, Address src) {
351 if (dst != rax) {
352 __ push(rax);
353 }
354 __ push(rcx);
355 __ push(rdx);
356 __ push(rdi);
357 __ push(rsi);
358 #ifdef _LP64
359 __ push(r8);
360 __ push(r9);
361 __ push(r10);
362 __ push(r11);
363 __ push(r12);
364 __ push(r13);
365 __ push(r14);
366 __ push(r15);
367 #endif
368
369 assert_different_registers(dst, rsi);
370 __ lea(rsi, src);
371 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native), dst, rsi);
372
373 #ifdef _LP64
374 __ pop(r15);
375 __ pop(r14);
376 __ pop(r13);
377 __ pop(r12);
378 __ pop(r11);
379 __ pop(r10);
380 __ pop(r9);
381 __ pop(r8);
382 #endif
383 __ pop(rsi);
384 __ pop(rdi);
385 __ pop(rdx);
386 __ pop(rcx);
387
388 if (dst != rax) {
389 __ movptr(dst, rax);
390 __ pop(rax);
391 }
497 // Move loaded oop to final destination
498 if (dst != result_dst) {
499 __ movptr(result_dst, dst);
500
501 if (!use_tmp1_for_dst) {
502 __ pop(dst);
503 }
504
505 dst = result_dst;
506 }
507 } else {
508 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
509 }
510
511 // 3: apply keep-alive barrier if needed
512 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
513 __ push_IU_state();
514 // That path can be reached from the c2i adapter with live fp
515 // arguments in registers.
516 LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));
517 __ subptr(rsp, 64);
518 __ movdbl(Address(rsp, 0), xmm0);
519 __ movdbl(Address(rsp, 8), xmm1);
520 __ movdbl(Address(rsp, 16), xmm2);
521 __ movdbl(Address(rsp, 24), xmm3);
522 __ movdbl(Address(rsp, 32), xmm4);
523 __ movdbl(Address(rsp, 40), xmm5);
524 __ movdbl(Address(rsp, 48), xmm6);
525 __ movdbl(Address(rsp, 56), xmm7);
526
527 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
528 assert_different_registers(dst, tmp1, tmp_thread);
529 if (!thread->is_valid()) {
530 thread = rdx;
531 }
532 NOT_LP64(__ get_thread(thread));
533 // Generate the SATB pre-barrier code to log the value of
534 // the referent field in an SATB buffer.
535 shenandoah_write_barrier_pre(masm /* masm */,
536 noreg /* obj */,
537 dst /* pre_val */,
538 thread /* thread */,
539 tmp1 /* tmp */,
540 true /* tosca_live */,
541 true /* expand_call */);
542 __ movdbl(xmm0, Address(rsp, 0));
543 __ movdbl(xmm1, Address(rsp, 8));
544 __ movdbl(xmm2, Address(rsp, 16));
545 __ movdbl(xmm3, Address(rsp, 24));
546 __ movdbl(xmm4, Address(rsp, 32));
547 __ movdbl(xmm5, Address(rsp, 40));
548 __ movdbl(xmm6, Address(rsp, 48));
549 __ movdbl(xmm7, Address(rsp, 56));
550 __ addptr(rsp, 64);
551 __ pop_IU_state();
552 }
553 }
554
555 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
556 Address dst, Register val, Register tmp1, Register tmp2) {
557
558 bool on_oop = is_reference_type(type);
559 bool in_heap = (decorators & IN_HEAP) != 0;
560 bool as_normal = (decorators & AS_NORMAL) != 0;
561 if (on_oop && in_heap) {
562 bool needs_pre_barrier = as_normal;
563
564 Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
565 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
566 // flatten object address if needed
567 // We do it regardless of precise because we need the registers
568 if (dst.index() == noreg && dst.disp() == 0) {
569 if (dst.base() != tmp1) {
570 __ movptr(tmp1, dst.base());
|
29 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
30 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
31 #include "gc/shenandoah/shenandoahRuntime.hpp"
32 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
33 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
34 #include "interpreter/interpreter.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "runtime/sharedRuntime.hpp"
37 #include "runtime/thread.hpp"
38 #include "utilities/macros.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 address ShenandoahBarrierSetAssembler::_shenandoah_lrb = NULL;
48
49 static void save_xmm_registers(MacroAssembler* masm) {
50 __ subptr(rsp, 64);
51 __ movdbl(Address(rsp, 0), xmm0);
52 __ movdbl(Address(rsp, 8), xmm1);
53 __ movdbl(Address(rsp, 16), xmm2);
54 __ movdbl(Address(rsp, 24), xmm3);
55 __ movdbl(Address(rsp, 32), xmm4);
56 __ movdbl(Address(rsp, 40), xmm5);
57 __ movdbl(Address(rsp, 48), xmm6);
58 __ movdbl(Address(rsp, 56), xmm7);
59 }
60
61 static void restore_xmm_registers(MacroAssembler* masm) {
62 __ movdbl(xmm0, Address(rsp, 0));
63 __ movdbl(xmm1, Address(rsp, 8));
64 __ movdbl(xmm2, Address(rsp, 16));
65 __ movdbl(xmm3, Address(rsp, 24));
66 __ movdbl(xmm4, Address(rsp, 32));
67 __ movdbl(xmm5, Address(rsp, 40));
68 __ movdbl(xmm6, Address(rsp, 48));
69 __ movdbl(xmm7, Address(rsp, 56));
70 __ addptr(rsp, 64);
71 }
72
73 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
74 Register src, Register dst, Register count) {
75
76 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
77
78 if (is_reference_type(type)) {
79
80 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahStoreValEnqueueBarrier || ShenandoahLoadRefBarrier) {
81 #ifdef _LP64
82 Register thread = r15_thread;
83 #else
84 Register thread = rax;
85 if (thread == src || thread == dst || thread == count) {
86 thread = rbx;
87 }
88 if (thread == src || thread == dst || thread == count) {
89 thread = rcx;
90 }
91 if (thread == src || thread == dst || thread == count) {
92 thread = rdx;
297 bool need_addr_setup = (src_addr != dst);
298
299 if (need_addr_setup) {
300 __ push(src_addr);
301 __ lea(src_addr, src);
302
303 if (dst != rax) {
304 // Move obj into rax and save rax
305 __ push(rax);
306 __ movptr(rax, dst);
307 }
308 } else {
309 // dst == rsi
310 __ push(rax);
311 __ movptr(rax, dst);
312
313 // we can clobber it, since it is outgoing register
314 __ lea(src_addr, src);
315 }
316
317 save_xmm_registers(masm);
318 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahBarrierSetAssembler::shenandoah_lrb())));
319 restore_xmm_registers(masm);
320
321 if (need_addr_setup) {
322 if (dst != rax) {
323 __ movptr(dst, rax);
324 __ pop(rax);
325 }
326 __ pop(src_addr);
327 } else {
328 __ movptr(dst, rax);
329 __ pop(rax);
330 }
331
332 __ bind(done);
333
334 #ifndef _LP64
335 __ pop(thread);
336 #endif
337 }
338
339 void ShenandoahBarrierSetAssembler::load_reference_barrier_native(MacroAssembler* masm, Register dst, Address src) {
377 if (dst != rax) {
378 __ push(rax);
379 }
380 __ push(rcx);
381 __ push(rdx);
382 __ push(rdi);
383 __ push(rsi);
384 #ifdef _LP64
385 __ push(r8);
386 __ push(r9);
387 __ push(r10);
388 __ push(r11);
389 __ push(r12);
390 __ push(r13);
391 __ push(r14);
392 __ push(r15);
393 #endif
394
395 assert_different_registers(dst, rsi);
396 __ lea(rsi, src);
397
398 save_xmm_registers(masm);
399 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_native), dst, rsi);
400 restore_xmm_registers(masm);
401
402 #ifdef _LP64
403 __ pop(r15);
404 __ pop(r14);
405 __ pop(r13);
406 __ pop(r12);
407 __ pop(r11);
408 __ pop(r10);
409 __ pop(r9);
410 __ pop(r8);
411 #endif
412 __ pop(rsi);
413 __ pop(rdi);
414 __ pop(rdx);
415 __ pop(rcx);
416
417 if (dst != rax) {
418 __ movptr(dst, rax);
419 __ pop(rax);
420 }
526 // Move loaded oop to final destination
527 if (dst != result_dst) {
528 __ movptr(result_dst, dst);
529
530 if (!use_tmp1_for_dst) {
531 __ pop(dst);
532 }
533
534 dst = result_dst;
535 }
536 } else {
537 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
538 }
539
540 // 3: apply keep-alive barrier if needed
541 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
542 __ push_IU_state();
543 // That path can be reached from the c2i adapter with live fp
544 // arguments in registers.
545 LP64_ONLY(assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call"));
546 save_xmm_registers(masm);
547
548 Register thread = NOT_LP64(tmp_thread) LP64_ONLY(r15_thread);
549 assert_different_registers(dst, tmp1, tmp_thread);
550 if (!thread->is_valid()) {
551 thread = rdx;
552 }
553 NOT_LP64(__ get_thread(thread));
554 // Generate the SATB pre-barrier code to log the value of
555 // the referent field in an SATB buffer.
556 shenandoah_write_barrier_pre(masm /* masm */,
557 noreg /* obj */,
558 dst /* pre_val */,
559 thread /* thread */,
560 tmp1 /* tmp */,
561 true /* tosca_live */,
562 true /* expand_call */);
563 restore_xmm_registers(masm);
564 __ pop_IU_state();
565 }
566 }
567
568 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
569 Address dst, Register val, Register tmp1, Register tmp2) {
570
571 bool on_oop = is_reference_type(type);
572 bool in_heap = (decorators & IN_HEAP) != 0;
573 bool as_normal = (decorators & AS_NORMAL) != 0;
574 if (on_oop && in_heap) {
575 bool needs_pre_barrier = as_normal;
576
577 Register tmp3 = LP64_ONLY(r8) NOT_LP64(rsi);
578 Register rthread = LP64_ONLY(r15_thread) NOT_LP64(rcx);
579 // flatten object address if needed
580 // We do it regardless of precise because we need the registers
581 if (dst.index() == noreg && dst.disp() == 0) {
582 if (dst.base() != tmp1) {
583 __ movptr(tmp1, dst.base());
|