37 #include "oops/klass.inline.hpp"
38 #include "opto/compile.hpp"
39 #include "opto/intrinsicnode.hpp"
40 #include "opto/matcher.hpp"
41 #include "prims/methodHandles.hpp"
42 #include "registerSaver_s390.hpp"
43 #include "runtime/biasedLocking.hpp"
44 #include "runtime/icache.hpp"
45 #include "runtime/interfaceSupport.inline.hpp"
46 #include "runtime/objectMonitor.hpp"
47 #include "runtime/os.hpp"
48 #include "runtime/safepoint.hpp"
49 #include "runtime/safepointMechanism.hpp"
50 #include "runtime/sharedRuntime.hpp"
51 #include "runtime/stubRoutines.hpp"
52 #include "utilities/events.hpp"
53 #include "utilities/macros.hpp"
54 #if INCLUDE_ALL_GCS
55 #include "gc/g1/g1BarrierSet.hpp"
56 #include "gc/g1/g1CardTable.hpp"
57 #include "gc/g1/heapRegion.hpp"
58 #endif
59
60 #include <ucontext.h>
61
62 #define BLOCK_COMMENT(str) block_comment(str)
63 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
64
65 // Move 32-bit register if destination and source are different.
66 void MacroAssembler::lr_if_needed(Register rd, Register rs) {
67 if (rs != rd) { z_lr(rd, rs); }
68 }
69
70 // Move register if destination and source are different.
71 void MacroAssembler::lgr_if_needed(Register rd, Register rs) {
72 if (rs != rd) { z_lgr(rd, rs); }
73 }
74
75 // Zero-extend 32-bit register into 64-bit register if destination and source are different.
76 void MacroAssembler::llgfr_if_needed(Register rd, Register rs) {
3553 // Note: Rpre_val needs special attention.
3554 // The flag pre_val_needed indicated that the caller of this emitter function
3555 // relies on Rpre_val containing the correct value, that is:
3556 // either the value it contained on entry to this code segment
3557 // or the value that was loaded into the register from (Robj+offset).
3558 //
3559 // Independent from this requirement, the contents of Rpre_val must survive
3560 // the push_frame() operation. push_frame() uses Z_R0_scratch by default
3561 // to temporarily remember the frame pointer.
3562 // If Rpre_val is assigned Z_R0_scratch by the caller, code must be emitted to
3563 // save it's value.
3564 void MacroAssembler::g1_write_barrier_pre(Register Robj,
3565 RegisterOrConstant offset,
3566 Register Rpre_val, // Ideally, this is a non-volatile register.
3567 Register Rval, // Will be preserved.
3568 Register Rtmp1, // If Rpre_val is volatile, either Rtmp1
3569 Register Rtmp2, // or Rtmp2 has to be non-volatile..
3570 bool pre_val_needed // Save Rpre_val across runtime call, caller uses it.
3571 ) {
3572 Label callRuntime, filtered;
3573 const int active_offset = in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_active());
3574 const int buffer_offset = in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_buf());
3575 const int index_offset = in_bytes(JavaThread::satb_mark_queue_offset() + SATBMarkQueue::byte_offset_of_index());
3576 assert_different_registers(Rtmp1, Rtmp2, Z_R0_scratch); // None of the Rtmp<i> must be Z_R0!!
3577 assert_different_registers(Robj, Z_R0_scratch); // Used for addressing. Furthermore, push_frame destroys Z_R0!!
3578 assert_different_registers(Rval, Z_R0_scratch); // push_frame destroys Z_R0!!
3579
3580 #ifdef ASSERT
3581 // make sure the register is not Z_R0. Used for addressing. Furthermore, would be destroyed by push_frame.
3582 if (offset.is_register() && offset.as_register()->encoding() == 0) {
3583 tty->print_cr("Roffset(g1_write_barrier_pre) = %%r%d", offset.as_register()->encoding());
3584 assert(false, "bad register for offset");
3585 }
3586 #endif
3587
3588 BLOCK_COMMENT("g1_write_barrier_pre {");
3589
3590 // Is marking active?
3591 // Note: value is loaded for test purposes only. No further use here.
3592 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
3593 load_and_test_int(Rtmp1, Address(Z_thread, active_offset));
3594 } else {
3595 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
3751
3752 // Filter young.
3753 assert((unsigned int)G1CardTable::g1_young_card_val() <= 255, "otherwise check this code");
3754 z_cli(0, Rcard_addr, (int)G1CardTable::g1_young_card_val());
3755 z_bre(filtered);
3756
3757 // Check the card value. If dirty, we're done.
3758 // This also avoids false sharing of the (already dirty) card.
3759 z_sync(); // Required to support concurrent cleaning.
3760 assert((unsigned int)CardTable::dirty_card_val() <= 255, "otherwise check this code");
3761 z_cli(0, Rcard_addr, CardTable::dirty_card_val()); // Reload after membar.
3762 z_bre(filtered);
3763
3764 // Storing a region crossing, non-NULL oop, card is clean.
3765 // Dirty card and log.
3766 z_mvi(0, Rcard_addr, CardTable::dirty_card_val());
3767
3768 Register Rcard_addr_x = Rcard_addr;
3769 Register Rqueue_index = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp1;
3770 Register Rqueue_buf = (Rtmp3 != Z_R0_scratch) ? Rtmp3 : Rtmp1;
3771 const int qidx_off = in_bytes(JavaThread::dirty_card_queue_offset() + SATBMarkQueue::byte_offset_of_index());
3772 const int qbuf_off = in_bytes(JavaThread::dirty_card_queue_offset() + SATBMarkQueue::byte_offset_of_buf());
3773 if ((Rcard_addr == Rqueue_buf) || (Rcard_addr == Rqueue_index)) {
3774 Rcard_addr_x = Z_R0_scratch; // Register shortage. We have to use Z_R0.
3775 }
3776 lgr_if_needed(Rcard_addr_x, Rcard_addr);
3777
3778 load_and_test_long(Rqueue_index, Address(Z_thread, qidx_off));
3779 z_bre(callRuntime); // Index == 0 then jump to runtime.
3780
3781 z_lg(Rqueue_buf, qbuf_off, Z_thread);
3782
3783 add2reg(Rqueue_index, -wordSize); // Decrement index.
3784 z_stg(Rqueue_index, qidx_off, Z_thread);
3785
3786 z_stg(Rcard_addr_x, 0, Rqueue_index, Rqueue_buf); // Store card.
3787 z_bru(filtered);
3788
3789 bind(callRuntime);
3790
3791 // TODO: do we need a frame? Introduced to be on the safe side.
3792 bool needs_frame = true;
|
37 #include "oops/klass.inline.hpp"
38 #include "opto/compile.hpp"
39 #include "opto/intrinsicnode.hpp"
40 #include "opto/matcher.hpp"
41 #include "prims/methodHandles.hpp"
42 #include "registerSaver_s390.hpp"
43 #include "runtime/biasedLocking.hpp"
44 #include "runtime/icache.hpp"
45 #include "runtime/interfaceSupport.inline.hpp"
46 #include "runtime/objectMonitor.hpp"
47 #include "runtime/os.hpp"
48 #include "runtime/safepoint.hpp"
49 #include "runtime/safepointMechanism.hpp"
50 #include "runtime/sharedRuntime.hpp"
51 #include "runtime/stubRoutines.hpp"
52 #include "utilities/events.hpp"
53 #include "utilities/macros.hpp"
54 #if INCLUDE_ALL_GCS
55 #include "gc/g1/g1BarrierSet.hpp"
56 #include "gc/g1/g1CardTable.hpp"
57 #include "gc/g1/g1ThreadLocalData.hpp"
58 #include "gc/g1/heapRegion.hpp"
59 #endif
60
61 #include <ucontext.h>
62
63 #define BLOCK_COMMENT(str) block_comment(str)
64 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
65
66 // Move 32-bit register if destination and source are different.
67 void MacroAssembler::lr_if_needed(Register rd, Register rs) {
68 if (rs != rd) { z_lr(rd, rs); }
69 }
70
71 // Move register if destination and source are different.
72 void MacroAssembler::lgr_if_needed(Register rd, Register rs) {
73 if (rs != rd) { z_lgr(rd, rs); }
74 }
75
76 // Zero-extend 32-bit register into 64-bit register if destination and source are different.
77 void MacroAssembler::llgfr_if_needed(Register rd, Register rs) {
3554 // Note: Rpre_val needs special attention.
3555 // The flag pre_val_needed indicated that the caller of this emitter function
3556 // relies on Rpre_val containing the correct value, that is:
3557 // either the value it contained on entry to this code segment
3558 // or the value that was loaded into the register from (Robj+offset).
3559 //
3560 // Independent from this requirement, the contents of Rpre_val must survive
3561 // the push_frame() operation. push_frame() uses Z_R0_scratch by default
3562 // to temporarily remember the frame pointer.
3563 // If Rpre_val is assigned Z_R0_scratch by the caller, code must be emitted to
3564 // save it's value.
3565 void MacroAssembler::g1_write_barrier_pre(Register Robj,
3566 RegisterOrConstant offset,
3567 Register Rpre_val, // Ideally, this is a non-volatile register.
3568 Register Rval, // Will be preserved.
3569 Register Rtmp1, // If Rpre_val is volatile, either Rtmp1
3570 Register Rtmp2, // or Rtmp2 has to be non-volatile..
3571 bool pre_val_needed // Save Rpre_val across runtime call, caller uses it.
3572 ) {
3573 Label callRuntime, filtered;
3574 const int active_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_active_offset());
3575 const int buffer_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_buffer_offset());
3576 const int index_offset = in_bytes(G1ThreadLocalData::satb_mark_queue_index_offset());
3577 assert_different_registers(Rtmp1, Rtmp2, Z_R0_scratch); // None of the Rtmp<i> must be Z_R0!!
3578 assert_different_registers(Robj, Z_R0_scratch); // Used for addressing. Furthermore, push_frame destroys Z_R0!!
3579 assert_different_registers(Rval, Z_R0_scratch); // push_frame destroys Z_R0!!
3580
3581 #ifdef ASSERT
3582 // make sure the register is not Z_R0. Used for addressing. Furthermore, would be destroyed by push_frame.
3583 if (offset.is_register() && offset.as_register()->encoding() == 0) {
3584 tty->print_cr("Roffset(g1_write_barrier_pre) = %%r%d", offset.as_register()->encoding());
3585 assert(false, "bad register for offset");
3586 }
3587 #endif
3588
3589 BLOCK_COMMENT("g1_write_barrier_pre {");
3590
3591 // Is marking active?
3592 // Note: value is loaded for test purposes only. No further use here.
3593 if (in_bytes(SATBMarkQueue::byte_width_of_active()) == 4) {
3594 load_and_test_int(Rtmp1, Address(Z_thread, active_offset));
3595 } else {
3596 guarantee(in_bytes(SATBMarkQueue::byte_width_of_active()) == 1, "Assumption");
3752
3753 // Filter young.
3754 assert((unsigned int)G1CardTable::g1_young_card_val() <= 255, "otherwise check this code");
3755 z_cli(0, Rcard_addr, (int)G1CardTable::g1_young_card_val());
3756 z_bre(filtered);
3757
3758 // Check the card value. If dirty, we're done.
3759 // This also avoids false sharing of the (already dirty) card.
3760 z_sync(); // Required to support concurrent cleaning.
3761 assert((unsigned int)CardTable::dirty_card_val() <= 255, "otherwise check this code");
3762 z_cli(0, Rcard_addr, CardTable::dirty_card_val()); // Reload after membar.
3763 z_bre(filtered);
3764
3765 // Storing a region crossing, non-NULL oop, card is clean.
3766 // Dirty card and log.
3767 z_mvi(0, Rcard_addr, CardTable::dirty_card_val());
3768
3769 Register Rcard_addr_x = Rcard_addr;
3770 Register Rqueue_index = (Rtmp2 != Z_R0_scratch) ? Rtmp2 : Rtmp1;
3771 Register Rqueue_buf = (Rtmp3 != Z_R0_scratch) ? Rtmp3 : Rtmp1;
3772 const int qidx_off = in_bytes(G1ThreadLocalData::dirty_card_queue_index_offset());
3773 const int qbuf_off = in_bytes(G1ThreadLocalData::dirty_card_queue_buffer_offset());
3774 if ((Rcard_addr == Rqueue_buf) || (Rcard_addr == Rqueue_index)) {
3775 Rcard_addr_x = Z_R0_scratch; // Register shortage. We have to use Z_R0.
3776 }
3777 lgr_if_needed(Rcard_addr_x, Rcard_addr);
3778
3779 load_and_test_long(Rqueue_index, Address(Z_thread, qidx_off));
3780 z_bre(callRuntime); // Index == 0 then jump to runtime.
3781
3782 z_lg(Rqueue_buf, qbuf_off, Z_thread);
3783
3784 add2reg(Rqueue_index, -wordSize); // Decrement index.
3785 z_stg(Rqueue_index, qidx_off, Z_thread);
3786
3787 z_stg(Rcard_addr_x, 0, Rqueue_index, Rqueue_buf); // Store card.
3788 z_bru(filtered);
3789
3790 bind(callRuntime);
3791
3792 // TODO: do we need a frame? Introduced to be on the safe side.
3793 bool needs_frame = true;
|