3554 if (do_load) {
3555 // We need to generate the load of the previous value
3556 assert(obj != NULL, "must have a base");
3557 assert(adr != NULL, "where are loading from?");
3558 assert(pre_val == NULL, "loaded already?");
3559 assert(val_type != NULL, "need a type");
3560 } else {
3561 // In this case both val_type and alias_idx are unused.
3562 assert(pre_val != NULL, "must be loaded already");
3563 assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
3564 }
3565 assert(bt == T_OBJECT, "or we shouldn't be here");
3566
3567 IdealKit ideal(this, true);
3568
3569 Node* tls = __ thread(); // ThreadLocalStorage
3570
3571 Node* no_ctrl = NULL;
3572 Node* no_base = __ top();
3573 Node* zero = __ ConI(0);
3574
3575 float likely = PROB_LIKELY(0.999);
3576 float unlikely = PROB_UNLIKELY(0.999);
3577
3578 BasicType active_type = in_bytes(PtrQueue::byte_width_of_active()) == 4 ? T_INT : T_BYTE;
3579 assert(in_bytes(PtrQueue::byte_width_of_active()) == 4 || in_bytes(PtrQueue::byte_width_of_active()) == 1, "flag width");
3580
3581 // Offsets into the thread
3582 const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 648
3583 PtrQueue::byte_offset_of_active());
3584 const int index_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 656
3585 PtrQueue::byte_offset_of_index());
3586 const int buffer_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 652
3587 PtrQueue::byte_offset_of_buf());
3588
3589 // Now the actual pointers into the thread
3590 Node* marking_adr = __ AddP(no_base, tls, __ ConX(marking_offset));
3591 Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
3592 Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
3593
3594 // Now some of the values
3595 Node* marking = __ load(__ ctrl(), marking_adr, TypeInt::INT, active_type, Compile::AliasIdxRaw);
3596
3597 // if (!marking)
3598 __ if_then(marking, BoolTest::ne, zero); {
3599 Node* index = __ load(__ ctrl(), index_adr, TypeInt::INT, T_INT, Compile::AliasIdxRaw);
3600
3601 if (do_load) {
3602 // load original value
3603 // alias_idx correct??
3604 pre_val = __ load(no_ctrl, adr, val_type, bt, alias_idx);
3605 }
3606
3607 // if (pre_val != NULL)
3608 __ if_then(pre_val, BoolTest::ne, null()); {
3609 Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
3610
3611 // is the queue for this thread full?
3612 __ if_then(index, BoolTest::ne, zero, likely); {
3613
3614 // decrement the index
3615 Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t)));
3616 Node* next_indexX = next_index;
3617 #ifdef _LP64
3618 // We could refine the type for what it's worth
3619 // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue);
3620 next_indexX = _gvn.transform( new (C) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) );
3621 #endif
3622
3623 // Now get the buffer location we will log the previous value into and store it
3624 Node *log_addr = __ AddP(no_base, buffer, next_indexX);
3625 __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw);
3626 // update the index
3627 __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw);
3628
3629 } __ else_(); {
3630
3631 // logging buffer is full, call the runtime
3632 const TypeFunc *tf = OptoRuntime::g1_wb_pre_Type();
3633 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", pre_val, tls);
3634 } __ end_if(); // (!index)
3635 } __ end_if(); // (pre_val != NULL)
3636 } __ end_if(); // (!marking)
3637
3638 // Final sync IdealKit and GraphKit.
3639 final_sync(ideal);
3640 }
3641
3642 //
3643 // Update the card table and add card address to the queue
3644 //
3645 void GraphKit::g1_mark_card(IdealKit& ideal,
3646 Node* card_adr,
3647 Node* oop_store,
3648 uint oop_alias_idx,
3649 Node* index,
3650 Node* index_adr,
3651 Node* buffer,
3652 const TypeFunc* tf) {
3653
3654 Node* zero = __ ConI(0);
3655 Node* no_base = __ top();
3656 BasicType card_bt = T_BYTE;
3657 // Smash zero into card. MUST BE ORDERED WRT TO STORE
3658 __ storeCM(__ ctrl(), card_adr, zero, oop_store, oop_alias_idx, card_bt, Compile::AliasIdxRaw);
3659
3660 // Now do the queue work
3661 __ if_then(index, BoolTest::ne, zero); {
3662
3663 Node* next_index = __ SubI(index, __ ConI(sizeof(intptr_t)));
3664 Node* next_indexX = next_index;
3665 #ifdef _LP64
3666 // We could refine the type for what it's worth
3667 // const TypeLong* lidxtype = TypeLong::make(CONST64(0), get_size_from_queue);
3668 next_indexX = _gvn.transform( new (C) ConvI2LNode(next_index, TypeLong::make(0, max_jlong, Type::WidenMax)) );
3669 #endif // _LP64
3670 Node* log_addr = __ AddP(no_base, buffer, next_indexX);
3671
3672 __ store(__ ctrl(), log_addr, card_adr, T_ADDRESS, Compile::AliasIdxRaw);
3673 __ store(__ ctrl(), index_adr, next_index, T_INT, Compile::AliasIdxRaw);
3674
3675 } __ else_(); {
3676 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), "g1_wb_post", card_adr, __ thread());
3677 } __ end_if();
3678
3679 }
3680
3681 void GraphKit::g1_write_barrier_post(Node* oop_store,
3682 Node* obj,
3683 Node* adr,
3684 uint alias_idx,
3685 Node* val,
3686 BasicType bt,
3687 bool use_precise) {
3688 // If we are writing a NULL then we need no post barrier
3689
3690 if (val != NULL && val->is_Con() && val->bottom_type() == TypePtr::NULL_PTR) {
3691 // Must be NULL
3692 const Type* t = val->bottom_type();
3693 assert(t == Type::TOP || t == TypePtr::NULL_PTR, "must be NULL");
3714
3715 // Get the alias_index for raw card-mark memory
3716 const TypePtr* card_type = TypeRawPtr::BOTTOM;
3717
3718 const TypeFunc *tf = OptoRuntime::g1_wb_post_Type();
3719
3720 // Offsets into the thread
3721 const int index_offset = in_bytes(JavaThread::dirty_card_queue_offset() +
3722 PtrQueue::byte_offset_of_index());
3723 const int buffer_offset = in_bytes(JavaThread::dirty_card_queue_offset() +
3724 PtrQueue::byte_offset_of_buf());
3725
3726 // Pointers into the thread
3727
3728 Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
3729 Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
3730
3731 // Now some values
3732 // Use ctrl to avoid hoisting these values past a safepoint, which could
3733 // potentially reset these fields in the JavaThread.
3734 Node* index = __ load(__ ctrl(), index_adr, TypeInt::INT, T_INT, Compile::AliasIdxRaw);
3735 Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
3736
3737 // Convert the store obj pointer to an int prior to doing math on it
3738 // Must use ctrl to prevent "integerized oop" existing across safepoint
3739 Node* cast = __ CastPX(__ ctrl(), adr);
3740
3741 // Divide pointer by card size
3742 Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) );
3743
3744 // Combine card table base and card offset
3745 Node* card_adr = __ AddP(no_base, byte_map_base_node(), card_offset );
3746
3747 // If we know the value being stored does it cross regions?
3748
3749 if (val != NULL) {
3750 // Does the store cause us to cross regions?
3751
3752 // Should be able to do an unsigned compare of region_size instead of
3753 // and extra shift. Do we have an unsigned compare??
3754 // Node* region_size = __ ConI(1 << HeapRegion::LogOfHRGrainBytes);
|
3554 if (do_load) {
3555 // We need to generate the load of the previous value
3556 assert(obj != NULL, "must have a base");
3557 assert(adr != NULL, "where are loading from?");
3558 assert(pre_val == NULL, "loaded already?");
3559 assert(val_type != NULL, "need a type");
3560 } else {
3561 // In this case both val_type and alias_idx are unused.
3562 assert(pre_val != NULL, "must be loaded already");
3563 assert(pre_val->bottom_type()->basic_type() == T_OBJECT, "or we shouldn't be here");
3564 }
3565 assert(bt == T_OBJECT, "or we shouldn't be here");
3566
3567 IdealKit ideal(this, true);
3568
3569 Node* tls = __ thread(); // ThreadLocalStorage
3570
3571 Node* no_ctrl = NULL;
3572 Node* no_base = __ top();
3573 Node* zero = __ ConI(0);
3574 Node* zeroX = __ ConX(0);
3575
3576 float likely = PROB_LIKELY(0.999);
3577 float unlikely = PROB_UNLIKELY(0.999);
3578
3579 BasicType active_type = in_bytes(PtrQueue::byte_width_of_active()) == 4 ? T_INT : T_BYTE;
3580 assert(in_bytes(PtrQueue::byte_width_of_active()) == 4 || in_bytes(PtrQueue::byte_width_of_active()) == 1, "flag width");
3581
3582 // Offsets into the thread
3583 const int marking_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 648
3584 PtrQueue::byte_offset_of_active());
3585 const int index_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 656
3586 PtrQueue::byte_offset_of_index());
3587 const int buffer_offset = in_bytes(JavaThread::satb_mark_queue_offset() + // 652
3588 PtrQueue::byte_offset_of_buf());
3589
3590 // Now the actual pointers into the thread
3591 Node* marking_adr = __ AddP(no_base, tls, __ ConX(marking_offset));
3592 Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
3593 Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
3594
3595 // Now some of the values
3596 Node* marking = __ load(__ ctrl(), marking_adr, TypeInt::INT, active_type, Compile::AliasIdxRaw);
3597
3598 // if (!marking)
3599 __ if_then(marking, BoolTest::ne, zero); {
3600 BasicType index_bt = TypeX_X->basic_type();
3601 assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 PtrQueue::_index with wrong size.");
3602 Node* index = __ load(__ ctrl(), index_adr, TypeX_X, index_bt, Compile::AliasIdxRaw);
3603
3604 if (do_load) {
3605 // load original value
3606 // alias_idx correct??
3607 pre_val = __ load(no_ctrl, adr, val_type, bt, alias_idx);
3608 }
3609
3610 // if (pre_val != NULL)
3611 __ if_then(pre_val, BoolTest::ne, null()); {
3612 Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
3613
3614 // is the queue for this thread full?
3615 __ if_then(index, BoolTest::ne, zeroX, likely); {
3616
3617 // decrement the index
3618 Node* next_index = _gvn.transform(new (C) SubXNode(index, __ ConX(sizeof(intptr_t))));
3619
3620 // Now get the buffer location we will log the previous value into and store it
3621 Node *log_addr = __ AddP(no_base, buffer, next_index);
3622 __ store(__ ctrl(), log_addr, pre_val, T_OBJECT, Compile::AliasIdxRaw);
3623 // update the index
3624 __ store(__ ctrl(), index_adr, next_index, index_bt, Compile::AliasIdxRaw);
3625
3626 } __ else_(); {
3627
3628 // logging buffer is full, call the runtime
3629 const TypeFunc *tf = OptoRuntime::g1_wb_pre_Type();
3630 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), "g1_wb_pre", pre_val, tls);
3631 } __ end_if(); // (!index)
3632 } __ end_if(); // (pre_val != NULL)
3633 } __ end_if(); // (!marking)
3634
3635 // Final sync IdealKit and GraphKit.
3636 final_sync(ideal);
3637 }
3638
3639 //
3640 // Update the card table and add card address to the queue
3641 //
3642 void GraphKit::g1_mark_card(IdealKit& ideal,
3643 Node* card_adr,
3644 Node* oop_store,
3645 uint oop_alias_idx,
3646 Node* index,
3647 Node* index_adr,
3648 Node* buffer,
3649 const TypeFunc* tf) {
3650
3651 Node* zero = __ ConI(0);
3652 Node* zeroX = __ ConX(0);
3653 Node* no_base = __ top();
3654 BasicType card_bt = T_BYTE;
3655 // Smash zero into card. MUST BE ORDERED WRT TO STORE
3656 __ storeCM(__ ctrl(), card_adr, zero, oop_store, oop_alias_idx, card_bt, Compile::AliasIdxRaw);
3657
3658 // Now do the queue work
3659 __ if_then(index, BoolTest::ne, zeroX); {
3660
3661 Node* next_index = _gvn.transform(new (C) SubXNode(index, __ ConX(sizeof(intptr_t))));
3662 Node* log_addr = __ AddP(no_base, buffer, next_index);
3663
3664 __ store(__ ctrl(), log_addr, card_adr, T_ADDRESS, Compile::AliasIdxRaw);
3665 __ store(__ ctrl(), index_adr, next_index, TypeX_X->basic_type(), Compile::AliasIdxRaw);
3666
3667 } __ else_(); {
3668 __ make_leaf_call(tf, CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), "g1_wb_post", card_adr, __ thread());
3669 } __ end_if();
3670
3671 }
3672
3673 void GraphKit::g1_write_barrier_post(Node* oop_store,
3674 Node* obj,
3675 Node* adr,
3676 uint alias_idx,
3677 Node* val,
3678 BasicType bt,
3679 bool use_precise) {
3680 // If we are writing a NULL then we need no post barrier
3681
3682 if (val != NULL && val->is_Con() && val->bottom_type() == TypePtr::NULL_PTR) {
3683 // Must be NULL
3684 const Type* t = val->bottom_type();
3685 assert(t == Type::TOP || t == TypePtr::NULL_PTR, "must be NULL");
3706
3707 // Get the alias_index for raw card-mark memory
3708 const TypePtr* card_type = TypeRawPtr::BOTTOM;
3709
3710 const TypeFunc *tf = OptoRuntime::g1_wb_post_Type();
3711
3712 // Offsets into the thread
3713 const int index_offset = in_bytes(JavaThread::dirty_card_queue_offset() +
3714 PtrQueue::byte_offset_of_index());
3715 const int buffer_offset = in_bytes(JavaThread::dirty_card_queue_offset() +
3716 PtrQueue::byte_offset_of_buf());
3717
3718 // Pointers into the thread
3719
3720 Node* buffer_adr = __ AddP(no_base, tls, __ ConX(buffer_offset));
3721 Node* index_adr = __ AddP(no_base, tls, __ ConX(index_offset));
3722
3723 // Now some values
3724 // Use ctrl to avoid hoisting these values past a safepoint, which could
3725 // potentially reset these fields in the JavaThread.
3726 Node* index = __ load(__ ctrl(), index_adr, TypeX_X, TypeX_X->basic_type(), Compile::AliasIdxRaw);
3727 Node* buffer = __ load(__ ctrl(), buffer_adr, TypeRawPtr::NOTNULL, T_ADDRESS, Compile::AliasIdxRaw);
3728
3729 // Convert the store obj pointer to an int prior to doing math on it
3730 // Must use ctrl to prevent "integerized oop" existing across safepoint
3731 Node* cast = __ CastPX(__ ctrl(), adr);
3732
3733 // Divide pointer by card size
3734 Node* card_offset = __ URShiftX( cast, __ ConI(CardTableModRefBS::card_shift) );
3735
3736 // Combine card table base and card offset
3737 Node* card_adr = __ AddP(no_base, byte_map_base_node(), card_offset );
3738
3739 // If we know the value being stored does it cross regions?
3740
3741 if (val != NULL) {
3742 // Does the store cause us to cross regions?
3743
3744 // Should be able to do an unsigned compare of region_size instead of
3745 // and extra shift. Do we have an unsigned compare??
3746 // Node* region_size = __ ConI(1 << HeapRegion::LogOfHRGrainBytes);
|