2692 Node* in_val = barrier->in(LoadBarrierNode::Oop);
2693 Node* in_adr = barrier->in(LoadBarrierNode::Address);
2694
2695 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
2696 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
2697
2698 const uintptr_t in_cset_bit = 0x1;
2699 const unsigned char* in_cset_map = (unsigned char*)ZHeap::heap()->pagetable_addr();
2700 const int in_cset_shift = ZPageSizeMinShift;
2701
2702 const uintptr_t* addr_good_bit = &ZAddressGoodMask;
2703 const uintptr_t* addr_bad_bit = &ZAddressBadMask;
2704
2705 const uintptr_t remapped_bit = ZAddressMetadataRemapped;
2706 const uintptr_t offset_mask = ZAddressOffsetMask;
2707
2708 float unlikely = PROB_UNLIKELY(0.999);
2709 float likely = PROB_LIKELY(0.999);
2710 const Type* in_val_maybe_null_t = _igvn.type(in_val);
2711
2712 Node* bad_mask = NULL;
2713 Node* good_mask = NULL;
2714 #ifdef SPARC
2715 bad_mask = _igvn.transform(new AddrBadBitNode());
2716 #else
2717 if (C->directive()->UseR15TestInLoadBarrierOption) {
2718 Node* jthread = _igvn.transform(new ThreadLocalNode());
2719 Node* adr = basic_plus_adr(jthread, in_bytes(JavaThread::zaddress_bad_mask_offset()));
2720 bad_mask = _igvn.transform(LoadNode::make(_igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
2721 } else {
2722 bad_mask = _igvn.transform(LoadNode::make(_igvn, in_ctrl, in_mem, makecon(TypeRawPtr::make((address)addr_bad_bit)),
2723 TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
2724 }
2725 #endif
2726
2727 Node* cast = _igvn.transform(new CastP2XNode(in_ctrl, in_val));
2728 Node* obj_masked = _igvn.transform(new AndXNode(cast, bad_mask));
2729 Node* cmp = _igvn.transform(new CmpXNode(obj_masked, _igvn.zerocon(TypeX_X->basic_type())));
2730 Node *bol = _igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
2731 IfNode* iff = _igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
2732 Node* then = _igvn.transform(new IfTrueNode(iff));
2733 Node* elsen = _igvn.transform(new IfFalseNode(iff));
2734
2735 Node* result_region;
2736 Node* result_val;
2737
2738 result_region = new RegionNode(3);
2739 result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);
2740
2741 result_region->set_req(1, elsen);
2742 Node* res = _igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));
2743 res->init_req(0, elsen);
2744 result_val->set_req(1, res);
2745
2746 const TypeFunc *tf = OptoRuntime::load_barrier_Type();
2803 traverse( barrier, out_ctrl, out_res, -1 );
2804
2805 tty->print( "\nBefore barrier optimization: preceding_barrier_node\n" );
2806 traverse( preceding_barrier_node, out_ctrl, out_res, -1 );
2807 #endif
2808
2809 const uintptr_t in_cset_bit = 0x1;
2810 const unsigned char* in_cset_map = (unsigned char*)ZHeap::heap()->pagetable_addr();
2811 const int in_cset_shift = ZPageSizeMinShift;
2812
2813 const uintptr_t* addr_good_bit = &ZAddressGoodMask;
2814 const uintptr_t* addr_bad_bit = &ZAddressBadMask;
2815
2816 const uintptr_t remapped_bit = ZAddressMetadataRemapped;
2817 const uintptr_t offset_mask = ZAddressOffsetMask;
2818
2819 float unlikely = PROB_UNLIKELY(0.999);
2820 float likely = PROB_LIKELY(0.999);
2821 const Type* in_val_not_null_t = _igvn.type(in_val);
2822
2823 Node* bad_mask = NULL;
2824 if (C->directive()->UseR15TestInLoadBarrierOption) {
2825 Node* jthread = _igvn.transform(new ThreadLocalNode());
2826 Node* adr = basic_plus_adr(jthread, in_bytes(JavaThread::zaddress_bad_mask_offset()));
2827 bad_mask = _igvn.transform(LoadNode::make(_igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
2828 } else {
2829 bad_mask = _igvn.transform(LoadNode::make(_igvn, in_ctrl, in_mem, makecon(TypeRawPtr::make((address)addr_bad_bit)),
2830 TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
2831 }
2832
2833 Node* cast = _igvn.transform(new CastP2XNode(in_ctrl, in_val));
2834 Node* obj_masked = _igvn.transform(new AndXNode(cast, bad_mask));
2835 Node* cmp = _igvn.transform(new CmpXNode(obj_masked, _igvn.zerocon(TypeX_X->basic_type())));
2836 Node *bol = _igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
2837 IfNode* iff = _igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
2838 Node* then = _igvn.transform(new IfTrueNode(iff));
2839 Node* elsen = _igvn.transform(new IfFalseNode(iff));
2840
2841 Node* slow_path_surrogate;
2842 if (!barrier->is_weak()) {
2843 slow_path_surrogate = _igvn.transform( new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), (const TypePtr*) in_val->bottom_type(), MemNode::unordered) );
2844 } else {
2845 slow_path_surrogate = _igvn.transform( new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), (const TypePtr*) in_val->bottom_type(), MemNode::unordered) );
2846 }
2847
2848 Node *new_loadp;
2849 new_loadp = slow_path_surrogate;
2850 // create the final region/phi pair to converge cntl/data paths to downstream code
2851 Node* result_region = _igvn.transform(new RegionNode(3));
2852 result_region->set_req(1, then);
|
2692 Node* in_val = barrier->in(LoadBarrierNode::Oop);
2693 Node* in_adr = barrier->in(LoadBarrierNode::Address);
2694
2695 Node* out_ctrl = barrier->proj_out(LoadBarrierNode::Control);
2696 Node* out_res = barrier->proj_out(LoadBarrierNode::Oop);
2697
2698 const uintptr_t in_cset_bit = 0x1;
2699 const unsigned char* in_cset_map = (unsigned char*)ZHeap::heap()->pagetable_addr();
2700 const int in_cset_shift = ZPageSizeMinShift;
2701
2702 const uintptr_t* addr_good_bit = &ZAddressGoodMask;
2703 const uintptr_t* addr_bad_bit = &ZAddressBadMask;
2704
2705 const uintptr_t remapped_bit = ZAddressMetadataRemapped;
2706 const uintptr_t offset_mask = ZAddressOffsetMask;
2707
2708 float unlikely = PROB_UNLIKELY(0.999);
2709 float likely = PROB_LIKELY(0.999);
2710 const Type* in_val_maybe_null_t = _igvn.type(in_val);
2711
2712 #ifdef SPARC
2713 Node* bad_mask = _igvn.transform(new AddrBadBitNode());
2714 #else
2715 Node* jthread = _igvn.transform(new ThreadLocalNode());
2716 Node* adr = basic_plus_adr(jthread, in_bytes(JavaThread::zaddress_bad_mask_offset()));
2717 Node* bad_mask = _igvn.transform(LoadNode::make(_igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
2718 #endif
2719 Node* cast = _igvn.transform(new CastP2XNode(in_ctrl, in_val));
2720 Node* obj_masked = _igvn.transform(new AndXNode(cast, bad_mask));
2721 Node* cmp = _igvn.transform(new CmpXNode(obj_masked, _igvn.zerocon(TypeX_X->basic_type())));
2722 Node *bol = _igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
2723 IfNode* iff = _igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
2724 Node* then = _igvn.transform(new IfTrueNode(iff));
2725 Node* elsen = _igvn.transform(new IfFalseNode(iff));
2726
2727 Node* result_region;
2728 Node* result_val;
2729
2730 result_region = new RegionNode(3);
2731 result_val = new PhiNode(result_region, TypeInstPtr::BOTTOM);
2732
2733 result_region->set_req(1, elsen);
2734 Node* res = _igvn.transform(new CastPPNode(in_val, in_val_maybe_null_t));
2735 res->init_req(0, elsen);
2736 result_val->set_req(1, res);
2737
2738 const TypeFunc *tf = OptoRuntime::load_barrier_Type();
2795 traverse( barrier, out_ctrl, out_res, -1 );
2796
2797 tty->print( "\nBefore barrier optimization: preceding_barrier_node\n" );
2798 traverse( preceding_barrier_node, out_ctrl, out_res, -1 );
2799 #endif
2800
2801 const uintptr_t in_cset_bit = 0x1;
2802 const unsigned char* in_cset_map = (unsigned char*)ZHeap::heap()->pagetable_addr();
2803 const int in_cset_shift = ZPageSizeMinShift;
2804
2805 const uintptr_t* addr_good_bit = &ZAddressGoodMask;
2806 const uintptr_t* addr_bad_bit = &ZAddressBadMask;
2807
2808 const uintptr_t remapped_bit = ZAddressMetadataRemapped;
2809 const uintptr_t offset_mask = ZAddressOffsetMask;
2810
2811 float unlikely = PROB_UNLIKELY(0.999);
2812 float likely = PROB_LIKELY(0.999);
2813 const Type* in_val_not_null_t = _igvn.type(in_val);
2814
2815 Node* jthread = _igvn.transform(new ThreadLocalNode());
2816 Node* adr = basic_plus_adr(jthread, in_bytes(JavaThread::zaddress_bad_mask_offset()));
2817 Node* bad_mask = _igvn.transform(LoadNode::make(_igvn, in_ctrl, in_mem, adr, TypeRawPtr::BOTTOM, TypeX_X, TypeX_X->basic_type(), MemNode::unordered));
2818 Node* cast = _igvn.transform(new CastP2XNode(in_ctrl, in_val));
2819 Node* obj_masked = _igvn.transform(new AndXNode(cast, bad_mask));
2820 Node* cmp = _igvn.transform(new CmpXNode(obj_masked, _igvn.zerocon(TypeX_X->basic_type())));
2821 Node *bol = _igvn.transform(new BoolNode(cmp, BoolTest::ne))->as_Bool();
2822 IfNode* iff = _igvn.transform(new IfNode(in_ctrl, bol, unlikely, COUNT_UNKNOWN))->as_If();
2823 Node* then = _igvn.transform(new IfTrueNode(iff));
2824 Node* elsen = _igvn.transform(new IfFalseNode(iff));
2825
2826 Node* slow_path_surrogate;
2827 if (!barrier->is_weak()) {
2828 slow_path_surrogate = _igvn.transform( new LoadBarrierSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), (const TypePtr*) in_val->bottom_type(), MemNode::unordered) );
2829 } else {
2830 slow_path_surrogate = _igvn.transform( new LoadBarrierWeakSlowRegNode(then, in_mem, in_adr, in_val->adr_type(), (const TypePtr*) in_val->bottom_type(), MemNode::unordered) );
2831 }
2832
2833 Node *new_loadp;
2834 new_loadp = slow_path_surrogate;
2835 // create the final region/phi pair to converge cntl/data paths to downstream code
2836 Node* result_region = _igvn.transform(new RegionNode(3));
2837 result_region->set_req(1, then);
|