# HG changeset patch # Parent d5a916bb03b160f2fbf41a8c418c3ab081558333 diff -r d5a916bb03b1 src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Mon Mar 02 14:09:51 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp Mon Mar 02 15:26:07 2020 +0100 @@ -357,7 +357,7 @@ } Node* ShenandoahBarrierSetC2::shenandoah_enqueue_barrier(GraphKit* kit, Node* pre_val) const { - return kit->gvn().transform(new ShenandoahEnqueueBarrierNode(pre_val)); + return pre_val; //kit->gvn().transform(new ShenandoahEnqueueBarrierNode(pre_val)); } // Helper that guards and inserts a pre-barrier. @@ -513,11 +513,11 @@ ShenandoahBarrierSet::use_load_reference_barrier_native(decorators, type)); if (access.is_parse_access()) { load = static_cast(access).kit()->gvn().transform(load); - load = shenandoah_enqueue_barrier(static_cast(access).kit(), load); + //load = shenandoah_enqueue_barrier(static_cast(access).kit(), load); } else { load = static_cast(access).gvn().transform(load); - load = new ShenandoahEnqueueBarrierNode(load); - load = static_cast(access).gvn().transform(load); + //load = new ShenandoahEnqueueBarrierNode(load); + //load = static_cast(access).gvn().transform(load); } } diff -r d5a916bb03b1 src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp Mon Mar 02 14:09:51 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp Mon Mar 02 15:26:07 2020 +0100 @@ -917,6 +917,26 @@ assert(is_heap_state_test(heap_stable_iff, flag), "Should match the shape"); } +void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, Node* gc_state, + PhaseIdealLoop* phase, jint flag) { + IdealLoopTree* loop = phase->get_loop(ctrl); + Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(flag)); + phase->register_new_node(heap_stable_and, ctrl); + Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT)); + phase->register_new_node(heap_stable_cmp, ctrl); + Node* heap_stable_test = new BoolNode(heap_stable_cmp, BoolTest::ne); + phase->register_new_node(heap_stable_test, ctrl); + IfNode* heap_stable_iff = new IfNode(ctrl, heap_stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN); + phase->register_control(heap_stable_iff, loop, ctrl); + + heap_stable_ctrl = new IfFalseNode(heap_stable_iff); + phase->register_control(heap_stable_ctrl, loop, heap_stable_iff); + ctrl = new IfTrueNode(heap_stable_iff); + phase->register_control(ctrl, loop, heap_stable_iff); + + assert(is_heap_state_test(heap_stable_iff, flag), "Should match the shape"); +} + void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) { const Type* val_t = phase->igvn().type(val); if (val_t->meet(TypePtr::NULL_PTR) == val_t) { @@ -1450,19 +1470,146 @@ uncasted_val = val->in(1); } - Node* heap_stable_ctrl = NULL; - Node* null_ctrl = NULL; - assert(val->bottom_type()->make_oopptr(), "need oop"); assert(val->bottom_type()->make_oopptr()->const_oop() == NULL, "expect non-constant"); + // The heap-state-load is shared between conc-mark-barrier and LRB paths. + Node* thread = new ThreadLocalNode(); + phase->register_new_node(thread, ctrl); + Node* offset = phase->igvn().MakeConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset())); + phase->set_ctrl(offset, phase->C->root()); + Node* gc_state_addr = new AddPNode(phase->C->top(), thread, offset); + phase->register_new_node(gc_state_addr, ctrl); + uint gc_state_idx = Compile::AliasIdxRaw; + const TypePtr* gc_state_adr_type = NULL; // debug-mode-only argument + debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx)); + + Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered); + phase->register_new_node(gc_state, ctrl); + + if (lrb->need_concmark_barrier()) { + Node* heap_stable_ctrl = NULL; + Node* null_ctrl = NULL; + + enum { _heap_stable = 1, _heap_unstable, PATH_LIMIT }; + Node* region = new RegionNode(PATH_LIMIT); + Node* phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); + + enum { _fast_path = 1, _slow_path, _null_path, PATH_LIMIT2 }; + Node* region2 = new RegionNode(PATH_LIMIT2); + Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); + + // Stable path. + test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, gc_state, phase, ShenandoahHeap::MARKING | ShenandoahHeap::TRAVERSAL); + region->init_req(_heap_stable, heap_stable_ctrl); + phi->init_req(_heap_stable, raw_mem); + + // Null path + Node* reg2_ctrl = NULL; + test_null(ctrl, val, null_ctrl, phase); + if (null_ctrl != NULL) { + reg2_ctrl = null_ctrl->in(0); + region2->init_req(_null_path, null_ctrl); + phi2->init_req(_null_path, raw_mem); + } else { + region2->del_req(_null_path); + phi2->del_req(_null_path); + } + + const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()); + const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()); + Node* thread = new ThreadLocalNode(); + phase->register_new_node(thread, ctrl); + Node* buffer_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(buffer_offset)); + phase->register_new_node(buffer_adr, ctrl); + Node* index_adr = new AddPNode(phase->C->top(), thread, phase->igvn().MakeConX(index_offset)); + phase->register_new_node(index_adr, ctrl); + + BasicType index_bt = TypeX_X->basic_type(); + assert(sizeof(size_t) == type2aelembytes(index_bt), "Loading G1 SATBMarkQueue::_index with wrong size."); + const TypePtr* adr_type = TypeRawPtr::BOTTOM; + Node* index = new LoadXNode(ctrl, raw_mem, index_adr, adr_type, TypeX_X, MemNode::unordered); + phase->register_new_node(index, ctrl); + Node* index_cmp = new CmpXNode(index, phase->igvn().MakeConX(0)); + phase->register_new_node(index_cmp, ctrl); + Node* index_test = new BoolNode(index_cmp, BoolTest::ne); + phase->register_new_node(index_test, ctrl); + IfNode* queue_full_iff = new IfNode(ctrl, index_test, PROB_LIKELY(0.999), COUNT_UNKNOWN); + if (reg2_ctrl == NULL) reg2_ctrl = queue_full_iff; + phase->register_control(queue_full_iff, loop, ctrl); + Node* not_full = new IfTrueNode(queue_full_iff); + phase->register_control(not_full, loop, queue_full_iff); + Node* full = new IfFalseNode(queue_full_iff); + phase->register_control(full, loop, queue_full_iff); + + ctrl = not_full; + + Node* next_index = new SubXNode(index, phase->igvn().MakeConX(sizeof(intptr_t))); + phase->register_new_node(next_index, ctrl); + + Node* buffer = new LoadPNode(ctrl, raw_mem, buffer_adr, adr_type, TypeRawPtr::NOTNULL, MemNode::unordered); + phase->register_new_node(buffer, ctrl); + Node *log_addr = new AddPNode(phase->C->top(), buffer, next_index); + phase->register_new_node(log_addr, ctrl); + Node* log_store = new StorePNode(ctrl, raw_mem, log_addr, adr_type, uncasted_val, MemNode::unordered); + phase->register_new_node(log_store, ctrl); + // update the index + Node* index_update = new StoreXNode(ctrl, log_store, index_adr, adr_type, next_index, MemNode::unordered); + phase->register_new_node(index_update, ctrl); + + // Fast-path case + region2->init_req(_fast_path, ctrl); + phi2->init_req(_fast_path, index_update); + + ctrl = full; + + Node* base = find_bottom_mem(ctrl, phase); + + MergeMemNode* mm = MergeMemNode::make(base); + mm->set_memory_at(Compile::AliasIdxRaw, raw_mem); + phase->register_new_node(mm, ctrl); + + Node* call = new CallLeafNode(ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type(), CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry), "shenandoah_wb_pre", TypeRawPtr::BOTTOM); + call->init_req(TypeFunc::Control, ctrl); + call->init_req(TypeFunc::I_O, phase->C->top()); + call->init_req(TypeFunc::Memory, mm); + call->init_req(TypeFunc::FramePtr, phase->C->top()); + call->init_req(TypeFunc::ReturnAdr, phase->C->top()); + call->init_req(TypeFunc::Parms, uncasted_val); + call->init_req(TypeFunc::Parms+1, thread); + phase->register_control(call, loop, ctrl); + + Node* ctrl_proj = new ProjNode(call, TypeFunc::Control); + phase->register_control(ctrl_proj, loop, call); + Node* mem_proj = new ProjNode(call, TypeFunc::Memory); + phase->register_new_node(mem_proj, call); + + // Slow-path case + region2->init_req(_slow_path, ctrl_proj); + phi2->init_req(_slow_path, mem_proj); + + phase->register_control(region2, loop, reg2_ctrl); + phase->register_new_node(phi2, region2); + + region->init_req(_heap_unstable, region2); + phi->init_req(_heap_unstable, phi2); + + phase->register_control(region, loop, heap_stable_ctrl->in(0)); + phase->register_new_node(phi, region); + raw_mem = phi; + //init_raw_mem = raw_mem; + ctrl = region; + } + + Node* heap_stable_ctrl = NULL; + Node* null_ctrl = NULL; enum { _heap_stable = 1, _not_cset, _evac_path, _null_path, PATH_LIMIT }; Node* region = new RegionNode(PATH_LIMIT); Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()); Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); // Stable path. - test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase); + test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, gc_state, phase); IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If(); // Heap stable case @@ -1470,11 +1617,9 @@ val_phi->init_req(_heap_stable, uncasted_val); raw_mem_phi->init_req(_heap_stable, raw_mem); - Node* reg2_ctrl = NULL; // Null case test_null(ctrl, val, null_ctrl, phase); if (null_ctrl != NULL) { - reg2_ctrl = null_ctrl->in(0); region->init_req(_null_path, null_ctrl); val_phi->init_req(_null_path, uncasted_val); raw_mem_phi->init_req(_null_path, raw_mem); @@ -1489,7 +1634,6 @@ Node* not_cset_ctrl = NULL; in_cset_fast_test(ctrl, not_cset_ctrl, uncasted_val, raw_mem, phase); if (not_cset_ctrl != NULL) { - if (reg2_ctrl == NULL) reg2_ctrl = not_cset_ctrl->in(0); region->init_req(_not_cset, not_cset_ctrl); val_phi->init_req(_not_cset, uncasted_val); raw_mem_phi->init_req(_not_cset, raw_mem); @@ -2205,7 +2349,7 @@ } if (type->make_oopptr() && type->make_oopptr()->const_oop() != NULL) { // tty->print_cr("optimize barrier on constant"); - return false; // Needed, apparently + return false; } switch (n->Opcode()) { @@ -2324,6 +2468,86 @@ return useful; } +bool ShenandoahLoadReferenceBarrierNode::need_concmark_barrier() { + Unique_Node_List visited; + Node_Stack stack(0); + stack.push(this, 0); + + bool useful = false; + while (!useful && stack.size() > 0) { + Node* n = stack.node(); + if (visited.member(n)) { + stack.pop(); + continue; + } + visited.push(n); + bool visit_users = false; + switch (n->Opcode()) { + case Op_CallStaticJava: + if (n->as_CallStaticJava()->uncommon_trap_request() == 0) { + useful = true; // regular Java call + } + // else uncommon trap + break; + case Op_CallDynamicJava: + case Op_CompareAndExchangeN: + case Op_CompareAndExchangeP: + case Op_CompareAndSwapN: + case Op_CompareAndSwapP: + case Op_GetAndSetN: + case Op_GetAndSetP: + case Op_Return: + case Op_StoreN: + case Op_StoreP: + useful = true; + if (useful) { + //tty->print_cr("need concmark because of:"); + //n->dump(); + } + break; + case Op_AddP: + case Op_Allocate: + case Op_AllocateArray: + case Op_ArrayCopy: + case Op_CmpP: + case Op_LoadL: + case Op_SafePoint: + case Op_SubTypeCheck: + // useful = false; + break; + case Op_CastPP: + case Op_CheckCastPP: + case Op_CMoveN: + case Op_CMoveP: + case Op_EncodeP: + case Op_Phi: + case Op_ShenandoahLoadReferenceBarrier: + visit_users = true; + break; + default: { +#ifdef ASSERT + //fatal("Unknown node in get_barrier_strength: %s", NodeClassNames[n->Opcode()]); + useful = true; +#else + // Default to strong: better to have excess barriers, rather than miss some. + useful = true; +#endif + } + } + + stack.pop(); + if (visit_users) { + for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { + Node* user = n->fast_out(i); + if (user != NULL) { + stack.push(user, 0); + } + } + } + } + return useful; +} + Node* ShenandoahEnqueueBarrierNode::Identity(PhaseGVN* phase) { PhaseIterGVN* igvn = phase->is_IterGVN(); Node* value = in(1); diff -r d5a916bb03b1 src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp Mon Mar 02 14:09:51 2020 +0100 +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp Mon Mar 02 15:26:07 2020 +0100 @@ -61,6 +61,8 @@ static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase); static void test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, PhaseIdealLoop* phase, jint flag = ShenandoahHeap::HAS_FORWARDED); + static void test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl, Node* gc_state, + PhaseIdealLoop* phase, jint flag = ShenandoahHeap::HAS_FORWARDED); static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase); static Node* clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase); static void fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, Unique_Node_List& uses, @@ -262,9 +264,12 @@ Strength get_barrier_strength(); CallStaticJavaNode* pin_and_expand_null_check(PhaseIterGVN& igvn); + bool need_concmark_barrier(); + private: bool needs_barrier(PhaseGVN* phase, Node* n); bool needs_barrier_impl(PhaseGVN* phase, Node* n, Unique_Node_List &visited); + };