diff --git a/src/hotspot/share/classfile/classLoader.cpp b/src/hotspot/share/classfile/classLoader.cpp --- a/src/hotspot/share/classfile/classLoader.cpp +++ b/src/hotspot/share/classfile/classLoader.cpp @@ -1842,14 +1842,14 @@ HandleMark hm(THREAD); ResourceMark rm(THREAD); - assert(has_jrt_entry(), "Compile The World not supported with exploded module build"); - // Find bootstrap loader Handle system_class_loader (THREAD, SystemDictionary::java_system_loader()); jlong start = os::javaTimeMillis(); // Compile the world for the modular java runtime image + if (has_jrt_entry()) { _jrt_entry->compile_the_world(system_class_loader, CATCH); + } // Iterate over all bootstrap class path appended entries ClassPathEntry* e = _first_append_entry; diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -37,26 +37,65 @@ } ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena) - : _shenandoah_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) { + : _shenandoah_write_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)), + _shenandoah_enqueue_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)), + _shenandoah_read_barriers(new (comp_arena) GrowableArray(comp_arena, 8, 0, NULL)) { } -int ShenandoahBarrierSetC2State::shenandoah_barriers_count() const { - return _shenandoah_barriers->length(); +int ShenandoahBarrierSetC2State::shenandoah_write_barriers_count() const { + return _shenandoah_write_barriers->length(); } -ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_barrier(int idx) const { - return _shenandoah_barriers->at(idx); +int ShenandoahBarrierSetC2State::shenandoah_enqueue_barriers_count() const { + return _shenandoah_enqueue_barriers->length(); } - -void ShenandoahBarrierSetC2State::add_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { - assert(!_shenandoah_barriers->contains(n), "duplicate entry in barrier list"); - _shenandoah_barriers->append(n); +int ShenandoahBarrierSetC2State::shenandoah_read_barriers_count() const { + return _shenandoah_read_barriers->length(); } -void ShenandoahBarrierSetC2State::remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n) { - if (_shenandoah_barriers->contains(n)) { - _shenandoah_barriers->remove(n); +ShenandoahWriteBarrierNode* ShenandoahBarrierSetC2State::shenandoah_write_barrier(int idx) const { + return _shenandoah_write_barriers->at(idx); +} + +ShenandoahEnqueueBarrierNode* ShenandoahBarrierSetC2State::shenandoah_enqueue_barrier(int idx) const { + return _shenandoah_enqueue_barriers->at(idx); +} + +ShenandoahReadBarrierNode* ShenandoahBarrierSetC2State::shenandoah_read_barrier(int idx) const { + return _shenandoah_read_barriers->at(idx); +} + +void ShenandoahBarrierSetC2State::add_shenandoah_write_barrier(ShenandoahWriteBarrierNode * n) { + assert(!_shenandoah_write_barriers->contains(n), "duplicate entry in barrier list"); + _shenandoah_write_barriers->append(n); +} + +void ShenandoahBarrierSetC2State::add_shenandoah_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) { + assert(!_shenandoah_enqueue_barriers->contains(n), "duplicate entry in barrier list"); + _shenandoah_enqueue_barriers->append(n); +} + +void ShenandoahBarrierSetC2State::add_shenandoah_read_barrier(ShenandoahReadBarrierNode * n) { + assert(!_shenandoah_read_barriers->contains(n), "duplicate entry in barrier list"); + _shenandoah_read_barriers->append(n); +} + +void ShenandoahBarrierSetC2State::remove_shenandoah_write_barrier(ShenandoahWriteBarrierNode * n) { + if (_shenandoah_write_barriers->contains(n)) { + _shenandoah_write_barriers->remove(n); + } +} + +void ShenandoahBarrierSetC2State::remove_shenandoah_enqueue_barrier(ShenandoahEnqueueBarrierNode * n) { + if (_shenandoah_enqueue_barriers->contains(n)) { + _shenandoah_enqueue_barriers->remove(n); + } +} + +void ShenandoahBarrierSetC2State::remove_shenandoah_read_barrier(ShenandoahReadBarrierNode * n) { + if (_shenandoah_read_barriers->contains(n)) { + _shenandoah_read_barriers->remove(n); } } @@ -851,13 +890,21 @@ // Support for macro expanded GC barriers void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const { if (node->Opcode() == Op_ShenandoahWriteBarrier) { - state()->add_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); + state()->add_shenandoah_write_barrier((ShenandoahWriteBarrierNode*) node); + } else if (node->Opcode() == Op_ShenandoahEnqueueBarrier) { + state()->add_shenandoah_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node); + } else if (node->Opcode() == Op_ShenandoahReadBarrier) { + state()->add_shenandoah_read_barrier((ShenandoahReadBarrierNode*) node); } } void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const { if (node->Opcode() == Op_ShenandoahWriteBarrier) { - state()->remove_shenandoah_barrier((ShenandoahWriteBarrierNode*) node); + state()->remove_shenandoah_write_barrier((ShenandoahWriteBarrierNode*) node); + } else if (node->Opcode() == Op_ShenandoahEnqueueBarrier) { + state()->remove_shenandoah_enqueue_barrier((ShenandoahEnqueueBarrierNode*) node); + } else if (node->Opcode() == Op_ShenandoahReadBarrier) { + state()->remove_shenandoah_read_barrier((ShenandoahReadBarrierNode*) node); } } @@ -892,13 +939,24 @@ } void ShenandoahBarrierSetC2::eliminate_useless_gc_barriers(Unique_Node_List &useful) const { - for (int i = state()->shenandoah_barriers_count()-1; i >= 0; i--) { - ShenandoahWriteBarrierNode* n = state()->shenandoah_barrier(i); + for (int i = state()->shenandoah_write_barriers_count()-1; i >= 0; i--) { + ShenandoahWriteBarrierNode* n = state()->shenandoah_write_barrier(i); if (!useful.member(n)) { - state()->remove_shenandoah_barrier(n); + state()->remove_shenandoah_write_barrier(n); } } - + for (int i = state()->shenandoah_enqueue_barriers_count()-1; i >= 0; i--) { + ShenandoahEnqueueBarrierNode* n = state()->shenandoah_enqueue_barrier(i); + if (!useful.member(n)) { + state()->remove_shenandoah_enqueue_barrier(n); + } + } + for (int i = state()->shenandoah_read_barriers_count()-1; i >= 0; i--) { + ShenandoahReadBarrierNode* n = state()->shenandoah_read_barrier(i); + if (!useful.member(n)) { + state()->remove_shenandoah_read_barrier(n); + } + } } void ShenandoahBarrierSetC2::add_users_to_worklist(Unique_Node_List* worklist) const {} diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp @@ -26,16 +26,28 @@ #include "gc/shared/c2/barrierSetC2.hpp" +class ShenandoahEnqueueBarrierNode; + class ShenandoahBarrierSetC2State : public ResourceObj { private: - GrowableArray* _shenandoah_barriers; + GrowableArray* _shenandoah_write_barriers; + GrowableArray* _shenandoah_enqueue_barriers; + GrowableArray* _shenandoah_read_barriers; public: ShenandoahBarrierSetC2State(Arena* comp_arena); - int shenandoah_barriers_count() const; - ShenandoahWriteBarrierNode* shenandoah_barrier(int idx) const; - void add_shenandoah_barrier(ShenandoahWriteBarrierNode * n); - void remove_shenandoah_barrier(ShenandoahWriteBarrierNode * n); + int shenandoah_write_barriers_count() const; + int shenandoah_enqueue_barriers_count() const; + int shenandoah_read_barriers_count() const; + ShenandoahWriteBarrierNode* shenandoah_write_barrier(int idx) const; + ShenandoahEnqueueBarrierNode* shenandoah_enqueue_barrier(int idx) const; + ShenandoahReadBarrierNode* shenandoah_read_barrier(int idx) const; + void add_shenandoah_write_barrier(ShenandoahWriteBarrierNode * n); + void add_shenandoah_enqueue_barrier(ShenandoahEnqueueBarrierNode * n); + void add_shenandoah_read_barrier(ShenandoahReadBarrierNode * n); + void remove_shenandoah_write_barrier(ShenandoahWriteBarrierNode * n); + void remove_shenandoah_enqueue_barrier(ShenandoahEnqueueBarrierNode * n); + void remove_shenandoah_read_barrier(ShenandoahReadBarrierNode * n); }; class ShenandoahBarrierSetC2 : public BarrierSetC2 { diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -185,6 +185,22 @@ return true; } +ShenandoahReadBarrierNode::ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj) + : ShenandoahBarrierNode(ctrl, mem, obj, true) { + assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || + ShenandoahWriteBarrier || ShenandoahAcmpBarrier), + "should be enabled"); + ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_read_barrier(this); +} + +ShenandoahReadBarrierNode::ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj, bool allow_fromspace) + : ShenandoahBarrierNode(ctrl, mem, obj, allow_fromspace) { + assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || + ShenandoahWriteBarrier || ShenandoahAcmpBarrier), + "should be enabled"); + ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_read_barrier(this); +} + bool ShenandoahReadBarrierNode::dominates_memory_rb_impl(PhaseGVN* phase, Node* b1, Node* b2, @@ -369,7 +385,7 @@ ShenandoahWriteBarrierNode::ShenandoahWriteBarrierNode(Compile* C, Node* ctrl, Node* mem, Node* obj) : ShenandoahBarrierNode(ctrl, mem, obj, false) { assert(UseShenandoahGC && ShenandoahWriteBarrier, "should be enabled"); - ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_barrier(this); + ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_write_barrier(this); } @@ -424,7 +440,9 @@ bool ShenandoahWriteBarrierNode::expand(Compile* C, PhaseIterGVN& igvn, int& loop_opts_cnt) { if (UseShenandoahGC) { - if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() > 0 || (!ShenandoahWriteBarrier && ShenandoahStoreValEnqueueBarrier)) { + if (ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_write_barriers_count() + + ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_enqueue_barriers_count() + + ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_read_barriers_count() > 0) { bool attempt_more_loopopts = ShenandoahLoopOptsAfterExpansion; C->clear_major_progress(); PhaseIdealLoop ideal_loop(igvn, LoopOptsShenandoahExpand); @@ -1880,8 +1898,8 @@ Unique_Node_List uses; do { progress = false; - for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) { - ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i); + for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_write_barriers_count(); i++) { + ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_write_barrier(i); wb->try_move_before_loop(memory_graph_fixers, phase, include_lsm, uses); @@ -2094,7 +2112,7 @@ } } -CallStaticJavaNode* ShenandoahWriteBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) { +CallStaticJavaNode* ShenandoahBarrierNode::pin_and_expand_null_check(PhaseIterGVN& igvn) { Node* val = in(ValueIn); const Type* val_t = igvn.type(val); @@ -2117,7 +2135,7 @@ return NULL; } -void ShenandoahWriteBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray& memory_graph_fixers, Unique_Node_List& uses) { +void ShenandoahBarrierNode::pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray& memory_graph_fixers, Unique_Node_List& uses) { Node* unc = pin_and_expand_null_check(phase->igvn()); Node* val = in(ValueIn); @@ -2139,7 +2157,6 @@ return; } - RegionNode* r = new RegionNode(3); IfNode* iff = unc_ctrl->in(0)->as_If(); @@ -2147,7 +2164,9 @@ Node* unc_ctrl_clone = unc_ctrl->clone(); phase->register_control(unc_ctrl_clone, loop, iff); Node* c = unc_ctrl_clone; - Node* new_cast = clone_null_check(c, val, unc_ctrl_clone, r, 1, phase); + Node* new_cast = val; + Node* iffalse = clone_null_check(c, new_cast, unc_ctrl_clone, phase); + r->init_req(1, iffalse); phase->igvn().replace_input_of(unc_ctrl, 0, c->in(0)); phase->set_idom(unc_ctrl, c->in(0), phase->dom_depth(unc_ctrl)); @@ -2160,6 +2179,7 @@ fix_null_check(iff, unc, unc_ctrl_clone, r, uses, phase); Node* iff_proj = iff->proj_out(0); r->init_req(2, iff_proj); + phase->register_control(r, phase->ltree_root(), iff); Node* new_bol = new_iff->in(1)->clone(); Node* new_cmp = new_bol->in(1)->clone(); @@ -2196,6 +2216,26 @@ Node* new_ctrl = unc_ctrl_clone; + Node* mem = in(Memory); + if (phase->has_ctrl(mem) && phase->get_ctrl(mem) == unc_ctrl) { + uses.clear(); + uses.push(mem); + for (uint i = 0; i < uses.size(); i++) { + Node* n = uses.at(i); + phase->set_ctrl(n, new_ctrl); + if (n->in(0) == unc_ctrl) { + phase->igvn().replace_input_of(n, 0, new_ctrl); + } + for (uint j = 0; j < n->req(); j++) { + Node* in = n->in(j); + if (in != NULL && phase->has_ctrl(in) && phase->get_ctrl(in) == unc_ctrl) { + uses.push(in); + } + } + } + } + + if (Opcode() == Op_ShenandoahWriteBarrier) { int alias = phase->C->get_alias_index(adr_type()); MemoryGraphFixer* fixer = find_fixer(memory_graph_fixers, alias); if (fixer == NULL) { @@ -2212,15 +2252,22 @@ } phase->set_ctrl_and_loop(this, new_ctrl); + phase->igvn().replace_input_of(this, Control, new_ctrl); phase->set_ctrl_and_loop(proj, new_ctrl); - fixer->fix_mem(new_ctrl, new_ctrl, mem, mem, proj, uses); + } else { + phase->set_ctrl_and_loop(this, new_ctrl); + } } } -void ShenandoahWriteBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) { +void ShenandoahBarrierNode::pin_and_expand_helper(PhaseIdealLoop* phase) { Node* val = in(ValueIn); + if (val->uncast()->is_ShenandoahBarrier()) { + return; + } + CallStaticJavaNode* unc = pin_and_expand_null_check(phase->igvn()); Node* rep = this; Node* ctrl = phase->get_ctrl(this); @@ -2262,7 +2309,7 @@ } } else { Node* c = phase->ctrl_or_self(u); - if (is_dominator(ctrl, c, this, u, phase)) { + if (u != c && is_dominator(ctrl, c, this, u, phase)) { phase->igvn().rehash_node_delayed(u); int nb = u->replace_edge(val, rep); assert(nb > 0, "no update?"); @@ -2432,8 +2479,8 @@ phase->register_control(evac_in_progress, loop, evacuation_iff); } -Node* ShenandoahWriteBarrierNode::clone_null_check(Node*& c, Node* val, Node* unc_ctrl, - Node* unc_region, uint input, PhaseIdealLoop* phase) { +Node* ShenandoahBarrierNode::clone_null_check(Node*& c, Node*& val, Node* unc_ctrl, + PhaseIdealLoop* phase) { IdealLoopTree *loop = phase->get_loop(c); Node* iff = unc_ctrl->in(0); assert(iff->is_If(), "broken"); @@ -2451,11 +2498,10 @@ val = new CastPPNode(uncasted_val, t); val->init_req(0, c); phase->register_new_node(val, c); - unc_region->init_req(input, iffalse); - return val; + return iffalse; } -void ShenandoahWriteBarrierNode::fix_null_check(Node* dom, Node* unc, Node* unc_ctrl, Node* unc_region, +void ShenandoahBarrierNode::fix_null_check(Node* dom, Node* unc, Node* unc_ctrl, Node* unc_region, Unique_Node_List& uses, PhaseIdealLoop* phase) { IfNode* iff = unc_ctrl->in(0)->as_If(); Node* proj = iff->proj_out(0); @@ -2502,14 +2548,14 @@ phase->igvn().rehash_node_delayed(use); int nb = use->replace_edge(proj, unc_region); assert(nb == 1, "only use expected"); - phase->register_control(unc_region, phase->ltree_root(), dom); } void ShenandoahWriteBarrierNode::evacuation_not_in_progress_null_check(Node*& c, Node*& val, Node* unc_ctrl, Node*& unc_region, PhaseIdealLoop* phase) { if (unc_ctrl != NULL) { // Clone the null check in this branch to allow implicit null check unc_region = new RegionNode(3); - val = clone_null_check(c, val, unc_ctrl, unc_region, 1, phase); + Node* iffalse = clone_null_check(c, val, unc_ctrl, phase); + unc_region->init_req(1, iffalse); } } @@ -2543,12 +2589,16 @@ Node* unc_region, Unique_Node_List& uses, PhaseIdealLoop* phase) { if (unc != NULL) { // Clone the null check in this branch to allow implicit null check - val = clone_null_check(c, val, unc_ctrl, unc_region, 2, phase); + Node* ctrl = clone_null_check(c, val, unc_ctrl, phase); + unc_region->init_req(2, ctrl); fix_null_check(evacuation_iff, unc, unc_ctrl, unc_region, uses, phase); + phase->register_control(unc_region, phase->ltree_root(), evacuation_iff); IfNode* iff = unc_ctrl->in(0)->as_If(); phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); + // phase->lazy_update(iff, iff->in(0)); + // phase->lazy_replace(unc_ctrl, iff->in(0)); } } @@ -2640,6 +2690,7 @@ void ShenandoahWriteBarrierNode::fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase) { Node* ctrl = phase->get_ctrl(barrier); Node* init_raw_mem = fixer.find_mem(ctrl, barrier); + Node* mem = barrier->in(Memory); // Update the control of all nodes that should be after the // barrier control flow @@ -2652,6 +2703,9 @@ if (phase->has_ctrl(init_raw_mem) && phase->get_ctrl(init_raw_mem) == ctrl && !init_raw_mem->is_Phi()) { uses_to_ignore.push(init_raw_mem); } + // if (phase->get_ctrl(mem) == ctrl && !mem->is_Phi()) { + // uses_to_ignore.push(mem); + // } for (uint next = 0; next < uses_to_ignore.size(); next++) { Node *n = uses_to_ignore.at(next); for (uint i = 0; i < n->req(); i++) { @@ -2692,24 +2746,6 @@ void ShenandoahWriteBarrierNode::pin_and_expand(PhaseIdealLoop* phase) { - Node_List enqueue_barriers; - if (ShenandoahStoreValEnqueueBarrier) { - Unique_Node_List wq; - wq.push(phase->C->root()); - for (uint i = 0; i < wq.size(); i++) { - Node* n = wq.at(i); - if (n->Opcode() == Op_ShenandoahEnqueueBarrier) { - enqueue_barriers.push(n); - } - for (uint i = 0; i < n->req(); i++) { - Node* in = n->in(i); - if (in != NULL) { - wq.push(in); - } - } - } - } - const bool trace = false; // Collect raw memory state at CFG points in the entire graph and @@ -2722,8 +2758,8 @@ optimize_before_expansion(phase, memory_graph_fixers, true); Unique_Node_List uses; - for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i++) { - ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i); + for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_write_barriers_count(); i++) { + ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_write_barrier(i); Node* ctrl = phase->get_ctrl(wb); Node* val = wb->in(ValueIn); @@ -2746,8 +2782,170 @@ MemoryGraphFixer fixer(Compile::AliasIdxRaw, true, phase); Unique_Node_List uses_to_ignore; - for (uint i = 0; i < enqueue_barriers.size(); i++) { - Node* barrier = enqueue_barriers.at(i); + + phase->C->print_method(PHASE_CUSTOM, 2); + + // for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_read_barriers_count(); i++) { + // ShenandoahReadBarrierNode* rb1 = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_read_barrier(i); + // if (rb1->in(0) != NULL) { + // Node* ctrl1 = rb1->in(0); + // Node* val1 = rb1->in(ValueIn); + // Node* mem1 = rb1->in(Memory); + // for (int j = 0; j < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_read_barriers_count(); j++) { + // ShenandoahReadBarrierNode* rb2 = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_read_barrier(j); + // if (rb2->in(0) == NULL) { + // Node* ctrl2 = phase->get_ctrl(rb2); + // Node* val2 = rb2->in(ValueIn); + // Node* mem2 = rb2->in(Memory); + // if (val1 == val2 && mem1 == mem2 && phase->is_dominator(ctrl1, ctrl2)) { + // if (UseNewCode) { + // tty->print("XXX Replacing"); rb2->dump(); + // ctrl2->dump(); + // tty->print("XXX by"); rb1->dump(); + // phase->get_ctrl(rb1)->dump(); + // } + // phase->igvn().replace_node(rb2, rb1); + // phase->set_ctrl(rb1, phase->dom_lca(phase->get_ctrl(rb1), ctrl2)); + // } + // } + // } + // } + // } + + for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_read_barriers_count(); i++) { + ShenandoahReadBarrierNode* rb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_read_barrier(i); + Node* ctrl = phase->get_ctrl(rb); + Node* val = rb->in(ValueIn); + + if (UseNewCode) { + tty->print("XXX"); rb->dump(); + } + + CallStaticJavaNode* unc = rb->pin_and_expand_null_check(phase->igvn()); + if (unc != NULL) { + Node* unc_ctrl = val->in(0); + Node* mem = rb->in(Memory); + Node* mem_c = phase->get_ctrl(mem); + if (phase->is_dominator(mem_c, unc_ctrl) && + (rb->in(Control) == NULL || phase->is_dominator(rb->in(Control), unc_ctrl))) { + if (UseNewCode) { + tty->print_cr("XXXXXX unc"); + } + rb->pin_and_expand_move_barrier(phase, memory_graph_fixers, uses); + } + } + + rb->pin_and_expand_helper(phase); + } + + for (int i = 0; i < ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_read_barriers_count(); i++) { + ShenandoahReadBarrierNode* rb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_read_barrier(i); + Node* ctrl = phase->get_ctrl(rb); + Node* val = rb->in(ValueIn); + + if ((ctrl->is_Proj() && ctrl->in(0)->is_CallJava()) || ctrl->is_Call() || ctrl->is_MemBar()) { + if (UseNewCode) { + tty->print_cr("XXXXXX skip"); + } + continue; + } + + CallStaticJavaNode* unc = rb->pin_and_expand_null_check(phase->igvn()); + + uint last = phase->C->unique(); + + Node* raw_mem = fixer.find_mem(ctrl, rb); + Node* raw_mem_for_ctrl = fixer.find_mem(ctrl, NULL); + int alias = phase->C->get_alias_index(rb->adr_type()); + IdealLoopTree *loop = phase->get_loop(ctrl); + + Node* unc_ctrl = NULL; + if (unc != NULL) { + if (val->in(0) != ctrl) { + unc = NULL; + } else { + unc_ctrl = val->in(0); + } + } + + Node* uncasted_val = val; + if (unc != NULL) { + uncasted_val = val->in(1); + } + + Node* heap_stable_ctrl = NULL; + Node* heap_not_stable_ctrl = NULL; + Node* unused_gc_state = NULL; + + test_heap_stable(ctrl, raw_mem, unused_gc_state, heap_stable_ctrl, heap_not_stable_ctrl, phase); + + Node* region = new RegionNode(3); + Node* val_phi = new PhiNode(region, uncasted_val->bottom_type()->is_oopptr()->cast_to_nonconst()); + + region->init_req(1, heap_stable_ctrl); + val_phi->init_req(1, uncasted_val); + + if (unc_ctrl != NULL) { + Node* iffalse = clone_null_check(heap_not_stable_ctrl, val, unc_ctrl, phase); + phase->igvn().replace_input_of(rb, ValueIn, val); + fix_null_check(iffalse, unc, unc_ctrl, iffalse, uses, phase); + } + + region->init_req(2, heap_not_stable_ctrl); + val_phi->init_req(2, rb); + + IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If(); + phase->register_control(region, loop, heap_stable_iff); + phase->register_new_node(val_phi, region); + + fix_ctrl(rb, region, fixer, uses, uses_to_ignore, last, phase); + if (unc != NULL) { + for (DUIterator_Fast imax, i = val->fast_outs(imax); i < imax; i++) { + Node* u = val->fast_out(i); + Node* c = phase->ctrl_or_self(u); + if (u != rb && (c != ctrl || is_dominator_same_ctrl(c, rb, u, phase))) { + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(val, val_phi); + --i, imax -= nb; + } + } + if (val->outcnt() == 0) { + phase->igvn()._worklist.push(val); + } + IfNode* iff = unc_ctrl->in(0)->as_If(); + phase->igvn().replace_input_of(iff, 1, phase->igvn().intcon(1)); + } + phase->igvn().replace_input_of(rb, 0, heap_not_stable_ctrl); + phase->set_ctrl(rb, heap_not_stable_ctrl); + + for (DUIterator_Fast jmax, j = rb->fast_outs(jmax); j < jmax; j++) { + Node* u = rb->fast_out(j); + if (u != val_phi) { + phase->igvn().rehash_node_delayed(u); + int nb = u->replace_edge(rb, val_phi); + assert(nb > 0, "should have updated use"); + --j, jmax -= nb; + } + } + + follow_barrier_uses(val_phi, ctrl, uses, phase); + for(uint next = 0; next < uses.size(); next++ ) { + Node *n = uses.at(next); + assert(phase->get_ctrl(n) == ctrl, "bad control"); + assert(n != raw_mem, "should leave input raw mem above the barrier"); + phase->set_ctrl(n, region); + follow_barrier_uses(n, ctrl, uses, phase); + } + + fixer.fix_mem(ctrl, region, raw_mem, raw_mem_for_ctrl); + + phase->C->print_method(PHASE_CUSTOM, 2); + } + + for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_enqueue_barriers_count(); i > 0; i--) { + int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_enqueue_barriers_count(); + ShenandoahEnqueueBarrierNode* barrier = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_enqueue_barrier(i-1); + Node* pre_val = barrier->in(1); if (phase->igvn().type(pre_val)->higher_equal(TypePtr::NULL_PTR)) { @@ -2884,11 +3082,13 @@ fixer.fix_mem(init_ctrl, stable_test_region, init_raw_mem, raw_mem_for_ctrl, stable_test_phi, uses); phase->igvn().replace_node(barrier, pre_val); + assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_enqueue_barriers_count() == cnt - 1, "not replaced"); } - - for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); i > 0; i--) { - int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count(); - ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barrier(i-1); + assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_enqueue_barriers_count() == 0, "all enqueue barrier nodes should have been replaced"); + + for (int i = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_write_barriers_count(); i > 0; i--) { + int cnt = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_write_barriers_count(); + ShenandoahWriteBarrierNode* wb = ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_write_barrier(i-1); uint last = phase->C->unique(); Node* ctrl = phase->get_ctrl(wb); @@ -2994,11 +3194,10 @@ } } if (val->outcnt() == 0) { - phase->lazy_update(val, out_val); phase->igvn()._worklist.push(val); } } - phase->lazy_replace(wb, out_val); + phase->igvn().replace_node(wb, out_val); follow_barrier_uses(mem_phi, ctrl, uses, phase); follow_barrier_uses(out_val, ctrl, uses, phase); @@ -3018,10 +3217,12 @@ // collected in memory_nodes to fix the memory graph. Update that // memory state as we go. fixer.fix_mem(ctrl, region, init_raw_mem, raw_mem_for_ctrl, raw_mem_phi, uses); - assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == cnt - 1, "not replaced"); + assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_write_barriers_count() == cnt - 1, "not replaced"); } - assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_barriers_count() == 0, "all write barrier nodes should have been replaced"); + phase->C->print_method(PHASE_CUSTOM, 2); + + assert(ShenandoahBarrierSetC2::bsc2()->state()->shenandoah_write_barriers_count() == 0, "all write barrier nodes should have been replaced"); } void ShenandoahWriteBarrierNode::move_evacuation_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { @@ -3060,6 +3261,8 @@ } void ShenandoahWriteBarrierNode::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) { + phase->C->print_method(PHASE_CUSTOM, 2); + assert(is_evacuation_in_progress_test(n) || is_heap_stable_test(n), "no other tests"); if (phase->identical_backtoback_ifs(n)) { Node* n_ctrl = is_evacuation_in_progress_test(n) ? ShenandoahWriteBarrierNode::evacuation_in_progress_test_ctrl(n) : n->in(0); @@ -3288,6 +3491,11 @@ } #endif +ShenandoahEnqueueBarrierNode::ShenandoahEnqueueBarrierNode(Node* val) + : Node(NULL, val) { + ShenandoahBarrierSetC2::bsc2()->state()->add_shenandoah_enqueue_barrier(this); +} + const Type* ShenandoahEnqueueBarrierNode::bottom_type() const { if (in(1) == NULL || in(1)->is_top()) { return Type::TOP; @@ -3738,6 +3946,37 @@ return false; } +void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl) { + if (mem_for_ctrl != mem) { + Node* old = mem_for_ctrl; + Node* prev = NULL; + while (old != mem) { + prev = old; + if (old->is_Store() || old->is_ClearArray() || old->is_LoadStore()) { + assert(_alias == Compile::AliasIdxRaw, ""); + old = old->in(MemNode::Memory); + } else if (old->Opcode() == Op_SCMemProj) { + assert(_alias == Compile::AliasIdxRaw, ""); + old = old->in(0); + } else if (old->Opcode() == Op_ShenandoahWBMemProj) { + assert(_alias != Compile::AliasIdxRaw, ""); + old = old->in(0); + } else if (old->Opcode() == Op_ShenandoahWriteBarrier) { + assert(_alias != Compile::AliasIdxRaw, ""); + old = old->in(ShenandoahBarrierNode::Memory); + } else { + ShouldNotReachHere(); + } + } + assert(prev != NULL, ""); + if (new_ctrl != ctrl) { + _memory_nodes.map(ctrl->_idx, mem); + _memory_nodes.map(new_ctrl->_idx, mem_for_ctrl); + } + } +} + + void MemoryGraphFixer::fix_mem(Node* ctrl, Node* new_ctrl, Node* mem, Node* mem_for_ctrl, Node* new_mem, Unique_Node_List& uses) { assert(_phase->ctrl_or_self(new_mem) == new_ctrl, ""); const bool trace = false; diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp @@ -53,8 +53,6 @@ #endif public: - -public: enum { Control, Memory, ValueIn @@ -118,6 +116,13 @@ static bool is_dominator_same_ctrl(Node* c, Node* d, Node* n, PhaseIdealLoop* phase); static Node* no_branches(Node* c, Node* dom, bool allow_one_proj, PhaseIdealLoop* phase); + CallStaticJavaNode* pin_and_expand_null_check(PhaseIterGVN& igvn); + void pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray& memory_graph_fixers, Unique_Node_List& uses); + static Node* clone_null_check(Node*& c, Node*& val, Node* unc_ctrl, PhaseIdealLoop* phase); + static void fix_null_check(Node* dom, Node* unc, Node* unc_ctrl, Node* unc_region, Unique_Node_List& uses, + PhaseIdealLoop* phase); + void pin_and_expand_helper(PhaseIdealLoop* phase); + protected: uint hash() const; uint cmp(const Node& n) const; @@ -133,18 +138,8 @@ class ShenandoahReadBarrierNode : public ShenandoahBarrierNode { public: - ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj) - : ShenandoahBarrierNode(ctrl, mem, obj, true) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || - ShenandoahWriteBarrier || ShenandoahAcmpBarrier), - "should be enabled"); - } - ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj, bool allow_fromspace) - : ShenandoahBarrierNode(ctrl, mem, obj, allow_fromspace) { - assert(UseShenandoahGC && (ShenandoahReadBarrier || ShenandoahStoreValReadBarrier || - ShenandoahWriteBarrier || ShenandoahAcmpBarrier), - "should be enabled"); - } + ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj); + ShenandoahReadBarrierNode(Node* ctrl, Node* mem, Node* obj, bool allow_fromspace); virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); virtual Node* Identity(PhaseGVN* phase); @@ -189,9 +184,6 @@ void try_move_before_loop(GrowableArray& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses); void try_move_before_loop_helper(LoopNode* cl, Node* val_ctrl, GrowableArray& memory_graph_fixers, PhaseIdealLoop* phase, bool include_lsm, Unique_Node_List& uses); static void pin_and_expand(PhaseIdealLoop* phase); - CallStaticJavaNode* pin_and_expand_null_check(PhaseIterGVN& igvn); - void pin_and_expand_move_barrier(PhaseIdealLoop* phase, GrowableArray& memory_graph_fixers, Unique_Node_List& uses); - void pin_and_expand_helper(PhaseIdealLoop* phase); static Node* find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase); static void follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase); static void test_heap_stable(Node* ctrl, Node* raw_mem, Node*& gc_state, Node*& heap_stable, @@ -209,10 +201,6 @@ PhaseIdealLoop* phase); static void heap_stable(Node* c, Node* val, Node* unc_ctrl, Node* raw_mem, Node* wb_mem, Node* region, Node* val_phi, Node* mem_phi, Node* raw_mem_phi, Node* unc_region, PhaseIdealLoop* phase); - static Node* clone_null_check(Node*& c, Node* val, Node* unc_ctrl, Node* unc_region, uint input, - PhaseIdealLoop* phase); - static void fix_null_check(Node* dom, Node* unc, Node* unc_ctrl, Node* unc_region, Unique_Node_List& uses, - PhaseIdealLoop* phase); static void evacuation_not_in_progress_null_check(Node*& c, Node*& val, Node* unc_ctrl, Node*& unc_region, PhaseIdealLoop* phase); static void evacuation_in_progress_null_check(Node*& c, Node*& val, Node* evacuation_iff, Node* unc, Node* unc_ctrl, @@ -260,8 +248,7 @@ class ShenandoahEnqueueBarrierNode : public Node { public: - ShenandoahEnqueueBarrierNode(Node* val) : Node(NULL, val) { - } + ShenandoahEnqueueBarrierNode(Node* val); const Type *bottom_type() const; const Type* Value(PhaseGVN* phase) const; @@ -302,6 +289,7 @@ Node* find_mem(Node* ctrl, Node* n) const; void fix_mem(Node* ctrl, Node* region, Node* mem, Node* mem_for_ctrl, Node* mem_phi, Unique_Node_List& uses); + void fix_mem(Node* ctrl, Node* region, Node* mem, Node* mem_for_ctrl); int alias() const { return _alias; } void remove(Node* n); }; diff --git a/src/hotspot/share/opto/ifnode.cpp b/src/hotspot/share/opto/ifnode.cpp --- a/src/hotspot/share/opto/ifnode.cpp +++ b/src/hotspot/share/opto/ifnode.cpp @@ -130,6 +130,38 @@ return NULL; } + if (r->req() == 3) { + Node* in1 = r->in(1); + Node* in2 = r->in(2); + while (in1 != NULL && + in1->is_Proj() && + in1->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) { + in1 = in1->in(0); + if (in1 != NULL) { + in1 = in1->in(0); + } + } + while (in2 != NULL && + in2->is_Proj() && + in2->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) { + in2 = in2->in(0); + if (in2 != NULL) { + in2 = in2->in(0); + } + } + if (in1 != NULL && + in2 != NULL && + in1->is_IfProj() && + in2->is_IfProj() && + in1->in(0) == in2->in(0)) { + Node* iff = in1->in(0); + if (iff != NULL && ShenandoahWriteBarrierNode::is_heap_stable_test(iff)) { + return NULL; + } + } + } + + // Make sure we can determine where all the uses of merged values go for (DUIterator_Fast jmax, j = r->fast_outs(jmax); j < jmax; j++) { Node* u = r->fast_out(j); diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -85,6 +85,9 @@ if (BreakAtNode != 0 && (_debug_idx == BreakAtNode || (int)_idx == BreakAtNode)) { tty->print_cr("BreakAtNode: _idx=%d _debug_idx=%d", _idx, _debug_idx); BREAKPOINT; + if (Compile::current()->method() != NULL) { + os::message_box("xxx", "yyy"); + } } #if OPTO_DU_ITERATOR_ASSERT _last_del = NULL; diff --git a/src/hotspot/share/opto/phasetype.hpp b/src/hotspot/share/opto/phasetype.hpp --- a/src/hotspot/share/opto/phasetype.hpp +++ b/src/hotspot/share/opto/phasetype.hpp @@ -56,6 +56,7 @@ PHASE_BEFORE_MACRO_EXPANSION, PHASE_END, PHASE_FAILURE, + PHASE_CUSTOM, PHASE_NUM_TYPES }; @@ -94,6 +95,7 @@ case PHASE_BEFORE_MACRO_EXPANSION: return "Before macro expansion"; case PHASE_END: return "End"; case PHASE_FAILURE: return "Failure"; + case PHASE_CUSTOM: return "custom"; default: ShouldNotReachHere(); return NULL;