16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/brooksPointer.hpp"
26 #include "gc/shenandoah/shenandoahHeap.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
28 #include "opto/arraycopynode.hpp"
29 #include "opto/block.hpp"
30 #include "opto/callnode.hpp"
31 #include "opto/castnode.hpp"
32 #include "opto/movenode.hpp"
33 #include "opto/phaseX.hpp"
34 #include "opto/rootnode.hpp"
35 #include "opto/runtime.hpp"
36 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
37 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
38 #include "opto/subnode.hpp"
39
40 Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) {
41 if (n == NULL) {
42 return NULL;
43 } else if (n->is_ShenandoahBarrier()) {
44 return n->in(ValueIn);
45 } else if (n->is_Phi() &&
46 n->req() == 3 &&
47 n->in(1) != NULL &&
48 n->in(1)->is_ShenandoahBarrier() &&
49 n->in(2) != NULL &&
50 n->in(2)->bottom_type() == TypePtr::NULL_PTR &&
51 n->in(0) != NULL &&
52 n->in(0)->in(1) != NULL &&
53 n->in(0)->in(1)->is_IfProj() &&
54 n->in(0)->in(2) != NULL &&
55 n->in(0)->in(2)->is_IfProj() &&
2515 }
2516 }
2517 } else if (n->is_ConstraintCast() || (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_CheckCastPP)) {
2518 Node* m = n->in(1);
2519 if (m != NULL) {
2520 wq.push(m);
2521 }
2522 } else {
2523 const TypeInstPtr* tn = n->bottom_type()->isa_instptr();
2524 if (tn != NULL) {
2525 if (tn->const_oop() != NULL) {
2526 if (const_oop == NULL) {
2527 const_oop = tn->const_oop();
2528 } else if (const_oop != tn->const_oop()) {
2529 const_oop = NULL;
2530 break;
2531 }
2532 } else {
2533 if (n->is_Proj()) {
2534 if (n->in(0)->Opcode() == Op_CallLeafNoFP) {
2535 if (n->in(0)->as_Call()->entry_point() != StubRoutines::shenandoah_wb_C()) {
2536 const_oop = NULL;
2537 break;
2538 }
2539 } else if (n->in(0)->is_MachCallLeaf()) {
2540 if (n->in(0)->as_MachCall()->entry_point() != StubRoutines::shenandoah_wb_C()) {
2541 const_oop = NULL;
2542 break;
2543 }
2544 }
2545 } else {
2546 fatal("2 different static fields being accessed with a single AddP");
2547 const_oop = NULL;
2548 break;
2549 }
2550 }
2551 } else {
2552 assert(n->bottom_type() == Type::TOP, "not an instance ptr?");
2553 }
2554 }
2555 }
2556 if (const_oop != NULL) {
2557 res = ti->cast_to_const(const_oop);
2558 }
2559 }
2560 }
3959 Node* raw_mem_phi, Node* unc_region, int alias, Unique_Node_List& uses,
3960 PhaseIdealLoop* phase) {
3961 evacuation_in_progress_null_check(c, val, evacuation_iff, unc, unc_ctrl, unc_region, uses, phase);
3962
3963 IdealLoopTree *loop = phase->get_loop(c);
3964 Node* rbtrue = new ShenandoahReadBarrierNode(c, wb_mem, val);
3965 phase->register_new_node(rbtrue, c);
3966
3967 Node* in_cset_fast_test_failure = NULL;
3968 in_cset_fast_test(c, rbtrue, raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, phase);
3969
3970 // The slow path stub consumes and produces raw memory in addition
3971 // to the existing memory edges
3972 Node* base = find_bottom_mem(c, phase);
3973
3974 MergeMemNode* mm = MergeMemNode::make(base);
3975 mm->set_memory_at(alias, wb_mem);
3976 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
3977 phase->register_new_node(mm, c);
3978
3979 Node* call = new CallLeafNoFPNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), StubRoutines::shenandoah_wb_C(), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
3980 call->init_req(TypeFunc::Control, c);
3981 call->init_req(TypeFunc::I_O, phase->C->top());
3982 call->init_req(TypeFunc::Memory, mm);
3983 call->init_req(TypeFunc::FramePtr, phase->C->top());
3984 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
3985 call->init_req(TypeFunc::Parms, rbtrue);
3986 phase->register_control(call, loop, c);
3987 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
3988 phase->register_control(ctrl_proj, loop, call);
3989 Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
3990 phase->register_new_node(mem_proj, call);
3991 Node* res_proj = new ProjNode(call, TypeFunc::Parms);
3992 phase->register_new_node(res_proj, call);
3993 Node* res = new CheckCastPPNode(ctrl_proj, res_proj, phase->igvn().type(val)->is_oopptr()->cast_to_nonconst());
3994 phase->register_new_node(res, ctrl_proj);
3995 region->init_req(2, ctrl_proj);
3996 val_phi->init_req(2, res);
3997 mem_phi->init_req(2, mem_proj);
3998 raw_mem_phi->init_req(2, mem_proj);
3999 }
4373 }
4374 phase->do_unswitching(loop, old_new, true);
4375 }
4376 }
4377 }
4378 }
4379 }
4380 }
4381
4382 #ifdef ASSERT
4383 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) {
4384 const bool trace = false;
4385 ResourceMark rm;
4386 Unique_Node_List nodes;
4387 Unique_Node_List controls;
4388 Unique_Node_List memories;
4389
4390 nodes.push(root);
4391 for (uint next = 0; next < nodes.size(); next++) {
4392 Node *n = nodes.at(next);
4393 if (n->Opcode() == Op_CallLeafNoFP && n->as_Call()->_entry_point == StubRoutines::shenandoah_wb_C()) {
4394 controls.push(n);
4395 if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
4396 for (uint next2 = 0; next2 < controls.size(); next2++) {
4397 Node *m = controls.at(next2);
4398 if (!m->is_Loop() || controls.member(m->in(LoopNode::EntryControl)) || 1) {
4399 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
4400 Node* u = m->fast_out(i);
4401 if (u->is_CFG() && !u->is_Root() &&
4402 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
4403 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
4404 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
4405 controls.push(u);
4406 }
4407 }
4408 }
4409 }
4410 memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
4411 for (uint next2 = 0; next2 < memories.size(); next2++) {
4412 Node *m = memories.at(next2);
4413 assert(m->bottom_type() == Type::MEMORY, "");
|
16 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 * or visit www.oracle.com if you need additional information or have any
20 * questions.
21 *
22 */
23
24 #include "precompiled.hpp"
25 #include "gc/shenandoah/brooksPointer.hpp"
26 #include "gc/shenandoah/shenandoahHeap.hpp"
27 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
28 #include "opto/arraycopynode.hpp"
29 #include "opto/block.hpp"
30 #include "opto/callnode.hpp"
31 #include "opto/castnode.hpp"
32 #include "opto/movenode.hpp"
33 #include "opto/phaseX.hpp"
34 #include "opto/rootnode.hpp"
35 #include "opto/runtime.hpp"
36 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
37 #include "gc/shenandoah/c2/shenandoahSupport.hpp"
38 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
39 #include "opto/subnode.hpp"
40
41 Node* ShenandoahBarrierNode::skip_through_barrier(Node* n) {
42 if (n == NULL) {
43 return NULL;
44 } else if (n->is_ShenandoahBarrier()) {
45 return n->in(ValueIn);
46 } else if (n->is_Phi() &&
47 n->req() == 3 &&
48 n->in(1) != NULL &&
49 n->in(1)->is_ShenandoahBarrier() &&
50 n->in(2) != NULL &&
51 n->in(2)->bottom_type() == TypePtr::NULL_PTR &&
52 n->in(0) != NULL &&
53 n->in(0)->in(1) != NULL &&
54 n->in(0)->in(1)->is_IfProj() &&
55 n->in(0)->in(2) != NULL &&
56 n->in(0)->in(2)->is_IfProj() &&
2516 }
2517 }
2518 } else if (n->is_ConstraintCast() || (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_CheckCastPP)) {
2519 Node* m = n->in(1);
2520 if (m != NULL) {
2521 wq.push(m);
2522 }
2523 } else {
2524 const TypeInstPtr* tn = n->bottom_type()->isa_instptr();
2525 if (tn != NULL) {
2526 if (tn->const_oop() != NULL) {
2527 if (const_oop == NULL) {
2528 const_oop = tn->const_oop();
2529 } else if (const_oop != tn->const_oop()) {
2530 const_oop = NULL;
2531 break;
2532 }
2533 } else {
2534 if (n->is_Proj()) {
2535 if (n->in(0)->Opcode() == Op_CallLeafNoFP) {
2536 if (!(ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) ||
2537 n->in(0)->as_Call()->entry_point() != ShenandoahBarrierSetAssembler::shenandoah_wb_C()) {
2538 const_oop = NULL;
2539 break;
2540 }
2541 } else if (n->in(0)->is_MachCallLeaf()) {
2542 if (!(ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) ||
2543 n->in(0)->as_MachCall()->entry_point() != ShenandoahBarrierSetAssembler::shenandoah_wb_C()) {
2544 const_oop = NULL;
2545 break;
2546 }
2547 }
2548 } else {
2549 fatal("2 different static fields being accessed with a single AddP");
2550 const_oop = NULL;
2551 break;
2552 }
2553 }
2554 } else {
2555 assert(n->bottom_type() == Type::TOP, "not an instance ptr?");
2556 }
2557 }
2558 }
2559 if (const_oop != NULL) {
2560 res = ti->cast_to_const(const_oop);
2561 }
2562 }
2563 }
3962 Node* raw_mem_phi, Node* unc_region, int alias, Unique_Node_List& uses,
3963 PhaseIdealLoop* phase) {
3964 evacuation_in_progress_null_check(c, val, evacuation_iff, unc, unc_ctrl, unc_region, uses, phase);
3965
3966 IdealLoopTree *loop = phase->get_loop(c);
3967 Node* rbtrue = new ShenandoahReadBarrierNode(c, wb_mem, val);
3968 phase->register_new_node(rbtrue, c);
3969
3970 Node* in_cset_fast_test_failure = NULL;
3971 in_cset_fast_test(c, rbtrue, raw_mem, wb_mem, region, val_phi, mem_phi, raw_mem_phi, phase);
3972
3973 // The slow path stub consumes and produces raw memory in addition
3974 // to the existing memory edges
3975 Node* base = find_bottom_mem(c, phase);
3976
3977 MergeMemNode* mm = MergeMemNode::make(base);
3978 mm->set_memory_at(alias, wb_mem);
3979 mm->set_memory_at(Compile::AliasIdxRaw, raw_mem);
3980 phase->register_new_node(mm, c);
3981
3982 Node* call = new CallLeafNoFPNode(ShenandoahBarrierSetC2::shenandoah_write_barrier_Type(), ShenandoahBarrierSetAssembler::shenandoah_wb_C(), "shenandoah_write_barrier", TypeRawPtr::BOTTOM);
3983 call->init_req(TypeFunc::Control, c);
3984 call->init_req(TypeFunc::I_O, phase->C->top());
3985 call->init_req(TypeFunc::Memory, mm);
3986 call->init_req(TypeFunc::FramePtr, phase->C->top());
3987 call->init_req(TypeFunc::ReturnAdr, phase->C->top());
3988 call->init_req(TypeFunc::Parms, rbtrue);
3989 phase->register_control(call, loop, c);
3990 Node* ctrl_proj = new ProjNode(call, TypeFunc::Control);
3991 phase->register_control(ctrl_proj, loop, call);
3992 Node* mem_proj = new ProjNode(call, TypeFunc::Memory);
3993 phase->register_new_node(mem_proj, call);
3994 Node* res_proj = new ProjNode(call, TypeFunc::Parms);
3995 phase->register_new_node(res_proj, call);
3996 Node* res = new CheckCastPPNode(ctrl_proj, res_proj, phase->igvn().type(val)->is_oopptr()->cast_to_nonconst());
3997 phase->register_new_node(res, ctrl_proj);
3998 region->init_req(2, ctrl_proj);
3999 val_phi->init_req(2, res);
4000 mem_phi->init_req(2, mem_proj);
4001 raw_mem_phi->init_req(2, mem_proj);
4002 }
4376 }
4377 phase->do_unswitching(loop, old_new, true);
4378 }
4379 }
4380 }
4381 }
4382 }
4383 }
4384
4385 #ifdef ASSERT
4386 void ShenandoahBarrierNode::verify_raw_mem(RootNode* root) {
4387 const bool trace = false;
4388 ResourceMark rm;
4389 Unique_Node_List nodes;
4390 Unique_Node_List controls;
4391 Unique_Node_List memories;
4392
4393 nodes.push(root);
4394 for (uint next = 0; next < nodes.size(); next++) {
4395 Node *n = nodes.at(next);
4396 if (n->Opcode() == Op_CallLeafNoFP &&
4397 (ShenandoahWriteBarrier || ShenandoahStoreValEnqueueBarrier) &&
4398 n->as_Call()->_entry_point == ShenandoahBarrierSetAssembler::shenandoah_wb_C()) {
4399 controls.push(n);
4400 if (trace) { tty->print("XXXXXX verifying"); n->dump(); }
4401 for (uint next2 = 0; next2 < controls.size(); next2++) {
4402 Node *m = controls.at(next2);
4403 if (!m->is_Loop() || controls.member(m->in(LoopNode::EntryControl)) || 1) {
4404 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
4405 Node* u = m->fast_out(i);
4406 if (u->is_CFG() && !u->is_Root() &&
4407 !(u->Opcode() == Op_CProj && u->in(0)->Opcode() == Op_NeverBranch && u->as_Proj()->_con == 1) &&
4408 !(u->is_Region() && u->unique_ctrl_out()->Opcode() == Op_Halt)) {
4409 if (trace) { tty->print("XXXXXX pushing control"); u->dump(); }
4410 controls.push(u);
4411 }
4412 }
4413 }
4414 }
4415 memories.push(n->as_Call()->proj_out(TypeFunc::Memory));
4416 for (uint next2 = 0; next2 < memories.size(); next2++) {
4417 Node *m = memories.at(next2);
4418 assert(m->bottom_type() == Type::MEMORY, "");
|