63 // Returns the number of replacements made.
64 //
65 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
66 int nreplacements = 0;
67 uint req = use->req();
68 for (uint j = 0; j < use->len(); j++) {
69 Node *uin = use->in(j);
70 if (uin == oldref) {
71 if (j < req)
72 use->set_req(j, newref);
73 else
74 use->set_prec(j, newref);
75 nreplacements++;
76 } else if (j >= req && uin == NULL) {
77 break;
78 }
79 }
80 return nreplacements;
81 }
82
83 void PhaseMacroExpand::copy_call_debug_info(CallNode *oldcall, CallNode * newcall) {
84 // Copy debug information and adjust JVMState information
85 uint old_dbg_start = oldcall->tf()->domain_sig()->cnt();
86 uint new_dbg_start = newcall->tf()->domain_sig()->cnt();
87 int jvms_adj = new_dbg_start - old_dbg_start;
88 assert (new_dbg_start == newcall->req(), "argument count mismatch");
89
90 // SafePointScalarObject node could be referenced several times in debug info.
91 // Use Dict to record cloned nodes.
92 Dict* sosn_map = new Dict(cmpkey,hashkey);
93 for (uint i = old_dbg_start; i < oldcall->req(); i++) {
94 Node* old_in = oldcall->in(i);
95 // Clone old SafePointScalarObjectNodes, adjusting their field contents.
96 if (old_in != NULL && old_in->is_SafePointScalarObject()) {
97 SafePointScalarObjectNode* old_sosn = old_in->as_SafePointScalarObject();
98 uint old_unique = C->unique();
99 Node* new_in = old_sosn->clone(sosn_map);
100 if (old_unique != C->unique()) { // New node?
101 new_in->set_req(0, C->root()); // reset control edge
102 new_in = transform_later(new_in); // Register new node.
103 }
104 old_in = new_in;
105 }
106 newcall->add_req(old_in);
107 }
108
109 // JVMS may be shared so clone it before we modify it
110 newcall->set_jvms(oldcall->jvms() != NULL ? oldcall->jvms()->clone_deep(C) : NULL);
111 for (JVMState *jvms = newcall->jvms(); jvms != NULL; jvms = jvms->caller()) {
112 jvms->set_map(newcall);
113 jvms->set_locoff(jvms->locoff()+jvms_adj);
114 jvms->set_stkoff(jvms->stkoff()+jvms_adj);
115 jvms->set_monoff(jvms->monoff()+jvms_adj);
116 jvms->set_scloff(jvms->scloff()+jvms_adj);
117 jvms->set_endoff(jvms->endoff()+jvms_adj);
118 }
119 }
120
121 Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) {
122 Node* cmp;
123 if (mask != 0) {
124 Node* and_node = transform_later(new AndXNode(word, MakeConX(mask)));
125 cmp = transform_later(new CmpXNode(and_node, MakeConX(bits)));
126 } else {
127 cmp = word;
128 }
129 Node* bol = transform_later(new BoolNode(cmp, BoolTest::ne));
130 IfNode* iff = new IfNode( ctrl, bol, PROB_MIN, COUNT_UNKNOWN );
131 transform_later(iff);
132
133 // Fast path taken.
134 Node *fast_taken = transform_later(new IfFalseNode(iff));
135
136 // Fast path not-taken, i.e. slow path
137 Node *slow_taken = transform_later(new IfTrueNode(iff));
138
139 if (return_fast_path) {
140 region->init_req(edge, slow_taken); // Capture slow-control
153 call->init_req( TypeFunc::Memory , oldcall->in( TypeFunc::Memory ) ); // ?????
154 call->init_req( TypeFunc::ReturnAdr, oldcall->in( TypeFunc::ReturnAdr ) );
155 call->init_req( TypeFunc::FramePtr, oldcall->in( TypeFunc::FramePtr ) );
156 }
157
158 //------------------------------make_slow_call---------------------------------
159 CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type,
160 address slow_call, const char* leaf_name, Node* slow_path,
161 Node* parm0, Node* parm1, Node* parm2) {
162
163 // Slow-path call
164 CallNode *call = leaf_name
165 ? (CallNode*)new CallLeafNode ( slow_call_type, slow_call, leaf_name, TypeRawPtr::BOTTOM )
166 : (CallNode*)new CallStaticJavaNode( slow_call_type, slow_call, OptoRuntime::stub_name(slow_call), oldcall->jvms()->bci(), TypeRawPtr::BOTTOM );
167
168 // Slow path call has no side-effects, uses few values
169 copy_predefined_input_for_runtime_call(slow_path, oldcall, call );
170 if (parm0 != NULL) call->init_req(TypeFunc::Parms+0, parm0);
171 if (parm1 != NULL) call->init_req(TypeFunc::Parms+1, parm1);
172 if (parm2 != NULL) call->init_req(TypeFunc::Parms+2, parm2);
173 copy_call_debug_info(oldcall, call);
174 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
175 _igvn.replace_node(oldcall, call);
176 transform_later(call);
177
178 return call;
179 }
180
181 void PhaseMacroExpand::extract_call_projections(CallNode *call) {
182 _fallthroughproj = NULL;
183 _fallthroughcatchproj = NULL;
184 _ioproj_fallthrough = NULL;
185 _ioproj_catchall = NULL;
186 _catchallcatchproj = NULL;
187 _memproj_fallthrough = NULL;
188 _memproj_catchall = NULL;
189 _resproj = NULL;
190 for (DUIterator_Fast imax, i = call->fast_outs(imax); i < imax; i++) {
191 ProjNode *pn = call->fast_out(i)->as_Proj();
192 switch (pn->_con) {
193 case TypeFunc::Control:
1575 }
1576
1577 // Generate slow-path call
1578 CallNode *call = new CallStaticJavaNode(slow_call_type, slow_call_address,
1579 OptoRuntime::stub_name(slow_call_address),
1580 alloc->jvms()->bci(),
1581 TypePtr::BOTTOM);
1582 call->init_req( TypeFunc::Control, slow_region );
1583 call->init_req( TypeFunc::I_O , top() ) ; // does no i/o
1584 call->init_req( TypeFunc::Memory , slow_mem ); // may gc ptrs
1585 call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) );
1586 call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) );
1587
1588 call->init_req(TypeFunc::Parms+0, klass_node);
1589 if (length != NULL) {
1590 call->init_req(TypeFunc::Parms+1, length);
1591 }
1592
1593 // Copy debug information and adjust JVMState information, then replace
1594 // allocate node with the call
1595 copy_call_debug_info((CallNode *) alloc, call);
1596 if (!always_slow) {
1597 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
1598 } else {
1599 // Hook i_o projection to avoid its elimination during allocation
1600 // replacement (when only a slow call is generated).
1601 call->set_req(TypeFunc::I_O, result_phi_i_o);
1602 }
1603 _igvn.replace_node(alloc, call);
1604 transform_later(call);
1605
1606 // Identify the output projections from the allocate node and
1607 // adjust any references to them.
1608 // The control and io projections look like:
1609 //
1610 // v---Proj(ctrl) <-----+ v---CatchProj(ctrl)
1611 // Allocate Catch
1612 // ^---Proj(io) <-------+ ^---CatchProj(io)
1613 //
1614 // We are interested in the CatchProj nodes.
1615 //
2455 Node* value_mask = _igvn.MakeConX(markWord::always_locked_pattern);
2456 Node* is_value = _igvn.transform(new AndXNode(mark, value_mask));
2457 Node* cmp = _igvn.transform(new CmpXNode(is_value, value_mask));
2458 Node* bol = _igvn.transform(new BoolNode(cmp, BoolTest::eq));
2459 Node* unc_ctrl = generate_slow_guard(&slow_path, bol, NULL);
2460
2461 int trap_request = Deoptimization::make_trap_request(Deoptimization::Reason_class_check, Deoptimization::Action_none);
2462 address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point();
2463 const TypePtr* no_memory_effects = NULL;
2464 JVMState* jvms = lock->jvms();
2465 CallNode* unc = new CallStaticJavaNode(OptoRuntime::uncommon_trap_Type(), call_addr, "uncommon_trap",
2466 jvms->bci(), no_memory_effects);
2467
2468 unc->init_req(TypeFunc::Control, unc_ctrl);
2469 unc->init_req(TypeFunc::I_O, lock->i_o());
2470 unc->init_req(TypeFunc::Memory, mem); // may gc ptrs
2471 unc->init_req(TypeFunc::FramePtr, lock->in(TypeFunc::FramePtr));
2472 unc->init_req(TypeFunc::ReturnAdr, lock->in(TypeFunc::ReturnAdr));
2473 unc->init_req(TypeFunc::Parms+0, _igvn.intcon(trap_request));
2474 unc->set_cnt(PROB_UNLIKELY_MAG(4));
2475 copy_call_debug_info(lock, unc);
2476
2477 assert(unc->peek_monitor_box() == box, "wrong monitor");
2478 assert(unc->peek_monitor_obj() == obj, "wrong monitor");
2479
2480 // pop monitor and push obj back on stack: we trap before the monitorenter
2481 unc->pop_monitor();
2482 unc->grow_stack(unc->jvms(), 1);
2483 unc->set_stack(unc->jvms(), unc->jvms()->stk_size()-1, obj);
2484
2485 _igvn.register_new_node_with_optimizer(unc);
2486
2487 Node* ctrl = _igvn.transform(new ProjNode(unc, TypeFunc::Control));
2488 Node* halt = _igvn.transform(new HaltNode(ctrl, lock->in(TypeFunc::FramePtr)));
2489 C->root()->add_req(halt);
2490 }
2491
2492 // Make slow path call
2493 CallNode *call = make_slow_call((CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(),
2494 OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path,
2495 obj, box, NULL);
2725 handler_call->init_req(TypeFunc::I_O, top());
2726 handler_call->init_req(TypeFunc::FramePtr, call->in(TypeFunc::FramePtr));
2727 handler_call->init_req(TypeFunc::ReturnAdr, top());
2728 handler_call->init_req(TypeFunc::Parms, pack_handler);
2729 handler_call->init_req(TypeFunc::Parms+1, old_top);
2730
2731 // We don't know how many values are returned. This assumes the
2732 // worst case, that all available registers are used.
2733 for (uint i = TypeFunc::Parms+1; i < domain->cnt(); i++) {
2734 if (domain->field_at(i) == Type::HALF) {
2735 slow_call->init_req(i, top());
2736 handler_call->init_req(i+1, top());
2737 continue;
2738 }
2739 Node* proj = transform_later(new ProjNode(call, i));
2740 slow_call->init_req(i, proj);
2741 handler_call->init_req(i+1, proj);
2742 }
2743
2744 // We can safepoint at that new call
2745 copy_call_debug_info(call, slow_call);
2746 transform_later(slow_call);
2747 transform_later(handler_call);
2748
2749 Node* handler_ctl = transform_later(new ProjNode(handler_call, TypeFunc::Control));
2750 rawmem = transform_later(new ProjNode(handler_call, TypeFunc::Memory));
2751 Node* slowpath_false_res = transform_later(new ProjNode(handler_call, TypeFunc::Parms));
2752
2753 MergeMemNode* slowpath_false_mem = MergeMemNode::make(mem);
2754 slowpath_false_mem->set_memory_at(Compile::AliasIdxRaw, rawmem);
2755 transform_later(slowpath_false_mem);
2756
2757 Node* r = new RegionNode(4);
2758 Node* mem_phi = new PhiNode(r, Type::MEMORY, TypePtr::BOTTOM);
2759 Node* io_phi = new PhiNode(r, Type::ABIO);
2760 Node* res_phi = new PhiNode(r, TypeInstPtr::BOTTOM);
2761
2762 r->init_req(1, no_allocation_ctl);
2763 mem_phi->init_req(1, mem);
2764 io_phi->init_req(1, io);
2765 res_phi->init_req(1, no_allocation_res);
|
63 // Returns the number of replacements made.
64 //
65 int PhaseMacroExpand::replace_input(Node *use, Node *oldref, Node *newref) {
66 int nreplacements = 0;
67 uint req = use->req();
68 for (uint j = 0; j < use->len(); j++) {
69 Node *uin = use->in(j);
70 if (uin == oldref) {
71 if (j < req)
72 use->set_req(j, newref);
73 else
74 use->set_prec(j, newref);
75 nreplacements++;
76 } else if (j >= req && uin == NULL) {
77 break;
78 }
79 }
80 return nreplacements;
81 }
82
83 Node* PhaseMacroExpand::opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path) {
84 Node* cmp;
85 if (mask != 0) {
86 Node* and_node = transform_later(new AndXNode(word, MakeConX(mask)));
87 cmp = transform_later(new CmpXNode(and_node, MakeConX(bits)));
88 } else {
89 cmp = word;
90 }
91 Node* bol = transform_later(new BoolNode(cmp, BoolTest::ne));
92 IfNode* iff = new IfNode( ctrl, bol, PROB_MIN, COUNT_UNKNOWN );
93 transform_later(iff);
94
95 // Fast path taken.
96 Node *fast_taken = transform_later(new IfFalseNode(iff));
97
98 // Fast path not-taken, i.e. slow path
99 Node *slow_taken = transform_later(new IfTrueNode(iff));
100
101 if (return_fast_path) {
102 region->init_req(edge, slow_taken); // Capture slow-control
115 call->init_req( TypeFunc::Memory , oldcall->in( TypeFunc::Memory ) ); // ?????
116 call->init_req( TypeFunc::ReturnAdr, oldcall->in( TypeFunc::ReturnAdr ) );
117 call->init_req( TypeFunc::FramePtr, oldcall->in( TypeFunc::FramePtr ) );
118 }
119
120 //------------------------------make_slow_call---------------------------------
121 CallNode* PhaseMacroExpand::make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type,
122 address slow_call, const char* leaf_name, Node* slow_path,
123 Node* parm0, Node* parm1, Node* parm2) {
124
125 // Slow-path call
126 CallNode *call = leaf_name
127 ? (CallNode*)new CallLeafNode ( slow_call_type, slow_call, leaf_name, TypeRawPtr::BOTTOM )
128 : (CallNode*)new CallStaticJavaNode( slow_call_type, slow_call, OptoRuntime::stub_name(slow_call), oldcall->jvms()->bci(), TypeRawPtr::BOTTOM );
129
130 // Slow path call has no side-effects, uses few values
131 copy_predefined_input_for_runtime_call(slow_path, oldcall, call );
132 if (parm0 != NULL) call->init_req(TypeFunc::Parms+0, parm0);
133 if (parm1 != NULL) call->init_req(TypeFunc::Parms+1, parm1);
134 if (parm2 != NULL) call->init_req(TypeFunc::Parms+2, parm2);
135 call->copy_call_debug_info(&_igvn, oldcall);
136 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
137 _igvn.replace_node(oldcall, call);
138 transform_later(call);
139
140 return call;
141 }
142
143 void PhaseMacroExpand::extract_call_projections(CallNode *call) {
144 _fallthroughproj = NULL;
145 _fallthroughcatchproj = NULL;
146 _ioproj_fallthrough = NULL;
147 _ioproj_catchall = NULL;
148 _catchallcatchproj = NULL;
149 _memproj_fallthrough = NULL;
150 _memproj_catchall = NULL;
151 _resproj = NULL;
152 for (DUIterator_Fast imax, i = call->fast_outs(imax); i < imax; i++) {
153 ProjNode *pn = call->fast_out(i)->as_Proj();
154 switch (pn->_con) {
155 case TypeFunc::Control:
1537 }
1538
1539 // Generate slow-path call
1540 CallNode *call = new CallStaticJavaNode(slow_call_type, slow_call_address,
1541 OptoRuntime::stub_name(slow_call_address),
1542 alloc->jvms()->bci(),
1543 TypePtr::BOTTOM);
1544 call->init_req( TypeFunc::Control, slow_region );
1545 call->init_req( TypeFunc::I_O , top() ) ; // does no i/o
1546 call->init_req( TypeFunc::Memory , slow_mem ); // may gc ptrs
1547 call->init_req( TypeFunc::ReturnAdr, alloc->in(TypeFunc::ReturnAdr) );
1548 call->init_req( TypeFunc::FramePtr, alloc->in(TypeFunc::FramePtr) );
1549
1550 call->init_req(TypeFunc::Parms+0, klass_node);
1551 if (length != NULL) {
1552 call->init_req(TypeFunc::Parms+1, length);
1553 }
1554
1555 // Copy debug information and adjust JVMState information, then replace
1556 // allocate node with the call
1557 call->copy_call_debug_info(&_igvn, alloc);
1558 if (!always_slow) {
1559 call->set_cnt(PROB_UNLIKELY_MAG(4)); // Same effect as RC_UNCOMMON.
1560 } else {
1561 // Hook i_o projection to avoid its elimination during allocation
1562 // replacement (when only a slow call is generated).
1563 call->set_req(TypeFunc::I_O, result_phi_i_o);
1564 }
1565 _igvn.replace_node(alloc, call);
1566 transform_later(call);
1567
1568 // Identify the output projections from the allocate node and
1569 // adjust any references to them.
1570 // The control and io projections look like:
1571 //
1572 // v---Proj(ctrl) <-----+ v---CatchProj(ctrl)
1573 // Allocate Catch
1574 // ^---Proj(io) <-------+ ^---CatchProj(io)
1575 //
1576 // We are interested in the CatchProj nodes.
1577 //
2417 Node* value_mask = _igvn.MakeConX(markWord::always_locked_pattern);
2418 Node* is_value = _igvn.transform(new AndXNode(mark, value_mask));
2419 Node* cmp = _igvn.transform(new CmpXNode(is_value, value_mask));
2420 Node* bol = _igvn.transform(new BoolNode(cmp, BoolTest::eq));
2421 Node* unc_ctrl = generate_slow_guard(&slow_path, bol, NULL);
2422
2423 int trap_request = Deoptimization::make_trap_request(Deoptimization::Reason_class_check, Deoptimization::Action_none);
2424 address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point();
2425 const TypePtr* no_memory_effects = NULL;
2426 JVMState* jvms = lock->jvms();
2427 CallNode* unc = new CallStaticJavaNode(OptoRuntime::uncommon_trap_Type(), call_addr, "uncommon_trap",
2428 jvms->bci(), no_memory_effects);
2429
2430 unc->init_req(TypeFunc::Control, unc_ctrl);
2431 unc->init_req(TypeFunc::I_O, lock->i_o());
2432 unc->init_req(TypeFunc::Memory, mem); // may gc ptrs
2433 unc->init_req(TypeFunc::FramePtr, lock->in(TypeFunc::FramePtr));
2434 unc->init_req(TypeFunc::ReturnAdr, lock->in(TypeFunc::ReturnAdr));
2435 unc->init_req(TypeFunc::Parms+0, _igvn.intcon(trap_request));
2436 unc->set_cnt(PROB_UNLIKELY_MAG(4));
2437 unc->copy_call_debug_info(&_igvn, lock);
2438
2439 assert(unc->peek_monitor_box() == box, "wrong monitor");
2440 assert(unc->peek_monitor_obj() == obj, "wrong monitor");
2441
2442 // pop monitor and push obj back on stack: we trap before the monitorenter
2443 unc->pop_monitor();
2444 unc->grow_stack(unc->jvms(), 1);
2445 unc->set_stack(unc->jvms(), unc->jvms()->stk_size()-1, obj);
2446
2447 _igvn.register_new_node_with_optimizer(unc);
2448
2449 Node* ctrl = _igvn.transform(new ProjNode(unc, TypeFunc::Control));
2450 Node* halt = _igvn.transform(new HaltNode(ctrl, lock->in(TypeFunc::FramePtr)));
2451 C->root()->add_req(halt);
2452 }
2453
2454 // Make slow path call
2455 CallNode *call = make_slow_call((CallNode *) lock, OptoRuntime::complete_monitor_enter_Type(),
2456 OptoRuntime::complete_monitor_locking_Java(), NULL, slow_path,
2457 obj, box, NULL);
2687 handler_call->init_req(TypeFunc::I_O, top());
2688 handler_call->init_req(TypeFunc::FramePtr, call->in(TypeFunc::FramePtr));
2689 handler_call->init_req(TypeFunc::ReturnAdr, top());
2690 handler_call->init_req(TypeFunc::Parms, pack_handler);
2691 handler_call->init_req(TypeFunc::Parms+1, old_top);
2692
2693 // We don't know how many values are returned. This assumes the
2694 // worst case, that all available registers are used.
2695 for (uint i = TypeFunc::Parms+1; i < domain->cnt(); i++) {
2696 if (domain->field_at(i) == Type::HALF) {
2697 slow_call->init_req(i, top());
2698 handler_call->init_req(i+1, top());
2699 continue;
2700 }
2701 Node* proj = transform_later(new ProjNode(call, i));
2702 slow_call->init_req(i, proj);
2703 handler_call->init_req(i+1, proj);
2704 }
2705
2706 // We can safepoint at that new call
2707 slow_call->copy_call_debug_info(&_igvn, call);
2708 transform_later(slow_call);
2709 transform_later(handler_call);
2710
2711 Node* handler_ctl = transform_later(new ProjNode(handler_call, TypeFunc::Control));
2712 rawmem = transform_later(new ProjNode(handler_call, TypeFunc::Memory));
2713 Node* slowpath_false_res = transform_later(new ProjNode(handler_call, TypeFunc::Parms));
2714
2715 MergeMemNode* slowpath_false_mem = MergeMemNode::make(mem);
2716 slowpath_false_mem->set_memory_at(Compile::AliasIdxRaw, rawmem);
2717 transform_later(slowpath_false_mem);
2718
2719 Node* r = new RegionNode(4);
2720 Node* mem_phi = new PhiNode(r, Type::MEMORY, TypePtr::BOTTOM);
2721 Node* io_phi = new PhiNode(r, Type::ABIO);
2722 Node* res_phi = new PhiNode(r, TypeInstPtr::BOTTOM);
2723
2724 r->init_req(1, no_allocation_ctl);
2725 mem_phi->init_req(1, mem);
2726 io_phi->init_req(1, io);
2727 res_phi->init_req(1, no_allocation_res);
|