6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "memory/allocation.inline.hpp"
27 #include "memory/resourceArea.hpp"
28 #include "opto/ad.hpp"
29 #include "opto/addnode.hpp"
30 #include "opto/callnode.hpp"
31 #include "opto/idealGraphPrinter.hpp"
32 #include "opto/matcher.hpp"
33 #include "opto/memnode.hpp"
34 #include "opto/movenode.hpp"
35 #include "opto/opcodes.hpp"
36 #include "opto/regmask.hpp"
37 #include "opto/rootnode.hpp"
38 #include "opto/runtime.hpp"
39 #include "opto/type.hpp"
40 #include "opto/vectornode.hpp"
41 #include "runtime/os.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "utilities/align.hpp"
44 #if INCLUDE_ZGC
45 #include "gc/z/zBarrierSetRuntime.hpp"
46 #endif // INCLUDE_ZGC
47
48 OptoReg::Name OptoReg::c_frame_pointer;
49
50 const RegMask *Matcher::idealreg2regmask[_last_machine_leaf];
51 RegMask Matcher::mreg2regmask[_last_Mach_Reg];
52 RegMask Matcher::STACK_ONLY_mask;
53 RegMask Matcher::c_frame_ptr_mask;
54 const uint Matcher::_begin_rematerialize = _BEGIN_REMATERIALIZE;
55 const uint Matcher::_end_rematerialize = _END_REMATERIALIZE;
56
57 //---------------------------Matcher-------------------------------------------
58 Matcher::Matcher()
59 : PhaseTransform( Phase::Ins_Select ),
60 _states_arena(Chunk::medium_size, mtCompiler),
61 _visited(&_states_arena),
62 _shared(&_states_arena),
63 _dontcare(&_states_arena),
64 _reduceOp(reduceOp), _leftOp(leftOp), _rightOp(rightOp),
65 _swallowed(swallowed),
66 _begin_inst_chain_rule(_BEGIN_INST_CHAIN_RULE),
2054 uint nop = n->Opcode();
2055 if (nstate == Pre_Visit) {
2056 if (address_visited.test(n->_idx)) { // Visited in address already?
2057 // Flag as visited and shared now.
2058 set_visited(n);
2059 }
2060 if (is_visited(n)) { // Visited already?
2061 // Node is shared and has no reason to clone. Flag it as shared.
2062 // This causes it to match into a register for the sharing.
2063 set_shared(n); // Flag as shared and
2064 mstack.pop(); // remove node from stack
2065 continue;
2066 }
2067 nstate = Visit; // Not already visited; so visit now
2068 }
2069 if (nstate == Visit) {
2070 mstack.set_state(Post_Visit);
2071 set_visited(n); // Flag as visited now
2072 bool mem_op = false;
2073 int mem_addr_idx = MemNode::Address;
2074
2075 switch( nop ) { // Handle some opcodes special
2076 case Op_Phi: // Treat Phis as shared roots
2077 case Op_Parm:
2078 case Op_Proj: // All handled specially during matching
2079 case Op_SafePointScalarObject:
2080 set_shared(n);
2081 set_dontcare(n);
2082 break;
2083 case Op_If:
2084 case Op_CountedLoopEnd:
2085 mstack.set_state(Alt_Post_Visit); // Alternative way
2086 // Convert (If (Bool (CmpX A B))) into (If (Bool) (CmpX A B)). Helps
2087 // with matching cmp/branch in 1 instruction. The Matcher needs the
2088 // Bool and CmpX side-by-side, because it can only get at constants
2089 // that are at the leaves of Match trees, and the Bool's condition acts
2090 // as a constant here.
2091 mstack.push(n->in(1), Visit); // Clone the Bool
2092 mstack.push(n->in(0), Pre_Visit); // Visit control input
2093 continue; // while (mstack.is_nonempty())
2094 case Op_ConvI2D: // These forms efficiently match with a prior
2095 case Op_ConvI2F: // Load but not a following Store
2096 if( n->in(1)->is_Load() && // Prior load
2097 n->outcnt() == 1 && // Not already shared
2098 n->unique_out()->is_Store() ) // Following store
2099 set_shared(n); // Force it to be a root
2100 break;
2101 case Op_ReverseBytesI:
2102 case Op_ReverseBytesL:
2103 if( n->in(1)->is_Load() && // Prior load
2104 n->outcnt() == 1 ) // Not already shared
2105 set_shared(n); // Force it to be a root
2106 break;
2107 case Op_BoxLock: // Cant match until we get stack-regs in ADLC
2108 case Op_IfFalse:
2109 case Op_IfTrue:
2110 case Op_MachProj:
2111 case Op_MergeMem:
2112 case Op_Catch:
2113 case Op_CatchProj:
2114 case Op_CProj:
2115 case Op_JumpProj:
2116 case Op_JProj:
2117 case Op_NeverBranch:
2118 set_dontcare(n);
2119 break;
2120 case Op_Jump:
2121 mstack.push(n->in(1), Pre_Visit); // Switch Value (could be shared)
2122 mstack.push(n->in(0), Pre_Visit); // Visit Control input
2123 continue; // while (mstack.is_nonempty())
2124 case Op_StrComp:
2125 case Op_StrEquals:
2126 case Op_StrIndexOf:
2127 case Op_StrIndexOfChar:
2128 case Op_AryEq:
2129 case Op_HasNegatives:
2130 case Op_StrInflatedCopy:
2131 case Op_StrCompressedCopy:
2132 case Op_EncodeISOArray:
2133 case Op_FmaD:
2134 case Op_FmaF:
2135 case Op_FmaVD:
2136 case Op_FmaVF:
2137 set_shared(n); // Force result into register (it will be anyways)
2138 break;
2139 case Op_ConP: { // Convert pointers above the centerline to NUL
2140 TypeNode *tn = n->as_Type(); // Constants derive from type nodes
2141 const TypePtr* tp = tn->type()->is_ptr();
2142 if (tp->_ptr == TypePtr::AnyNull) {
2143 tn->set_type(TypePtr::NULL_PTR);
2144 }
2145 break;
2146 }
2147 case Op_ConN: { // Convert narrow pointers above the centerline to NUL
2148 TypeNode *tn = n->as_Type(); // Constants derive from type nodes
2149 const TypePtr* tp = tn->type()->make_ptr();
2150 if (tp && tp->_ptr == TypePtr::AnyNull) {
2151 tn->set_type(TypeNarrowOop::NULL_PTR);
2152 }
2153 break;
2154 }
2155 case Op_Binary: // These are introduced in the Post_Visit state.
2156 ShouldNotReachHere();
2157 break;
2158 case Op_ClearArray:
2159 case Op_SafePoint:
2160 mem_op = true;
2161 break;
2162 #if INCLUDE_ZGC
2163 case Op_CallLeaf:
2164 if (UseZGC) {
2165 if (n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() ||
2166 n->as_Call()->entry_point() == ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr()) {
2167 mem_op = true;
2168 mem_addr_idx = TypeFunc::Parms+1;
2169 }
2170 break;
2171 }
2172 #endif
2173 default:
2174 if( n->is_Store() ) {
2175 // Do match stores, despite no ideal reg
2176 mem_op = true;
2177 break;
2178 }
2179 if( n->is_Mem() ) { // Loads and LoadStores
2180 mem_op = true;
2181 // Loads must be root of match tree due to prior load conflict
2182 if( C->subsume_loads() == false )
2183 set_shared(n);
2184 }
2185 // Fall into default case
2186 if( !n->ideal_reg() )
2187 set_dontcare(n); // Unmatchable Nodes
2188 } // end_switch
2189
2190 for(int i = n->req() - 1; i >= 0; --i) { // For my children
2191 Node *m = n->in(i); // Get ith input
2192 if (m == NULL) continue; // Ignore NULLs
2193 uint mop = m->Opcode();
2194
2195 // Must clone all producers of flags, or we will not match correctly.
2196 // Suppose a compare setting int-flags is shared (e.g., a switch-tree)
2197 // then it will match into an ideal Op_RegFlags. Alas, the fp-flags
2198 // are also there, so we may match a float-branch to int-flags and
2199 // expect the allocator to haul the flags from the int-side to the
2200 // fp-side. No can do.
2201 if( _must_clone[mop] ) {
2202 mstack.push(m, Visit);
2203 continue; // for(int i = ...)
2204 }
2205
2206 if( mop == Op_AddP && m->in(AddPNode::Base)->is_DecodeNarrowPtr()) {
2207 // Bases used in addresses must be shared but since
2208 // they are shared through a DecodeN they may appear
2209 // to have a single use so force sharing here.
2210 set_shared(m->in(AddPNode::Base)->in(1));
2211 }
2212
2213 // if 'n' and 'm' are part of a graph for BMI instruction, clone this node.
2214 #ifdef X86
2215 if (UseBMI1Instructions && is_bmi_pattern(n, m)) {
2216 mstack.push(m, Visit);
2217 continue;
2218 }
2219 #endif
2220
2221 // Clone addressing expressions as they are "free" in memory access instructions
2222 if (mem_op && i == mem_addr_idx && mop == Op_AddP &&
2223 // When there are other uses besides address expressions
2224 // put it on stack and mark as shared.
2225 !is_visited(m)) {
2226 // Some inputs for address expression are not put on stack
2227 // to avoid marking them as shared and forcing them into register
2228 // if they are used only in address expressions.
2229 // But they should be marked as shared if there are other uses
2230 // besides address expressions.
2231
2232 if (clone_address_expressions(m->as_AddP(), mstack, address_visited)) {
2233 continue;
2234 }
2235 } // if( mem_op &&
2236 mstack.push(m, Pre_Visit);
2237 } // for(int i = ...)
2238 }
2239 else if (nstate == Alt_Post_Visit) {
2240 mstack.pop(); // Remove node from stack
2241 // We cannot remove the Cmp input from the Bool here, as the Bool may be
2242 // shared and all users of the Bool need to move the Cmp in parallel.
2243 // This leaves both the Bool and the If pointing at the Cmp. To
2244 // prevent the Matcher from trying to Match the Cmp along both paths
2245 // BoolNode::match_edge always returns a zero.
2246
2247 // We reorder the Op_If in a pre-order manner, so we can visit without
2248 // accidentally sharing the Cmp (the Bool and the If make 2 users).
2249 n->add_req( n->in(1)->in(1) ); // Add the Cmp next to the Bool
2250 }
2251 else if (nstate == Post_Visit) {
2252 mstack.pop(); // Remove node from stack
2253
2254 // Now hack a few special opcodes
2255 switch( n->Opcode() ) { // Handle some opcodes special
2256 case Op_StorePConditional:
2257 case Op_StoreIConditional:
2258 case Op_StoreLConditional:
2259 case Op_CompareAndExchangeB:
2260 case Op_CompareAndExchangeS:
2261 case Op_CompareAndExchangeI:
2262 case Op_CompareAndExchangeL:
2263 case Op_CompareAndExchangeP:
2264 case Op_CompareAndExchangeN:
2265 case Op_WeakCompareAndSwapB:
2266 case Op_WeakCompareAndSwapS:
2267 case Op_WeakCompareAndSwapI:
2268 case Op_WeakCompareAndSwapL:
2269 case Op_WeakCompareAndSwapP:
2270 case Op_WeakCompareAndSwapN:
2271 case Op_CompareAndSwapB:
2272 case Op_CompareAndSwapS:
2273 case Op_CompareAndSwapI:
2274 case Op_CompareAndSwapL:
2275 case Op_CompareAndSwapP:
2276 case Op_CompareAndSwapN: { // Convert trinary to binary-tree
2277 Node *newval = n->in(MemNode::ValueIn );
2278 Node *oldval = n->in(LoadStoreConditionalNode::ExpectedIn);
2279 Node *pair = new BinaryNode( oldval, newval );
2280 n->set_req(MemNode::ValueIn,pair);
2281 n->del_req(LoadStoreConditionalNode::ExpectedIn);
2282 break;
2283 }
2284 case Op_CMoveD: // Convert trinary to binary-tree
2285 case Op_CMoveF:
2286 case Op_CMoveI:
2287 case Op_CMoveL:
2288 case Op_CMoveN:
2289 case Op_CMoveP:
2290 case Op_CMoveVF:
2291 case Op_CMoveVD: {
2292 // Restructure into a binary tree for Matching. It's possible that
2293 // we could move this code up next to the graph reshaping for IfNodes
2294 // or vice-versa, but I do not want to debug this for Ladybird.
2295 // 10/2/2000 CNC.
2296 Node *pair1 = new BinaryNode(n->in(1),n->in(1)->in(1));
2297 n->set_req(1,pair1);
2298 Node *pair2 = new BinaryNode(n->in(2),n->in(3));
2299 n->set_req(2,pair2);
2300 n->del_req(3);
2301 break;
2302 }
2303 case Op_LoopLimit: {
2304 Node *pair1 = new BinaryNode(n->in(1),n->in(2));
2305 n->set_req(1,pair1);
2306 n->set_req(2,n->in(3));
2307 n->del_req(3);
2308 break;
2309 }
2310 case Op_StrEquals:
2311 case Op_StrIndexOfChar: {
2312 Node *pair1 = new BinaryNode(n->in(2),n->in(3));
2313 n->set_req(2,pair1);
2314 n->set_req(3,n->in(4));
2315 n->del_req(4);
2316 break;
2317 }
2318 case Op_StrComp:
2319 case Op_StrIndexOf: {
2320 Node *pair1 = new BinaryNode(n->in(2),n->in(3));
2321 n->set_req(2,pair1);
2322 Node *pair2 = new BinaryNode(n->in(4),n->in(5));
2323 n->set_req(3,pair2);
2324 n->del_req(5);
2325 n->del_req(4);
2326 break;
2327 }
2328 case Op_StrCompressedCopy:
2329 case Op_StrInflatedCopy:
2330 case Op_EncodeISOArray: {
2331 // Restructure into a binary tree for Matching.
2332 Node* pair = new BinaryNode(n->in(3), n->in(4));
2333 n->set_req(3, pair);
2334 n->del_req(4);
2335 break;
2336 }
2337 case Op_FmaD:
2338 case Op_FmaF:
2339 case Op_FmaVD:
2340 case Op_FmaVF: {
2341 // Restructure into a binary tree for Matching.
2342 Node* pair = new BinaryNode(n->in(1), n->in(2));
2343 n->set_req(2, pair);
2344 n->set_req(1, n->in(3));
2345 n->del_req(3);
2346 break;
2347 }
2348 default:
2349 break;
2350 }
2351 }
2352 else {
2353 ShouldNotReachHere();
2354 }
2355 } // end of while (mstack.is_nonempty())
2356 }
2357
2358 #ifdef ASSERT
2359 // machine-independent root to machine-dependent root
2360 void Matcher::dump_old2new_map() {
2361 _old2new_map.dump();
2362 }
2363 #endif
2364
2365 //---------------------------collect_null_checks-------------------------------
2366 // Find null checks in the ideal graph; write a machine-specific node for
2367 // it. Used by later implicit-null-check handling. Actually collects
2368 // either an IfTrue or IfFalse for the common NOT-null path, AND the ideal
2369 // value being tested.
2370 void Matcher::collect_null_checks( Node *proj, Node *orig_proj ) {
2371 Node *iff = proj->in(0);
2372 if( iff->Opcode() == Op_If ) {
2373 // During matching If's have Bool & Cmp side-by-side
2374 BoolNode *b = iff->in(1)->as_Bool();
2375 Node *cmp = iff->in(2);
2499 // that a monitor exit operation contains a serializing instruction.
2500
2501 if (xop == Op_MemBarVolatile ||
2502 xop == Op_CompareAndExchangeB ||
2503 xop == Op_CompareAndExchangeS ||
2504 xop == Op_CompareAndExchangeI ||
2505 xop == Op_CompareAndExchangeL ||
2506 xop == Op_CompareAndExchangeP ||
2507 xop == Op_CompareAndExchangeN ||
2508 xop == Op_WeakCompareAndSwapB ||
2509 xop == Op_WeakCompareAndSwapS ||
2510 xop == Op_WeakCompareAndSwapL ||
2511 xop == Op_WeakCompareAndSwapP ||
2512 xop == Op_WeakCompareAndSwapN ||
2513 xop == Op_WeakCompareAndSwapI ||
2514 xop == Op_CompareAndSwapB ||
2515 xop == Op_CompareAndSwapS ||
2516 xop == Op_CompareAndSwapL ||
2517 xop == Op_CompareAndSwapP ||
2518 xop == Op_CompareAndSwapN ||
2519 xop == Op_CompareAndSwapI) {
2520 return true;
2521 }
2522
2523 // Op_FastLock previously appeared in the Op_* list above.
2524 // With biased locking we're no longer guaranteed that a monitor
2525 // enter operation contains a serializing instruction.
2526 if ((xop == Op_FastLock) && !UseBiasedLocking) {
2527 return true;
2528 }
2529
2530 if (x->is_MemBar()) {
2531 // We must retain this membar if there is an upcoming volatile
2532 // load, which will be followed by acquire membar.
2533 if (xop == Op_MemBarAcquire || xop == Op_LoadFence) {
2534 return false;
2535 } else {
2536 // For other kinds of barriers, check by pretending we
2537 // are them, and seeing if we can be removed.
2538 return post_store_load_barrier(x->as_MemBar());
2539 }
|
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "gc/shared/barrierSet.hpp"
27 #include "gc/shared/c2/barrierSetC2.hpp"
28 #include "memory/allocation.inline.hpp"
29 #include "memory/resourceArea.hpp"
30 #include "opto/ad.hpp"
31 #include "opto/addnode.hpp"
32 #include "opto/callnode.hpp"
33 #include "opto/idealGraphPrinter.hpp"
34 #include "opto/matcher.hpp"
35 #include "opto/memnode.hpp"
36 #include "opto/movenode.hpp"
37 #include "opto/opcodes.hpp"
38 #include "opto/regmask.hpp"
39 #include "opto/rootnode.hpp"
40 #include "opto/runtime.hpp"
41 #include "opto/type.hpp"
42 #include "opto/vectornode.hpp"
43 #include "runtime/os.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "utilities/align.hpp"
46
47 OptoReg::Name OptoReg::c_frame_pointer;
48
49 const RegMask *Matcher::idealreg2regmask[_last_machine_leaf];
50 RegMask Matcher::mreg2regmask[_last_Mach_Reg];
51 RegMask Matcher::STACK_ONLY_mask;
52 RegMask Matcher::c_frame_ptr_mask;
53 const uint Matcher::_begin_rematerialize = _BEGIN_REMATERIALIZE;
54 const uint Matcher::_end_rematerialize = _END_REMATERIALIZE;
55
56 //---------------------------Matcher-------------------------------------------
57 Matcher::Matcher()
58 : PhaseTransform( Phase::Ins_Select ),
59 _states_arena(Chunk::medium_size, mtCompiler),
60 _visited(&_states_arena),
61 _shared(&_states_arena),
62 _dontcare(&_states_arena),
63 _reduceOp(reduceOp), _leftOp(leftOp), _rightOp(rightOp),
64 _swallowed(swallowed),
65 _begin_inst_chain_rule(_BEGIN_INST_CHAIN_RULE),
2053 uint nop = n->Opcode();
2054 if (nstate == Pre_Visit) {
2055 if (address_visited.test(n->_idx)) { // Visited in address already?
2056 // Flag as visited and shared now.
2057 set_visited(n);
2058 }
2059 if (is_visited(n)) { // Visited already?
2060 // Node is shared and has no reason to clone. Flag it as shared.
2061 // This causes it to match into a register for the sharing.
2062 set_shared(n); // Flag as shared and
2063 mstack.pop(); // remove node from stack
2064 continue;
2065 }
2066 nstate = Visit; // Not already visited; so visit now
2067 }
2068 if (nstate == Visit) {
2069 mstack.set_state(Post_Visit);
2070 set_visited(n); // Flag as visited now
2071 bool mem_op = false;
2072 int mem_addr_idx = MemNode::Address;
2073 bool gc_handled = BarrierSet::barrier_set()->barrier_set_c2()->matcher_find_shared_visit(this, mstack, n, nop, mem_op, mem_addr_idx);
2074 if (!gc_handled) {
2075 if (find_shared_visit(mstack, n, nop, mem_op, mem_addr_idx)) {
2076 continue;
2077 }
2078 }
2079 for(int i = n->req() - 1; i >= 0; --i) { // For my children
2080 Node *m = n->in(i); // Get ith input
2081 if (m == NULL) continue; // Ignore NULLs
2082 uint mop = m->Opcode();
2083
2084 // Must clone all producers of flags, or we will not match correctly.
2085 // Suppose a compare setting int-flags is shared (e.g., a switch-tree)
2086 // then it will match into an ideal Op_RegFlags. Alas, the fp-flags
2087 // are also there, so we may match a float-branch to int-flags and
2088 // expect the allocator to haul the flags from the int-side to the
2089 // fp-side. No can do.
2090 if( _must_clone[mop] ) {
2091 mstack.push(m, Visit);
2092 continue; // for(int i = ...)
2093 }
2094
2095 if( mop == Op_AddP && m->in(AddPNode::Base)->is_DecodeNarrowPtr()) {
2096 // Bases used in addresses must be shared but since
2097 // they are shared through a DecodeN they may appear
2098 // to have a single use so force sharing here.
2099 set_shared(m->in(AddPNode::Base)->in(1));
2100 }
2101
2102 // if 'n' and 'm' are part of a graph for BMI instruction, clone this node.
2103 #ifdef X86
2104 if (UseBMI1Instructions && is_bmi_pattern(n, m)) {
2105 mstack.push(m, Visit);
2106 continue;
2107 }
2108 #endif
2109
2110 // Clone addressing expressions as they are "free" in memory access instructions
2111 if (mem_op && i == mem_addr_idx && mop == Op_AddP &&
2112 // When there are other uses besides address expressions
2113 // put it on stack and mark as shared.
2114 !is_visited(m)) {
2115 // Some inputs for address expression are not put on stack
2116 // to avoid marking them as shared and forcing them into register
2117 // if they are used only in address expressions.
2118 // But they should be marked as shared if there are other uses
2119 // besides address expressions.
2120
2121 if (clone_address_expressions(m->as_AddP(), mstack, address_visited)) {
2122 continue;
2123 }
2124 } // if( mem_op &&
2125 mstack.push(m, Pre_Visit);
2126 } // for(int i = ...)
2127 }
2128 else if (nstate == Alt_Post_Visit) {
2129 mstack.pop(); // Remove node from stack
2130 // We cannot remove the Cmp input from the Bool here, as the Bool may be
2131 // shared and all users of the Bool need to move the Cmp in parallel.
2132 // This leaves both the Bool and the If pointing at the Cmp. To
2133 // prevent the Matcher from trying to Match the Cmp along both paths
2134 // BoolNode::match_edge always returns a zero.
2135
2136 // We reorder the Op_If in a pre-order manner, so we can visit without
2137 // accidentally sharing the Cmp (the Bool and the If make 2 users).
2138 n->add_req( n->in(1)->in(1) ); // Add the Cmp next to the Bool
2139 }
2140 else if (nstate == Post_Visit) {
2141 mstack.pop(); // Remove node from stack
2142
2143 // Now hack a few special opcodes
2144 uint opcode = n->Opcode();
2145 bool gc_handled = BarrierSet::barrier_set()->barrier_set_c2()->matcher_find_shared_post_visit(this, n, opcode);
2146 if (!gc_handled) {
2147 find_shared_post_visit(n, opcode);
2148 }
2149 }
2150 else {
2151 ShouldNotReachHere();
2152 }
2153 } // end of while (mstack.is_nonempty())
2154 }
2155
2156 bool Matcher::find_shared_visit(MStack& mstack, Node* n, uint opcode, bool& mem_op, int& mem_addr_idx) {
2157 switch(opcode) { // Handle some opcodes special
2158 case Op_Phi: // Treat Phis as shared roots
2159 case Op_Parm:
2160 case Op_Proj: // All handled specially during matching
2161 case Op_SafePointScalarObject:
2162 set_shared(n);
2163 set_dontcare(n);
2164 break;
2165 case Op_If:
2166 case Op_CountedLoopEnd:
2167 mstack.set_state(Alt_Post_Visit); // Alternative way
2168 // Convert (If (Bool (CmpX A B))) into (If (Bool) (CmpX A B)). Helps
2169 // with matching cmp/branch in 1 instruction. The Matcher needs the
2170 // Bool and CmpX side-by-side, because it can only get at constants
2171 // that are at the leaves of Match trees, and the Bool's condition acts
2172 // as a constant here.
2173 mstack.push(n->in(1), Visit); // Clone the Bool
2174 mstack.push(n->in(0), Pre_Visit); // Visit control input
2175 return true; // while (mstack.is_nonempty())
2176 case Op_ConvI2D: // These forms efficiently match with a prior
2177 case Op_ConvI2F: // Load but not a following Store
2178 if( n->in(1)->is_Load() && // Prior load
2179 n->outcnt() == 1 && // Not already shared
2180 n->unique_out()->is_Store() ) // Following store
2181 set_shared(n); // Force it to be a root
2182 break;
2183 case Op_ReverseBytesI:
2184 case Op_ReverseBytesL:
2185 if( n->in(1)->is_Load() && // Prior load
2186 n->outcnt() == 1 ) // Not already shared
2187 set_shared(n); // Force it to be a root
2188 break;
2189 case Op_BoxLock: // Cant match until we get stack-regs in ADLC
2190 case Op_IfFalse:
2191 case Op_IfTrue:
2192 case Op_MachProj:
2193 case Op_MergeMem:
2194 case Op_Catch:
2195 case Op_CatchProj:
2196 case Op_CProj:
2197 case Op_JumpProj:
2198 case Op_JProj:
2199 case Op_NeverBranch:
2200 set_dontcare(n);
2201 break;
2202 case Op_Jump:
2203 mstack.push(n->in(1), Pre_Visit); // Switch Value (could be shared)
2204 mstack.push(n->in(0), Pre_Visit); // Visit Control input
2205 return true; // while (mstack.is_nonempty())
2206 case Op_StrComp:
2207 case Op_StrEquals:
2208 case Op_StrIndexOf:
2209 case Op_StrIndexOfChar:
2210 case Op_AryEq:
2211 case Op_HasNegatives:
2212 case Op_StrInflatedCopy:
2213 case Op_StrCompressedCopy:
2214 case Op_EncodeISOArray:
2215 case Op_FmaD:
2216 case Op_FmaF:
2217 case Op_FmaVD:
2218 case Op_FmaVF:
2219 set_shared(n); // Force result into register (it will be anyways)
2220 break;
2221 case Op_ConP: { // Convert pointers above the centerline to NUL
2222 TypeNode *tn = n->as_Type(); // Constants derive from type nodes
2223 const TypePtr* tp = tn->type()->is_ptr();
2224 if (tp->_ptr == TypePtr::AnyNull) {
2225 tn->set_type(TypePtr::NULL_PTR);
2226 }
2227 break;
2228 }
2229 case Op_ConN: { // Convert narrow pointers above the centerline to NUL
2230 TypeNode *tn = n->as_Type(); // Constants derive from type nodes
2231 const TypePtr* tp = tn->type()->make_ptr();
2232 if (tp && tp->_ptr == TypePtr::AnyNull) {
2233 tn->set_type(TypeNarrowOop::NULL_PTR);
2234 }
2235 break;
2236 }
2237 case Op_Binary: // These are introduced in the Post_Visit state.
2238 ShouldNotReachHere();
2239 break;
2240 case Op_ClearArray:
2241 case Op_SafePoint:
2242 mem_op = true;
2243 break;
2244 default:
2245 if( n->is_Store() ) {
2246 // Do match stores, despite no ideal reg
2247 mem_op = true;
2248 break;
2249 }
2250 if( n->is_Mem() ) { // Loads and LoadStores
2251 mem_op = true;
2252 // Loads must be root of match tree due to prior load conflict
2253 if( C->subsume_loads() == false )
2254 set_shared(n);
2255 }
2256 // Fall into default case
2257 if( !n->ideal_reg() )
2258 set_dontcare(n); // Unmatchable Nodes
2259 } // end_switch
2260 return false;
2261 }
2262
2263 void Matcher::find_shared_post_visit(Node* n, uint opcode) {
2264 switch(opcode) { // Handle some opcodes special
2265 case Op_StorePConditional:
2266 case Op_StoreIConditional:
2267 case Op_StoreLConditional:
2268 case Op_CompareAndExchangeB:
2269 case Op_CompareAndExchangeS:
2270 case Op_CompareAndExchangeI:
2271 case Op_CompareAndExchangeL:
2272 case Op_CompareAndExchangeP:
2273 case Op_CompareAndExchangeN:
2274 case Op_WeakCompareAndSwapB:
2275 case Op_WeakCompareAndSwapS:
2276 case Op_WeakCompareAndSwapI:
2277 case Op_WeakCompareAndSwapL:
2278 case Op_WeakCompareAndSwapP:
2279 case Op_WeakCompareAndSwapN:
2280 case Op_CompareAndSwapB:
2281 case Op_CompareAndSwapS:
2282 case Op_CompareAndSwapI:
2283 case Op_CompareAndSwapL:
2284 case Op_CompareAndSwapP:
2285 case Op_CompareAndSwapN: { // Convert trinary to binary-tree
2286 Node* newval = n->in(MemNode::ValueIn);
2287 Node* oldval = n->in(LoadStoreConditionalNode::ExpectedIn);
2288 Node* pair = new BinaryNode(oldval, newval);
2289 n->set_req(MemNode::ValueIn, pair);
2290 n->del_req(LoadStoreConditionalNode::ExpectedIn);
2291 break;
2292 }
2293 case Op_CMoveD: // Convert trinary to binary-tree
2294 case Op_CMoveF:
2295 case Op_CMoveI:
2296 case Op_CMoveL:
2297 case Op_CMoveN:
2298 case Op_CMoveP:
2299 case Op_CMoveVF:
2300 case Op_CMoveVD: {
2301 // Restructure into a binary tree for Matching. It's possible that
2302 // we could move this code up next to the graph reshaping for IfNodes
2303 // or vice-versa, but I do not want to debug this for Ladybird.
2304 // 10/2/2000 CNC.
2305 Node* pair1 = new BinaryNode(n->in(1), n->in(1)->in(1));
2306 n->set_req(1, pair1);
2307 Node* pair2 = new BinaryNode(n->in(2), n->in(3));
2308 n->set_req(2, pair2);
2309 n->del_req(3);
2310 break;
2311 }
2312 case Op_LoopLimit: {
2313 Node* pair1 = new BinaryNode(n->in(1), n->in(2));
2314 n->set_req(1, pair1);
2315 n->set_req(2, n->in(3));
2316 n->del_req(3);
2317 break;
2318 }
2319 case Op_StrEquals:
2320 case Op_StrIndexOfChar: {
2321 Node* pair1 = new BinaryNode(n->in(2), n->in(3));
2322 n->set_req(2, pair1);
2323 n->set_req(3, n->in(4));
2324 n->del_req(4);
2325 break;
2326 }
2327 case Op_StrComp:
2328 case Op_StrIndexOf: {
2329 Node* pair1 = new BinaryNode(n->in(2), n->in(3));
2330 n->set_req(2, pair1);
2331 Node* pair2 = new BinaryNode(n->in(4),n->in(5));
2332 n->set_req(3, pair2);
2333 n->del_req(5);
2334 n->del_req(4);
2335 break;
2336 }
2337 case Op_StrCompressedCopy:
2338 case Op_StrInflatedCopy:
2339 case Op_EncodeISOArray: {
2340 // Restructure into a binary tree for Matching.
2341 Node* pair = new BinaryNode(n->in(3), n->in(4));
2342 n->set_req(3, pair);
2343 n->del_req(4);
2344 break;
2345 }
2346 case Op_FmaD:
2347 case Op_FmaF:
2348 case Op_FmaVD:
2349 case Op_FmaVF: {
2350 // Restructure into a binary tree for Matching.
2351 Node* pair = new BinaryNode(n->in(1), n->in(2));
2352 n->set_req(2, pair);
2353 n->set_req(1, n->in(3));
2354 n->del_req(3);
2355 break;
2356 }
2357 default:
2358 break;
2359 }
2360 }
2361
2362 #ifdef ASSERT
2363 // machine-independent root to machine-dependent root
2364 void Matcher::dump_old2new_map() {
2365 _old2new_map.dump();
2366 }
2367 #endif
2368
2369 //---------------------------collect_null_checks-------------------------------
2370 // Find null checks in the ideal graph; write a machine-specific node for
2371 // it. Used by later implicit-null-check handling. Actually collects
2372 // either an IfTrue or IfFalse for the common NOT-null path, AND the ideal
2373 // value being tested.
2374 void Matcher::collect_null_checks( Node *proj, Node *orig_proj ) {
2375 Node *iff = proj->in(0);
2376 if( iff->Opcode() == Op_If ) {
2377 // During matching If's have Bool & Cmp side-by-side
2378 BoolNode *b = iff->in(1)->as_Bool();
2379 Node *cmp = iff->in(2);
2503 // that a monitor exit operation contains a serializing instruction.
2504
2505 if (xop == Op_MemBarVolatile ||
2506 xop == Op_CompareAndExchangeB ||
2507 xop == Op_CompareAndExchangeS ||
2508 xop == Op_CompareAndExchangeI ||
2509 xop == Op_CompareAndExchangeL ||
2510 xop == Op_CompareAndExchangeP ||
2511 xop == Op_CompareAndExchangeN ||
2512 xop == Op_WeakCompareAndSwapB ||
2513 xop == Op_WeakCompareAndSwapS ||
2514 xop == Op_WeakCompareAndSwapL ||
2515 xop == Op_WeakCompareAndSwapP ||
2516 xop == Op_WeakCompareAndSwapN ||
2517 xop == Op_WeakCompareAndSwapI ||
2518 xop == Op_CompareAndSwapB ||
2519 xop == Op_CompareAndSwapS ||
2520 xop == Op_CompareAndSwapL ||
2521 xop == Op_CompareAndSwapP ||
2522 xop == Op_CompareAndSwapN ||
2523 xop == Op_CompareAndSwapI ||
2524 BarrierSet::barrier_set()->barrier_set_c2()->matcher_is_store_load_barrier(x, xop)) {
2525 return true;
2526 }
2527
2528 // Op_FastLock previously appeared in the Op_* list above.
2529 // With biased locking we're no longer guaranteed that a monitor
2530 // enter operation contains a serializing instruction.
2531 if ((xop == Op_FastLock) && !UseBiasedLocking) {
2532 return true;
2533 }
2534
2535 if (x->is_MemBar()) {
2536 // We must retain this membar if there is an upcoming volatile
2537 // load, which will be followed by acquire membar.
2538 if (xop == Op_MemBarAcquire || xop == Op_LoadFence) {
2539 return false;
2540 } else {
2541 // For other kinds of barriers, check by pretending we
2542 // are them, and seeing if we can be removed.
2543 return post_store_load_barrier(x->as_MemBar());
2544 }
|