1905 } 1906 } 1907 } 1908 1909 1910 // ------------------------------------------------------------------------- 1911 // Java-Java calling convention 1912 // (what you use when Java calls Java) 1913 1914 //------------------------------find_receiver---------------------------------- 1915 // For a given signature, return the OptoReg for parameter 0. 1916 OptoReg::Name Matcher::find_receiver( bool is_outgoing ) { 1917 VMRegPair regs; 1918 BasicType sig_bt = T_OBJECT; 1919 calling_convention(&sig_bt, ®s, 1, is_outgoing); 1920 // Return argument 0 register. In the LP64 build pointers 1921 // take 2 registers, but the VM wants only the 'main' name. 1922 return OptoReg::as_OptoReg(regs.first()); 1923 } 1924 1925 // A method-klass-holder may be passed in the inline_cache_reg 1926 // and then expanded into the inline_cache_reg and a method_oop register 1927 // defined in ad_<arch>.cpp 1928 1929 1930 //------------------------------find_shared------------------------------------ 1931 // Set bits if Node is shared or otherwise a root 1932 void Matcher::find_shared( Node *n ) { 1933 // Allocate stack of size C->unique() * 2 to avoid frequent realloc 1934 MStack mstack(C->unique() * 2); 1935 // Mark nodes as address_visited if they are inputs to an address expression 1936 VectorSet address_visited(Thread::current()->resource_area()); 1937 mstack.push(n, Visit); // Don't need to pre-visit root node 1938 while (mstack.is_nonempty()) { 1939 n = mstack.node(); // Leave node on stack 1940 Node_State nstate = mstack.state(); 1941 uint nop = n->Opcode(); 1942 if (nstate == Pre_Visit) { 1943 if (address_visited.test(n->_idx)) { // Visited in address already? 1944 // Flag as visited and shared now. 2061 uint mop = m->Opcode(); 2062 2063 // Must clone all producers of flags, or we will not match correctly. 2064 // Suppose a compare setting int-flags is shared (e.g., a switch-tree) 2065 // then it will match into an ideal Op_RegFlags. Alas, the fp-flags 2066 // are also there, so we may match a float-branch to int-flags and 2067 // expect the allocator to haul the flags from the int-side to the 2068 // fp-side. No can do. 2069 if( _must_clone[mop] ) { 2070 mstack.push(m, Visit); 2071 continue; // for(int i = ...) 2072 } 2073 2074 if( mop == Op_AddP && m->in(AddPNode::Base)->is_DecodeNarrowPtr()) { 2075 // Bases used in addresses must be shared but since 2076 // they are shared through a DecodeN they may appear 2077 // to have a single use so force sharing here. 2078 set_shared(m->in(AddPNode::Base)->in(1)); 2079 } 2080 2081 // Clone addressing expressions as they are "free" in memory access instructions 2082 if( mem_op && i == MemNode::Address && mop == Op_AddP ) { 2083 // Some inputs for address expression are not put on stack 2084 // to avoid marking them as shared and forcing them into register 2085 // if they are used only in address expressions. 2086 // But they should be marked as shared if there are other uses 2087 // besides address expressions. 2088 2089 Node *off = m->in(AddPNode::Offset); 2090 if( off->is_Con() && 2091 // When there are other uses besides address expressions 2092 // put it on stack and mark as shared. 2093 !is_visited(m) ) { 2094 address_visited.test_set(m->_idx); // Flag as address_visited 2095 Node *adr = m->in(AddPNode::Address); 2096 2097 // Intel, ARM and friends can handle 2 adds in addressing mode 2098 if( clone_shift_expressions && adr->is_AddP() && 2099 // AtomicAdd is not an addressing expression. 2100 // Cheap to find it by looking for screwy base. | 1905 } 1906 } 1907 } 1908 1909 1910 // ------------------------------------------------------------------------- 1911 // Java-Java calling convention 1912 // (what you use when Java calls Java) 1913 1914 //------------------------------find_receiver---------------------------------- 1915 // For a given signature, return the OptoReg for parameter 0. 1916 OptoReg::Name Matcher::find_receiver( bool is_outgoing ) { 1917 VMRegPair regs; 1918 BasicType sig_bt = T_OBJECT; 1919 calling_convention(&sig_bt, ®s, 1, is_outgoing); 1920 // Return argument 0 register. In the LP64 build pointers 1921 // take 2 registers, but the VM wants only the 'main' name. 1922 return OptoReg::as_OptoReg(regs.first()); 1923 } 1924 1925 // This function identifies sub-graphs in which a 'load' node is 1926 // input to two different nodes, and such that it can be matched 1927 // with BMI instructions like blsi, blsr, etc. 1928 // Example : for b = -a[i] & a[i] can be matched to blsi r32, m32. 1929 // The graph is (AndL (SubL (Con0 LoadL*) LoadL*)), where LoadL* 1930 // refers to the same node. 1931 #if defined(TARGET_ARCH_MODEL_x86_32) || defined(TARGET_ARCH_MODEL_x86_64) 1932 1933 // Match the generic fused operations pattern (op1 (op2 Con{ConType} mop) mop) 1934 // This is a temporary solution until we make DAGs expressible in ADL. 1935 template<typename ConType> 1936 class FusedPatternMatcher { 1937 Node* _op1_node; 1938 Node* _mop_node; 1939 int _con_op; 1940 1941 static int match_next(Node* n, int next_op, int next_op_idx) { 1942 if (n->in(1) == NULL || n->in(2) == NULL) { 1943 return -1; 1944 } 1945 1946 if (next_op_idx == -1) { // n is commutative, try rotations 1947 if (n->in(1)->Opcode() == next_op) { 1948 return 1; 1949 } else if (n->in(2)->Opcode() == next_op) { 1950 return 2; 1951 } 1952 } else { 1953 assert(next_op_idx > 0 && next_op_idx <= 2, "Bad argument index"); 1954 if (n->in(next_op_idx)->Opcode() == next_op) { 1955 return next_op_idx; 1956 } 1957 } 1958 return -1; 1959 } 1960 public: 1961 FusedPatternMatcher(Node* op1_node, Node *mop_node, int con_op) : 1962 _op1_node(op1_node), _mop_node(mop_node), _con_op(con_op) { } 1963 1964 bool match(int op1, int op1_op2_idx, // op1 and the index of the op1->op2 edge, -1 if op1 is commutative 1965 int op2, int op2_con_idx, // op2 and the index of the op2->con edge, -1 if op2 is commutative 1966 typename ConType::NativeType con_value) { 1967 if (_op1_node->Opcode() != op1) { 1968 return false; 1969 } 1970 if (_mop_node->outcnt() > 2) { 1971 return false; 1972 } 1973 op1_op2_idx = match_next(_op1_node, op2, op1_op2_idx); 1974 if (op1_op2_idx == -1) { 1975 return false; 1976 } 1977 // Memory operation must be the other edge 1978 int op1_mop_idx = op1_op2_idx % 2 + 1; 1979 1980 // Check that the mop node is really what we want 1981 if (_op1_node->in(op1_mop_idx) == _mop_node) { 1982 Node *op2_node = _op1_node->in(op1_op2_idx); 1983 if (op2_node->outcnt() > 1) { 1984 return false; 1985 } 1986 assert(op2_node->Opcode() == op2, "Should be"); 1987 op2_con_idx = match_next(op2_node, _con_op, op2_con_idx); 1988 if (op2_con_idx == -1) { 1989 return false; 1990 } 1991 // Memory operation must be the other edge 1992 int op2_mop_idx = op2_con_idx % 2 + 1; 1993 // Check that the memory operation is the same node 1994 if (op2_node->in(op2_mop_idx) == _mop_node) { 1995 // Now check the constant 1996 const Type* con_type = op2_node->in(op2_con_idx)->bottom_type(); 1997 if (con_type != Type::TOP && ConType::as_self(con_type)->get_con() == con_value) { 1998 return true; 1999 } 2000 } 2001 } 2002 return false; 2003 } 2004 }; 2005 2006 2007 bool Matcher::is_bmi_pattern(Node *n, Node *m) { 2008 if (n != NULL && m != NULL) { 2009 if (m->Opcode() == Op_LoadI) { 2010 FusedPatternMatcher<TypeInt> bmii(n, m, Op_ConI); 2011 return bmii.match(Op_AndI, -1, Op_SubI, 1, 0) || 2012 bmii.match(Op_AndI, -1, Op_AddI, -1, -1) || 2013 bmii.match(Op_XorI, -1, Op_AddI, -1, -1); 2014 } else if (m->Opcode() == Op_LoadL) { 2015 FusedPatternMatcher<TypeLong> bmil(n, m, Op_ConL); 2016 return bmil.match(Op_AndL, -1, Op_SubL, 1, 0) || 2017 bmil.match(Op_AndL, -1, Op_AddL, -1, -1) || 2018 bmil.match(Op_XorL, -1, Op_AddL, -1, -1); 2019 } 2020 } 2021 return false; 2022 } 2023 #endif 2024 2025 // A method-klass-holder may be passed in the inline_cache_reg 2026 // and then expanded into the inline_cache_reg and a method_oop register 2027 // defined in ad_<arch>.cpp 2028 2029 2030 //------------------------------find_shared------------------------------------ 2031 // Set bits if Node is shared or otherwise a root 2032 void Matcher::find_shared( Node *n ) { 2033 // Allocate stack of size C->unique() * 2 to avoid frequent realloc 2034 MStack mstack(C->unique() * 2); 2035 // Mark nodes as address_visited if they are inputs to an address expression 2036 VectorSet address_visited(Thread::current()->resource_area()); 2037 mstack.push(n, Visit); // Don't need to pre-visit root node 2038 while (mstack.is_nonempty()) { 2039 n = mstack.node(); // Leave node on stack 2040 Node_State nstate = mstack.state(); 2041 uint nop = n->Opcode(); 2042 if (nstate == Pre_Visit) { 2043 if (address_visited.test(n->_idx)) { // Visited in address already? 2044 // Flag as visited and shared now. 2161 uint mop = m->Opcode(); 2162 2163 // Must clone all producers of flags, or we will not match correctly. 2164 // Suppose a compare setting int-flags is shared (e.g., a switch-tree) 2165 // then it will match into an ideal Op_RegFlags. Alas, the fp-flags 2166 // are also there, so we may match a float-branch to int-flags and 2167 // expect the allocator to haul the flags from the int-side to the 2168 // fp-side. No can do. 2169 if( _must_clone[mop] ) { 2170 mstack.push(m, Visit); 2171 continue; // for(int i = ...) 2172 } 2173 2174 if( mop == Op_AddP && m->in(AddPNode::Base)->is_DecodeNarrowPtr()) { 2175 // Bases used in addresses must be shared but since 2176 // they are shared through a DecodeN they may appear 2177 // to have a single use so force sharing here. 2178 set_shared(m->in(AddPNode::Base)->in(1)); 2179 } 2180 2181 // if 'n' and 'm' are part of a graph for BMI instruction, clone this node. 2182 #if defined(TARGET_ARCH_MODEL_x86_32) || defined(TARGET_ARCH_MODEL_x86_64) 2183 if (UseBMI1Instructions && is_bmi_pattern(n, m)) { 2184 mstack.push(m, Visit); 2185 continue; 2186 } 2187 #endif 2188 2189 // Clone addressing expressions as they are "free" in memory access instructions 2190 if( mem_op && i == MemNode::Address && mop == Op_AddP ) { 2191 // Some inputs for address expression are not put on stack 2192 // to avoid marking them as shared and forcing them into register 2193 // if they are used only in address expressions. 2194 // But they should be marked as shared if there are other uses 2195 // besides address expressions. 2196 2197 Node *off = m->in(AddPNode::Offset); 2198 if( off->is_Con() && 2199 // When there are other uses besides address expressions 2200 // put it on stack and mark as shared. 2201 !is_visited(m) ) { 2202 address_visited.test_set(m->_idx); // Flag as address_visited 2203 Node *adr = m->in(AddPNode::Address); 2204 2205 // Intel, ARM and friends can handle 2 adds in addressing mode 2206 if( clone_shift_expressions && adr->is_AddP() && 2207 // AtomicAdd is not an addressing expression. 2208 // Cheap to find it by looking for screwy base. |