1858
1859 return ac;
1860 }
1861
1862 void ArrayCopyNode::connect_outputs(GraphKit* kit) {
1863 kit->set_all_memory_call(this, true);
1864 kit->set_control(kit->gvn().transform(new ProjNode(this,TypeFunc::Control)));
1865 kit->set_i_o(kit->gvn().transform(new ProjNode(this, TypeFunc::I_O)));
1866 kit->make_slow_call_ex(this, kit->env()->Throwable_klass(), true);
1867 kit->set_all_memory_call(this);
1868 }
1869
1870 #ifndef PRODUCT
1871 const char* ArrayCopyNode::_kind_names[] = {"arraycopy", "arraycopy, validated arguments", "clone", "oop array clone", "CopyOf", "CopyOfRange"};
1872 void ArrayCopyNode::dump_spec(outputStream *st) const {
1873 CallNode::dump_spec(st);
1874 st->print(" (%s%s)", _kind_names[_kind], _alloc_tightly_coupled ? ", tightly coupled allocation" : "");
1875 }
1876 #endif
1877
1878 int ArrayCopyNode::get_count(PhaseGVN *phase) const {
1879 Node* src = in(ArrayCopyNode::Src);
1880 const Type* src_type = phase->type(src);
1881
1882 assert(is_clonebasic(), "unexpected arraycopy type");
1883 if (src_type->isa_instptr()) {
1884 const TypeInstPtr* inst_src = src_type->is_instptr();
1885 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
1886 // ciInstanceKlass::nof_nonstatic_fields() doesn't take injected
1887 // fields into account. They are rare anyway so easier to simply
1888 // skip instances with injected fields.
1889 if ((!inst_src->klass_is_exact() && (ik->is_interface() || ik->has_subklass())) || ik->has_injected_fields()) {
1890 return -1;
1891 }
1892 int nb_fields = ik->nof_nonstatic_fields();
1893 return nb_fields;
1894 }
1895 return -1;
1896 }
1897
1898 Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) {
1899 assert(is_clonebasic(), "unexpected arraycopy type");
1900
1901 Node* src = in(ArrayCopyNode::Src);
1902 Node* dest = in(ArrayCopyNode::Dest);
1903 Node* ctl = in(TypeFunc::Control);
1904 Node* in_mem = in(TypeFunc::Memory);
1905
1906 const Type* src_type = phase->type(src);
1907 const Type* dest_type = phase->type(dest);
1908
1909 assert(src->is_AddP(), "should be base + off");
1910 assert(dest->is_AddP(), "should be base + off");
1911 Node* base_src = src->in(AddPNode::Base);
1912 Node* base_dest = dest->in(AddPNode::Base);
1913
1914 MergeMemNode* mem = MergeMemNode::make(in_mem);
1915
1916 const TypeInstPtr* inst_src = src_type->is_instptr();
1917
1918 if (!inst_src->klass_is_exact()) {
1919 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
1920 assert(!ik->is_interface() && !ik->has_subklass(), "inconsistent klass hierarchy");
1921 phase->C->dependencies()->assert_leaf_type(ik);
1922 }
1923
1924 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
1925 assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields");
1926
1927 for (int i = 0; i < count; i++) {
1928 ciField* field = ik->nonstatic_field_at(i);
1929 int fieldidx = phase->C->alias_type(field)->index();
1930 const TypePtr* adr_type = phase->C->alias_type(field)->adr_type();
1931 Node* off = phase->MakeConX(field->offset());
1932 Node* next_src = phase->transform(new AddPNode(base_src,base_src,off));
1933 Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off));
1934 BasicType bt = field->layout_type();
1935
1936 const Type *type;
1942 type = TypeOopPtr::make_from_klass(field_klass->as_klass());
1943 }
1944 } else {
1945 type = Type::get_const_basic_type(bt);
1946 }
1947
1948 Node* v = LoadNode::make(*phase, ctl, mem->memory_at(fieldidx), next_src, adr_type, type, bt, MemNode::unordered);
1949 v = phase->transform(v);
1950 Node* s = StoreNode::make(*phase, ctl, mem->memory_at(fieldidx), next_dest, adr_type, v, bt, MemNode::unordered);
1951 s = phase->transform(s);
1952 mem->set_memory_at(fieldidx, s);
1953 }
1954
1955 if (!finish_transform(phase, can_reshape, ctl, mem)) {
1956 return NULL;
1957 }
1958
1959 return mem;
1960 }
1961
1962 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,
1963 Node* ctl, Node *mem) {
1964 if (can_reshape) {
1965 PhaseIterGVN* igvn = phase->is_IterGVN();
1966 assert(is_clonebasic(), "unexpected arraycopy type");
1967 Node* out_mem = proj_out(TypeFunc::Memory);
1968
1969 if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() ||
1970 out_mem->raw_out(0)->outcnt() != 1 || !out_mem->raw_out(0)->raw_out(0)->is_MemBar()) {
1971 assert(!GraphKit::use_ReduceInitialCardMarks(), "can only happen with card marking");
1972 return false;
1973 }
1974
1975 igvn->replace_node(out_mem->raw_out(0), mem);
1976
1977 Node* out_ctl = proj_out(TypeFunc::Control);
1978 igvn->replace_node(out_ctl, ctl);
1979 }
1980 return true;
1981 }
1982
1983
1984 Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) {
1985
1986 if (StressArrayCopyMacroNode && !can_reshape) return NULL;
1987
1988 // See if it's a small array copy and we can inline it as
1989 // loads/stores
1990 // Here we can only do:
1991 // - clone for which we don't need to do card marking
1992
1993 if (!is_clonebasic()) {
1994 return NULL;
1995 }
1996
1997 if (in(TypeFunc::Control)->is_top() || in(TypeFunc::Memory)->is_top()) {
1998 return NULL;
1999 }
2000
2001 int count = get_count(phase);
2002
2003 if (count < 0 || count > ArrayCopyLoadStoreMaxElem) {
2004 return NULL;
2005 }
2006
2007 Node* mem = try_clone_instance(phase, can_reshape, count);
2008 return mem;
2009 }
|
1858
1859 return ac;
1860 }
1861
1862 void ArrayCopyNode::connect_outputs(GraphKit* kit) {
1863 kit->set_all_memory_call(this, true);
1864 kit->set_control(kit->gvn().transform(new ProjNode(this,TypeFunc::Control)));
1865 kit->set_i_o(kit->gvn().transform(new ProjNode(this, TypeFunc::I_O)));
1866 kit->make_slow_call_ex(this, kit->env()->Throwable_klass(), true);
1867 kit->set_all_memory_call(this);
1868 }
1869
1870 #ifndef PRODUCT
1871 const char* ArrayCopyNode::_kind_names[] = {"arraycopy", "arraycopy, validated arguments", "clone", "oop array clone", "CopyOf", "CopyOfRange"};
1872 void ArrayCopyNode::dump_spec(outputStream *st) const {
1873 CallNode::dump_spec(st);
1874 st->print(" (%s%s)", _kind_names[_kind], _alloc_tightly_coupled ? ", tightly coupled allocation" : "");
1875 }
1876 #endif
1877
1878 intptr_t ArrayCopyNode::get_length_if_constant(PhaseGVN *phase) const {
1879 // check that length is constant
1880 Node* length = in(ArrayCopyNode::Length);
1881 const Type* length_type = phase->type(length);
1882
1883 if (length_type == Type::TOP) {
1884 return -1;
1885 }
1886
1887 intptr_t length_const = -1;
1888 if (is_clonebasic()) {
1889 const TypeX* length_int_type = length_type->is_intptr_t();
1890 if (length_int_type->is_con()) {
1891 length_const = length_int_type->get_con();
1892 }
1893 } else if (is_arraycopy() || is_copyof() || is_copyofrange()) {
1894 const TypeInt* length_int_type = length_type->is_int();
1895 if (length_int_type->is_con()) {
1896 length_const = length_int_type->get_con();
1897 }
1898 } else {
1899 fatal("unexpected array copy type");
1900 }
1901
1902 return length_const;
1903 }
1904
1905 int ArrayCopyNode::get_count(PhaseGVN *phase) const {
1906 Node* src = in(ArrayCopyNode::Src);
1907 const Type* src_type = phase->type(src);
1908
1909 if (is_clonebasic()) {
1910 if (src_type->isa_instptr()) {
1911 const TypeInstPtr* inst_src = src_type->is_instptr();
1912 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
1913 // ciInstanceKlass::nof_nonstatic_fields() doesn't take injected
1914 // fields into account. They are rare anyway so easier to simply
1915 // skip instances with injected fields.
1916 if ((!inst_src->klass_is_exact() && (ik->is_interface() || ik->has_subklass())) || ik->has_injected_fields()) {
1917 return -1;
1918 }
1919 int nb_fields = ik->nof_nonstatic_fields();
1920 return nb_fields;
1921 } else {
1922 const TypeAryPtr* ary_src = src_type->isa_aryptr();
1923 assert (ary_src != NULL, "not an array or instance?");
1924 // clone passes a length as a rounded number of longs. If we're
1925 // cloning an array we'll do it element by element. If the
1926 // length input to ArrayCopyNode is constant, length of input
1927 // array must be too.
1928
1929 assert((get_length_if_constant(phase) == -1) == !ary_src->size()->is_con(), "inconsistent");
1930
1931 if (ary_src->size()->is_con()) {
1932 return ary_src->size()->get_con();
1933 }
1934 return -1;
1935 }
1936 }
1937
1938 return get_length_if_constant(phase);
1939 }
1940
1941 Node* ArrayCopyNode::try_clone_instance(PhaseGVN *phase, bool can_reshape, int count) {
1942 if (!is_clonebasic()) {
1943 return NULL;
1944 }
1945
1946 Node* src = in(ArrayCopyNode::Src);
1947 Node* dest = in(ArrayCopyNode::Dest);
1948 Node* ctl = in(TypeFunc::Control);
1949 Node* in_mem = in(TypeFunc::Memory);
1950
1951 const Type* src_type = phase->type(src);
1952 const Type* dest_type = phase->type(dest);
1953
1954 assert(src->is_AddP(), "should be base + off");
1955 assert(dest->is_AddP(), "should be base + off");
1956 Node* base_src = src->in(AddPNode::Base);
1957 Node* base_dest = dest->in(AddPNode::Base);
1958
1959 MergeMemNode* mem = MergeMemNode::make(in_mem);
1960
1961 const TypeInstPtr* inst_src = src_type->isa_instptr();
1962
1963 if (inst_src == NULL) {
1964 return NULL;
1965 }
1966
1967 if (!inst_src->klass_is_exact()) {
1968 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
1969 assert(!ik->is_interface() && !ik->has_subklass(), "inconsistent klass hierarchy");
1970 phase->C->dependencies()->assert_leaf_type(ik);
1971 }
1972
1973 ciInstanceKlass* ik = inst_src->klass()->as_instance_klass();
1974 assert(ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem, "too many fields");
1975
1976 for (int i = 0; i < count; i++) {
1977 ciField* field = ik->nonstatic_field_at(i);
1978 int fieldidx = phase->C->alias_type(field)->index();
1979 const TypePtr* adr_type = phase->C->alias_type(field)->adr_type();
1980 Node* off = phase->MakeConX(field->offset());
1981 Node* next_src = phase->transform(new AddPNode(base_src,base_src,off));
1982 Node* next_dest = phase->transform(new AddPNode(base_dest,base_dest,off));
1983 BasicType bt = field->layout_type();
1984
1985 const Type *type;
1991 type = TypeOopPtr::make_from_klass(field_klass->as_klass());
1992 }
1993 } else {
1994 type = Type::get_const_basic_type(bt);
1995 }
1996
1997 Node* v = LoadNode::make(*phase, ctl, mem->memory_at(fieldidx), next_src, adr_type, type, bt, MemNode::unordered);
1998 v = phase->transform(v);
1999 Node* s = StoreNode::make(*phase, ctl, mem->memory_at(fieldidx), next_dest, adr_type, v, bt, MemNode::unordered);
2000 s = phase->transform(s);
2001 mem->set_memory_at(fieldidx, s);
2002 }
2003
2004 if (!finish_transform(phase, can_reshape, ctl, mem)) {
2005 return NULL;
2006 }
2007
2008 return mem;
2009 }
2010
2011 Node* ArrayCopyNode::conv_I2X_offset(PhaseGVN *phase, Node* offset, const TypeAryPtr* ary_t) {
2012 #ifdef _LP64
2013 // see comment in GraphKit::array_element_address
2014 int index_max = ary_t->size()->_hi - 1;
2015 const TypeLong* lidxtype = TypeLong::make(CONST64(0), index_max, Type::WidenMax);
2016 return phase->transform(new ConvI2LNode(offset, lidxtype));
2017 #else
2018 return offset;
2019 #endif
2020 }
2021
2022 bool ArrayCopyNode::prepare_array_copy(PhaseGVN *phase, bool can_reshape,
2023 Node*& adr_src,
2024 Node*& base_src,
2025 Node*& adr_dest,
2026 Node*& base_dest,
2027 BasicType& copy_type,
2028 const Type*& value_type,
2029 bool& disjoint_bases) {
2030 Node* src = in(ArrayCopyNode::Src);
2031 Node* dest = in(ArrayCopyNode::Dest);
2032 const Type* src_type = phase->type(src);
2033 const TypeAryPtr* ary_src = src_type->isa_aryptr();
2034
2035 if (is_arraycopy() || is_copyofrange() || is_copyof()) {
2036 const Type* dest_type = phase->type(dest);
2037 const TypeAryPtr* ary_dest = dest_type->isa_aryptr();
2038 Node* src_offset = in(ArrayCopyNode::SrcPos);
2039 Node* dest_offset = in(ArrayCopyNode::DestPos);
2040
2041 // newly allocated object is guaranteed to not overlap with source object
2042 disjoint_bases = is_alloc_tightly_coupled();
2043
2044 if (ary_src == NULL || ary_src->klass() == NULL ||
2045 ary_dest == NULL || ary_dest->klass() == NULL) {
2046 // We don't know if arguments are arrays
2047 return false;
2048 }
2049
2050 BasicType src_elem = ary_src->klass()->as_array_klass()->element_type()->basic_type();
2051 BasicType dest_elem = ary_dest->klass()->as_array_klass()->element_type()->basic_type();
2052 if (src_elem == T_ARRAY) src_elem = T_OBJECT;
2053 if (dest_elem == T_ARRAY) dest_elem = T_OBJECT;
2054
2055 if (src_elem != dest_elem || dest_elem == T_VOID) {
2056 // We don't know if arguments are arrays of the same type
2057 return false;
2058 }
2059
2060 if (dest_elem == T_OBJECT && (!is_alloc_tightly_coupled() || !GraphKit::use_ReduceInitialCardMarks())) {
2061 // It's an object array copy but we can't emit the card marking
2062 // that is needed
2063 return false;
2064 }
2065
2066 value_type = ary_src->elem();
2067
2068 base_src = src;
2069 base_dest = dest;
2070
2071 uint shift = exact_log2(type2aelembytes(dest_elem));
2072 uint header = arrayOopDesc::base_offset_in_bytes(dest_elem);
2073
2074 adr_src = src;
2075 adr_dest = dest;
2076
2077 src_offset = conv_I2X_offset(phase, src_offset, ary_src);
2078 dest_offset = conv_I2X_offset(phase, dest_offset, ary_src);
2079
2080 Node* src_scale = phase->transform(new LShiftXNode(src_offset, phase->intcon(shift)));
2081 Node* dest_scale = phase->transform(new LShiftXNode(dest_offset, phase->intcon(shift)));
2082
2083 adr_src = phase->transform(new AddPNode(base_src, adr_src, src_scale));
2084 adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, dest_scale));
2085
2086 adr_src = new AddPNode(base_src, adr_src, phase->MakeConX(header));
2087 adr_dest = new AddPNode(base_dest, adr_dest, phase->MakeConX(header));
2088
2089 adr_src = phase->transform(adr_src);
2090 adr_dest = phase->transform(adr_dest);
2091
2092 copy_type = dest_elem;
2093 } else {
2094 assert (is_clonebasic(), "should be");
2095
2096 disjoint_bases = true;
2097 assert(src->is_AddP(), "should be base + off");
2098 assert(dest->is_AddP(), "should be base + off");
2099 adr_src = src;
2100 base_src = src->in(AddPNode::Base);
2101 adr_dest = dest;
2102 base_dest = dest->in(AddPNode::Base);
2103
2104 assert(phase->type(src->in(AddPNode::Offset))->is_intptr_t()->get_con() == phase->type(dest->in(AddPNode::Offset))->is_intptr_t()->get_con(), "same start offset?");
2105 BasicType elem = ary_src->klass()->as_array_klass()->element_type()->basic_type();
2106 if (elem == T_ARRAY) elem = T_OBJECT;
2107
2108 int diff = arrayOopDesc::base_offset_in_bytes(elem) - phase->type(src->in(AddPNode::Offset))->is_intptr_t()->get_con();
2109 assert(diff >= 0, "clone should not start after 1st array element");
2110 if (diff > 0) {
2111 adr_src = phase->transform(new AddPNode(base_src, adr_src, phase->MakeConX(diff)));
2112 adr_dest = phase->transform(new AddPNode(base_dest, adr_dest, phase->MakeConX(diff)));
2113 }
2114
2115 copy_type = elem;
2116 value_type = ary_src->elem();
2117 }
2118 return true;
2119 }
2120
2121 const TypePtr* ArrayCopyNode::get_address_type(PhaseGVN *phase, Node* n) {
2122 const Type* at = phase->type(n);
2123 assert(at != Type::TOP, "unexpected type");
2124 const TypePtr* atp = at->isa_ptr();
2125 // adjust atp to be the correct array element address type
2126 atp = atp->add_offset(Type::OffsetBot);
2127 return atp;
2128 }
2129
2130 void ArrayCopyNode::array_copy_test_overlap(PhaseGVN *phase, bool can_reshape, bool disjoint_bases, Node*& forward_ctl, Node*& backward_ctl) {
2131 Node* ctl = in(TypeFunc::Control);
2132 if (!disjoint_bases) {
2133 Node* src_offset = in(ArrayCopyNode::SrcPos);
2134 Node* dest_offset = in(ArrayCopyNode::DestPos);
2135 assert(src_offset != NULL && dest_offset != NULL, "should be");
2136 Node* cmp = phase->transform(new CmpINode(src_offset, dest_offset));
2137 Node *bol = phase->transform(new BoolNode(cmp, BoolTest::lt));
2138 IfNode *iff = new IfNode(ctl, bol, PROB_FAIR, COUNT_UNKNOWN);
2139
2140 phase->transform(iff);
2141
2142 forward_ctl = phase->transform(new IfFalseNode(iff));
2143 backward_ctl = phase->transform(new IfTrueNode(iff));
2144 } else {
2145 forward_ctl = ctl;
2146 }
2147 }
2148
2149 Node* ArrayCopyNode::array_copy_forward(PhaseGVN *phase,
2150 bool can_reshape,
2151 Node* forward_ctl,
2152 Node* start_mem_src,
2153 Node* start_mem_dest,
2154 const TypePtr* atp_src,
2155 const TypePtr* atp_dest,
2156 Node* adr_src,
2157 Node* base_src,
2158 Node* adr_dest,
2159 Node* base_dest,
2160 BasicType copy_type,
2161 const Type* value_type,
2162 int count) {
2163 Node* mem = phase->C->top();
2164 if (!forward_ctl->is_top()) {
2165 // copy forward
2166 mem = start_mem_dest;
2167
2168 if (count > 0) {
2169 Node* v = LoadNode::make(*phase, forward_ctl, start_mem_src, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
2170 v = phase->transform(v);
2171 mem = StoreNode::make(*phase, forward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
2172 mem = phase->transform(mem);
2173 for (int i = 1; i < count; i++) {
2174 Node* off = phase->MakeConX(type2aelembytes(copy_type) * i);
2175 Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
2176 Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
2177 v = LoadNode::make(*phase, forward_ctl, mem, next_src, atp_src, value_type, copy_type, MemNode::unordered);
2178 v = phase->transform(v);
2179 mem = StoreNode::make(*phase, forward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
2180 mem = phase->transform(mem);
2181 }
2182 } else if(can_reshape) {
2183 PhaseIterGVN* igvn = phase->is_IterGVN();
2184 igvn->_worklist.push(adr_src);
2185 igvn->_worklist.push(adr_dest);
2186 }
2187 }
2188 return mem;
2189 }
2190
2191 Node* ArrayCopyNode::array_copy_backward(PhaseGVN *phase,
2192 bool can_reshape,
2193 Node* backward_ctl,
2194 Node* start_mem_src,
2195 Node* start_mem_dest,
2196 const TypePtr* atp_src,
2197 const TypePtr* atp_dest,
2198 Node* adr_src,
2199 Node* base_src,
2200 Node* adr_dest,
2201 Node* base_dest,
2202 BasicType copy_type,
2203 const Type* value_type,
2204 int count) {
2205 Node* mem = phase->C->top();
2206 if (!backward_ctl->is_top()) {
2207 // copy backward
2208 mem = start_mem_dest;
2209
2210 if (count > 0) {
2211 for (int i = count-1; i >= 1; i--) {
2212 Node* off = phase->MakeConX(type2aelembytes(copy_type) * i);
2213 Node* next_src = phase->transform(new AddPNode(base_src,adr_src,off));
2214 Node* next_dest = phase->transform(new AddPNode(base_dest,adr_dest,off));
2215 Node* v = LoadNode::make(*phase, backward_ctl, mem, next_src, atp_src, value_type, copy_type, MemNode::unordered);
2216 v = phase->transform(v);
2217 mem = StoreNode::make(*phase, backward_ctl,mem,next_dest,atp_dest,v, copy_type, MemNode::unordered);
2218 mem = phase->transform(mem);
2219 }
2220 Node* v = LoadNode::make(*phase, backward_ctl, mem, adr_src, atp_src, value_type, copy_type, MemNode::unordered);
2221 v = phase->transform(v);
2222 mem = StoreNode::make(*phase, backward_ctl, mem, adr_dest, atp_dest, v, copy_type, MemNode::unordered);
2223 mem = phase->transform(mem);
2224 } else if(can_reshape) {
2225 PhaseIterGVN* igvn = phase->is_IterGVN();
2226 igvn->_worklist.push(adr_src);
2227 igvn->_worklist.push(adr_dest);
2228 }
2229 }
2230 return mem;
2231 }
2232
2233 bool ArrayCopyNode::finish_transform(PhaseGVN *phase, bool can_reshape,
2234 Node* ctl, Node *mem) {
2235 if (can_reshape) {
2236 PhaseIterGVN* igvn = phase->is_IterGVN();
2237 igvn->set_delay_transform(false);
2238 if (is_clonebasic()) {
2239 Node* out_mem = proj_out(TypeFunc::Memory);
2240
2241 if (out_mem->outcnt() != 1 || !out_mem->raw_out(0)->is_MergeMem() ||
2242 out_mem->raw_out(0)->outcnt() != 1 || !out_mem->raw_out(0)->raw_out(0)->is_MemBar()) {
2243 assert(!GraphKit::use_ReduceInitialCardMarks(), "can only happen with card marking");
2244 return false;
2245 }
2246
2247 igvn->replace_node(out_mem->raw_out(0), mem);
2248
2249 Node* out_ctl = proj_out(TypeFunc::Control);
2250 igvn->replace_node(out_ctl, ctl);
2251 } else {
2252 // replace fallthrough projections of the ArrayCopyNode by the
2253 // new memory, control and the input IO.
2254 CallProjections callprojs;
2255 extract_projections(&callprojs, true);
2256
2257 igvn->replace_node(callprojs.fallthrough_ioproj, in(TypeFunc::I_O));
2258 igvn->replace_node(callprojs.fallthrough_memproj, mem);
2259 igvn->replace_node(callprojs.fallthrough_catchproj, ctl);
2260
2261 // The ArrayCopyNode is not disconnected. It still has the
2262 // projections for the exception case. Replace current
2263 // ArrayCopyNode with a dummy new one with a top() control so
2264 // that this part of the graph stays consistent but is
2265 // eventually removed.
2266
2267 set_req(0, phase->C->top());
2268 remove_dead_region(phase, can_reshape);
2269 }
2270 } else {
2271 if (in(TypeFunc::Control) != ctl) {
2272 // we can't return new memory and control from Ideal at parse time
2273 assert(!is_clonebasic(), "added control for clone?");
2274 return NULL;
2275 }
2276 }
2277 return true;
2278 }
2279
2280
2281 Node *ArrayCopyNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2282 if (StressArrayCopyMacroNode && !can_reshape) return NULL;
2283
2284 // See if it's a small array copy and we can inline it as
2285 // loads/stores
2286 // Here we can only do:
2287 // - arraycopy if all arguments were validated before and we don't
2288 // need card marking
2289 // - clone for which we don't need to do card marking
2290
2291 if (!is_clonebasic() && !is_arraycopy_validated() &&
2292 !is_copyofrange_validated() && !is_copyof_validated()) {
2293 return NULL;
2294 }
2295
2296 if (in(TypeFunc::Control)->is_top() || in(TypeFunc::Memory)->is_top()) {
2297 return NULL;
2298 }
2299
2300 int count = get_count(phase);
2301
2302 if (count < 0 || count > ArrayCopyLoadStoreMaxElem) {
2303 return NULL;
2304 }
2305
2306 Node* mem = try_clone_instance(phase, can_reshape, count);
2307 if (mem != NULL) {
2308 return mem;
2309 }
2310
2311 Node* adr_src = NULL;
2312 Node* base_src = NULL;
2313 Node* adr_dest = NULL;
2314 Node* base_dest = NULL;
2315 BasicType copy_type = T_ILLEGAL;
2316 const Type* value_type = NULL;
2317 bool disjoint_bases = false;
2318
2319 if (!prepare_array_copy(phase, can_reshape,
2320 adr_src, base_src, adr_dest, base_dest,
2321 copy_type, value_type, disjoint_bases)) {
2322 return NULL;
2323 }
2324
2325 Node* src = in(ArrayCopyNode::Src);
2326 Node* dest = in(ArrayCopyNode::Dest);
2327 const TypePtr* atp_src = get_address_type(phase, src);
2328 const TypePtr* atp_dest = get_address_type(phase, dest);
2329 uint alias_idx_src = phase->C->get_alias_index(atp_src);
2330 uint alias_idx_dest = phase->C->get_alias_index(atp_dest);
2331
2332 Node *in_mem = in(TypeFunc::Memory);
2333 Node *start_mem_src = in_mem;
2334 Node *start_mem_dest = in_mem;
2335 if (in_mem->is_MergeMem()) {
2336 start_mem_src = in_mem->as_MergeMem()->memory_at(alias_idx_src);
2337 start_mem_dest = in_mem->as_MergeMem()->memory_at(alias_idx_dest);
2338 }
2339
2340
2341 if (can_reshape) {
2342 assert(!phase->is_IterGVN()->delay_transform(), "cannot delay transforms");
2343 phase->is_IterGVN()->set_delay_transform(true);
2344 }
2345
2346 Node* backward_ctl = phase->C->top();
2347 Node* forward_ctl = phase->C->top();
2348 array_copy_test_overlap(phase, can_reshape, disjoint_bases, forward_ctl, backward_ctl);
2349
2350 Node* forward_mem = array_copy_forward(phase, can_reshape, forward_ctl,
2351 start_mem_src, start_mem_dest,
2352 atp_src, atp_dest,
2353 adr_src, base_src, adr_dest, base_dest,
2354 copy_type, value_type, count);
2355
2356 Node* backward_mem = array_copy_backward(phase, can_reshape, backward_ctl,
2357 start_mem_src, start_mem_dest,
2358 atp_src, atp_dest,
2359 adr_src, base_src, adr_dest, base_dest,
2360 copy_type, value_type, count);
2361
2362 Node* ctl = NULL;
2363 if (!forward_ctl->is_top() && !backward_ctl->is_top()) {
2364 ctl = new RegionNode(3);
2365 mem = new PhiNode(ctl, Type::MEMORY, atp_dest);
2366 ctl->init_req(1, forward_ctl);
2367 mem->init_req(1, forward_mem);
2368 ctl->init_req(2, backward_ctl);
2369 mem->init_req(2, backward_mem);
2370 ctl = phase->transform(ctl);
2371 mem = phase->transform(mem);
2372 } else if (!forward_ctl->is_top()) {
2373 ctl = forward_ctl;
2374 mem = forward_mem;
2375 } else {
2376 assert(!backward_ctl->is_top(), "no copy?");
2377 ctl = backward_ctl;
2378 mem = backward_mem;
2379 }
2380
2381 if (can_reshape) {
2382 assert(phase->is_IterGVN()->delay_transform(), "should be delaying transforms");
2383 phase->is_IterGVN()->set_delay_transform(false);
2384 }
2385
2386 MergeMemNode* out_mem = MergeMemNode::make(in_mem);
2387 out_mem->set_memory_at(alias_idx_dest, mem);
2388 mem = out_mem;
2389
2390 if (!finish_transform(phase, can_reshape, ctl, mem)) {
2391 return NULL;
2392 }
2393
2394 return mem;
2395 }
|