27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "memory/allocation.inline.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "oops/objArrayKlass.hpp"
33 #include "opto/addnode.hpp"
34 #include "opto/arraycopynode.hpp"
35 #include "opto/cfgnode.hpp"
36 #include "opto/compile.hpp"
37 #include "opto/connode.hpp"
38 #include "opto/convertnode.hpp"
39 #include "opto/loopnode.hpp"
40 #include "opto/machnode.hpp"
41 #include "opto/matcher.hpp"
42 #include "opto/memnode.hpp"
43 #include "opto/mulnode.hpp"
44 #include "opto/narrowptrnode.hpp"
45 #include "opto/phaseX.hpp"
46 #include "opto/regmask.hpp"
47 #include "utilities/align.hpp"
48 #include "utilities/copy.hpp"
49 #include "utilities/macros.hpp"
50 #include "utilities/vmError.hpp"
51 #if INCLUDE_ZGC
52 #include "gc/z/c2/zBarrierSetC2.hpp"
53 #endif
54
55 // Portions of code courtesy of Clifford Click
56
57 // Optimization - Graph Style
58
59 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
60
61 //=============================================================================
62 uint MemNode::size_of() const { return sizeof(*this); }
63
64 const TypePtr *MemNode::adr_type() const {
65 Node* adr = in(Address);
66 if (adr == NULL) return NULL; // node is dead
801 "use LoadKlassNode instead");
802 assert(!(adr_type->isa_aryptr() &&
803 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
804 "use LoadRangeNode instead");
805 // Check control edge of raw loads
806 assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
807 // oop will be recorded in oop map if load crosses safepoint
808 rt->isa_oopptr() || is_immutable_value(adr),
809 "raw memory operations should have control edge");
810 LoadNode* load = NULL;
811 switch (bt) {
812 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
813 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
814 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
815 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
816 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
817 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency); break;
818 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
819 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
820 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
821 case T_OBJECT:
822 #ifdef _LP64
823 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
824 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
825 } else
826 #endif
827 {
828 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
829 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
830 }
831 break;
832 default:
833 ShouldNotReachHere();
834 break;
835 }
836 assert(load != NULL, "LoadNode should have been created");
837 if (unaligned) {
838 load->set_unaligned_access();
839 }
840 if (mismatched) {
1063 // In other words, we are loading from a casted version of
1064 // the same pointer-and-offset that we stored to.
1065 // Thus, we are able to replace L by V.
1066 }
1067 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1068 if (store_Opcode() != st->Opcode())
1069 return NULL;
1070 return st->in(MemNode::ValueIn);
1071 }
1072
1073 // A load from a freshly-created object always returns zero.
1074 // (This can happen after LoadNode::Ideal resets the load's memory input
1075 // to find_captured_store, which returned InitializeNode::zero_memory.)
1076 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1077 (st->in(0) == ld_alloc) &&
1078 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1079 // return a zero value for the load's basic type
1080 // (This is one of the few places where a generic PhaseTransform
1081 // can create new nodes. Think of it as lazily manifesting
1082 // virtually pre-existing constants.)
1083 return phase->zerocon(memory_type());
1084 }
1085
1086 // A load from an initialization barrier can match a captured store.
1087 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1088 InitializeNode* init = st->in(0)->as_Initialize();
1089 AllocateNode* alloc = init->allocation();
1090 if ((alloc != NULL) && (alloc == ld_alloc)) {
1091 // examine a captured store value
1092 st = init->find_captured_store(ld_off, memory_size(), phase);
1093 if (st != NULL) {
1094 continue; // take one more trip around
1095 }
1096 }
1097 }
1098
1099 // Load boxed value from result of valueOf() call is input parameter.
1100 if (this->is_Load() && ld_adr->is_AddP() &&
1101 (tp != NULL) && tp->is_ptr_to_boxed_value()) {
1102 intptr_t ignore = 0;
1120 //----------------------is_instance_field_load_with_local_phi------------------
1121 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1122 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1123 in(Address)->is_AddP() ) {
1124 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1125 // Only instances and boxed values.
1126 if( t_oop != NULL &&
1127 (t_oop->is_ptr_to_boxed_value() ||
1128 t_oop->is_known_instance_field()) &&
1129 t_oop->offset() != Type::OffsetBot &&
1130 t_oop->offset() != Type::OffsetTop) {
1131 return true;
1132 }
1133 }
1134 return false;
1135 }
1136
1137 //------------------------------Identity---------------------------------------
1138 // Loads are identity if previous store is to same address
1139 Node* LoadNode::Identity(PhaseGVN* phase) {
1140 // If the previous store-maker is the right kind of Store, and the store is
1141 // to the same address, then we are equal to the value stored.
1142 Node* mem = in(Memory);
1143 Node* value = can_see_stored_value(mem, phase);
1144 if( value ) {
1145 // byte, short & char stores truncate naturally.
1146 // A load has to load the truncated value which requires
1147 // some sort of masking operation and that requires an
1148 // Ideal call instead of an Identity call.
1149 if (memory_size() < BytesPerInt) {
1150 // If the input to the store does not fit with the load's result type,
1151 // it must be truncated via an Ideal call.
1152 if (!phase->type(value)->higher_equal(phase->type(this)))
1153 return this;
1154 }
1155 // (This works even when value is a Con, but LoadNode::Value
1156 // usually runs first, producing the singleton type of the Con.)
1157 return value;
1158 }
1159
1647 // fold up, do so.
1648 Node* prev_mem = find_previous_store(phase);
1649 if (prev_mem != NULL) {
1650 Node* value = can_see_arraycopy_value(prev_mem, phase);
1651 if (value != NULL) {
1652 return value;
1653 }
1654 }
1655 // Steps (a), (b): Walk past independent stores to find an exact match.
1656 if (prev_mem != NULL && prev_mem != in(MemNode::Memory)) {
1657 // (c) See if we can fold up on the spot, but don't fold up here.
1658 // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
1659 // just return a prior value, which is done by Identity calls.
1660 if (can_see_stored_value(prev_mem, phase)) {
1661 // Make ready for step (d):
1662 set_req(MemNode::Memory, prev_mem);
1663 return this;
1664 }
1665 }
1666
1667 return progress ? this : NULL;
1668 }
1669
1670 // Helper to recognize certain Klass fields which are invariant across
1671 // some group of array types (e.g., int[] or all T[] where T < Object).
1672 const Type*
1673 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
1674 ciKlass* klass) const {
1675 if (tkls->offset() == in_bytes(Klass::modifier_flags_offset())) {
1676 // The field is Klass::_modifier_flags. Return its (constant) value.
1677 // (Folds up the 2nd indirection in aClassConstant.getModifiers().)
1678 assert(this->Opcode() == Op_LoadI, "must load an int from _modifier_flags");
1679 return TypeInt::make(klass->modifier_flags());
1680 }
1681 if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
1682 // The field is Klass::_access_flags. Return its (constant) value.
1683 // (Folds up the 2nd indirection in Reflection.getClassAccessFlags(aClassConstant).)
1684 assert(this->Opcode() == Op_LoadI, "must load an int from _access_flags");
1685 return TypeInt::make(klass->access_flags());
1686 }
1736 }
1737 }
1738
1739 // Don't do this for integer types. There is only potential profit if
1740 // the element type t is lower than _type; that is, for int types, if _type is
1741 // more restrictive than t. This only happens here if one is short and the other
1742 // char (both 16 bits), and in those cases we've made an intentional decision
1743 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1744 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1745 //
1746 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1747 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
1748 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1749 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
1750 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1751 // In fact, that could have been the original type of p1, and p1 could have
1752 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1753 // expression (LShiftL quux 3) independently optimized to the constant 8.
1754 if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1755 && (_type->isa_vect() == NULL)
1756 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1757 // t might actually be lower than _type, if _type is a unique
1758 // concrete subclass of abstract class t.
1759 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
1760 const Type* jt = t->join_speculative(_type);
1761 // In any case, do not allow the join, per se, to empty out the type.
1762 if (jt->empty() && !t->empty()) {
1763 // This can happen if a interface-typed array narrows to a class type.
1764 jt = _type;
1765 }
1766 #ifdef ASSERT
1767 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1768 // The pointers in the autobox arrays are always non-null
1769 Node* base = adr->in(AddPNode::Base);
1770 if ((base != NULL) && base->is_DecodeN()) {
1771 // Get LoadN node which loads IntegerCache.cache field
1772 base = base->in(1);
1773 }
1774 if ((base != NULL) && base->is_Con()) {
1775 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1776 if ((base_type != NULL) && base_type->is_autobox_cache()) {
1777 // It could be narrow oop
1778 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1779 }
1780 }
1781 }
1782 #endif
1783 return jt;
1784 }
1785 }
1786 } else if (tp->base() == Type::InstPtr) {
1787 assert( off != Type::OffsetBot ||
1788 // arrays can be cast to Objects
1789 tp->is_oopptr()->klass()->is_java_lang_Object() ||
1790 // unsafe field access may not have a constant offset
1791 C->has_unsafe_access(),
1792 "Field accesses must be precise" );
1793 // For oop loads, we expect the _type to be precise.
1794
1795 // Optimize loads from constant fields.
1796 const TypeInstPtr* tinst = tp->is_instptr();
1797 ciObject* const_oop = tinst->const_oop();
1798 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
1799 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), memory_type());
1800 if (con_type != NULL) {
1801 return con_type;
1802 }
1803 }
1804 } else if (tp->base() == Type::KlassPtr) {
1805 assert( off != Type::OffsetBot ||
1806 // arrays can be cast to Objects
1807 tp->is_klassptr()->klass()->is_java_lang_Object() ||
1808 // also allow array-loading from the primary supertype
1809 // array during subtype checks
1810 Opcode() == Op_LoadKlass,
1811 "Field accesses must be precise" );
1812 // For klass/static loads, we expect the _type to be precise
1813 } else if (tp->base() == Type::RawPtr && adr->is_Load() && off == 0) {
1814 /* With mirrors being an indirect in the Klass*
1815 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
1816 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
1817 *
1818 * So check the type and klass of the node before the LoadP.
1819 */
1820 Node* adr2 = adr->in(MemNode::Address);
1821 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
1822 if (tkls != NULL && !StressReflectiveCode) {
1823 ciKlass* klass = tkls->klass();
1824 if (klass->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
1825 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1826 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1827 return TypeInstPtr::make(klass->java_mirror());
1828 }
1829 }
1830 }
1831
1832 const TypeKlassPtr *tkls = tp->isa_klassptr();
1833 if (tkls != NULL && !StressReflectiveCode) {
1834 ciKlass* klass = tkls->klass();
1835 if (klass->is_loaded() && tkls->klass_is_exact()) {
1836 // We are loading a field from a Klass metaobject whose identity
1837 // is known at compile time (the type is "exact" or "precise").
1838 // Check for fields we know are maintained as constants by the VM.
1839 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
1840 // The field is Klass::_super_check_offset. Return its (constant) value.
1841 // (Folds up type checking code.)
1842 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
1843 return TypeInt::make(klass->super_check_offset());
1844 }
1845 // Compute index into primary_supers array
1846 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1847 // Check for overflowing; use unsigned compare to handle the negative case.
1848 if( depth < ciKlass::primary_super_limit() ) {
1849 // The field is an element of Klass::_primary_supers. Return its (constant) value.
1850 // (Folds up type checking code.)
1851 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1852 ciKlass *ss = klass->super_of_depth(depth);
1853 return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1854 }
1855 const Type* aift = load_array_final_field(tkls, klass);
1856 if (aift != NULL) return aift;
1857 }
1858
1859 // We can still check if we are loading from the primary_supers array at a
1860 // shallow enough depth. Even though the klass is not exact, entries less
1861 // than or equal to its super depth are correct.
1862 if (klass->is_loaded() ) {
1863 ciType *inner = klass;
1864 while( inner->is_obj_array_klass() )
1865 inner = inner->as_obj_array_klass()->base_element_type();
1866 if( inner->is_instance_klass() &&
1867 !inner->as_instance_klass()->flags().is_interface() ) {
1868 // Compute index into primary_supers array
1869 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1870 // Check for overflowing; use unsigned compare to handle the negative case.
1871 if( depth < ciKlass::primary_super_limit() &&
1872 depth <= klass->super_depth() ) { // allow self-depth checks to handle self-check case
1873 // The field is an element of Klass::_primary_supers. Return its (constant) value.
1874 // (Folds up type checking code.)
1875 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1876 ciKlass *ss = klass->super_of_depth(depth);
1877 return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1878 }
1879 }
1880 }
1881
1882 // If the type is enough to determine that the thing is not an array,
2129 if( !ik->is_loaded() )
2130 return _type; // Bail out if not loaded
2131 if (offset == oopDesc::klass_offset_in_bytes()) {
2132 if (tinst->klass_is_exact()) {
2133 return TypeKlassPtr::make(ik);
2134 }
2135 // See if we can become precise: no subklasses and no interface
2136 // (Note: We need to support verified interfaces.)
2137 if (!ik->is_interface() && !ik->has_subklass()) {
2138 //assert(!UseExactTypes, "this code should be useless with exact types");
2139 // Add a dependence; if any subclass added we need to recompile
2140 if (!ik->is_final()) {
2141 // %%% should use stronger assert_unique_concrete_subtype instead
2142 phase->C->dependencies()->assert_leaf_type(ik);
2143 }
2144 // Return precise klass
2145 return TypeKlassPtr::make(ik);
2146 }
2147
2148 // Return root of possible klass
2149 return TypeKlassPtr::make(TypePtr::NotNull, ik, 0/*offset*/);
2150 }
2151 }
2152
2153 // Check for loading klass from an array
2154 const TypeAryPtr *tary = tp->isa_aryptr();
2155 if( tary != NULL ) {
2156 ciKlass *tary_klass = tary->klass();
2157 if (tary_klass != NULL // can be NULL when at BOTTOM or TOP
2158 && tary->offset() == oopDesc::klass_offset_in_bytes()) {
2159 if (tary->klass_is_exact()) {
2160 return TypeKlassPtr::make(tary_klass);
2161 }
2162 ciArrayKlass *ak = tary->klass()->as_array_klass();
2163 // If the klass is an object array, we defer the question to the
2164 // array component klass.
2165 if( ak->is_obj_array_klass() ) {
2166 assert( ak->is_loaded(), "" );
2167 ciKlass *base_k = ak->as_obj_array_klass()->base_element_klass();
2168 if( base_k->is_loaded() && base_k->is_instance_klass() ) {
2169 ciInstanceKlass* ik = base_k->as_instance_klass();
2170 // See if we can become precise: no subklasses and no interface
2171 if (!ik->is_interface() && !ik->has_subklass()) {
2172 //assert(!UseExactTypes, "this code should be useless with exact types");
2173 // Add a dependence; if any subclass added we need to recompile
2174 if (!ik->is_final()) {
2175 phase->C->dependencies()->assert_leaf_type(ik);
2176 }
2177 // Return precise array klass
2178 return TypeKlassPtr::make(ak);
2179 }
2180 }
2181 return TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/);
2182 } else { // Found a type-array?
2183 //assert(!UseExactTypes, "this code should be useless with exact types");
2184 assert( ak->is_type_array_klass(), "" );
2185 return TypeKlassPtr::make(ak); // These are always precise
2186 }
2187 }
2188 }
2189
2190 // Check for loading klass from an array klass
2191 const TypeKlassPtr *tkls = tp->isa_klassptr();
2192 if (tkls != NULL && !StressReflectiveCode) {
2193 ciKlass* klass = tkls->klass();
2194 if( !klass->is_loaded() )
2195 return _type; // Bail out if not loaded
2196 if( klass->is_obj_array_klass() &&
2197 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2198 ciKlass* elem = klass->as_obj_array_klass()->element_klass();
2199 // // Always returning precise element type is incorrect,
2200 // // e.g., element type could be object and array may contain strings
2201 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2202
2203 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2204 // according to the element type's subclassing.
2205 return TypeKlassPtr::make(tkls->ptr(), elem, 0/*offset*/);
2206 }
2207 if( klass->is_instance_klass() && tkls->klass_is_exact() &&
2208 tkls->offset() == in_bytes(Klass::super_offset())) {
2209 ciKlass* sup = klass->as_instance_klass()->super();
2210 // The field is Klass::_super. Return its (constant) value.
2211 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2212 return sup ? TypeKlassPtr::make(sup) : TypePtr::NULL_PTR;
2213 }
2214 }
2215
2216 // Bailout case
2217 return LoadNode::Value(phase);
2218 }
2219
2220 //------------------------------Identity---------------------------------------
2221 // To clean up reflective code, simplify k.java_mirror.as_klass to plain k.
2222 // Also feed through the klass in Allocate(...klass...)._klass.
2223 Node* LoadKlassNode::Identity(PhaseGVN* phase) {
2224 return klass_identity_common(phase);
2225 }
2393 //=============================================================================
2394 //---------------------------StoreNode::make-----------------------------------
2395 // Polymorphic factory method:
2396 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo) {
2397 assert((mo == unordered || mo == release), "unexpected");
2398 Compile* C = gvn.C;
2399 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2400 ctl != NULL, "raw memory operations should have control edge");
2401
2402 switch (bt) {
2403 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2404 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2405 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2406 case T_CHAR:
2407 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2408 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo);
2409 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2410 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo);
2411 case T_METADATA:
2412 case T_ADDRESS:
2413 case T_OBJECT:
2414 #ifdef _LP64
2415 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2416 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2417 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2418 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2419 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2420 adr->bottom_type()->isa_rawptr())) {
2421 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2422 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2423 }
2424 #endif
2425 {
2426 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2427 }
2428 default:
2429 ShouldNotReachHere();
2430 return (StoreNode*)NULL;
2431 }
2432 }
2471 // since they must follow each StoreP operation. Redundant StoreCMs
2472 // are eliminated just before matching in final_graph_reshape.
2473 {
2474 Node* st = mem;
2475 // If Store 'st' has more than one use, we cannot fold 'st' away.
2476 // For example, 'st' might be the final state at a conditional
2477 // return. Or, 'st' might be used by some node which is live at
2478 // the same time 'st' is live, which might be unschedulable. So,
2479 // require exactly ONE user until such time as we clone 'mem' for
2480 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2481 // true).
2482 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2483 // Looking at a dead closed cycle of memory?
2484 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2485 assert(Opcode() == st->Opcode() ||
2486 st->Opcode() == Op_StoreVector ||
2487 Opcode() == Op_StoreVector ||
2488 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2489 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2490 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2491 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2492 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2493
2494 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2495 st->as_Store()->memory_size() <= this->memory_size()) {
2496 Node* use = st->raw_out(0);
2497 phase->igvn_rehash_node_delayed(use);
2498 if (can_reshape) {
2499 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase->is_IterGVN());
2500 } else {
2501 // It's OK to do this in the parser, since DU info is always accurate,
2502 // and the parser always refers to nodes via SafePointNode maps.
2503 use->set_req(MemNode::Memory, st->in(MemNode::Memory));
2504 }
2505 return this;
2506 }
2507 st = st->in(MemNode::Memory);
2508 }
2509 }
2510
2556 // Load then Store? Then the Store is useless
2557 if (val->is_Load() &&
2558 val->in(MemNode::Address)->eqv_uncast(adr) &&
2559 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2560 val->as_Load()->store_Opcode() == Opcode()) {
2561 result = mem;
2562 }
2563
2564 // Two stores in a row of the same value?
2565 if (result == this &&
2566 mem->is_Store() &&
2567 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2568 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2569 mem->Opcode() == Opcode()) {
2570 result = mem;
2571 }
2572
2573 // Store of zero anywhere into a freshly-allocated object?
2574 // Then the store is useless.
2575 // (It must already have been captured by the InitializeNode.)
2576 if (result == this &&
2577 ReduceFieldZeroing && phase->type(val)->is_zero_type()) {
2578 // a newly allocated object is already all-zeroes everywhere
2579 if (mem->is_Proj() && mem->in(0)->is_Allocate()) {
2580 result = mem;
2581 }
2582
2583 if (result == this) {
2584 // the store may also apply to zero-bits in an earlier object
2585 Node* prev_mem = find_previous_store(phase);
2586 // Steps (a), (b): Walk past independent stores to find an exact match.
2587 if (prev_mem != NULL) {
2588 Node* prev_val = can_see_stored_value(prev_mem, phase);
2589 if (prev_val != NULL && phase->eqv(prev_val, val)) {
2590 // prev_val and val might differ by a cast; it would be good
2591 // to keep the more informative of the two.
2592 result = mem;
2593 }
2594 }
2595 }
2596 }
2597
2598 if (result != this && phase->is_IterGVN() != NULL) {
2599 MemBarNode* trailing = trailing_membar();
2600 if (trailing != NULL) {
2601 #ifdef ASSERT
2602 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2603 assert(t_oop == NULL || t_oop->is_known_instance_field(), "only for non escaping objects");
2604 #endif
2605 PhaseIterGVN* igvn = phase->is_IterGVN();
2606 trailing->remove(igvn);
2607 }
2608 }
2609
2610 return result;
2611 }
2612
2613 //------------------------------match_edge-------------------------------------
2614 // Do we Match on this edge index or not? Match only memory & value
2615 uint StoreNode::match_edge(uint idx) const {
2616 return idx == MemNode::Address || idx == MemNode::ValueIn;
2880 // Clearing a short array is faster with stores
2881 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2882 // Already know this is a large node, do not try to ideal it
2883 if (!IdealizeClearArrayNode || _is_large) return NULL;
2884
2885 const int unit = BytesPerLong;
2886 const TypeX* t = phase->type(in(2))->isa_intptr_t();
2887 if (!t) return NULL;
2888 if (!t->is_con()) return NULL;
2889 intptr_t raw_count = t->get_con();
2890 intptr_t size = raw_count;
2891 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
2892 // Clearing nothing uses the Identity call.
2893 // Negative clears are possible on dead ClearArrays
2894 // (see jck test stmt114.stmt11402.val).
2895 if (size <= 0 || size % unit != 0) return NULL;
2896 intptr_t count = size / unit;
2897 // Length too long; communicate this to matchers and assemblers.
2898 // Assemblers are responsible to produce fast hardware clears for it.
2899 if (size > InitArrayShortSize) {
2900 return new ClearArrayNode(in(0), in(1), in(2), in(3), true);
2901 }
2902 Node *mem = in(1);
2903 if( phase->type(mem)==Type::TOP ) return NULL;
2904 Node *adr = in(3);
2905 const Type* at = phase->type(adr);
2906 if( at==Type::TOP ) return NULL;
2907 const TypePtr* atp = at->isa_ptr();
2908 // adjust atp to be the correct array element address type
2909 if (atp == NULL) atp = TypePtr::BOTTOM;
2910 else atp = atp->add_offset(Type::OffsetBot);
2911 // Get base for derived pointer purposes
2912 if( adr->Opcode() != Op_AddP ) Unimplemented();
2913 Node *base = adr->in(1);
2914
2915 Node *zero = phase->makecon(TypeLong::ZERO);
2916 Node *off = phase->MakeConX(BytesPerLong);
2917 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
2918 count--;
2919 while( count-- ) {
2920 mem = phase->transform(mem);
2921 adr = phase->transform(new AddPNode(base,adr,off));
2922 mem = new StoreLNode(in(0),mem,adr,atp,zero,MemNode::unordered,false);
2923 }
2924 return mem;
2925 }
2926
2927 //----------------------------step_through----------------------------------
2928 // Return allocation input memory edge if it is different instance
2929 // or itself if it is the one we are looking for.
2930 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
2931 Node* n = *np;
2932 assert(n->is_ClearArray(), "sanity");
2933 intptr_t offset;
2934 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
2935 // This method is called only before Allocate nodes are expanded
2936 // during macro nodes expansion. Before that ClearArray nodes are
2937 // only generated in PhaseMacroExpand::generate_arraycopy() (before
2938 // Allocate nodes are expanded) which follows allocations.
2939 assert(alloc != NULL, "should have allocation");
2940 if (alloc->_idx == instance_id) {
2941 // Can not bypass initialization of the instance we are looking for.
2942 return false;
2943 }
2944 // Otherwise skip it.
2945 InitializeNode* init = alloc->initialization();
2946 if (init != NULL)
2947 *np = init->in(TypeFunc::Memory);
2948 else
2949 *np = alloc->in(TypeFunc::Memory);
2950 return true;
2951 }
2952
2953 //----------------------------clear_memory-------------------------------------
2954 // Generate code to initialize object storage to zero.
2955 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
2956 intptr_t start_offset,
2957 Node* end_offset,
2958 PhaseGVN* phase) {
2959 intptr_t offset = start_offset;
2960
2961 int unit = BytesPerLong;
2962 if ((offset % unit) != 0) {
2963 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
2964 adr = phase->transform(adr);
2965 const TypePtr* atp = TypeRawPtr::BOTTOM;
2966 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
2967 mem = phase->transform(mem);
2968 offset += BytesPerInt;
2969 }
2970 assert((offset % unit) == 0, "");
2971
2972 // Initialize the remaining stuff, if any, with a ClearArray.
2973 return clear_memory(ctl, mem, dest, phase->MakeConX(offset), end_offset, phase);
2974 }
2975
2976 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
2977 Node* start_offset,
2978 Node* end_offset,
2979 PhaseGVN* phase) {
2980 if (start_offset == end_offset) {
2981 // nothing to do
2982 return mem;
2983 }
2984
2985 int unit = BytesPerLong;
2986 Node* zbase = start_offset;
2987 Node* zend = end_offset;
2988
2989 // Scale to the unit required by the CPU:
2990 if (!Matcher::init_array_count_is_in_bytes) {
2991 Node* shift = phase->intcon(exact_log2(unit));
2992 zbase = phase->transform(new URShiftXNode(zbase, shift) );
2993 zend = phase->transform(new URShiftXNode(zend, shift) );
2994 }
2995
2996 // Bulk clear double-words
2997 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
2998 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
2999 mem = new ClearArrayNode(ctl, mem, zsize, adr, false);
3000 return phase->transform(mem);
3001 }
3002
3003 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3004 intptr_t start_offset,
3005 intptr_t end_offset,
3006 PhaseGVN* phase) {
3007 if (start_offset == end_offset) {
3008 // nothing to do
3009 return mem;
3010 }
3011
3012 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3013 intptr_t done_offset = end_offset;
3014 if ((done_offset % BytesPerLong) != 0) {
3015 done_offset -= BytesPerInt;
3016 }
3017 if (done_offset > start_offset) {
3018 mem = clear_memory(ctl, mem, dest,
3019 start_offset, phase->MakeConX(done_offset), phase);
3020 }
3021 if (done_offset < end_offset) { // emit the final 32-bit store
3022 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3023 adr = phase->transform(adr);
3024 const TypePtr* atp = TypeRawPtr::BOTTOM;
3025 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3026 mem = phase->transform(mem);
3027 done_offset += BytesPerInt;
3028 }
3029 assert(done_offset == end_offset, "");
3030 return mem;
3031 }
3032
3033 //=============================================================================
3034 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3035 : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3036 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3037 #ifdef ASSERT
3038 , _pair_idx(0)
3039 #endif
3040 {
3041 init_class_id(Class_MemBar);
3042 Node* top = C->top();
3043 init_req(TypeFunc::I_O,top);
3044 init_req(TypeFunc::FramePtr,top);
3045 init_req(TypeFunc::ReturnAdr,top);
3154 PhaseIterGVN* igvn = phase->is_IterGVN();
3155 remove(igvn);
3156 // Must return either the original node (now dead) or a new node
3157 // (Do not return a top here, since that would break the uniqueness of top.)
3158 return new ConINode(TypeInt::ZERO);
3159 }
3160 }
3161 return progress ? this : NULL;
3162 }
3163
3164 //------------------------------Value------------------------------------------
3165 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3166 if( !in(0) ) return Type::TOP;
3167 if( phase->type(in(0)) == Type::TOP )
3168 return Type::TOP;
3169 return TypeTuple::MEMBAR;
3170 }
3171
3172 //------------------------------match------------------------------------------
3173 // Construct projections for memory.
3174 Node *MemBarNode::match( const ProjNode *proj, const Matcher *m ) {
3175 switch (proj->_con) {
3176 case TypeFunc::Control:
3177 case TypeFunc::Memory:
3178 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3179 }
3180 ShouldNotReachHere();
3181 return NULL;
3182 }
3183
3184 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3185 trailing->_kind = TrailingStore;
3186 leading->_kind = LeadingStore;
3187 #ifdef ASSERT
3188 trailing->_pair_idx = leading->_idx;
3189 leading->_pair_idx = leading->_idx;
3190 #endif
3191 }
3192
3193 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3194 trailing->_kind = TrailingLoadStore;
3440 return (req() > RawStores);
3441 }
3442
3443 void InitializeNode::set_complete(PhaseGVN* phase) {
3444 assert(!is_complete(), "caller responsibility");
3445 _is_complete = Complete;
3446
3447 // After this node is complete, it contains a bunch of
3448 // raw-memory initializations. There is no need for
3449 // it to have anything to do with non-raw memory effects.
3450 // Therefore, tell all non-raw users to re-optimize themselves,
3451 // after skipping the memory effects of this initialization.
3452 PhaseIterGVN* igvn = phase->is_IterGVN();
3453 if (igvn) igvn->add_users_to_worklist(this);
3454 }
3455
3456 // convenience function
3457 // return false if the init contains any stores already
3458 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3459 InitializeNode* init = initialization();
3460 if (init == NULL || init->is_complete()) return false;
3461 init->remove_extra_zeroes();
3462 // for now, if this allocation has already collected any inits, bail:
3463 if (init->is_non_zero()) return false;
3464 init->set_complete(phase);
3465 return true;
3466 }
3467
3468 void InitializeNode::remove_extra_zeroes() {
3469 if (req() == RawStores) return;
3470 Node* zmem = zero_memory();
3471 uint fill = RawStores;
3472 for (uint i = fill; i < req(); i++) {
3473 Node* n = in(i);
3474 if (n->is_top() || n == zmem) continue; // skip
3475 if (fill < i) set_req(fill, n); // compact
3476 ++fill;
3477 }
3478 // delete any empty spaces created:
3479 while (fill < req()) {
3480 del_req(fill);
4183 // z's_done 12 16 16 16 12 16 12
4184 // z's_needed 12 16 16 16 16 16 16
4185 // zsize 0 0 0 0 4 0 4
4186 if (next_full_store < 0) {
4187 // Conservative tack: Zero to end of current word.
4188 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4189 } else {
4190 // Zero to beginning of next fully initialized word.
4191 // Or, don't zero at all, if we are already in that word.
4192 assert(next_full_store >= zeroes_needed, "must go forward");
4193 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4194 zeroes_needed = next_full_store;
4195 }
4196 }
4197
4198 if (zeroes_needed > zeroes_done) {
4199 intptr_t zsize = zeroes_needed - zeroes_done;
4200 // Do some incremental zeroing on rawmem, in parallel with inits.
4201 zeroes_done = align_down(zeroes_done, BytesPerInt);
4202 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4203 zeroes_done, zeroes_needed,
4204 phase);
4205 zeroes_done = zeroes_needed;
4206 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4207 do_zeroing = false; // leave the hole, next time
4208 }
4209 }
4210
4211 // Collect the store and move on:
4212 st->set_req(MemNode::Memory, inits);
4213 inits = st; // put it on the linearized chain
4214 set_req(i, zmem); // unhook from previous position
4215
4216 if (zeroes_done == st_off)
4217 zeroes_done = next_init_off;
4218
4219 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4220
4221 #ifdef ASSERT
4222 // Various order invariants. Weaker than stores_are_sane because
4242 remove_extra_zeroes(); // clear out all the zmems left over
4243 add_req(inits);
4244
4245 if (!(UseTLAB && ZeroTLAB)) {
4246 // If anything remains to be zeroed, zero it all now.
4247 zeroes_done = align_down(zeroes_done, BytesPerInt);
4248 // if it is the last unused 4 bytes of an instance, forget about it
4249 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4250 if (zeroes_done + BytesPerLong >= size_limit) {
4251 AllocateNode* alloc = allocation();
4252 assert(alloc != NULL, "must be present");
4253 if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4254 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4255 ciKlass* k = phase->type(klass_node)->is_klassptr()->klass();
4256 if (zeroes_done == k->layout_helper())
4257 zeroes_done = size_limit;
4258 }
4259 }
4260 if (zeroes_done < size_limit) {
4261 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4262 zeroes_done, size_in_bytes, phase);
4263 }
4264 }
4265
4266 set_complete(phase);
4267 return rawmem;
4268 }
4269
4270
4271 #ifdef ASSERT
4272 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4273 if (is_complete())
4274 return true; // stores could be anything at this point
4275 assert(allocation() != NULL, "must be present");
4276 intptr_t last_off = allocation()->minimum_header_size();
4277 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4278 Node* st = in(i);
4279 intptr_t st_off = get_store_offset(st, phase);
4280 if (st_off < 0) continue; // ignore dead garbage
4281 if (last_off > st_off) {
|
27 #include "compiler/compileLog.hpp"
28 #include "gc/shared/barrierSet.hpp"
29 #include "gc/shared/c2/barrierSetC2.hpp"
30 #include "memory/allocation.inline.hpp"
31 #include "memory/resourceArea.hpp"
32 #include "oops/objArrayKlass.hpp"
33 #include "opto/addnode.hpp"
34 #include "opto/arraycopynode.hpp"
35 #include "opto/cfgnode.hpp"
36 #include "opto/compile.hpp"
37 #include "opto/connode.hpp"
38 #include "opto/convertnode.hpp"
39 #include "opto/loopnode.hpp"
40 #include "opto/machnode.hpp"
41 #include "opto/matcher.hpp"
42 #include "opto/memnode.hpp"
43 #include "opto/mulnode.hpp"
44 #include "opto/narrowptrnode.hpp"
45 #include "opto/phaseX.hpp"
46 #include "opto/regmask.hpp"
47 #include "opto/valuetypenode.hpp"
48 #include "utilities/align.hpp"
49 #include "utilities/copy.hpp"
50 #include "utilities/macros.hpp"
51 #include "utilities/vmError.hpp"
52 #if INCLUDE_ZGC
53 #include "gc/z/c2/zBarrierSetC2.hpp"
54 #endif
55
56 // Portions of code courtesy of Clifford Click
57
58 // Optimization - Graph Style
59
60 static Node *step_through_mergemem(PhaseGVN *phase, MergeMemNode *mmem, const TypePtr *tp, const TypePtr *adr_check, outputStream *st);
61
62 //=============================================================================
63 uint MemNode::size_of() const { return sizeof(*this); }
64
65 const TypePtr *MemNode::adr_type() const {
66 Node* adr = in(Address);
67 if (adr == NULL) return NULL; // node is dead
802 "use LoadKlassNode instead");
803 assert(!(adr_type->isa_aryptr() &&
804 adr_type->offset() == arrayOopDesc::length_offset_in_bytes()),
805 "use LoadRangeNode instead");
806 // Check control edge of raw loads
807 assert( ctl != NULL || C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
808 // oop will be recorded in oop map if load crosses safepoint
809 rt->isa_oopptr() || is_immutable_value(adr),
810 "raw memory operations should have control edge");
811 LoadNode* load = NULL;
812 switch (bt) {
813 case T_BOOLEAN: load = new LoadUBNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
814 case T_BYTE: load = new LoadBNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
815 case T_INT: load = new LoadINode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
816 case T_CHAR: load = new LoadUSNode(ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
817 case T_SHORT: load = new LoadSNode (ctl, mem, adr, adr_type, rt->is_int(), mo, control_dependency); break;
818 case T_LONG: load = new LoadLNode (ctl, mem, adr, adr_type, rt->is_long(), mo, control_dependency); break;
819 case T_FLOAT: load = new LoadFNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
820 case T_DOUBLE: load = new LoadDNode (ctl, mem, adr, adr_type, rt, mo, control_dependency); break;
821 case T_ADDRESS: load = new LoadPNode (ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency); break;
822 case T_VALUETYPE:
823 case T_OBJECT:
824 #ifdef _LP64
825 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
826 load = new LoadNNode(ctl, mem, adr, adr_type, rt->make_narrowoop(), mo, control_dependency);
827 } else
828 #endif
829 {
830 assert(!adr->bottom_type()->is_ptr_to_narrowoop() && !adr->bottom_type()->is_ptr_to_narrowklass(), "should have got back a narrow oop");
831 load = new LoadPNode(ctl, mem, adr, adr_type, rt->is_ptr(), mo, control_dependency);
832 }
833 break;
834 default:
835 ShouldNotReachHere();
836 break;
837 }
838 assert(load != NULL, "LoadNode should have been created");
839 if (unaligned) {
840 load->set_unaligned_access();
841 }
842 if (mismatched) {
1065 // In other words, we are loading from a casted version of
1066 // the same pointer-and-offset that we stored to.
1067 // Thus, we are able to replace L by V.
1068 }
1069 // Now prove that we have a LoadQ matched to a StoreQ, for some Q.
1070 if (store_Opcode() != st->Opcode())
1071 return NULL;
1072 return st->in(MemNode::ValueIn);
1073 }
1074
1075 // A load from a freshly-created object always returns zero.
1076 // (This can happen after LoadNode::Ideal resets the load's memory input
1077 // to find_captured_store, which returned InitializeNode::zero_memory.)
1078 if (st->is_Proj() && st->in(0)->is_Allocate() &&
1079 (st->in(0) == ld_alloc) &&
1080 (ld_off >= st->in(0)->as_Allocate()->minimum_header_size())) {
1081 // return a zero value for the load's basic type
1082 // (This is one of the few places where a generic PhaseTransform
1083 // can create new nodes. Think of it as lazily manifesting
1084 // virtually pre-existing constants.)
1085 assert(memory_type() != T_VALUETYPE, "should not be used for value types");
1086 Node* default_value = ld_alloc->in(AllocateNode::DefaultValue);
1087 if (default_value != NULL) {
1088 return default_value;
1089 }
1090 assert(ld_alloc->in(AllocateNode::RawDefaultValue) == NULL, "default value may not be null");
1091 return phase->zerocon(memory_type());
1092 }
1093
1094 // A load from an initialization barrier can match a captured store.
1095 if (st->is_Proj() && st->in(0)->is_Initialize()) {
1096 InitializeNode* init = st->in(0)->as_Initialize();
1097 AllocateNode* alloc = init->allocation();
1098 if ((alloc != NULL) && (alloc == ld_alloc)) {
1099 // examine a captured store value
1100 st = init->find_captured_store(ld_off, memory_size(), phase);
1101 if (st != NULL) {
1102 continue; // take one more trip around
1103 }
1104 }
1105 }
1106
1107 // Load boxed value from result of valueOf() call is input parameter.
1108 if (this->is_Load() && ld_adr->is_AddP() &&
1109 (tp != NULL) && tp->is_ptr_to_boxed_value()) {
1110 intptr_t ignore = 0;
1128 //----------------------is_instance_field_load_with_local_phi------------------
1129 bool LoadNode::is_instance_field_load_with_local_phi(Node* ctrl) {
1130 if( in(Memory)->is_Phi() && in(Memory)->in(0) == ctrl &&
1131 in(Address)->is_AddP() ) {
1132 const TypeOopPtr* t_oop = in(Address)->bottom_type()->isa_oopptr();
1133 // Only instances and boxed values.
1134 if( t_oop != NULL &&
1135 (t_oop->is_ptr_to_boxed_value() ||
1136 t_oop->is_known_instance_field()) &&
1137 t_oop->offset() != Type::OffsetBot &&
1138 t_oop->offset() != Type::OffsetTop) {
1139 return true;
1140 }
1141 }
1142 return false;
1143 }
1144
1145 //------------------------------Identity---------------------------------------
1146 // Loads are identity if previous store is to same address
1147 Node* LoadNode::Identity(PhaseGVN* phase) {
1148 // Loading from a ValueTypePtr? The ValueTypePtr has the values of
1149 // all fields as input. Look for the field with matching offset.
1150 Node* addr = in(Address);
1151 intptr_t offset;
1152 Node* base = AddPNode::Ideal_base_and_offset(addr, phase, offset);
1153 if (base != NULL && base->is_ValueTypePtr() && offset > oopDesc::klass_offset_in_bytes()) {
1154 Node* value = base->as_ValueTypePtr()->field_value_by_offset((int)offset, true);
1155 if (value->is_ValueType()) {
1156 // Non-flattened value type field
1157 ValueTypeNode* vt = value->as_ValueType();
1158 if (vt->is_allocated(phase)) {
1159 value = vt->get_oop();
1160 } else {
1161 // Not yet allocated, bail out
1162 value = NULL;
1163 }
1164 }
1165 if (value != NULL) {
1166 if (Opcode() == Op_LoadN) {
1167 // Encode oop value if we are loading a narrow oop
1168 assert(!phase->type(value)->isa_narrowoop(), "should already be decoded");
1169 value = phase->transform(new EncodePNode(value, bottom_type()));
1170 }
1171 return value;
1172 }
1173 }
1174
1175 // If the previous store-maker is the right kind of Store, and the store is
1176 // to the same address, then we are equal to the value stored.
1177 Node* mem = in(Memory);
1178 Node* value = can_see_stored_value(mem, phase);
1179 if( value ) {
1180 // byte, short & char stores truncate naturally.
1181 // A load has to load the truncated value which requires
1182 // some sort of masking operation and that requires an
1183 // Ideal call instead of an Identity call.
1184 if (memory_size() < BytesPerInt) {
1185 // If the input to the store does not fit with the load's result type,
1186 // it must be truncated via an Ideal call.
1187 if (!phase->type(value)->higher_equal(phase->type(this)))
1188 return this;
1189 }
1190 // (This works even when value is a Con, but LoadNode::Value
1191 // usually runs first, producing the singleton type of the Con.)
1192 return value;
1193 }
1194
1682 // fold up, do so.
1683 Node* prev_mem = find_previous_store(phase);
1684 if (prev_mem != NULL) {
1685 Node* value = can_see_arraycopy_value(prev_mem, phase);
1686 if (value != NULL) {
1687 return value;
1688 }
1689 }
1690 // Steps (a), (b): Walk past independent stores to find an exact match.
1691 if (prev_mem != NULL && prev_mem != in(MemNode::Memory)) {
1692 // (c) See if we can fold up on the spot, but don't fold up here.
1693 // Fold-up might require truncation (for LoadB/LoadS/LoadUS) or
1694 // just return a prior value, which is done by Identity calls.
1695 if (can_see_stored_value(prev_mem, phase)) {
1696 // Make ready for step (d):
1697 set_req(MemNode::Memory, prev_mem);
1698 return this;
1699 }
1700 }
1701
1702 AllocateNode* alloc = AllocateNode::Ideal_allocation(address, phase);
1703 if (alloc != NULL && mem->is_Proj() &&
1704 mem->in(0) != NULL &&
1705 mem->in(0) == alloc->initialization() &&
1706 Opcode() == Op_LoadX &&
1707 alloc->initialization()->proj_out_or_null(0) != NULL) {
1708 InitializeNode* init = alloc->initialization();
1709 Node* control = init->proj_out(0);
1710 return alloc->make_ideal_mark(phase, address, control, mem, NULL);
1711 }
1712
1713 return progress ? this : NULL;
1714 }
1715
1716 // Helper to recognize certain Klass fields which are invariant across
1717 // some group of array types (e.g., int[] or all T[] where T < Object).
1718 const Type*
1719 LoadNode::load_array_final_field(const TypeKlassPtr *tkls,
1720 ciKlass* klass) const {
1721 if (tkls->offset() == in_bytes(Klass::modifier_flags_offset())) {
1722 // The field is Klass::_modifier_flags. Return its (constant) value.
1723 // (Folds up the 2nd indirection in aClassConstant.getModifiers().)
1724 assert(this->Opcode() == Op_LoadI, "must load an int from _modifier_flags");
1725 return TypeInt::make(klass->modifier_flags());
1726 }
1727 if (tkls->offset() == in_bytes(Klass::access_flags_offset())) {
1728 // The field is Klass::_access_flags. Return its (constant) value.
1729 // (Folds up the 2nd indirection in Reflection.getClassAccessFlags(aClassConstant).)
1730 assert(this->Opcode() == Op_LoadI, "must load an int from _access_flags");
1731 return TypeInt::make(klass->access_flags());
1732 }
1782 }
1783 }
1784
1785 // Don't do this for integer types. There is only potential profit if
1786 // the element type t is lower than _type; that is, for int types, if _type is
1787 // more restrictive than t. This only happens here if one is short and the other
1788 // char (both 16 bits), and in those cases we've made an intentional decision
1789 // to use one kind of load over the other. See AndINode::Ideal and 4965907.
1790 // Also, do not try to narrow the type for a LoadKlass, regardless of offset.
1791 //
1792 // Yes, it is possible to encounter an expression like (LoadKlass p1:(AddP x x 8))
1793 // where the _gvn.type of the AddP is wider than 8. This occurs when an earlier
1794 // copy p0 of (AddP x x 8) has been proven equal to p1, and the p0 has been
1795 // subsumed by p1. If p1 is on the worklist but has not yet been re-transformed,
1796 // it is possible that p1 will have a type like Foo*[int+]:NotNull*+any.
1797 // In fact, that could have been the original type of p1, and p1 could have
1798 // had an original form like p1:(AddP x x (LShiftL quux 3)), where the
1799 // expression (LShiftL quux 3) independently optimized to the constant 8.
1800 if ((t->isa_int() == NULL) && (t->isa_long() == NULL)
1801 && (_type->isa_vect() == NULL)
1802 && t->isa_valuetype() == NULL
1803 && Opcode() != Op_LoadKlass && Opcode() != Op_LoadNKlass) {
1804 // t might actually be lower than _type, if _type is a unique
1805 // concrete subclass of abstract class t.
1806 if (off_beyond_header || off == Type::OffsetBot) { // is the offset beyond the header?
1807 const Type* jt = t->join_speculative(_type);
1808 // In any case, do not allow the join, per se, to empty out the type.
1809 if (jt->empty() && !t->empty()) {
1810 // This can happen if a interface-typed array narrows to a class type.
1811 jt = _type;
1812 }
1813 #ifdef ASSERT
1814 if (phase->C->eliminate_boxing() && adr->is_AddP()) {
1815 // The pointers in the autobox arrays are always non-null
1816 Node* base = adr->in(AddPNode::Base);
1817 if ((base != NULL) && base->is_DecodeN()) {
1818 // Get LoadN node which loads IntegerCache.cache field
1819 base = base->in(1);
1820 }
1821 if ((base != NULL) && base->is_Con()) {
1822 const TypeAryPtr* base_type = base->bottom_type()->isa_aryptr();
1823 if ((base_type != NULL) && base_type->is_autobox_cache()) {
1824 // It could be narrow oop
1825 assert(jt->make_ptr()->ptr() == TypePtr::NotNull,"sanity");
1826 }
1827 }
1828 }
1829 #endif
1830 return jt;
1831 }
1832 }
1833 } else if (tp->base() == Type::InstPtr) {
1834 assert( off != Type::OffsetBot ||
1835 // arrays can be cast to Objects
1836 tp->is_oopptr()->klass()->is_java_lang_Object() ||
1837 tp->is_oopptr()->klass() == ciEnv::current()->Class_klass() ||
1838 // unsafe field access may not have a constant offset
1839 C->has_unsafe_access(),
1840 "Field accesses must be precise" );
1841 // For oop loads, we expect the _type to be precise.
1842
1843 // Optimize loads from constant fields.
1844 const TypeInstPtr* tinst = tp->is_instptr();
1845 ciObject* const_oop = tinst->const_oop();
1846 if (!is_mismatched_access() && off != Type::OffsetBot && const_oop != NULL && const_oop->is_instance()) {
1847 BasicType bt = memory_type();
1848 ciType* mirror_type = const_oop->as_instance()->java_mirror_type();
1849 if (mirror_type != NULL && mirror_type->is_valuetype()) {
1850 ciValueKlass* vk = mirror_type->as_value_klass();
1851 if (off == vk->default_value_offset()) {
1852 // Loading a special hidden field that contains the oop of the default value type
1853 const Type* const_oop = TypeInstPtr::make(vk->default_value_instance());
1854 return (bt == T_NARROWOOP) ? const_oop->make_narrowoop() : const_oop;
1855 }
1856 }
1857 const Type* con_type = Type::make_constant_from_field(const_oop->as_instance(), off, is_unsigned(), bt);
1858 if (con_type != NULL) {
1859 return con_type;
1860 }
1861 }
1862 } else if (tp->base() == Type::KlassPtr) {
1863 assert( off != Type::OffsetBot ||
1864 // arrays can be cast to Objects
1865 tp->is_klassptr()->klass() == NULL ||
1866 tp->is_klassptr()->klass()->is_java_lang_Object() ||
1867 // also allow array-loading from the primary supertype
1868 // array during subtype checks
1869 Opcode() == Op_LoadKlass,
1870 "Field accesses must be precise" );
1871 // For klass/static loads, we expect the _type to be precise
1872 } else if (tp->base() == Type::RawPtr && !StressReflectiveCode) {
1873 if (adr->is_Load() && off == 0) {
1874 /* With mirrors being an indirect in the Klass*
1875 * the VM is now using two loads. LoadKlass(LoadP(LoadP(Klass, mirror_offset), zero_offset))
1876 * The LoadP from the Klass has a RawPtr type (see LibraryCallKit::load_mirror_from_klass).
1877 *
1878 * So check the type and klass of the node before the LoadP.
1879 */
1880 Node* adr2 = adr->in(MemNode::Address);
1881 const TypeKlassPtr* tkls = phase->type(adr2)->isa_klassptr();
1882 if (tkls != NULL) {
1883 ciKlass* klass = tkls->klass();
1884 if (klass != NULL && klass->is_loaded() && tkls->klass_is_exact() && tkls->offset() == in_bytes(Klass::java_mirror_offset())) {
1885 assert(adr->Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1886 assert(Opcode() == Op_LoadP, "must load an oop from _java_mirror");
1887 return TypeInstPtr::make(klass->java_mirror());
1888 }
1889 }
1890 } else {
1891 // Check for a load of the default value offset from the ValueKlassFixedBlock:
1892 // LoadI(LoadP(value_klass, adr_valueklass_fixed_block_offset), default_value_offset_offset)
1893 intptr_t offset = 0;
1894 Node* base = AddPNode::Ideal_base_and_offset(adr, phase, offset);
1895 if (base != NULL && base->is_Load() && offset == in_bytes(ValueKlass::default_value_offset_offset())) {
1896 const TypeKlassPtr* tkls = phase->type(base->in(MemNode::Address))->isa_klassptr();
1897 if (tkls != NULL && tkls->is_loaded() && tkls->klass_is_exact() && tkls->isa_valuetype() &&
1898 tkls->offset() == in_bytes(InstanceKlass::adr_valueklass_fixed_block_offset())) {
1899 assert(base->Opcode() == Op_LoadP, "must load an oop from klass");
1900 assert(Opcode() == Op_LoadI, "must load an int from fixed block");
1901 return TypeInt::make(tkls->klass()->as_value_klass()->default_value_offset());
1902 }
1903 }
1904 }
1905 }
1906
1907 const TypeKlassPtr *tkls = tp->isa_klassptr();
1908 if (tkls != NULL && !StressReflectiveCode) {
1909 ciKlass* klass = tkls->klass();
1910 if (tkls->is_loaded() && tkls->klass_is_exact()) {
1911 // We are loading a field from a Klass metaobject whose identity
1912 // is known at compile time (the type is "exact" or "precise").
1913 // Check for fields we know are maintained as constants by the VM.
1914 if (tkls->offset() == in_bytes(Klass::super_check_offset_offset())) {
1915 // The field is Klass::_super_check_offset. Return its (constant) value.
1916 // (Folds up type checking code.)
1917 assert(Opcode() == Op_LoadI, "must load an int from _super_check_offset");
1918 return TypeInt::make(klass->super_check_offset());
1919 }
1920 // Compute index into primary_supers array
1921 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1922 // Check for overflowing; use unsigned compare to handle the negative case.
1923 if( depth < ciKlass::primary_super_limit() ) {
1924 // The field is an element of Klass::_primary_supers. Return its (constant) value.
1925 // (Folds up type checking code.)
1926 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1927 ciKlass *ss = klass->super_of_depth(depth);
1928 return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1929 }
1930 const Type* aift = load_array_final_field(tkls, klass);
1931 if (aift != NULL) return aift;
1932 }
1933
1934 // We can still check if we are loading from the primary_supers array at a
1935 // shallow enough depth. Even though the klass is not exact, entries less
1936 // than or equal to its super depth are correct.
1937 if (tkls->is_loaded()) {
1938 ciType *inner = klass;
1939 while( inner->is_obj_array_klass() )
1940 inner = inner->as_obj_array_klass()->base_element_type();
1941 if( inner->is_instance_klass() &&
1942 !inner->as_instance_klass()->flags().is_interface() ) {
1943 // Compute index into primary_supers array
1944 juint depth = (tkls->offset() - in_bytes(Klass::primary_supers_offset())) / sizeof(Klass*);
1945 // Check for overflowing; use unsigned compare to handle the negative case.
1946 if( depth < ciKlass::primary_super_limit() &&
1947 depth <= klass->super_depth() ) { // allow self-depth checks to handle self-check case
1948 // The field is an element of Klass::_primary_supers. Return its (constant) value.
1949 // (Folds up type checking code.)
1950 assert(Opcode() == Op_LoadKlass, "must load a klass from _primary_supers");
1951 ciKlass *ss = klass->super_of_depth(depth);
1952 return ss ? TypeKlassPtr::make(ss) : TypePtr::NULL_PTR;
1953 }
1954 }
1955 }
1956
1957 // If the type is enough to determine that the thing is not an array,
2204 if( !ik->is_loaded() )
2205 return _type; // Bail out if not loaded
2206 if (offset == oopDesc::klass_offset_in_bytes()) {
2207 if (tinst->klass_is_exact()) {
2208 return TypeKlassPtr::make(ik);
2209 }
2210 // See if we can become precise: no subklasses and no interface
2211 // (Note: We need to support verified interfaces.)
2212 if (!ik->is_interface() && !ik->has_subklass()) {
2213 //assert(!UseExactTypes, "this code should be useless with exact types");
2214 // Add a dependence; if any subclass added we need to recompile
2215 if (!ik->is_final()) {
2216 // %%% should use stronger assert_unique_concrete_subtype instead
2217 phase->C->dependencies()->assert_leaf_type(ik);
2218 }
2219 // Return precise klass
2220 return TypeKlassPtr::make(ik);
2221 }
2222
2223 // Return root of possible klass
2224 return TypeKlassPtr::make(TypePtr::NotNull, ik, Type::Offset(0));
2225 }
2226 }
2227
2228 // Check for loading klass from an array
2229 const TypeAryPtr *tary = tp->isa_aryptr();
2230 if( tary != NULL ) {
2231 ciKlass *tary_klass = tary->klass();
2232 if (tary_klass != NULL // can be NULL when at BOTTOM or TOP
2233 && tary->offset() == oopDesc::klass_offset_in_bytes()) {
2234 if (tary->klass_is_exact()) {
2235 return TypeKlassPtr::make(tary_klass);
2236 }
2237 ciArrayKlass *ak = tary->klass()->as_array_klass();
2238 // If the klass is an object array, we defer the question to the
2239 // array component klass.
2240 if( ak->is_obj_array_klass() ) {
2241 assert( ak->is_loaded(), "" );
2242 ciKlass *base_k = ak->as_obj_array_klass()->base_element_klass();
2243 if( base_k->is_loaded() && base_k->is_instance_klass() ) {
2244 ciInstanceKlass* ik = base_k->as_instance_klass();
2245 // See if we can become precise: no subklasses and no interface
2246 if (!ik->is_interface() && !ik->has_subklass()) {
2247 //assert(!UseExactTypes, "this code should be useless with exact types");
2248 // Add a dependence; if any subclass added we need to recompile
2249 if (!ik->is_final()) {
2250 phase->C->dependencies()->assert_leaf_type(ik);
2251 }
2252 // Return precise array klass
2253 return TypeKlassPtr::make(ak);
2254 }
2255 }
2256 return TypeKlassPtr::make(TypePtr::NotNull, ak, Type::Offset(0));
2257 } else { // Found a type-array?
2258 //assert(!UseExactTypes, "this code should be useless with exact types");
2259 assert( ak->is_type_array_klass(), "" );
2260 return TypeKlassPtr::make(ak); // These are always precise
2261 }
2262 }
2263 }
2264
2265 // Check for loading klass from an array klass
2266 const TypeKlassPtr *tkls = tp->isa_klassptr();
2267 if (tkls != NULL && !StressReflectiveCode) {
2268 if (!tkls->is_loaded()) {
2269 return _type; // Bail out if not loaded
2270 }
2271 ciKlass* klass = tkls->klass();
2272 if( klass->is_obj_array_klass() &&
2273 tkls->offset() == in_bytes(ObjArrayKlass::element_klass_offset())) {
2274 ciKlass* elem = klass->as_obj_array_klass()->element_klass();
2275 // // Always returning precise element type is incorrect,
2276 // // e.g., element type could be object and array may contain strings
2277 // return TypeKlassPtr::make(TypePtr::Constant, elem, 0);
2278
2279 // The array's TypeKlassPtr was declared 'precise' or 'not precise'
2280 // according to the element type's subclassing.
2281 return TypeKlassPtr::make(tkls->ptr(), elem, Type::Offset(0));
2282 }
2283 if( klass->is_instance_klass() && tkls->klass_is_exact() &&
2284 tkls->offset() == in_bytes(Klass::super_offset())) {
2285 ciKlass* sup = klass->as_instance_klass()->super();
2286 // The field is Klass::_super. Return its (constant) value.
2287 // (Folds up the 2nd indirection in aClassConstant.getSuperClass().)
2288 return sup ? TypeKlassPtr::make(sup) : TypePtr::NULL_PTR;
2289 }
2290 }
2291
2292 // Bailout case
2293 return LoadNode::Value(phase);
2294 }
2295
2296 //------------------------------Identity---------------------------------------
2297 // To clean up reflective code, simplify k.java_mirror.as_klass to plain k.
2298 // Also feed through the klass in Allocate(...klass...)._klass.
2299 Node* LoadKlassNode::Identity(PhaseGVN* phase) {
2300 return klass_identity_common(phase);
2301 }
2469 //=============================================================================
2470 //---------------------------StoreNode::make-----------------------------------
2471 // Polymorphic factory method:
2472 StoreNode* StoreNode::make(PhaseGVN& gvn, Node* ctl, Node* mem, Node* adr, const TypePtr* adr_type, Node* val, BasicType bt, MemOrd mo) {
2473 assert((mo == unordered || mo == release), "unexpected");
2474 Compile* C = gvn.C;
2475 assert(C->get_alias_index(adr_type) != Compile::AliasIdxRaw ||
2476 ctl != NULL, "raw memory operations should have control edge");
2477
2478 switch (bt) {
2479 case T_BOOLEAN: val = gvn.transform(new AndINode(val, gvn.intcon(0x1))); // Fall through to T_BYTE case
2480 case T_BYTE: return new StoreBNode(ctl, mem, adr, adr_type, val, mo);
2481 case T_INT: return new StoreINode(ctl, mem, adr, adr_type, val, mo);
2482 case T_CHAR:
2483 case T_SHORT: return new StoreCNode(ctl, mem, adr, adr_type, val, mo);
2484 case T_LONG: return new StoreLNode(ctl, mem, adr, adr_type, val, mo);
2485 case T_FLOAT: return new StoreFNode(ctl, mem, adr, adr_type, val, mo);
2486 case T_DOUBLE: return new StoreDNode(ctl, mem, adr, adr_type, val, mo);
2487 case T_METADATA:
2488 case T_ADDRESS:
2489 case T_VALUETYPE:
2490 case T_OBJECT:
2491 #ifdef _LP64
2492 if (adr->bottom_type()->is_ptr_to_narrowoop()) {
2493 val = gvn.transform(new EncodePNode(val, val->bottom_type()->make_narrowoop()));
2494 return new StoreNNode(ctl, mem, adr, adr_type, val, mo);
2495 } else if (adr->bottom_type()->is_ptr_to_narrowklass() ||
2496 (UseCompressedClassPointers && val->bottom_type()->isa_klassptr() &&
2497 adr->bottom_type()->isa_rawptr())) {
2498 val = gvn.transform(new EncodePKlassNode(val, val->bottom_type()->make_narrowklass()));
2499 return new StoreNKlassNode(ctl, mem, adr, adr_type, val, mo);
2500 }
2501 #endif
2502 {
2503 return new StorePNode(ctl, mem, adr, adr_type, val, mo);
2504 }
2505 default:
2506 ShouldNotReachHere();
2507 return (StoreNode*)NULL;
2508 }
2509 }
2548 // since they must follow each StoreP operation. Redundant StoreCMs
2549 // are eliminated just before matching in final_graph_reshape.
2550 {
2551 Node* st = mem;
2552 // If Store 'st' has more than one use, we cannot fold 'st' away.
2553 // For example, 'st' might be the final state at a conditional
2554 // return. Or, 'st' might be used by some node which is live at
2555 // the same time 'st' is live, which might be unschedulable. So,
2556 // require exactly ONE user until such time as we clone 'mem' for
2557 // each of 'mem's uses (thus making the exactly-1-user-rule hold
2558 // true).
2559 while (st->is_Store() && st->outcnt() == 1 && st->Opcode() != Op_StoreCM) {
2560 // Looking at a dead closed cycle of memory?
2561 assert(st != st->in(MemNode::Memory), "dead loop in StoreNode::Ideal");
2562 assert(Opcode() == st->Opcode() ||
2563 st->Opcode() == Op_StoreVector ||
2564 Opcode() == Op_StoreVector ||
2565 phase->C->get_alias_index(adr_type()) == Compile::AliasIdxRaw ||
2566 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreI) || // expanded ClearArrayNode
2567 (Opcode() == Op_StoreI && st->Opcode() == Op_StoreL) || // initialization by arraycopy
2568 (Opcode() == Op_StoreL && st->Opcode() == Op_StoreN) ||
2569 (is_mismatched_access() || st->as_Store()->is_mismatched_access()),
2570 "no mismatched stores, except on raw memory: %s %s", NodeClassNames[Opcode()], NodeClassNames[st->Opcode()]);
2571
2572 if (st->in(MemNode::Address)->eqv_uncast(address) &&
2573 st->as_Store()->memory_size() <= this->memory_size()) {
2574 Node* use = st->raw_out(0);
2575 phase->igvn_rehash_node_delayed(use);
2576 if (can_reshape) {
2577 use->set_req_X(MemNode::Memory, st->in(MemNode::Memory), phase->is_IterGVN());
2578 } else {
2579 // It's OK to do this in the parser, since DU info is always accurate,
2580 // and the parser always refers to nodes via SafePointNode maps.
2581 use->set_req(MemNode::Memory, st->in(MemNode::Memory));
2582 }
2583 return this;
2584 }
2585 st = st->in(MemNode::Memory);
2586 }
2587 }
2588
2634 // Load then Store? Then the Store is useless
2635 if (val->is_Load() &&
2636 val->in(MemNode::Address)->eqv_uncast(adr) &&
2637 val->in(MemNode::Memory )->eqv_uncast(mem) &&
2638 val->as_Load()->store_Opcode() == Opcode()) {
2639 result = mem;
2640 }
2641
2642 // Two stores in a row of the same value?
2643 if (result == this &&
2644 mem->is_Store() &&
2645 mem->in(MemNode::Address)->eqv_uncast(adr) &&
2646 mem->in(MemNode::ValueIn)->eqv_uncast(val) &&
2647 mem->Opcode() == Opcode()) {
2648 result = mem;
2649 }
2650
2651 // Store of zero anywhere into a freshly-allocated object?
2652 // Then the store is useless.
2653 // (It must already have been captured by the InitializeNode.)
2654 if (result == this && ReduceFieldZeroing) {
2655 // a newly allocated object is already all-zeroes everywhere
2656 if (mem->is_Proj() && mem->in(0)->is_Allocate() &&
2657 (phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == val)) {
2658 assert(!phase->type(val)->is_zero_type() || mem->in(0)->in(AllocateNode::DefaultValue) == NULL, "storing null to value array is forbidden");
2659 result = mem;
2660 }
2661
2662 if (result == this) {
2663 // the store may also apply to zero-bits in an earlier object
2664 Node* prev_mem = find_previous_store(phase);
2665 // Steps (a), (b): Walk past independent stores to find an exact match.
2666 if (prev_mem != NULL) {
2667 Node* prev_val = can_see_stored_value(prev_mem, phase);
2668 if (prev_val != NULL && phase->eqv(prev_val, val)) {
2669 // prev_val and val might differ by a cast; it would be good
2670 // to keep the more informative of the two.
2671 if (phase->type(val)->is_zero_type()) {
2672 result = mem;
2673 } else if (prev_mem->is_Proj() && prev_mem->in(0)->is_Initialize()) {
2674 InitializeNode* init = prev_mem->in(0)->as_Initialize();
2675 AllocateNode* alloc = init->allocation();
2676 if (alloc != NULL && alloc->in(AllocateNode::DefaultValue) == val) {
2677 result = mem;
2678 }
2679 }
2680 }
2681 }
2682 }
2683 }
2684
2685 if (result != this && phase->is_IterGVN() != NULL) {
2686 MemBarNode* trailing = trailing_membar();
2687 if (trailing != NULL) {
2688 #ifdef ASSERT
2689 const TypeOopPtr* t_oop = phase->type(in(Address))->isa_oopptr();
2690 assert(t_oop == NULL || t_oop->is_known_instance_field(), "only for non escaping objects");
2691 #endif
2692 PhaseIterGVN* igvn = phase->is_IterGVN();
2693 trailing->remove(igvn);
2694 }
2695 }
2696
2697 return result;
2698 }
2699
2700 //------------------------------match_edge-------------------------------------
2701 // Do we Match on this edge index or not? Match only memory & value
2702 uint StoreNode::match_edge(uint idx) const {
2703 return idx == MemNode::Address || idx == MemNode::ValueIn;
2967 // Clearing a short array is faster with stores
2968 Node *ClearArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) {
2969 // Already know this is a large node, do not try to ideal it
2970 if (!IdealizeClearArrayNode || _is_large) return NULL;
2971
2972 const int unit = BytesPerLong;
2973 const TypeX* t = phase->type(in(2))->isa_intptr_t();
2974 if (!t) return NULL;
2975 if (!t->is_con()) return NULL;
2976 intptr_t raw_count = t->get_con();
2977 intptr_t size = raw_count;
2978 if (!Matcher::init_array_count_is_in_bytes) size *= unit;
2979 // Clearing nothing uses the Identity call.
2980 // Negative clears are possible on dead ClearArrays
2981 // (see jck test stmt114.stmt11402.val).
2982 if (size <= 0 || size % unit != 0) return NULL;
2983 intptr_t count = size / unit;
2984 // Length too long; communicate this to matchers and assemblers.
2985 // Assemblers are responsible to produce fast hardware clears for it.
2986 if (size > InitArrayShortSize) {
2987 return new ClearArrayNode(in(0), in(1), in(2), in(3), in(4), true);
2988 }
2989 Node *mem = in(1);
2990 if( phase->type(mem)==Type::TOP ) return NULL;
2991 Node *adr = in(3);
2992 const Type* at = phase->type(adr);
2993 if( at==Type::TOP ) return NULL;
2994 const TypePtr* atp = at->isa_ptr();
2995 // adjust atp to be the correct array element address type
2996 if (atp == NULL) atp = TypePtr::BOTTOM;
2997 else atp = atp->add_offset(Type::OffsetBot);
2998 // Get base for derived pointer purposes
2999 if( adr->Opcode() != Op_AddP ) Unimplemented();
3000 Node *base = adr->in(1);
3001
3002 Node *val = in(4);
3003 Node *off = phase->MakeConX(BytesPerLong);
3004 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3005 count--;
3006 while( count-- ) {
3007 mem = phase->transform(mem);
3008 adr = phase->transform(new AddPNode(base,adr,off));
3009 mem = new StoreLNode(in(0), mem, adr, atp, val, MemNode::unordered, false);
3010 }
3011 return mem;
3012 }
3013
3014 //----------------------------step_through----------------------------------
3015 // Return allocation input memory edge if it is different instance
3016 // or itself if it is the one we are looking for.
3017 bool ClearArrayNode::step_through(Node** np, uint instance_id, PhaseTransform* phase) {
3018 Node* n = *np;
3019 assert(n->is_ClearArray(), "sanity");
3020 intptr_t offset;
3021 AllocateNode* alloc = AllocateNode::Ideal_allocation(n->in(3), phase, offset);
3022 // This method is called only before Allocate nodes are expanded
3023 // during macro nodes expansion. Before that ClearArray nodes are
3024 // only generated in PhaseMacroExpand::generate_arraycopy() (before
3025 // Allocate nodes are expanded) which follows allocations.
3026 assert(alloc != NULL, "should have allocation");
3027 if (alloc->_idx == instance_id) {
3028 // Can not bypass initialization of the instance we are looking for.
3029 return false;
3030 }
3031 // Otherwise skip it.
3032 InitializeNode* init = alloc->initialization();
3033 if (init != NULL)
3034 *np = init->in(TypeFunc::Memory);
3035 else
3036 *np = alloc->in(TypeFunc::Memory);
3037 return true;
3038 }
3039
3040 //----------------------------clear_memory-------------------------------------
3041 // Generate code to initialize object storage to zero.
3042 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3043 Node* val,
3044 Node* raw_val,
3045 intptr_t start_offset,
3046 Node* end_offset,
3047 PhaseGVN* phase) {
3048 intptr_t offset = start_offset;
3049
3050 int unit = BytesPerLong;
3051 if ((offset % unit) != 0) {
3052 Node* adr = new AddPNode(dest, dest, phase->MakeConX(offset));
3053 adr = phase->transform(adr);
3054 const TypePtr* atp = TypeRawPtr::BOTTOM;
3055 if (val != NULL) {
3056 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3057 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3058 } else {
3059 assert(raw_val == NULL, "val may not be null");
3060 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3061 }
3062 mem = phase->transform(mem);
3063 offset += BytesPerInt;
3064 }
3065 assert((offset % unit) == 0, "");
3066
3067 // Initialize the remaining stuff, if any, with a ClearArray.
3068 return clear_memory(ctl, mem, dest, raw_val, phase->MakeConX(offset), end_offset, phase);
3069 }
3070
3071 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3072 Node* raw_val,
3073 Node* start_offset,
3074 Node* end_offset,
3075 PhaseGVN* phase) {
3076 if (start_offset == end_offset) {
3077 // nothing to do
3078 return mem;
3079 }
3080
3081 int unit = BytesPerLong;
3082 Node* zbase = start_offset;
3083 Node* zend = end_offset;
3084
3085 // Scale to the unit required by the CPU:
3086 if (!Matcher::init_array_count_is_in_bytes) {
3087 Node* shift = phase->intcon(exact_log2(unit));
3088 zbase = phase->transform(new URShiftXNode(zbase, shift) );
3089 zend = phase->transform(new URShiftXNode(zend, shift) );
3090 }
3091
3092 // Bulk clear double-words
3093 Node* zsize = phase->transform(new SubXNode(zend, zbase) );
3094 Node* adr = phase->transform(new AddPNode(dest, dest, start_offset) );
3095 if (raw_val == NULL) {
3096 raw_val = phase->MakeConX(0);
3097 }
3098 mem = new ClearArrayNode(ctl, mem, zsize, adr, raw_val, false);
3099 return phase->transform(mem);
3100 }
3101
3102 Node* ClearArrayNode::clear_memory(Node* ctl, Node* mem, Node* dest,
3103 Node* val,
3104 Node* raw_val,
3105 intptr_t start_offset,
3106 intptr_t end_offset,
3107 PhaseGVN* phase) {
3108 if (start_offset == end_offset) {
3109 // nothing to do
3110 return mem;
3111 }
3112
3113 assert((end_offset % BytesPerInt) == 0, "odd end offset");
3114 intptr_t done_offset = end_offset;
3115 if ((done_offset % BytesPerLong) != 0) {
3116 done_offset -= BytesPerInt;
3117 }
3118 if (done_offset > start_offset) {
3119 mem = clear_memory(ctl, mem, dest, val, raw_val,
3120 start_offset, phase->MakeConX(done_offset), phase);
3121 }
3122 if (done_offset < end_offset) { // emit the final 32-bit store
3123 Node* adr = new AddPNode(dest, dest, phase->MakeConX(done_offset));
3124 adr = phase->transform(adr);
3125 const TypePtr* atp = TypeRawPtr::BOTTOM;
3126 if (val != NULL) {
3127 assert(phase->type(val)->isa_narrowoop(), "should be narrow oop");
3128 mem = new StoreNNode(ctl, mem, adr, atp, val, MemNode::unordered);
3129 } else {
3130 assert(raw_val == NULL, "val may not be null");
3131 mem = StoreNode::make(*phase, ctl, mem, adr, atp, phase->zerocon(T_INT), T_INT, MemNode::unordered);
3132 }
3133 mem = phase->transform(mem);
3134 done_offset += BytesPerInt;
3135 }
3136 assert(done_offset == end_offset, "");
3137 return mem;
3138 }
3139
3140 //=============================================================================
3141 MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent)
3142 : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)),
3143 _adr_type(C->get_adr_type(alias_idx)), _kind(Standalone)
3144 #ifdef ASSERT
3145 , _pair_idx(0)
3146 #endif
3147 {
3148 init_class_id(Class_MemBar);
3149 Node* top = C->top();
3150 init_req(TypeFunc::I_O,top);
3151 init_req(TypeFunc::FramePtr,top);
3152 init_req(TypeFunc::ReturnAdr,top);
3261 PhaseIterGVN* igvn = phase->is_IterGVN();
3262 remove(igvn);
3263 // Must return either the original node (now dead) or a new node
3264 // (Do not return a top here, since that would break the uniqueness of top.)
3265 return new ConINode(TypeInt::ZERO);
3266 }
3267 }
3268 return progress ? this : NULL;
3269 }
3270
3271 //------------------------------Value------------------------------------------
3272 const Type* MemBarNode::Value(PhaseGVN* phase) const {
3273 if( !in(0) ) return Type::TOP;
3274 if( phase->type(in(0)) == Type::TOP )
3275 return Type::TOP;
3276 return TypeTuple::MEMBAR;
3277 }
3278
3279 //------------------------------match------------------------------------------
3280 // Construct projections for memory.
3281 Node *MemBarNode::match(const ProjNode *proj, const Matcher *m, const RegMask* mask) {
3282 switch (proj->_con) {
3283 case TypeFunc::Control:
3284 case TypeFunc::Memory:
3285 return new MachProjNode(this,proj->_con,RegMask::Empty,MachProjNode::unmatched_proj);
3286 }
3287 ShouldNotReachHere();
3288 return NULL;
3289 }
3290
3291 void MemBarNode::set_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3292 trailing->_kind = TrailingStore;
3293 leading->_kind = LeadingStore;
3294 #ifdef ASSERT
3295 trailing->_pair_idx = leading->_idx;
3296 leading->_pair_idx = leading->_idx;
3297 #endif
3298 }
3299
3300 void MemBarNode::set_load_store_pair(MemBarNode* leading, MemBarNode* trailing) {
3301 trailing->_kind = TrailingLoadStore;
3547 return (req() > RawStores);
3548 }
3549
3550 void InitializeNode::set_complete(PhaseGVN* phase) {
3551 assert(!is_complete(), "caller responsibility");
3552 _is_complete = Complete;
3553
3554 // After this node is complete, it contains a bunch of
3555 // raw-memory initializations. There is no need for
3556 // it to have anything to do with non-raw memory effects.
3557 // Therefore, tell all non-raw users to re-optimize themselves,
3558 // after skipping the memory effects of this initialization.
3559 PhaseIterGVN* igvn = phase->is_IterGVN();
3560 if (igvn) igvn->add_users_to_worklist(this);
3561 }
3562
3563 // convenience function
3564 // return false if the init contains any stores already
3565 bool AllocateNode::maybe_set_complete(PhaseGVN* phase) {
3566 InitializeNode* init = initialization();
3567 if (init == NULL || init->is_complete()) {
3568 return false;
3569 }
3570 init->remove_extra_zeroes();
3571 // for now, if this allocation has already collected any inits, bail:
3572 if (init->is_non_zero()) return false;
3573 init->set_complete(phase);
3574 return true;
3575 }
3576
3577 void InitializeNode::remove_extra_zeroes() {
3578 if (req() == RawStores) return;
3579 Node* zmem = zero_memory();
3580 uint fill = RawStores;
3581 for (uint i = fill; i < req(); i++) {
3582 Node* n = in(i);
3583 if (n->is_top() || n == zmem) continue; // skip
3584 if (fill < i) set_req(fill, n); // compact
3585 ++fill;
3586 }
3587 // delete any empty spaces created:
3588 while (fill < req()) {
3589 del_req(fill);
4292 // z's_done 12 16 16 16 12 16 12
4293 // z's_needed 12 16 16 16 16 16 16
4294 // zsize 0 0 0 0 4 0 4
4295 if (next_full_store < 0) {
4296 // Conservative tack: Zero to end of current word.
4297 zeroes_needed = align_up(zeroes_needed, BytesPerInt);
4298 } else {
4299 // Zero to beginning of next fully initialized word.
4300 // Or, don't zero at all, if we are already in that word.
4301 assert(next_full_store >= zeroes_needed, "must go forward");
4302 assert((next_full_store & (BytesPerInt-1)) == 0, "even boundary");
4303 zeroes_needed = next_full_store;
4304 }
4305 }
4306
4307 if (zeroes_needed > zeroes_done) {
4308 intptr_t zsize = zeroes_needed - zeroes_done;
4309 // Do some incremental zeroing on rawmem, in parallel with inits.
4310 zeroes_done = align_down(zeroes_done, BytesPerInt);
4311 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4312 allocation()->in(AllocateNode::DefaultValue),
4313 allocation()->in(AllocateNode::RawDefaultValue),
4314 zeroes_done, zeroes_needed,
4315 phase);
4316 zeroes_done = zeroes_needed;
4317 if (zsize > InitArrayShortSize && ++big_init_gaps > 2)
4318 do_zeroing = false; // leave the hole, next time
4319 }
4320 }
4321
4322 // Collect the store and move on:
4323 st->set_req(MemNode::Memory, inits);
4324 inits = st; // put it on the linearized chain
4325 set_req(i, zmem); // unhook from previous position
4326
4327 if (zeroes_done == st_off)
4328 zeroes_done = next_init_off;
4329
4330 assert(!do_zeroing || zeroes_done >= next_init_off, "don't miss any");
4331
4332 #ifdef ASSERT
4333 // Various order invariants. Weaker than stores_are_sane because
4353 remove_extra_zeroes(); // clear out all the zmems left over
4354 add_req(inits);
4355
4356 if (!(UseTLAB && ZeroTLAB)) {
4357 // If anything remains to be zeroed, zero it all now.
4358 zeroes_done = align_down(zeroes_done, BytesPerInt);
4359 // if it is the last unused 4 bytes of an instance, forget about it
4360 intptr_t size_limit = phase->find_intptr_t_con(size_in_bytes, max_jint);
4361 if (zeroes_done + BytesPerLong >= size_limit) {
4362 AllocateNode* alloc = allocation();
4363 assert(alloc != NULL, "must be present");
4364 if (alloc != NULL && alloc->Opcode() == Op_Allocate) {
4365 Node* klass_node = alloc->in(AllocateNode::KlassNode);
4366 ciKlass* k = phase->type(klass_node)->is_klassptr()->klass();
4367 if (zeroes_done == k->layout_helper())
4368 zeroes_done = size_limit;
4369 }
4370 }
4371 if (zeroes_done < size_limit) {
4372 rawmem = ClearArrayNode::clear_memory(rawctl, rawmem, rawptr,
4373 allocation()->in(AllocateNode::DefaultValue),
4374 allocation()->in(AllocateNode::RawDefaultValue),
4375 zeroes_done, size_in_bytes, phase);
4376 }
4377 }
4378
4379 set_complete(phase);
4380 return rawmem;
4381 }
4382
4383
4384 #ifdef ASSERT
4385 bool InitializeNode::stores_are_sane(PhaseTransform* phase) {
4386 if (is_complete())
4387 return true; // stores could be anything at this point
4388 assert(allocation() != NULL, "must be present");
4389 intptr_t last_off = allocation()->minimum_header_size();
4390 for (uint i = InitializeNode::RawStores; i < req(); i++) {
4391 Node* st = in(i);
4392 intptr_t st_off = get_store_offset(st, phase);
4393 if (st_off < 0) continue; // ignore dead garbage
4394 if (last_off > st_off) {
|