68 ProjNode *_fallthroughproj;
69 ProjNode *_fallthroughcatchproj;
70 ProjNode *_ioproj_fallthrough;
71 ProjNode *_ioproj_catchall;
72 ProjNode *_catchallcatchproj;
73 ProjNode *_memproj_fallthrough;
74 ProjNode *_memproj_catchall;
75 ProjNode *_resproj;
76
77 // Additional data collected during macro expansion
78 bool _has_locks;
79
80 void expand_allocate(AllocateNode *alloc);
81 void expand_allocate_array(AllocateArrayNode *alloc);
82 void expand_allocate_common(AllocateNode* alloc,
83 Node* length,
84 const TypeFunc* slow_call_type,
85 address slow_call_address);
86 Node *value_from_mem(Node *mem, Node *ctl, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc);
87 Node *value_from_mem_phi(Node *mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level);
88
89 bool eliminate_boxing_node(CallStaticJavaNode *boxing);
90 bool eliminate_allocate_node(AllocateNode *alloc);
91 bool can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints);
92 bool scalar_replacement(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints_done);
93 void process_users_of_allocation(CallNode *alloc);
94
95 void eliminate_gc_barrier(Node *p2x);
96 void mark_eliminated_box(Node* box, Node* obj);
97 void mark_eliminated_locking_nodes(AbstractLockNode *alock);
98 bool eliminate_locking_node(AbstractLockNode *alock);
99 void expand_lock_node(LockNode *lock);
100 void expand_unlock_node(UnlockNode *unlock);
101
102 // More helper methods modeled after GraphKit for array copy
103 void insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent = NULL);
104 Node* array_element_address(Node* ary, Node* idx, BasicType elembt);
105 Node* ConvI2L(Node* offset);
106 Node* make_leaf_call(Node* ctrl, Node* mem,
107 const TypeFunc* call_type, address call_addr,
108 const char* call_name,
109 const TypePtr* adr_type,
110 Node* parm0 = NULL, Node* parm1 = NULL,
111 Node* parm2 = NULL, Node* parm3 = NULL,
112 Node* parm4 = NULL, Node* parm5 = NULL,
113 Node* parm6 = NULL, Node* parm7 = NULL);
114
115 // helper methods modeled after LibraryCallKit for array copy
116 Node* generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob);
117 Node* generate_slow_guard(Node** ctrl, Node* test, RegionNode* region);
118 void generate_negative_guard(Node** ctrl, Node* index, RegionNode* region);
119 void generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region);
120
121 // More helper methods for array copy
122 Node* generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative);
123 void finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type);
124 address basictype2arraycopy(BasicType t,
125 Node* src_offset,
126 Node* dest_offset,
127 bool disjoint_bases,
128 const char* &name,
129 bool dest_uninitialized);
130 Node* generate_arraycopy(ArrayCopyNode *ac,
131 AllocateArrayNode* alloc,
132 Node** ctrl, MergeMemNode* mem, Node** io,
133 const TypePtr* adr_type,
134 BasicType basic_elem_type,
135 Node* src, Node* src_offset,
136 Node* dest, Node* dest_offset,
137 Node* copy_length,
138 bool disjoint_bases = false,
139 bool length_never_negative = false,
140 RegionNode* slow_region = NULL);
141 void generate_clear_array(Node* ctrl, MergeMemNode* merge_mem,
142 const TypePtr* adr_type,
143 Node* dest,
144 BasicType basic_elem_type,
145 Node* slice_idx,
146 Node* slice_len,
147 Node* dest_size);
148 bool generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io,
149 const TypePtr* adr_type,
150 BasicType basic_elem_type,
151 AllocateNode* alloc,
152 Node* src, Node* src_offset,
153 Node* dest, Node* dest_offset,
154 Node* dest_size, bool dest_uninitialized);
155 MergeMemNode* generate_slow_arraycopy(ArrayCopyNode *ac,
156 Node** ctrl, Node* mem, Node** io,
157 const TypePtr* adr_type,
158 Node* src, Node* src_offset,
159 Node* dest, Node* dest_offset,
160 Node* copy_length, bool dest_uninitialized);
161 Node* generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem,
162 const TypePtr* adr_type,
163 Node* dest_elem_klass,
164 Node* src, Node* src_offset,
165 Node* dest, Node* dest_offset,
166 Node* copy_length, bool dest_uninitialized);
167 Node* generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem,
168 const TypePtr* adr_type,
169 Node* src, Node* src_offset,
170 Node* dest, Node* dest_offset,
171 Node* copy_length, bool dest_uninitialized);
172 void generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem,
173 const TypePtr* adr_type,
174 BasicType basic_elem_type,
175 bool disjoint_bases,
176 Node* src, Node* src_offset,
177 Node* dest, Node* dest_offset,
178 Node* copy_length, bool dest_uninitialized);
179
180 void expand_arraycopy_node(ArrayCopyNode *ac);
181
182 int replace_input(Node *use, Node *oldref, Node *newref);
183 void copy_call_debug_info(CallNode *oldcall, CallNode * newcall);
184 Node* opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path = false);
185 void copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call);
186 CallNode* make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call,
187 const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1,
188 Node* parm2);
189 void extract_call_projections(CallNode *call);
190
191 Node* initialize_object(AllocateNode* alloc,
192 Node* control, Node* rawmem, Node* object,
193 Node* klass_node, Node* length,
194 Node* size_in_bytes);
195
196 Node* make_arraycopy_load(ArrayCopyNode* ac, intptr_t offset, Node* ctl, Node* mem, BasicType ft, const Type *ftype, AllocateNode *alloc);
197
198 public:
199 PhaseMacroExpand(PhaseIterGVN &igvn) : Phase(Macro_Expand), _igvn(igvn), _has_locks(false) {
200 _igvn.set_delay_transform(true);
201 }
202 void eliminate_macro_nodes();
203 bool expand_macro_nodes();
204
205 PhaseIterGVN &igvn() const { return _igvn; }
206
207 // Members accessed from BarrierSetC2
208 void replace_node(Node* source, Node* target) { _igvn.replace_node(source, target); }
209 Node* intcon(jint con) const { return _igvn.intcon(con); }
210 Node* longcon(jlong con) const { return _igvn.longcon(con); }
211 Node* makecon(const Type *t) const { return _igvn.makecon(t); }
212 Node* top() const { return C->top(); }
213
214 Node* prefetch_allocation(Node* i_o,
215 Node*& needgc_false, Node*& contended_phi_rawmem,
216 Node* old_eden_top, Node* new_eden_top,
|
68 ProjNode *_fallthroughproj;
69 ProjNode *_fallthroughcatchproj;
70 ProjNode *_ioproj_fallthrough;
71 ProjNode *_ioproj_catchall;
72 ProjNode *_catchallcatchproj;
73 ProjNode *_memproj_fallthrough;
74 ProjNode *_memproj_catchall;
75 ProjNode *_resproj;
76
77 // Additional data collected during macro expansion
78 bool _has_locks;
79
80 void expand_allocate(AllocateNode *alloc);
81 void expand_allocate_array(AllocateArrayNode *alloc);
82 void expand_allocate_common(AllocateNode* alloc,
83 Node* length,
84 const TypeFunc* slow_call_type,
85 address slow_call_address);
86 Node *value_from_mem(Node *mem, Node *ctl, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc);
87 Node *value_from_mem_phi(Node *mem, BasicType ft, const Type *ftype, const TypeOopPtr *adr_t, AllocateNode *alloc, Node_Stack *value_phis, int level);
88 Node* value_type_from_mem(Node* mem, Node* ctl, ciValueKlass* vk, const TypeAryPtr* adr_type, int offset, AllocateNode* alloc);
89
90 bool eliminate_boxing_node(CallStaticJavaNode *boxing);
91 bool eliminate_allocate_node(AllocateNode *alloc);
92 bool can_eliminate_allocation(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints);
93 bool scalar_replacement(AllocateNode *alloc, GrowableArray <SafePointNode *>& safepoints_done);
94 void process_users_of_allocation(CallNode *alloc);
95
96 void eliminate_gc_barrier(Node *p2x);
97 void mark_eliminated_box(Node* box, Node* obj);
98 void mark_eliminated_locking_nodes(AbstractLockNode *alock);
99 bool eliminate_locking_node(AbstractLockNode *alock);
100 void expand_lock_node(LockNode *lock);
101 void expand_unlock_node(UnlockNode *unlock);
102 void expand_mh_intrinsic_return(CallStaticJavaNode* call);
103
104 // More helper methods modeled after GraphKit for array copy
105 void insert_mem_bar(Node** ctrl, Node** mem, int opcode, Node* precedent = NULL);
106 Node* array_element_address(Node* ary, Node* idx, BasicType elembt);
107 Node* ConvI2L(Node* offset);
108 Node* make_leaf_call(Node* ctrl, Node* mem,
109 const TypeFunc* call_type, address call_addr,
110 const char* call_name,
111 const TypePtr* adr_type,
112 Node* parm0 = NULL, Node* parm1 = NULL,
113 Node* parm2 = NULL, Node* parm3 = NULL,
114 Node* parm4 = NULL, Node* parm5 = NULL,
115 Node* parm6 = NULL, Node* parm7 = NULL);
116
117 // helper methods modeled after LibraryCallKit for array copy
118 Node* generate_guard(Node** ctrl, Node* test, RegionNode* region, float true_prob);
119 Node* generate_slow_guard(Node** ctrl, Node* test, RegionNode* region);
120 Node* generate_fair_guard(Node** ctrl, Node* test, RegionNode* region);
121 void generate_negative_guard(Node** ctrl, Node* index, RegionNode* region);
122 void generate_limit_guard(Node** ctrl, Node* offset, Node* subseq_length, Node* array_length, RegionNode* region);
123
124 // More helper methods for array copy
125 Node* generate_nonpositive_guard(Node** ctrl, Node* index, bool never_negative);
126 Node* generate_flattened_array_guard(Node** ctrl, Node* mem, Node* obj, RegionNode* region);
127 Node* generate_object_array_guard(Node** ctrl, Node* mem, Node* obj, RegionNode* region);
128 Node* generate_array_guard(Node** ctrl, Node* mem, Node* obj, RegionNode* region, jint lh_con);
129
130 void finish_arraycopy_call(Node* call, Node** ctrl, MergeMemNode** mem, const TypePtr* adr_type);
131 address basictype2arraycopy(BasicType t,
132 Node* src_offset,
133 Node* dest_offset,
134 bool disjoint_bases,
135 const char* &name,
136 bool dest_uninitialized);
137 Node* generate_arraycopy(ArrayCopyNode *ac,
138 AllocateArrayNode* alloc,
139 Node** ctrl, MergeMemNode* mem, Node** io,
140 const TypePtr* adr_type,
141 BasicType basic_elem_type,
142 Node* src, Node* src_offset,
143 Node* dest, Node* dest_offset,
144 Node* copy_length,
145 Node* dest_length,
146 bool disjoint_bases = false,
147 bool length_never_negative = false,
148 RegionNode* slow_region = NULL);
149 void generate_clear_array(Node* ctrl, MergeMemNode* merge_mem,
150 const TypePtr* adr_type,
151 Node* dest,
152 Node* val,
153 Node* raw_val,
154 BasicType basic_elem_type,
155 Node* slice_idx,
156 Node* slice_len,
157 Node* dest_size);
158 bool generate_block_arraycopy(Node** ctrl, MergeMemNode** mem, Node* io,
159 const TypePtr* adr_type,
160 BasicType basic_elem_type,
161 AllocateNode* alloc,
162 Node* src, Node* src_offset,
163 Node* dest, Node* dest_offset,
164 Node* dest_size, bool dest_uninitialized);
165 MergeMemNode* generate_slow_arraycopy(ArrayCopyNode *ac,
166 Node** ctrl, Node* mem, Node** io,
167 const TypePtr* adr_type,
168 Node* src, Node* src_offset,
169 Node* dest, Node* dest_offset,
170 Node* copy_length, bool dest_uninitialized);
171 Node* generate_checkcast_arraycopy(Node** ctrl, MergeMemNode** mem,
172 const TypePtr* adr_type,
173 Node* dest_elem_klass,
174 Node* src, Node* src_offset,
175 Node* dest, Node* dest_offset,
176 Node* copy_length, bool dest_uninitialized);
177 Node* generate_generic_arraycopy(Node** ctrl, MergeMemNode** mem,
178 const TypePtr* adr_type,
179 Node* src, Node* src_offset,
180 Node* dest, Node* dest_offset,
181 Node* copy_length, bool dest_uninitialized);
182 void generate_unchecked_arraycopy(Node** ctrl, MergeMemNode** mem,
183 const TypePtr* adr_type,
184 BasicType basic_elem_type,
185 bool disjoint_bases,
186 Node* src, Node* src_offset,
187 Node* dest, Node* dest_offset,
188 Node* copy_length, bool dest_uninitialized);
189 const TypePtr* adjust_parameters_for_vt(const TypeAryPtr* top_dest, Node*& src_offset,
190 Node*& dest_offset, Node*& length, BasicType& dest_elem,
191 Node*& dest_length);
192 void expand_arraycopy_node(ArrayCopyNode *ac);
193
194 int replace_input(Node *use, Node *oldref, Node *newref);
195 void copy_call_debug_info(CallNode *oldcall, CallNode * newcall);
196 Node* opt_bits_test(Node* ctrl, Node* region, int edge, Node* word, int mask, int bits, bool return_fast_path = false);
197 void copy_predefined_input_for_runtime_call(Node * ctrl, CallNode* oldcall, CallNode* call);
198 CallNode* make_slow_call(CallNode *oldcall, const TypeFunc* slow_call_type, address slow_call,
199 const char* leaf_name, Node* slow_path, Node* parm0, Node* parm1,
200 Node* parm2);
201 void extract_call_projections(CallNode *call);
202
203 Node* initialize_object(AllocateNode* alloc,
204 Node* control, Node* rawmem, Node* object,
205 Node* klass_node, Node* length,
206 Node* size_in_bytes);
207
208 Node* make_arraycopy_load(ArrayCopyNode* ac, intptr_t offset, Node* ctl, Node* mem, BasicType ft, const Type *ftype, AllocateNode *alloc);
209
210 bool can_try_zeroing_elimination(AllocateArrayNode* alloc, Node* src, Node* dest) const;
211
212 public:
213 PhaseMacroExpand(PhaseIterGVN &igvn) : Phase(Macro_Expand), _igvn(igvn), _has_locks(false) {
214 _igvn.set_delay_transform(true);
215 }
216 void eliminate_macro_nodes();
217 bool expand_macro_nodes();
218
219 PhaseIterGVN &igvn() const { return _igvn; }
220
221 // Members accessed from BarrierSetC2
222 void replace_node(Node* source, Node* target) { _igvn.replace_node(source, target); }
223 Node* intcon(jint con) const { return _igvn.intcon(con); }
224 Node* longcon(jlong con) const { return _igvn.longcon(con); }
225 Node* makecon(const Type *t) const { return _igvn.makecon(t); }
226 Node* top() const { return C->top(); }
227
228 Node* prefetch_allocation(Node* i_o,
229 Node*& needgc_false, Node*& contended_phi_rawmem,
230 Node* old_eden_top, Node* new_eden_top,
|