16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "opto/castnode.hpp"
27 #include "opto/graphKit.hpp"
28 #include "opto/phaseX.hpp"
29 #include "opto/rootnode.hpp"
30 #include "opto/vector.hpp"
31 #include "utilities/macros.hpp"
32
33 void PhaseVector::optimize_vector_boxes() {
34 Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]);
35
36 assert(C->inlining_incrementally() == false, "sanity");
37
38 C->set_inlining_incrementally(true); // FIXME another way to signal GraphKit it's post-parsing phase?
39
40 C->for_igvn()->clear();
41 C->initial_gvn()->replace_with(&_igvn);
42
43 expand_vunbox_nodes();
44 scalarize_vbox_nodes();
45
46 C->inline_vector_reboxing_calls();
47
48 expand_vbox_nodes();
49 eliminate_vbox_alloc_nodes();
50
51 C->set_inlining_incrementally(false); // FIXME another way to signal GraphKit it's post-parsing phase?
52
53 do_cleanup();
54 }
55
56 void PhaseVector::do_cleanup() {
57 if (C->failing()) return;
58 {
59 Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]);
60 ResourceMark rm;
61 PhaseRemoveUseless pru(C->initial_gvn(), C->for_igvn());
62 }
63
64 if (C->failing()) return;
65
66 {
67 Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]);
68 _igvn = PhaseIterGVN(C->initial_gvn());
69 _igvn.optimize();
70 }
71 }
72
73 void PhaseVector::scalarize_vbox_nodes() {
74 if (C->failing()) return;
75
76 if (!EnableVectorReboxing) {
77 return; // don't scalarize vector boxes
78 }
79
80 int macro_idx = C->macro_count() - 1;
81 while (macro_idx >= 0) {
82 Node * n = C->macro_node(macro_idx);
83 assert(n->is_macro(), "only macro nodes expected here");
84 if (n->Opcode() == Op_VectorBox) {
85 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
86 scalarize_vbox_node(vbox);
87 if (C->failing()) return;
88 C->print_method(PHASE_SCALARIZE_VBOX, vbox, 3);
89 }
90 if (C->failing()) return;
91 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
92 }
93 }
94
95 void PhaseVector::expand_vbox_nodes() {
96 if (C->failing()) return;
97
98 int macro_idx = C->macro_count() - 1;
99 while (macro_idx >= 0) {
100 Node * n = C->macro_node(macro_idx);
101 assert(n->is_macro(), "only macro nodes expected here");
102 if (n->Opcode() == Op_VectorBox) {
103 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
104 expand_vbox_node(vbox);
105 if (C->failing()) return;
106 C->print_method(PHASE_EXPAND_VBOX, vbox, 3);
107 }
108 if (C->failing()) return;
109 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
110 }
111 }
112
113 void PhaseVector::expand_vunbox_nodes() {
114 if (C->failing()) return;
115
116 int macro_idx = C->macro_count() - 1;
117 while (macro_idx >= 0) {
118 Node * n = C->macro_node(macro_idx);
119 assert(n->is_macro(), "only macro nodes expected here");
120 if (n->Opcode() == Op_VectorUnbox) {
121 VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n);
122 expand_vunbox_node(vec_unbox);
123 if (C->failing()) return;
124 C->print_method(PHASE_EXPAND_VUNBOX, vec_unbox, 3);
125 }
126 if (C->failing()) return;
255
256 JVMState *jvms = sfpt->jvms();
257
258 jvms->set_endoff(sfpt->req());
259 // Now make a pass over the debug information replacing any references
260 // to the allocated object with "sobj"
261 int start = jvms->debug_start();
262 int end = jvms->debug_end();
263 sfpt->replace_edges_in_range(vec_box, sobj, start, end);
264
265 C->record_for_igvn(sfpt);
266 }
267 }
268
269 void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) {
270 if (vec_box->outcnt() > 0) {
271 Node* vbox = vec_box->in(VectorBoxNode::Box);
272 Node* vect = vec_box->in(VectorBoxNode::Value);
273 Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(), vec_box->vec_type());
274 C->gvn_replace_by(vec_box, result);
275 }
276 C->remove_macro_node(vec_box);
277 }
278
279 Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
280 Node* vect,
281 const TypeInstPtr* box_type,
282 const TypeVect* vect_type) {
283 if (vbox->is_Phi() && vect->is_Phi()) {
284 assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), "");
285 Node* new_phi = new PhiNode(vbox->as_Phi()->region(), box_type);
286 for (uint i = 1; i < vbox->req(); i++) {
287 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i), box_type, vect_type);
288 new_phi->set_req(i, new_box);
289 }
290 new_phi = C->initial_gvn()->transform(new_phi);
291 return new_phi;
292 } else if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) {
293 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0));
294 return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type);
295 } else {
296 assert(!vbox->is_Phi(), "");
297 // TODO: ensure that expanded vbox is initialized with the same value (vect).
298 return vbox; // already expanded
299 }
300 }
301
302 static bool is_vector_mask(ciKlass* klass) {
303 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
304 }
305
306 static bool is_vector_shuffle(ciKlass* klass) {
307 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
308 }
309
310 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
311 Node* value,
312 const TypeInstPtr* box_type,
313 const TypeVect* vect_type) {
314 JVMState* jvms = clone_jvms(C, vbox_alloc);
315 GraphKit kit(jvms);
316 PhaseGVN& gvn = kit.gvn();
317
318 ciInstanceKlass* box_klass = box_type->klass()->as_instance_klass();
319 BasicType bt = vect_type->element_basic_type();
320 int num_elem = vect_type->length();
321
322 bool is_mask = is_vector_mask(box_klass);
323 if (is_mask && bt != T_BOOLEAN) {
324 value = gvn.transform(new VectorStoreMaskNode(value, bt, num_elem));
325 // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array.
326 bt = T_BOOLEAN;
327 assert(value->as_Vector()->bottom_type()->is_vect()->element_basic_type() == bt,
328 "must be consistent with mask representation");
329 }
330
331 // Generate array allocation for the field which holds the values.
332 const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt));
333 Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1);
334
335 // Store the vector value into the array.
336 // (The store should be captured by InitializeNode and turned into initialized store later.)
337 Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt);
338 const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr();
339 Node* arr_mem = kit.memory(arr_adr);
340 Node* vstore = gvn.transform(StoreVectorNode::make(0,
341 kit.control(),
342 arr_mem,
343 arr_adr,
344 arr_adr_type,
356 // Store the allocated array into object.
357 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbol::payload_name(),
358 ciSymbol::object_signature(),
359 false);
360 assert(field != NULL, "");
361 Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes());
362 const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr();
363
364 // The store should be captured by InitializeNode and turned into initialized store later.
365 Node* field_store = gvn.transform(kit.access_store_at(vec_obj,
366 vec_field,
367 vec_adr_type,
368 arr,
369 TypeOopPtr::make_from_klass(field->type()->as_klass()),
370 T_OBJECT,
371 IN_HEAP));
372 kit.set_memory(field_store, vec_adr_type);
373
374 kit.replace_call(vbox_alloc, vec_obj, true);
375 C->remove_macro_node(vbox_alloc);
376 return vec_obj;
377 }
378
379 void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) {
380 if (vec_unbox->outcnt() > 0) {
381 GraphKit kit;
382 PhaseGVN& gvn = kit.gvn();
383
384 Node* obj = vec_unbox->obj();
385 const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr();
386 ciInstanceKlass* from_kls = tinst->klass()->as_instance_klass();
387 BasicType bt = vec_unbox->vect_type()->element_basic_type();
388 BasicType masktype = bt;
389 BasicType elem_bt;
390
391 if (is_vector_mask(from_kls)) {
392 bt = T_BOOLEAN;
393 } else if (is_vector_shuffle(from_kls)) {
394 if (vec_unbox->is_shuffle_to_vector() == true) {
395 elem_bt = bt;
|
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "opto/castnode.hpp"
27 #include "opto/graphKit.hpp"
28 #include "opto/phaseX.hpp"
29 #include "opto/rootnode.hpp"
30 #include "opto/vector.hpp"
31 #include "utilities/macros.hpp"
32
33 void PhaseVector::optimize_vector_boxes() {
34 Compile::TracePhase tp("vector_elimination", &timers[_t_vector_elimination]);
35
36 // Signal GraphKit it's post-parse phase.
37 assert(C->inlining_incrementally() == false, "sanity");
38 C->set_inlining_incrementally(true);
39
40 C->for_igvn()->clear();
41 C->initial_gvn()->replace_with(&_igvn);
42
43 expand_vunbox_nodes();
44 scalarize_vbox_nodes();
45
46 C->inline_vector_reboxing_calls();
47
48 expand_vbox_nodes();
49 eliminate_vbox_alloc_nodes();
50
51 C->set_inlining_incrementally(false);
52
53 do_cleanup();
54 }
55
56 void PhaseVector::do_cleanup() {
57 if (C->failing()) return;
58 {
59 Compile::TracePhase tp("vector_pru", &timers[_t_vector_pru]);
60 ResourceMark rm;
61 PhaseRemoveUseless pru(C->initial_gvn(), C->for_igvn());
62 if (C->failing()) return;
63 }
64 {
65 Compile::TracePhase tp("incrementalInline_igvn", &timers[_t_vector_igvn]);
66 _igvn = PhaseIterGVN(C->initial_gvn());
67 _igvn.optimize();
68 if (C->failing()) return;
69 }
70 C->print_method(PHASE_ITER_GVN_BEFORE_EA, 3);
71 }
72
73 void PhaseVector::scalarize_vbox_nodes() {
74 if (C->failing()) return;
75
76 if (!EnableVectorReboxing) {
77 return; // don't scalarize vector boxes
78 }
79
80 int macro_idx = C->macro_count() - 1;
81 while (macro_idx >= 0) {
82 Node * n = C->macro_node(macro_idx);
83 assert(n->is_macro(), "only macro nodes expected here");
84 if (n->Opcode() == Op_VectorBox) {
85 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
86 scalarize_vbox_node(vbox);
87 if (C->failing()) return;
88 C->print_method(PHASE_SCALARIZE_VBOX, vbox, 3);
89 }
90 if (C->failing()) return;
91 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
92 }
93 }
94
95 void PhaseVector::expand_vbox_nodes() {
96 if (C->failing()) return;
97
98 int macro_idx = C->macro_count() - 1;
99 while (macro_idx >= 0) {
100 Node * n = C->macro_node(macro_idx);
101 assert(n->is_macro(), "only macro nodes expected here");
102 if (n->Opcode() == Op_VectorBox) {
103 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n);
104 expand_vbox_node(vbox);
105 if (C->failing()) return;
106 }
107 if (C->failing()) return;
108 macro_idx = MIN2(macro_idx - 1, C->macro_count() - 1);
109 }
110 }
111
112 void PhaseVector::expand_vunbox_nodes() {
113 if (C->failing()) return;
114
115 int macro_idx = C->macro_count() - 1;
116 while (macro_idx >= 0) {
117 Node * n = C->macro_node(macro_idx);
118 assert(n->is_macro(), "only macro nodes expected here");
119 if (n->Opcode() == Op_VectorUnbox) {
120 VectorUnboxNode* vec_unbox = static_cast<VectorUnboxNode*>(n);
121 expand_vunbox_node(vec_unbox);
122 if (C->failing()) return;
123 C->print_method(PHASE_EXPAND_VUNBOX, vec_unbox, 3);
124 }
125 if (C->failing()) return;
254
255 JVMState *jvms = sfpt->jvms();
256
257 jvms->set_endoff(sfpt->req());
258 // Now make a pass over the debug information replacing any references
259 // to the allocated object with "sobj"
260 int start = jvms->debug_start();
261 int end = jvms->debug_end();
262 sfpt->replace_edges_in_range(vec_box, sobj, start, end);
263
264 C->record_for_igvn(sfpt);
265 }
266 }
267
268 void PhaseVector::expand_vbox_node(VectorBoxNode* vec_box) {
269 if (vec_box->outcnt() > 0) {
270 Node* vbox = vec_box->in(VectorBoxNode::Box);
271 Node* vect = vec_box->in(VectorBoxNode::Value);
272 Node* result = expand_vbox_node_helper(vbox, vect, vec_box->box_type(), vec_box->vec_type());
273 C->gvn_replace_by(vec_box, result);
274 C->print_method(PHASE_EXPAND_VBOX, vec_box, 3);
275 }
276 C->remove_macro_node(vec_box);
277 }
278
279 Node* PhaseVector::expand_vbox_node_helper(Node* vbox,
280 Node* vect,
281 const TypeInstPtr* box_type,
282 const TypeVect* vect_type) {
283 if (vbox->is_Phi() && vect->is_Phi()) {
284 assert(vbox->as_Phi()->region() == vect->as_Phi()->region(), "");
285 Node* new_phi = new PhiNode(vbox->as_Phi()->region(), box_type);
286 for (uint i = 1; i < vbox->req(); i++) {
287 Node* new_box = expand_vbox_node_helper(vbox->in(i), vect->in(i), box_type, vect_type);
288 new_phi->set_req(i, new_box);
289 }
290 new_phi = C->initial_gvn()->transform(new_phi);
291 return new_phi;
292 } else if (vbox->is_Proj() && vbox->in(0)->Opcode() == Op_VectorBoxAllocate) {
293 VectorBoxAllocateNode* vbox_alloc = static_cast<VectorBoxAllocateNode*>(vbox->in(0));
294 return expand_vbox_alloc_node(vbox_alloc, vect, box_type, vect_type);
295 } else {
296 assert(!vbox->is_Phi(), "");
297 // TODO: assert that expanded vbox is initialized with the same value (vect).
298 return vbox; // already expanded
299 }
300 }
301
302 static bool is_vector_mask(ciKlass* klass) {
303 return klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass());
304 }
305
306 static bool is_vector_shuffle(ciKlass* klass) {
307 return klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass());
308 }
309
310 Node* PhaseVector::expand_vbox_alloc_node(VectorBoxAllocateNode* vbox_alloc,
311 Node* value,
312 const TypeInstPtr* box_type,
313 const TypeVect* vect_type) {
314 JVMState* jvms = clone_jvms(C, vbox_alloc);
315 GraphKit kit(jvms);
316 PhaseGVN& gvn = kit.gvn();
317
318 ciInstanceKlass* box_klass = box_type->klass()->as_instance_klass();
319 BasicType bt = vect_type->element_basic_type();
320 int num_elem = vect_type->length();
321
322 bool is_mask = is_vector_mask(box_klass);
323 if (is_mask && bt != T_BOOLEAN) {
324 value = gvn.transform(VectorStoreMaskNode::make(gvn, value, bt, num_elem));
325 // Although type of mask depends on its definition, in terms of storage everything is stored in boolean array.
326 bt = T_BOOLEAN;
327 assert(value->as_Vector()->bottom_type()->is_vect()->element_basic_type() == bt,
328 "must be consistent with mask representation");
329 }
330
331 // Generate array allocation for the field which holds the values.
332 const TypeKlassPtr* array_klass = TypeKlassPtr::make(ciTypeArrayKlass::make(bt));
333 Node* arr = kit.new_array(kit.makecon(array_klass), kit.intcon(num_elem), 1);
334
335 // Store the vector value into the array.
336 // (The store should be captured by InitializeNode and turned into initialized store later.)
337 Node* arr_adr = kit.array_element_address(arr, kit.intcon(0), bt);
338 const TypePtr* arr_adr_type = arr_adr->bottom_type()->is_ptr();
339 Node* arr_mem = kit.memory(arr_adr);
340 Node* vstore = gvn.transform(StoreVectorNode::make(0,
341 kit.control(),
342 arr_mem,
343 arr_adr,
344 arr_adr_type,
356 // Store the allocated array into object.
357 ciField* field = ciEnv::current()->vector_VectorPayload_klass()->get_field_by_name(ciSymbol::payload_name(),
358 ciSymbol::object_signature(),
359 false);
360 assert(field != NULL, "");
361 Node* vec_field = kit.basic_plus_adr(vec_obj, field->offset_in_bytes());
362 const TypePtr* vec_adr_type = vec_field->bottom_type()->is_ptr();
363
364 // The store should be captured by InitializeNode and turned into initialized store later.
365 Node* field_store = gvn.transform(kit.access_store_at(vec_obj,
366 vec_field,
367 vec_adr_type,
368 arr,
369 TypeOopPtr::make_from_klass(field->type()->as_klass()),
370 T_OBJECT,
371 IN_HEAP));
372 kit.set_memory(field_store, vec_adr_type);
373
374 kit.replace_call(vbox_alloc, vec_obj, true);
375 C->remove_macro_node(vbox_alloc);
376
377 return vec_obj;
378 }
379
380 void PhaseVector::expand_vunbox_node(VectorUnboxNode* vec_unbox) {
381 if (vec_unbox->outcnt() > 0) {
382 GraphKit kit;
383 PhaseGVN& gvn = kit.gvn();
384
385 Node* obj = vec_unbox->obj();
386 const TypeInstPtr* tinst = gvn.type(obj)->isa_instptr();
387 ciInstanceKlass* from_kls = tinst->klass()->as_instance_klass();
388 BasicType bt = vec_unbox->vect_type()->element_basic_type();
389 BasicType masktype = bt;
390 BasicType elem_bt;
391
392 if (is_vector_mask(from_kls)) {
393 bt = T_BOOLEAN;
394 } else if (is_vector_shuffle(from_kls)) {
395 if (vec_unbox->is_shuffle_to_vector() == true) {
396 elem_bt = bt;
|