222 GrowableArray<Node*> _iteration_last; // nodes in the generation that has deps to phi
223 GrowableArray<SWNodeInfo> _node_info; // Info needed per node
224 CloneMap& _clone_map; // map of nodes created in cloning
225
226 MemNode* _align_to_ref; // Memory reference that pre-loop will align to
227
228 GrowableArray<OrderedPair> _disjoint_ptrs; // runtime disambiguated pointer pairs
229
230 DepGraph _dg; // Dependence graph
231
232 // Scratch pads
233 VectorSet _visited; // Visited set
234 VectorSet _post_visited; // Post-visited set
235 Node_Stack _n_idx_list; // List of (node,index) pairs
236 GrowableArray<Node*> _nlist; // List of nodes
237 GrowableArray<Node*> _stk; // Stack of nodes
238
239 public:
240 SuperWord(PhaseIdealLoop* phase);
241
242 void transform_loop(IdealLoopTree* lpt);
243
244 // Accessors for SWPointer
245 PhaseIdealLoop* phase() { return _phase; }
246 IdealLoopTree* lpt() { return _lpt; }
247 PhiNode* iv() { return _iv; }
248
249 private:
250 IdealLoopTree* _lpt; // Current loop tree node
251 LoopNode* _lp; // Current LoopNode
252 Node* _bb; // Current basic block
253 PhiNode* _iv; // Induction var
254 bool _race_possible; // In cases where SDMU is true
255 bool _do_vector_loop; // whether to do vectorization/simd style
256 bool _vector_loop_debug; // provide more printing in debug mode
257 int _num_work_vecs; // Number of non memory vector operations
258 int _num_reductions; // Number of reduction expressions applied
259 int _ii_first; // generation with direct deps from mem phi
260 int _ii_last; // generation with direct deps to mem phi
261 GrowableArray<int> _ii_order;
262
263 // Accessors
264 Arena* arena() { return _arena; }
265
266 Node* bb() { return _bb; }
267 void set_bb(Node* bb) { _bb = bb; }
268
269 void set_lpt(IdealLoopTree* lpt) { _lpt = lpt; }
270
271 LoopNode* lp() { return _lp; }
272 void set_lp(LoopNode* lp) { _lp = lp;
273 _iv = lp->as_CountedLoop()->phi()->as_Phi(); }
274 int iv_stride() { return lp()->as_CountedLoop()->stride_con(); }
453 char* blank(uint depth);
454
455 void packset_sort(int n);
456 };
457
458
459
460 //------------------------------SWPointer---------------------------
461 // Information about an address for dependence checking and vector alignment
462 class SWPointer VALUE_OBJ_CLASS_SPEC {
463 protected:
464 MemNode* _mem; // My memory reference node
465 SuperWord* _slp; // SuperWord class
466
467 Node* _base; // NULL if unsafe nonheap reference
468 Node* _adr; // address pointer
469 jint _scale; // multiplier for iv (in bytes), 0 if no loop iv
470 jint _offset; // constant offset (in bytes)
471 Node* _invar; // invariant offset (in bytes), NULL if none
472 bool _negate_invar; // if true then use: (0 - _invar)
473
474 PhaseIdealLoop* phase() { return _slp->phase(); }
475 IdealLoopTree* lpt() { return _slp->lpt(); }
476 PhiNode* iv() { return _slp->iv(); } // Induction var
477
478 bool invariant(Node* n) {
479 Node *n_c = phase()->get_ctrl(n);
480 return !lpt()->is_member(phase()->get_loop(n_c));
481 }
482
483 // Match: k*iv + offset
484 bool scaled_iv_plus_offset(Node* n);
485 // Match: k*iv where k is a constant that's not zero
486 bool scaled_iv(Node* n);
487 // Match: offset is (k [+/- invariant])
488 bool offset_plus_k(Node* n, bool negate = false);
489
490 public:
491 enum CMP {
492 Less = 1,
493 Greater = 2,
494 Equal = 4,
495 NotEqual = (Less | Greater),
496 NotComparable = (Less | Greater | Equal)
497 };
498
499 SWPointer(MemNode* mem, SuperWord* slp);
500 // Following is used to create a temporary object during
501 // the pattern match of an address expression.
502 SWPointer(SWPointer* p);
503
504 bool valid() { return _adr != NULL; }
505 bool has_iv() { return _scale != 0; }
506
507 Node* base() { return _base; }
508 Node* adr() { return _adr; }
509 MemNode* mem() { return _mem; }
510 int scale_in_bytes() { return _scale; }
511 Node* invar() { return _invar; }
512 bool negate_invar() { return _negate_invar; }
513 int offset_in_bytes() { return _offset; }
514 int memory_size() { return _mem->memory_size(); }
515
516 // Comparable?
517 int cmp(SWPointer& q) {
518 if (valid() && q.valid() &&
519 (_adr == q._adr || _base == _adr && q._base == q._adr) &&
520 _scale == q._scale &&
521 _invar == q._invar &&
522 _negate_invar == q._negate_invar) {
523 bool overlap = q._offset < _offset + memory_size() &&
524 _offset < q._offset + q.memory_size();
525 return overlap ? Equal : (_offset < q._offset ? Less : Greater);
526 } else {
527 return NotComparable;
528 }
529 }
530
531 bool not_equal(SWPointer& q) { return not_equal(cmp(q)); }
532 bool equal(SWPointer& q) { return equal(cmp(q)); }
533 bool comparable(SWPointer& q) { return comparable(cmp(q)); }
534 static bool not_equal(int cmp) { return cmp <= NotEqual; }
|
222 GrowableArray<Node*> _iteration_last; // nodes in the generation that has deps to phi
223 GrowableArray<SWNodeInfo> _node_info; // Info needed per node
224 CloneMap& _clone_map; // map of nodes created in cloning
225
226 MemNode* _align_to_ref; // Memory reference that pre-loop will align to
227
228 GrowableArray<OrderedPair> _disjoint_ptrs; // runtime disambiguated pointer pairs
229
230 DepGraph _dg; // Dependence graph
231
232 // Scratch pads
233 VectorSet _visited; // Visited set
234 VectorSet _post_visited; // Post-visited set
235 Node_Stack _n_idx_list; // List of (node,index) pairs
236 GrowableArray<Node*> _nlist; // List of nodes
237 GrowableArray<Node*> _stk; // Stack of nodes
238
239 public:
240 SuperWord(PhaseIdealLoop* phase);
241
242 void transform_loop(IdealLoopTree* lpt, bool do_optimization);
243
244 void unrolling_analysis(CountedLoopNode *cl, int &local_loop_unroll_factor);
245
246 // Accessors for SWPointer
247 PhaseIdealLoop* phase() { return _phase; }
248 IdealLoopTree* lpt() { return _lpt; }
249 PhiNode* iv() { return _iv; }
250 bool early_return() { return _early_return; }
251
252 private:
253 IdealLoopTree* _lpt; // Current loop tree node
254 LoopNode* _lp; // Current LoopNode
255 Node* _bb; // Current basic block
256 PhiNode* _iv; // Induction var
257 bool _race_possible; // In cases where SDMU is true
258 bool _early_return; // True if we do not initialize
259 bool _do_vector_loop; // whether to do vectorization/simd style
260 bool _vector_loop_debug; // provide more printing in debug mode
261 int _num_work_vecs; // Number of non memory vector operations
262 int _num_reductions; // Number of reduction expressions applied
263 int _ii_first; // generation with direct deps from mem phi
264 int _ii_last; // generation with direct deps to mem phi
265 GrowableArray<int> _ii_order;
266
267 // Accessors
268 Arena* arena() { return _arena; }
269
270 Node* bb() { return _bb; }
271 void set_bb(Node* bb) { _bb = bb; }
272
273 void set_lpt(IdealLoopTree* lpt) { _lpt = lpt; }
274
275 LoopNode* lp() { return _lp; }
276 void set_lp(LoopNode* lp) { _lp = lp;
277 _iv = lp->as_CountedLoop()->phi()->as_Phi(); }
278 int iv_stride() { return lp()->as_CountedLoop()->stride_con(); }
457 char* blank(uint depth);
458
459 void packset_sort(int n);
460 };
461
462
463
464 //------------------------------SWPointer---------------------------
465 // Information about an address for dependence checking and vector alignment
466 class SWPointer VALUE_OBJ_CLASS_SPEC {
467 protected:
468 MemNode* _mem; // My memory reference node
469 SuperWord* _slp; // SuperWord class
470
471 Node* _base; // NULL if unsafe nonheap reference
472 Node* _adr; // address pointer
473 jint _scale; // multiplier for iv (in bytes), 0 if no loop iv
474 jint _offset; // constant offset (in bytes)
475 Node* _invar; // invariant offset (in bytes), NULL if none
476 bool _negate_invar; // if true then use: (0 - _invar)
477 Node_Stack* _nstack; // stack used to record a swpointer trace of variants
478 bool _analyze_only; // Used in loop unrolling only for swpointer trace
479 uint _stack_idx; // Used in loop unrolling only for swpointer trace
480
481 PhaseIdealLoop* phase() { return _slp->phase(); }
482 IdealLoopTree* lpt() { return _slp->lpt(); }
483 PhiNode* iv() { return _slp->iv(); } // Induction var
484
485 bool invariant(Node* n) {
486 Node *n_c = phase()->get_ctrl(n);
487 return !lpt()->is_member(phase()->get_loop(n_c));
488 }
489
490 // Match: k*iv + offset
491 bool scaled_iv_plus_offset(Node* n);
492 // Match: k*iv where k is a constant that's not zero
493 bool scaled_iv(Node* n);
494 // Match: offset is (k [+/- invariant])
495 bool offset_plus_k(Node* n, bool negate = false);
496
497 public:
498 enum CMP {
499 Less = 1,
500 Greater = 2,
501 Equal = 4,
502 NotEqual = (Less | Greater),
503 NotComparable = (Less | Greater | Equal)
504 };
505
506 SWPointer(MemNode* mem, SuperWord* slp, Node_Stack *nstack, bool analyze_only);
507 // Following is used to create a temporary object during
508 // the pattern match of an address expression.
509 SWPointer(SWPointer* p);
510
511 bool valid() { return _adr != NULL; }
512 bool has_iv() { return _scale != 0; }
513
514 Node* base() { return _base; }
515 Node* adr() { return _adr; }
516 MemNode* mem() { return _mem; }
517 int scale_in_bytes() { return _scale; }
518 Node* invar() { return _invar; }
519 bool negate_invar() { return _negate_invar; }
520 int offset_in_bytes() { return _offset; }
521 int memory_size() { return _mem->memory_size(); }
522 Node_Stack* node_stack() { return _nstack; }
523
524 // Comparable?
525 int cmp(SWPointer& q) {
526 if (valid() && q.valid() &&
527 (_adr == q._adr || _base == _adr && q._base == q._adr) &&
528 _scale == q._scale &&
529 _invar == q._invar &&
530 _negate_invar == q._negate_invar) {
531 bool overlap = q._offset < _offset + memory_size() &&
532 _offset < q._offset + q.memory_size();
533 return overlap ? Equal : (_offset < q._offset ? Less : Greater);
534 } else {
535 return NotComparable;
536 }
537 }
538
539 bool not_equal(SWPointer& q) { return not_equal(cmp(q)); }
540 bool equal(SWPointer& q) { return equal(cmp(q)); }
541 bool comparable(SWPointer& q) { return comparable(cmp(q)); }
542 static bool not_equal(int cmp) { return cmp <= NotEqual; }
|