133 tty->print("lpt->_head %d", lpt->_head->_idx); lpt->_head->dump();
134 lpt->dump_head();
135 }
136 #endif
137 return;
138 }
139
140 // Make sure the are no extra control users of the loop backedge
141 if (cl->back_control()->outcnt() != 1) {
142 return;
143 }
144
145 // Skip any loops already optimized by slp
146 if (cl->is_vectorized_loop()) return;
147
148 if (cl->is_main_loop()) {
149 // Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit))))
150 CountedLoopEndNode* pre_end = get_pre_loop_end(cl);
151 if (pre_end == NULL) return;
152 Node *pre_opaq1 = pre_end->limit();
153 if (pre_opaq1->Opcode() != Op_Opaque1) return;
154 }
155
156 init(); // initialize data structures
157
158 set_lpt(lpt);
159 set_lp(cl);
160
161 // For now, define one block which is the entire loop body
162 set_bb(cl);
163
164 if (do_optimization) {
165 assert(_packset.length() == 0, "packset must be empty");
166 SLP_extract();
167 if (PostLoopMultiversioning && Matcher::has_predicated_vectors()) {
168 if (cl->is_vectorized_loop() && cl->is_main_loop() && !cl->is_reduction_loop()) {
169 IdealLoopTree *lpt_next = lpt->_next;
170 CountedLoopNode *cl_next = lpt_next->_head->as_CountedLoop();
171 _phase->has_range_checks(lpt_next);
172 if (cl_next->is_post_loop() && !cl_next->range_checks_present()) {
173 if (!cl_next->is_vectorized_loop()) {
1019 Node* prev = NULL;
1020 while (true) {
1021 NOT_PRODUCT( if(is_trace_mem_slice()) tty->print_cr("SuperWord::mem_slice_preds: n %d", n->_idx);)
1022 assert(in_bb(n), "must be in block");
1023 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1024 Node* out = n->fast_out(i);
1025 if (out->is_Load()) {
1026 if (in_bb(out)) {
1027 preds.push(out);
1028 if (TraceSuperWord && Verbose) {
1029 tty->print_cr("SuperWord::mem_slice_preds: added pred(%d)", out->_idx);
1030 }
1031 }
1032 } else {
1033 // FIXME
1034 if (out->is_MergeMem() && !in_bb(out)) {
1035 // Either unrolling is causing a memory edge not to disappear,
1036 // or need to run igvn.optimize() again before SLP
1037 } else if (out->is_Phi() && out->bottom_type() == Type::MEMORY && !in_bb(out)) {
1038 // Ditto. Not sure what else to check further.
1039 } else if (out->Opcode() == Op_StoreCM && out->in(MemNode::OopStore) == n) {
1040 // StoreCM has an input edge used as a precedence edge.
1041 // Maybe an issue when oop stores are vectorized.
1042 } else {
1043 assert(out == prev || prev == NULL, "no branches off of store slice");
1044 }
1045 }//else
1046 }//for
1047 if (n == stop) break;
1048 preds.push(n);
1049 if (TraceSuperWord && Verbose) {
1050 tty->print_cr("SuperWord::mem_slice_preds: added pred(%d)", n->_idx);
1051 }
1052 prev = n;
1053 assert(n->is_Mem(), "unexpected node %s", n->Name());
1054 n = n->in(MemNode::Memory);
1055 }
1056 }
1057
1058 //------------------------------stmts_can_pack---------------------------
1059 // Can s1 and s2 be in a pack with s1 immediately preceding s2 and
1780 else {
1781 return false;
1782 }
1783
1784 for (uint j = 1; j < cmpd_pk->size(); j++) {
1785 if (cmpd_pk->at(j)->in(1) != cmovd_pk->at(j)->as_CMove()->in(cmovd_ind1)
1786 || cmpd_pk->at(j)->in(2) != cmovd_pk->at(j)->as_CMove()->in(cmovd_ind2)) {
1787 return false;
1788 }//if
1789 }
1790 NOT_PRODUCT(if(_sw->is_trace_cmov()) { tty->print("CMoveKit::test_cmpd_pack: cmpd pack for 1st CmpD %d is OK for vectorization: ", cmpd0->_idx); cmpd0->dump(); })
1791 return true;
1792 }
1793
1794 //------------------------------implemented---------------------------
1795 // Can code be generated for pack p?
1796 bool SuperWord::implemented(Node_List* p) {
1797 bool retValue = false;
1798 Node* p0 = p->at(0);
1799 if (p0 != NULL) {
1800 int opc = p0->Opcode();
1801 uint size = p->size();
1802 if (p0->is_reduction()) {
1803 const Type *arith_type = p0->bottom_type();
1804 // Length 2 reductions of INT/LONG do not offer performance benefits
1805 if (((arith_type->basic_type() == T_INT) || (arith_type->basic_type() == T_LONG)) && (size == 2)) {
1806 retValue = false;
1807 } else {
1808 retValue = ReductionNode::implemented(opc, size, arith_type->basic_type());
1809 }
1810 } else {
1811 retValue = VectorNode::implemented(opc, size, velt_basic_type(p0));
1812 }
1813 if (!retValue) {
1814 if (is_cmov_pack(p)) {
1815 NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::implemented: found cmpd pack"); print_pack(p);})
1816 return true;
1817 }
1818 }
1819 }
1820 return retValue;
2168
2169 if (do_reserve_copy() && !make_reversable.has_reserved()) {
2170 NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: loop was not reserved correctly, exiting SuperWord");})
2171 return;
2172 }
2173
2174 for (int i = 0; i < _block.length(); i++) {
2175 Node* n = _block.at(i);
2176 Node_List* p = my_pack(n);
2177 if (p && n == executed_last(p)) {
2178 uint vlen = p->size();
2179 uint vlen_in_bytes = 0;
2180 Node* vn = NULL;
2181 Node* low_adr = p->at(0);
2182 Node* first = executed_first(p);
2183 if (can_process_post_loop) {
2184 // override vlen with the main loops vector length
2185 vlen = cl->slp_max_unroll();
2186 }
2187 NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: %d executed first, %d executed last in pack", first->_idx, n->_idx); print_pack(p);})
2188 int opc = n->Opcode();
2189 if (n->is_Load()) {
2190 Node* ctl = n->in(MemNode::Control);
2191 Node* mem = first->in(MemNode::Memory);
2192 SWPointer p1(n->as_Mem(), this, NULL, false);
2193 // Identify the memory dependency for the new loadVector node by
2194 // walking up through memory chain.
2195 // This is done to give flexibility to the new loadVector node so that
2196 // it can move above independent storeVector nodes.
2197 while (mem->is_StoreVector()) {
2198 SWPointer p2(mem->as_Mem(), this, NULL, false);
2199 int cmp = p1.cmp(p2);
2200 if (SWPointer::not_equal(cmp) || !SWPointer::comparable(cmp)) {
2201 mem = mem->in(MemNode::Memory);
2202 } else {
2203 break; // dependent memory
2204 }
2205 }
2206 Node* adr = low_adr->in(MemNode::Address);
2207 const TypePtr* atyp = n->adr_type();
2208 vn = LoadVectorNode::make(opc, ctl, mem, adr, atyp, vlen, velt_basic_type(n), control_dependency(p));
2250 ShouldNotReachHere();
2251 }
2252 if (VectorNode::is_invariant_vector(in1) && (node_isa_reduction == false) && (n->is_Add() || n->is_Mul())) {
2253 // Move invariant vector input into second position to avoid register spilling.
2254 Node* tmp = in1;
2255 in1 = in2;
2256 in2 = tmp;
2257 }
2258 if (node_isa_reduction) {
2259 const Type *arith_type = n->bottom_type();
2260 vn = ReductionNode::make(opc, NULL, in1, in2, arith_type->basic_type());
2261 if (in2->is_Load()) {
2262 vlen_in_bytes = in2->as_LoadVector()->memory_size();
2263 } else {
2264 vlen_in_bytes = in2->as_Vector()->length_in_bytes();
2265 }
2266 } else {
2267 vn = VectorNode::make(opc, in1, in2, vlen, velt_basic_type(n));
2268 vlen_in_bytes = vn->as_Vector()->length_in_bytes();
2269 }
2270 } else if (opc == Op_SqrtD || opc == Op_AbsF || opc == Op_AbsD || opc == Op_NegF || opc == Op_NegD) {
2271 // Promote operand to vector (Sqrt/Abs/Neg are 2 address instructions)
2272 Node* in = vector_opd(p, 1);
2273 vn = VectorNode::make(opc, in, NULL, vlen, velt_basic_type(n));
2274 vlen_in_bytes = vn->as_Vector()->length_in_bytes();
2275 } else if (is_cmov_pack(p)) {
2276 if (can_process_post_loop) {
2277 // do not refactor of flow in post loop context
2278 return;
2279 }
2280 if (!n->is_CMove()) {
2281 continue;
2282 }
2283 // place here CMoveVDNode
2284 NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: print before CMove vectorization"); print_loop(false);})
2285 Node* bol = n->in(CMoveNode::Condition);
2286 if (!bol->is_Bool() && bol->Opcode() == Op_ExtractI && bol->req() > 1 ) {
2287 NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: %d is not Bool node, trying its in(1) node %d", bol->_idx, bol->in(1)->_idx); bol->dump(); bol->in(1)->dump();})
2288 bol = bol->in(1); //may be ExtractNode
2289 }
2290
2291 assert(bol->is_Bool(), "should be BoolNode - too late to bail out!");
2292 if (!bol->is_Bool()) {
2293 if (do_reserve_copy()) {
2294 NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: expected %d bool node, exiting SuperWord", bol->_idx); bol->dump();})
2295 return; //and reverse to backup IG
2296 }
2297 ShouldNotReachHere();
2298 }
2299
2300 int cond = (int)bol->as_Bool()->_test._test;
2301 Node* in_cc = _igvn.intcon(cond);
2302 NOT_PRODUCT(if(is_trace_cmov()) {tty->print("SWPointer::output: created intcon in_cc node %d", in_cc->_idx); in_cc->dump();})
2303 Node* cc = bol->clone();
2304 cc->set_req(1, in_cc);
2305 NOT_PRODUCT(if(is_trace_cmov()) {tty->print("SWPointer::output: created bool cc node %d", cc->_idx); cc->dump();})
2306
2862 bool same_type = true;
2863 for (DUIterator_Fast kmax, k = in->fast_outs(kmax); k < kmax; k++) {
2864 Node *use = in->fast_out(k);
2865 if (!in_bb(use) || !same_velt_type(use, n)) {
2866 same_type = false;
2867 break;
2868 }
2869 }
2870 if (same_type) {
2871 // For right shifts of small integer types (bool, byte, char, short)
2872 // we need precise information about sign-ness. Only Load nodes have
2873 // this information because Store nodes are the same for signed and
2874 // unsigned values. And any arithmetic operation after a load may
2875 // expand a value to signed Int so such right shifts can't be used
2876 // because vector elements do not have upper bits of Int.
2877 const Type* vt = vtn;
2878 if (VectorNode::is_shift(in)) {
2879 Node* load = in->in(1);
2880 if (load->is_Load() && in_bb(load) && (velt_type(load)->basic_type() == T_INT)) {
2881 vt = velt_type(load);
2882 } else if (in->Opcode() != Op_LShiftI) {
2883 // Widen type to Int to avoid creation of right shift vector
2884 // (align + data_size(s1) check in stmts_can_pack() will fail).
2885 // Note, left shifts work regardless type.
2886 vt = TypeInt::INT;
2887 }
2888 }
2889 set_velt_type(in, vt);
2890 }
2891 }
2892 }
2893 }
2894 }
2895 #ifndef PRODUCT
2896 if (TraceSuperWord && Verbose) {
2897 for (int i = 0; i < _block.length(); i++) {
2898 Node* n = _block.at(i);
2899 velt_type(n)->dump();
2900 tty->print("\t");
2901 n->dump();
2902 }
2926 int offset = p.offset_in_bytes();
2927 offset += iv_adjust*p.memory_size();
2928 int off_rem = offset % vw;
2929 int off_mod = off_rem >= 0 ? off_rem : off_rem + vw;
2930 if (TraceSuperWord && Verbose) {
2931 tty->print_cr("SWPointer::memory_alignment: off_rem = %d, off_mod = %d", off_rem, off_mod);
2932 }
2933 return off_mod;
2934 }
2935
2936 //---------------------------container_type---------------------------
2937 // Smallest type containing range of values
2938 const Type* SuperWord::container_type(Node* n) {
2939 if (n->is_Mem()) {
2940 BasicType bt = n->as_Mem()->memory_type();
2941 if (n->is_Store() && (bt == T_CHAR)) {
2942 // Use T_SHORT type instead of T_CHAR for stored values because any
2943 // preceding arithmetic operation extends values to signed Int.
2944 bt = T_SHORT;
2945 }
2946 if (n->Opcode() == Op_LoadUB) {
2947 // Adjust type for unsigned byte loads, it is important for right shifts.
2948 // T_BOOLEAN is used because there is no basic type representing type
2949 // TypeInt::UBYTE. Use of T_BOOLEAN for vectors is fine because only
2950 // size (one byte) and sign is important.
2951 bt = T_BOOLEAN;
2952 }
2953 return Type::get_const_basic_type(bt);
2954 }
2955 const Type* t = _igvn.type(n);
2956 if (t->basic_type() == T_INT) {
2957 // A narrow type of arithmetic operations will be determined by
2958 // propagating the type of memory operations.
2959 return TypeInt::INT;
2960 }
2961 return t;
2962 }
2963
2964 bool SuperWord::same_velt_type(Node* n1, Node* n2) {
2965 const Type* vt1 = velt_type(n1);
2966 const Type* vt2 = velt_type(n2);
3066 Node* n = p->at(i);
3067 assert(n->is_Load(), "only meaningful for loads");
3068 if (!n->depends_only_on_test()) {
3069 dep = LoadNode::Pinned;
3070 }
3071 }
3072 return dep;
3073 }
3074
3075
3076 //----------------------------align_initial_loop_index---------------------------
3077 // Adjust pre-loop limit so that in main loop, a load/store reference
3078 // to align_to_ref will be a position zero in the vector.
3079 // (iv + k) mod vector_align == 0
3080 void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
3081 CountedLoopNode *main_head = lp()->as_CountedLoop();
3082 assert(main_head->is_main_loop(), "");
3083 CountedLoopEndNode* pre_end = get_pre_loop_end(main_head);
3084 assert(pre_end != NULL, "we must have a correct pre-loop");
3085 Node *pre_opaq1 = pre_end->limit();
3086 assert(pre_opaq1->Opcode() == Op_Opaque1, "");
3087 Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
3088 Node *lim0 = pre_opaq->in(1);
3089
3090 // Where we put new limit calculations
3091 Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
3092
3093 // Ensure the original loop limit is available from the
3094 // pre-loop Opaque1 node.
3095 Node *orig_limit = pre_opaq->original_loop_limit();
3096 assert(orig_limit != NULL && _igvn.type(orig_limit) != Type::TOP, "");
3097
3098 SWPointer align_to_ref_p(align_to_ref, this, NULL, false);
3099 assert(align_to_ref_p.valid(), "sanity");
3100
3101 // Given:
3102 // lim0 == original pre loop limit
3103 // V == v_align (power of 2)
3104 // invar == extra invariant piece of the address expression
3105 // e == offset [ +/- invar ]
3106 //
3420 return !lpt()->is_member(phase()->get_loop(n_c));
3421 }
3422 //------------------------scaled_iv_plus_offset--------------------
3423 // Match: k*iv + offset
3424 // where: k is a constant that maybe zero, and
3425 // offset is (k2 [+/- invariant]) where k2 maybe zero and invariant is optional
3426 bool SWPointer::scaled_iv_plus_offset(Node* n) {
3427 NOT_PRODUCT(Tracer::Depth ddd;)
3428 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_1(n);)
3429
3430 if (scaled_iv(n)) {
3431 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_2(n);)
3432 return true;
3433 }
3434
3435 if (offset_plus_k(n)) {
3436 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_3(n);)
3437 return true;
3438 }
3439
3440 int opc = n->Opcode();
3441 if (opc == Op_AddI) {
3442 if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2))) {
3443 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_4(n);)
3444 return true;
3445 }
3446 if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) {
3447 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_5(n);)
3448 return true;
3449 }
3450 } else if (opc == Op_SubI) {
3451 if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2), true)) {
3452 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_6(n);)
3453 return true;
3454 }
3455 if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) {
3456 _scale *= -1;
3457 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_7(n);)
3458 return true;
3459 }
3460 }
3461
3462 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_8(n);)
3463 return false;
3464 }
3465
3466 //----------------------------scaled_iv------------------------
3467 // Match: k*iv where k is a constant that's not zero
3468 bool SWPointer::scaled_iv(Node* n) {
3469 NOT_PRODUCT(Tracer::Depth ddd;)
3470 NOT_PRODUCT(_tracer.scaled_iv_1(n);)
3471
3472 if (_scale != 0) { // already found a scale
3473 NOT_PRODUCT(_tracer.scaled_iv_2(n, _scale);)
3474 return false;
3475 }
3476
3477 if (n == iv()) {
3478 _scale = 1;
3479 NOT_PRODUCT(_tracer.scaled_iv_3(n, _scale);)
3480 return true;
3481 }
3482 if (_analyze_only && (invariant(n) == false)) {
3483 _nstack->push(n, _stack_idx++);
3484 }
3485
3486 int opc = n->Opcode();
3487 if (opc == Op_MulI) {
3488 if (n->in(1) == iv() && n->in(2)->is_Con()) {
3489 _scale = n->in(2)->get_int();
3490 NOT_PRODUCT(_tracer.scaled_iv_4(n, _scale);)
3491 return true;
3492 } else if (n->in(2) == iv() && n->in(1)->is_Con()) {
3493 _scale = n->in(1)->get_int();
3494 NOT_PRODUCT(_tracer.scaled_iv_5(n, _scale);)
3495 return true;
3496 }
3497 } else if (opc == Op_LShiftI) {
3498 if (n->in(1) == iv() && n->in(2)->is_Con()) {
3499 _scale = 1 << n->in(2)->get_int();
3500 NOT_PRODUCT(_tracer.scaled_iv_6(n, _scale);)
3501 return true;
3502 }
3503 } else if (opc == Op_ConvI2L) {
3504 if (n->in(1)->Opcode() == Op_CastII &&
3505 n->in(1)->as_CastII()->has_range_check()) {
3506 // Skip range check dependent CastII nodes
3507 n = n->in(1);
3508 }
3509 if (scaled_iv_plus_offset(n->in(1))) {
3510 NOT_PRODUCT(_tracer.scaled_iv_7(n);)
3511 return true;
3512 }
3513 } else if (opc == Op_LShiftL) {
3514 if (!has_iv() && _invar == NULL) {
3515 // Need to preserve the current _offset value, so
3516 // create a temporary object for this expression subtree.
3517 // Hacky, so should re-engineer the address pattern match.
3518 NOT_PRODUCT(Tracer::Depth dddd;)
3519 SWPointer tmp(this);
3520 NOT_PRODUCT(_tracer.scaled_iv_8(n, &tmp);)
3521
3522 if (tmp.scaled_iv_plus_offset(n->in(1))) {
3523 if (tmp._invar == NULL || _slp->do_vector_loop()) {
3524 int mult = 1 << n->in(2)->get_int();
3525 _scale = tmp._scale * mult;
3526 _offset += tmp._offset * mult;
3527 NOT_PRODUCT(_tracer.scaled_iv_9(n, _scale, _offset, mult);)
3528 return true;
3529 }
3530 }
3531 }
3532 }
3533 NOT_PRODUCT(_tracer.scaled_iv_10(n);)
3534 return false;
3535 }
3536
3537 //----------------------------offset_plus_k------------------------
3538 // Match: offset is (k [+/- invariant])
3539 // where k maybe zero and invariant is optional, but not both.
3540 bool SWPointer::offset_plus_k(Node* n, bool negate) {
3541 NOT_PRODUCT(Tracer::Depth ddd;)
3542 NOT_PRODUCT(_tracer.offset_plus_k_1(n);)
3543
3544 int opc = n->Opcode();
3545 if (opc == Op_ConI) {
3546 _offset += negate ? -(n->get_int()) : n->get_int();
3547 NOT_PRODUCT(_tracer.offset_plus_k_2(n, _offset);)
3548 return true;
3549 } else if (opc == Op_ConL) {
3550 // Okay if value fits into an int
3551 const TypeLong* t = n->find_long_type();
3552 if (t->higher_equal(TypeLong::INT)) {
3553 jlong loff = n->get_long();
3554 jint off = (jint)loff;
3555 _offset += negate ? -off : loff;
3556 NOT_PRODUCT(_tracer.offset_plus_k_3(n, _offset);)
3557 return true;
3558 }
3559 NOT_PRODUCT(_tracer.offset_plus_k_4(n);)
3560 return false;
3561 }
3562 if (_invar != NULL) { // already has an invariant
3563 NOT_PRODUCT(_tracer.offset_plus_k_5(n, _invar);)
3564 return false;
3565 }
3566
3567 if (_analyze_only && (invariant(n) == false)) {
3568 _nstack->push(n, _stack_idx++);
3569 }
3570 if (opc == Op_AddI) {
3571 if (n->in(2)->is_Con() && invariant(n->in(1))) {
3572 _negate_invar = negate;
3573 _invar = n->in(1);
3574 _offset += negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
3575 NOT_PRODUCT(_tracer.offset_plus_k_6(n, _invar, _negate_invar, _offset);)
3576 return true;
3577 } else if (n->in(1)->is_Con() && invariant(n->in(2))) {
3578 _offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int();
3579 _negate_invar = negate;
3580 _invar = n->in(2);
3581 NOT_PRODUCT(_tracer.offset_plus_k_7(n, _invar, _negate_invar, _offset);)
3582 return true;
3583 }
3584 }
3585 if (opc == Op_SubI) {
3586 if (n->in(2)->is_Con() && invariant(n->in(1))) {
3587 _negate_invar = negate;
3588 _invar = n->in(1);
3589 _offset += !negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
3590 NOT_PRODUCT(_tracer.offset_plus_k_8(n, _invar, _negate_invar, _offset);)
3591 return true;
3592 } else if (n->in(1)->is_Con() && invariant(n->in(2))) {
3593 _offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int();
3594 _negate_invar = !negate;
3595 _invar = n->in(2);
3596 NOT_PRODUCT(_tracer.offset_plus_k_9(n, _invar, _negate_invar, _offset);)
3597 return true;
3598 }
3599 }
3600 if (invariant(n)) {
3601 if (opc == Op_ConvI2L) {
3602 n = n->in(1);
3603 if (n->Opcode() == Op_CastII &&
3604 n->as_CastII()->has_range_check()) {
3605 // Skip range check dependent CastII nodes
3606 assert(invariant(n), "sanity");
3607 n = n->in(1);
3608 }
3609 }
3610 if (n->bottom_type()->isa_int()) {
3611 _negate_invar = negate;
3612 _invar = n;
3613 NOT_PRODUCT(_tracer.offset_plus_k_10(n, _invar, _negate_invar, _offset);)
3614 return true;
3615 }
3616 }
3617
3618 NOT_PRODUCT(_tracer.offset_plus_k_11(n);)
3619 return false;
3620 }
3621
3622 //----------------------------print------------------------
3623 void SWPointer::print() {
|
133 tty->print("lpt->_head %d", lpt->_head->_idx); lpt->_head->dump();
134 lpt->dump_head();
135 }
136 #endif
137 return;
138 }
139
140 // Make sure the are no extra control users of the loop backedge
141 if (cl->back_control()->outcnt() != 1) {
142 return;
143 }
144
145 // Skip any loops already optimized by slp
146 if (cl->is_vectorized_loop()) return;
147
148 if (cl->is_main_loop()) {
149 // Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit))))
150 CountedLoopEndNode* pre_end = get_pre_loop_end(cl);
151 if (pre_end == NULL) return;
152 Node *pre_opaq1 = pre_end->limit();
153 if (pre_opaq1->Opcode() != Opcodes::Op_Opaque1) return;
154 }
155
156 init(); // initialize data structures
157
158 set_lpt(lpt);
159 set_lp(cl);
160
161 // For now, define one block which is the entire loop body
162 set_bb(cl);
163
164 if (do_optimization) {
165 assert(_packset.length() == 0, "packset must be empty");
166 SLP_extract();
167 if (PostLoopMultiversioning && Matcher::has_predicated_vectors()) {
168 if (cl->is_vectorized_loop() && cl->is_main_loop() && !cl->is_reduction_loop()) {
169 IdealLoopTree *lpt_next = lpt->_next;
170 CountedLoopNode *cl_next = lpt_next->_head->as_CountedLoop();
171 _phase->has_range_checks(lpt_next);
172 if (cl_next->is_post_loop() && !cl_next->range_checks_present()) {
173 if (!cl_next->is_vectorized_loop()) {
1019 Node* prev = NULL;
1020 while (true) {
1021 NOT_PRODUCT( if(is_trace_mem_slice()) tty->print_cr("SuperWord::mem_slice_preds: n %d", n->_idx);)
1022 assert(in_bb(n), "must be in block");
1023 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
1024 Node* out = n->fast_out(i);
1025 if (out->is_Load()) {
1026 if (in_bb(out)) {
1027 preds.push(out);
1028 if (TraceSuperWord && Verbose) {
1029 tty->print_cr("SuperWord::mem_slice_preds: added pred(%d)", out->_idx);
1030 }
1031 }
1032 } else {
1033 // FIXME
1034 if (out->is_MergeMem() && !in_bb(out)) {
1035 // Either unrolling is causing a memory edge not to disappear,
1036 // or need to run igvn.optimize() again before SLP
1037 } else if (out->is_Phi() && out->bottom_type() == Type::MEMORY && !in_bb(out)) {
1038 // Ditto. Not sure what else to check further.
1039 } else if (out->Opcode() == Opcodes::Op_StoreCM && out->in(MemNode::OopStore) == n) {
1040 // StoreCM has an input edge used as a precedence edge.
1041 // Maybe an issue when oop stores are vectorized.
1042 } else {
1043 assert(out == prev || prev == NULL, "no branches off of store slice");
1044 }
1045 }//else
1046 }//for
1047 if (n == stop) break;
1048 preds.push(n);
1049 if (TraceSuperWord && Verbose) {
1050 tty->print_cr("SuperWord::mem_slice_preds: added pred(%d)", n->_idx);
1051 }
1052 prev = n;
1053 assert(n->is_Mem(), "unexpected node %s", n->Name());
1054 n = n->in(MemNode::Memory);
1055 }
1056 }
1057
1058 //------------------------------stmts_can_pack---------------------------
1059 // Can s1 and s2 be in a pack with s1 immediately preceding s2 and
1780 else {
1781 return false;
1782 }
1783
1784 for (uint j = 1; j < cmpd_pk->size(); j++) {
1785 if (cmpd_pk->at(j)->in(1) != cmovd_pk->at(j)->as_CMove()->in(cmovd_ind1)
1786 || cmpd_pk->at(j)->in(2) != cmovd_pk->at(j)->as_CMove()->in(cmovd_ind2)) {
1787 return false;
1788 }//if
1789 }
1790 NOT_PRODUCT(if(_sw->is_trace_cmov()) { tty->print("CMoveKit::test_cmpd_pack: cmpd pack for 1st CmpD %d is OK for vectorization: ", cmpd0->_idx); cmpd0->dump(); })
1791 return true;
1792 }
1793
1794 //------------------------------implemented---------------------------
1795 // Can code be generated for pack p?
1796 bool SuperWord::implemented(Node_List* p) {
1797 bool retValue = false;
1798 Node* p0 = p->at(0);
1799 if (p0 != NULL) {
1800 Opcodes opc = p0->Opcode();
1801 uint size = p->size();
1802 if (p0->is_reduction()) {
1803 const Type *arith_type = p0->bottom_type();
1804 // Length 2 reductions of INT/LONG do not offer performance benefits
1805 if (((arith_type->basic_type() == T_INT) || (arith_type->basic_type() == T_LONG)) && (size == 2)) {
1806 retValue = false;
1807 } else {
1808 retValue = ReductionNode::implemented(opc, size, arith_type->basic_type());
1809 }
1810 } else {
1811 retValue = VectorNode::implemented(opc, size, velt_basic_type(p0));
1812 }
1813 if (!retValue) {
1814 if (is_cmov_pack(p)) {
1815 NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::implemented: found cmpd pack"); print_pack(p);})
1816 return true;
1817 }
1818 }
1819 }
1820 return retValue;
2168
2169 if (do_reserve_copy() && !make_reversable.has_reserved()) {
2170 NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: loop was not reserved correctly, exiting SuperWord");})
2171 return;
2172 }
2173
2174 for (int i = 0; i < _block.length(); i++) {
2175 Node* n = _block.at(i);
2176 Node_List* p = my_pack(n);
2177 if (p && n == executed_last(p)) {
2178 uint vlen = p->size();
2179 uint vlen_in_bytes = 0;
2180 Node* vn = NULL;
2181 Node* low_adr = p->at(0);
2182 Node* first = executed_first(p);
2183 if (can_process_post_loop) {
2184 // override vlen with the main loops vector length
2185 vlen = cl->slp_max_unroll();
2186 }
2187 NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: %d executed first, %d executed last in pack", first->_idx, n->_idx); print_pack(p);})
2188 Opcodes opc = n->Opcode();
2189 if (n->is_Load()) {
2190 Node* ctl = n->in(MemNode::Control);
2191 Node* mem = first->in(MemNode::Memory);
2192 SWPointer p1(n->as_Mem(), this, NULL, false);
2193 // Identify the memory dependency for the new loadVector node by
2194 // walking up through memory chain.
2195 // This is done to give flexibility to the new loadVector node so that
2196 // it can move above independent storeVector nodes.
2197 while (mem->is_StoreVector()) {
2198 SWPointer p2(mem->as_Mem(), this, NULL, false);
2199 int cmp = p1.cmp(p2);
2200 if (SWPointer::not_equal(cmp) || !SWPointer::comparable(cmp)) {
2201 mem = mem->in(MemNode::Memory);
2202 } else {
2203 break; // dependent memory
2204 }
2205 }
2206 Node* adr = low_adr->in(MemNode::Address);
2207 const TypePtr* atyp = n->adr_type();
2208 vn = LoadVectorNode::make(opc, ctl, mem, adr, atyp, vlen, velt_basic_type(n), control_dependency(p));
2250 ShouldNotReachHere();
2251 }
2252 if (VectorNode::is_invariant_vector(in1) && (node_isa_reduction == false) && (n->is_Add() || n->is_Mul())) {
2253 // Move invariant vector input into second position to avoid register spilling.
2254 Node* tmp = in1;
2255 in1 = in2;
2256 in2 = tmp;
2257 }
2258 if (node_isa_reduction) {
2259 const Type *arith_type = n->bottom_type();
2260 vn = ReductionNode::make(opc, NULL, in1, in2, arith_type->basic_type());
2261 if (in2->is_Load()) {
2262 vlen_in_bytes = in2->as_LoadVector()->memory_size();
2263 } else {
2264 vlen_in_bytes = in2->as_Vector()->length_in_bytes();
2265 }
2266 } else {
2267 vn = VectorNode::make(opc, in1, in2, vlen, velt_basic_type(n));
2268 vlen_in_bytes = vn->as_Vector()->length_in_bytes();
2269 }
2270 } else if (opc == Opcodes::Op_SqrtD || opc == Opcodes::Op_AbsF || opc == Opcodes::Op_AbsD || opc == Opcodes::Op_NegF || opc == Opcodes::Op_NegD) {
2271 // Promote operand to vector (Sqrt/Abs/Neg are 2 address instructions)
2272 Node* in = vector_opd(p, 1);
2273 vn = VectorNode::make(opc, in, NULL, vlen, velt_basic_type(n));
2274 vlen_in_bytes = vn->as_Vector()->length_in_bytes();
2275 } else if (is_cmov_pack(p)) {
2276 if (can_process_post_loop) {
2277 // do not refactor of flow in post loop context
2278 return;
2279 }
2280 if (!n->is_CMove()) {
2281 continue;
2282 }
2283 // place here CMoveVDNode
2284 NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: print before CMove vectorization"); print_loop(false);})
2285 Node* bol = n->in(CMoveNode::Condition);
2286 if (!bol->is_Bool() && bol->Opcode() == Opcodes::Op_ExtractI && bol->req() > 1 ) {
2287 NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: %d is not Bool node, trying its in(1) node %d", bol->_idx, bol->in(1)->_idx); bol->dump(); bol->in(1)->dump();})
2288 bol = bol->in(1); //may be ExtractNode
2289 }
2290
2291 assert(bol->is_Bool(), "should be BoolNode - too late to bail out!");
2292 if (!bol->is_Bool()) {
2293 if (do_reserve_copy()) {
2294 NOT_PRODUCT(if(is_trace_loop_reverse() || TraceLoopOpts) {tty->print_cr("SWPointer::output: expected %d bool node, exiting SuperWord", bol->_idx); bol->dump();})
2295 return; //and reverse to backup IG
2296 }
2297 ShouldNotReachHere();
2298 }
2299
2300 int cond = (int)bol->as_Bool()->_test._test;
2301 Node* in_cc = _igvn.intcon(cond);
2302 NOT_PRODUCT(if(is_trace_cmov()) {tty->print("SWPointer::output: created intcon in_cc node %d", in_cc->_idx); in_cc->dump();})
2303 Node* cc = bol->clone();
2304 cc->set_req(1, in_cc);
2305 NOT_PRODUCT(if(is_trace_cmov()) {tty->print("SWPointer::output: created bool cc node %d", cc->_idx); cc->dump();})
2306
2862 bool same_type = true;
2863 for (DUIterator_Fast kmax, k = in->fast_outs(kmax); k < kmax; k++) {
2864 Node *use = in->fast_out(k);
2865 if (!in_bb(use) || !same_velt_type(use, n)) {
2866 same_type = false;
2867 break;
2868 }
2869 }
2870 if (same_type) {
2871 // For right shifts of small integer types (bool, byte, char, short)
2872 // we need precise information about sign-ness. Only Load nodes have
2873 // this information because Store nodes are the same for signed and
2874 // unsigned values. And any arithmetic operation after a load may
2875 // expand a value to signed Int so such right shifts can't be used
2876 // because vector elements do not have upper bits of Int.
2877 const Type* vt = vtn;
2878 if (VectorNode::is_shift(in)) {
2879 Node* load = in->in(1);
2880 if (load->is_Load() && in_bb(load) && (velt_type(load)->basic_type() == T_INT)) {
2881 vt = velt_type(load);
2882 } else if (in->Opcode() != Opcodes::Op_LShiftI) {
2883 // Widen type to Int to avoid creation of right shift vector
2884 // (align + data_size(s1) check in stmts_can_pack() will fail).
2885 // Note, left shifts work regardless type.
2886 vt = TypeInt::INT;
2887 }
2888 }
2889 set_velt_type(in, vt);
2890 }
2891 }
2892 }
2893 }
2894 }
2895 #ifndef PRODUCT
2896 if (TraceSuperWord && Verbose) {
2897 for (int i = 0; i < _block.length(); i++) {
2898 Node* n = _block.at(i);
2899 velt_type(n)->dump();
2900 tty->print("\t");
2901 n->dump();
2902 }
2926 int offset = p.offset_in_bytes();
2927 offset += iv_adjust*p.memory_size();
2928 int off_rem = offset % vw;
2929 int off_mod = off_rem >= 0 ? off_rem : off_rem + vw;
2930 if (TraceSuperWord && Verbose) {
2931 tty->print_cr("SWPointer::memory_alignment: off_rem = %d, off_mod = %d", off_rem, off_mod);
2932 }
2933 return off_mod;
2934 }
2935
2936 //---------------------------container_type---------------------------
2937 // Smallest type containing range of values
2938 const Type* SuperWord::container_type(Node* n) {
2939 if (n->is_Mem()) {
2940 BasicType bt = n->as_Mem()->memory_type();
2941 if (n->is_Store() && (bt == T_CHAR)) {
2942 // Use T_SHORT type instead of T_CHAR for stored values because any
2943 // preceding arithmetic operation extends values to signed Int.
2944 bt = T_SHORT;
2945 }
2946 if (n->Opcode() == Opcodes::Op_LoadUB) {
2947 // Adjust type for unsigned byte loads, it is important for right shifts.
2948 // T_BOOLEAN is used because there is no basic type representing type
2949 // TypeInt::UBYTE. Use of T_BOOLEAN for vectors is fine because only
2950 // size (one byte) and sign is important.
2951 bt = T_BOOLEAN;
2952 }
2953 return Type::get_const_basic_type(bt);
2954 }
2955 const Type* t = _igvn.type(n);
2956 if (t->basic_type() == T_INT) {
2957 // A narrow type of arithmetic operations will be determined by
2958 // propagating the type of memory operations.
2959 return TypeInt::INT;
2960 }
2961 return t;
2962 }
2963
2964 bool SuperWord::same_velt_type(Node* n1, Node* n2) {
2965 const Type* vt1 = velt_type(n1);
2966 const Type* vt2 = velt_type(n2);
3066 Node* n = p->at(i);
3067 assert(n->is_Load(), "only meaningful for loads");
3068 if (!n->depends_only_on_test()) {
3069 dep = LoadNode::Pinned;
3070 }
3071 }
3072 return dep;
3073 }
3074
3075
3076 //----------------------------align_initial_loop_index---------------------------
3077 // Adjust pre-loop limit so that in main loop, a load/store reference
3078 // to align_to_ref will be a position zero in the vector.
3079 // (iv + k) mod vector_align == 0
3080 void SuperWord::align_initial_loop_index(MemNode* align_to_ref) {
3081 CountedLoopNode *main_head = lp()->as_CountedLoop();
3082 assert(main_head->is_main_loop(), "");
3083 CountedLoopEndNode* pre_end = get_pre_loop_end(main_head);
3084 assert(pre_end != NULL, "we must have a correct pre-loop");
3085 Node *pre_opaq1 = pre_end->limit();
3086 assert(pre_opaq1->Opcode() == Opcodes::Op_Opaque1, "");
3087 Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
3088 Node *lim0 = pre_opaq->in(1);
3089
3090 // Where we put new limit calculations
3091 Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
3092
3093 // Ensure the original loop limit is available from the
3094 // pre-loop Opaque1 node.
3095 Node *orig_limit = pre_opaq->original_loop_limit();
3096 assert(orig_limit != NULL && _igvn.type(orig_limit) != Type::TOP, "");
3097
3098 SWPointer align_to_ref_p(align_to_ref, this, NULL, false);
3099 assert(align_to_ref_p.valid(), "sanity");
3100
3101 // Given:
3102 // lim0 == original pre loop limit
3103 // V == v_align (power of 2)
3104 // invar == extra invariant piece of the address expression
3105 // e == offset [ +/- invar ]
3106 //
3420 return !lpt()->is_member(phase()->get_loop(n_c));
3421 }
3422 //------------------------scaled_iv_plus_offset--------------------
3423 // Match: k*iv + offset
3424 // where: k is a constant that maybe zero, and
3425 // offset is (k2 [+/- invariant]) where k2 maybe zero and invariant is optional
3426 bool SWPointer::scaled_iv_plus_offset(Node* n) {
3427 NOT_PRODUCT(Tracer::Depth ddd;)
3428 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_1(n);)
3429
3430 if (scaled_iv(n)) {
3431 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_2(n);)
3432 return true;
3433 }
3434
3435 if (offset_plus_k(n)) {
3436 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_3(n);)
3437 return true;
3438 }
3439
3440 Opcodes opc = n->Opcode();
3441 if (opc == Opcodes::Op_AddI) {
3442 if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2))) {
3443 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_4(n);)
3444 return true;
3445 }
3446 if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) {
3447 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_5(n);)
3448 return true;
3449 }
3450 } else if (opc == Opcodes::Op_SubI) {
3451 if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2), true)) {
3452 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_6(n);)
3453 return true;
3454 }
3455 if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) {
3456 _scale *= -1;
3457 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_7(n);)
3458 return true;
3459 }
3460 }
3461
3462 NOT_PRODUCT(_tracer.scaled_iv_plus_offset_8(n);)
3463 return false;
3464 }
3465
3466 //----------------------------scaled_iv------------------------
3467 // Match: k*iv where k is a constant that's not zero
3468 bool SWPointer::scaled_iv(Node* n) {
3469 NOT_PRODUCT(Tracer::Depth ddd;)
3470 NOT_PRODUCT(_tracer.scaled_iv_1(n);)
3471
3472 if (_scale != 0) { // already found a scale
3473 NOT_PRODUCT(_tracer.scaled_iv_2(n, _scale);)
3474 return false;
3475 }
3476
3477 if (n == iv()) {
3478 _scale = 1;
3479 NOT_PRODUCT(_tracer.scaled_iv_3(n, _scale);)
3480 return true;
3481 }
3482 if (_analyze_only && (invariant(n) == false)) {
3483 _nstack->push(n, _stack_idx++);
3484 }
3485
3486 Opcodes opc = n->Opcode();
3487 if (opc == Opcodes::Op_MulI) {
3488 if (n->in(1) == iv() && n->in(2)->is_Con()) {
3489 _scale = n->in(2)->get_int();
3490 NOT_PRODUCT(_tracer.scaled_iv_4(n, _scale);)
3491 return true;
3492 } else if (n->in(2) == iv() && n->in(1)->is_Con()) {
3493 _scale = n->in(1)->get_int();
3494 NOT_PRODUCT(_tracer.scaled_iv_5(n, _scale);)
3495 return true;
3496 }
3497 } else if (opc == Opcodes::Op_LShiftI) {
3498 if (n->in(1) == iv() && n->in(2)->is_Con()) {
3499 _scale = 1 << n->in(2)->get_int();
3500 NOT_PRODUCT(_tracer.scaled_iv_6(n, _scale);)
3501 return true;
3502 }
3503 } else if (opc == Opcodes::Op_ConvI2L) {
3504 if (n->in(1)->Opcode() == Opcodes::Op_CastII &&
3505 n->in(1)->as_CastII()->has_range_check()) {
3506 // Skip range check dependent CastII nodes
3507 n = n->in(1);
3508 }
3509 if (scaled_iv_plus_offset(n->in(1))) {
3510 NOT_PRODUCT(_tracer.scaled_iv_7(n);)
3511 return true;
3512 }
3513 } else if (opc == Opcodes::Op_LShiftL) {
3514 if (!has_iv() && _invar == NULL) {
3515 // Need to preserve the current _offset value, so
3516 // create a temporary object for this expression subtree.
3517 // Hacky, so should re-engineer the address pattern match.
3518 NOT_PRODUCT(Tracer::Depth dddd;)
3519 SWPointer tmp(this);
3520 NOT_PRODUCT(_tracer.scaled_iv_8(n, &tmp);)
3521
3522 if (tmp.scaled_iv_plus_offset(n->in(1))) {
3523 if (tmp._invar == NULL || _slp->do_vector_loop()) {
3524 int mult = 1 << n->in(2)->get_int();
3525 _scale = tmp._scale * mult;
3526 _offset += tmp._offset * mult;
3527 NOT_PRODUCT(_tracer.scaled_iv_9(n, _scale, _offset, mult);)
3528 return true;
3529 }
3530 }
3531 }
3532 }
3533 NOT_PRODUCT(_tracer.scaled_iv_10(n);)
3534 return false;
3535 }
3536
3537 //----------------------------offset_plus_k------------------------
3538 // Match: offset is (k [+/- invariant])
3539 // where k maybe zero and invariant is optional, but not both.
3540 bool SWPointer::offset_plus_k(Node* n, bool negate) {
3541 NOT_PRODUCT(Tracer::Depth ddd;)
3542 NOT_PRODUCT(_tracer.offset_plus_k_1(n);)
3543
3544 Opcodes opc = n->Opcode();
3545 if (opc == Opcodes::Op_ConI) {
3546 _offset += negate ? -(n->get_int()) : n->get_int();
3547 NOT_PRODUCT(_tracer.offset_plus_k_2(n, _offset);)
3548 return true;
3549 } else if (opc == Opcodes::Op_ConL) {
3550 // Okay if value fits into an int
3551 const TypeLong* t = n->find_long_type();
3552 if (t->higher_equal(TypeLong::INT)) {
3553 jlong loff = n->get_long();
3554 jint off = (jint)loff;
3555 _offset += negate ? -off : loff;
3556 NOT_PRODUCT(_tracer.offset_plus_k_3(n, _offset);)
3557 return true;
3558 }
3559 NOT_PRODUCT(_tracer.offset_plus_k_4(n);)
3560 return false;
3561 }
3562 if (_invar != NULL) { // already has an invariant
3563 NOT_PRODUCT(_tracer.offset_plus_k_5(n, _invar);)
3564 return false;
3565 }
3566
3567 if (_analyze_only && (invariant(n) == false)) {
3568 _nstack->push(n, _stack_idx++);
3569 }
3570 if (opc == Opcodes::Op_AddI) {
3571 if (n->in(2)->is_Con() && invariant(n->in(1))) {
3572 _negate_invar = negate;
3573 _invar = n->in(1);
3574 _offset += negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
3575 NOT_PRODUCT(_tracer.offset_plus_k_6(n, _invar, _negate_invar, _offset);)
3576 return true;
3577 } else if (n->in(1)->is_Con() && invariant(n->in(2))) {
3578 _offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int();
3579 _negate_invar = negate;
3580 _invar = n->in(2);
3581 NOT_PRODUCT(_tracer.offset_plus_k_7(n, _invar, _negate_invar, _offset);)
3582 return true;
3583 }
3584 }
3585 if (opc == Opcodes::Op_SubI) {
3586 if (n->in(2)->is_Con() && invariant(n->in(1))) {
3587 _negate_invar = negate;
3588 _invar = n->in(1);
3589 _offset += !negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
3590 NOT_PRODUCT(_tracer.offset_plus_k_8(n, _invar, _negate_invar, _offset);)
3591 return true;
3592 } else if (n->in(1)->is_Con() && invariant(n->in(2))) {
3593 _offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int();
3594 _negate_invar = !negate;
3595 _invar = n->in(2);
3596 NOT_PRODUCT(_tracer.offset_plus_k_9(n, _invar, _negate_invar, _offset);)
3597 return true;
3598 }
3599 }
3600 if (invariant(n)) {
3601 if (opc == Opcodes::Op_ConvI2L) {
3602 n = n->in(1);
3603 if (n->Opcode() == Opcodes::Op_CastII &&
3604 n->as_CastII()->has_range_check()) {
3605 // Skip range check dependent CastII nodes
3606 assert(invariant(n), "sanity");
3607 n = n->in(1);
3608 }
3609 }
3610 if (n->bottom_type()->isa_int()) {
3611 _negate_invar = negate;
3612 _invar = n;
3613 NOT_PRODUCT(_tracer.offset_plus_k_10(n, _invar, _negate_invar, _offset);)
3614 return true;
3615 }
3616 }
3617
3618 NOT_PRODUCT(_tracer.offset_plus_k_11(n);)
3619 return false;
3620 }
3621
3622 //----------------------------print------------------------
3623 void SWPointer::print() {
|