< prev index next >
src/share/vm/opto/superword.cpp
Print this page
*** 148,158 ****
if (cl->is_main_loop()) {
// Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit))))
CountedLoopEndNode* pre_end = get_pre_loop_end(cl);
if (pre_end == NULL) return;
Node *pre_opaq1 = pre_end->limit();
! if (pre_opaq1->Opcode() != Op_Opaque1) return;
}
init(); // initialize data structures
set_lpt(lpt);
--- 148,158 ----
if (cl->is_main_loop()) {
// Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit))))
CountedLoopEndNode* pre_end = get_pre_loop_end(cl);
if (pre_end == NULL) return;
Node *pre_opaq1 = pre_end->limit();
! if (pre_opaq1->Opcode() != Opcodes::Op_Opaque1) return;
}
init(); // initialize data structures
set_lpt(lpt);
*** 1034,1044 ****
if (out->is_MergeMem() && !in_bb(out)) {
// Either unrolling is causing a memory edge not to disappear,
// or need to run igvn.optimize() again before SLP
} else if (out->is_Phi() && out->bottom_type() == Type::MEMORY && !in_bb(out)) {
// Ditto. Not sure what else to check further.
! } else if (out->Opcode() == Op_StoreCM && out->in(MemNode::OopStore) == n) {
// StoreCM has an input edge used as a precedence edge.
// Maybe an issue when oop stores are vectorized.
} else {
assert(out == prev || prev == NULL, "no branches off of store slice");
}
--- 1034,1044 ----
if (out->is_MergeMem() && !in_bb(out)) {
// Either unrolling is causing a memory edge not to disappear,
// or need to run igvn.optimize() again before SLP
} else if (out->is_Phi() && out->bottom_type() == Type::MEMORY && !in_bb(out)) {
// Ditto. Not sure what else to check further.
! } else if (out->Opcode() == Opcodes::Op_StoreCM && out->in(MemNode::OopStore) == n) {
// StoreCM has an input edge used as a precedence edge.
// Maybe an issue when oop stores are vectorized.
} else {
assert(out == prev || prev == NULL, "no branches off of store slice");
}
*** 1795,1805 ****
// Can code be generated for pack p?
bool SuperWord::implemented(Node_List* p) {
bool retValue = false;
Node* p0 = p->at(0);
if (p0 != NULL) {
! int opc = p0->Opcode();
uint size = p->size();
if (p0->is_reduction()) {
const Type *arith_type = p0->bottom_type();
// Length 2 reductions of INT/LONG do not offer performance benefits
if (((arith_type->basic_type() == T_INT) || (arith_type->basic_type() == T_LONG)) && (size == 2)) {
--- 1795,1805 ----
// Can code be generated for pack p?
bool SuperWord::implemented(Node_List* p) {
bool retValue = false;
Node* p0 = p->at(0);
if (p0 != NULL) {
! Opcodes opc = p0->Opcode();
uint size = p->size();
if (p0->is_reduction()) {
const Type *arith_type = p0->bottom_type();
// Length 2 reductions of INT/LONG do not offer performance benefits
if (((arith_type->basic_type() == T_INT) || (arith_type->basic_type() == T_LONG)) && (size == 2)) {
*** 2183,2193 ****
if (can_process_post_loop) {
// override vlen with the main loops vector length
vlen = cl->slp_max_unroll();
}
NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: %d executed first, %d executed last in pack", first->_idx, n->_idx); print_pack(p);})
! int opc = n->Opcode();
if (n->is_Load()) {
Node* ctl = n->in(MemNode::Control);
Node* mem = first->in(MemNode::Memory);
SWPointer p1(n->as_Mem(), this, NULL, false);
// Identify the memory dependency for the new loadVector node by
--- 2183,2193 ----
if (can_process_post_loop) {
// override vlen with the main loops vector length
vlen = cl->slp_max_unroll();
}
NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: %d executed first, %d executed last in pack", first->_idx, n->_idx); print_pack(p);})
! Opcodes opc = n->Opcode();
if (n->is_Load()) {
Node* ctl = n->in(MemNode::Control);
Node* mem = first->in(MemNode::Memory);
SWPointer p1(n->as_Mem(), this, NULL, false);
// Identify the memory dependency for the new loadVector node by
*** 2265,2275 ****
}
} else {
vn = VectorNode::make(opc, in1, in2, vlen, velt_basic_type(n));
vlen_in_bytes = vn->as_Vector()->length_in_bytes();
}
! } else if (opc == Op_SqrtD || opc == Op_AbsF || opc == Op_AbsD || opc == Op_NegF || opc == Op_NegD) {
// Promote operand to vector (Sqrt/Abs/Neg are 2 address instructions)
Node* in = vector_opd(p, 1);
vn = VectorNode::make(opc, in, NULL, vlen, velt_basic_type(n));
vlen_in_bytes = vn->as_Vector()->length_in_bytes();
} else if (is_cmov_pack(p)) {
--- 2265,2275 ----
}
} else {
vn = VectorNode::make(opc, in1, in2, vlen, velt_basic_type(n));
vlen_in_bytes = vn->as_Vector()->length_in_bytes();
}
! } else if (opc == Opcodes::Op_SqrtD || opc == Opcodes::Op_AbsF || opc == Opcodes::Op_AbsD || opc == Opcodes::Op_NegF || opc == Opcodes::Op_NegD) {
// Promote operand to vector (Sqrt/Abs/Neg are 2 address instructions)
Node* in = vector_opd(p, 1);
vn = VectorNode::make(opc, in, NULL, vlen, velt_basic_type(n));
vlen_in_bytes = vn->as_Vector()->length_in_bytes();
} else if (is_cmov_pack(p)) {
*** 2281,2291 ****
continue;
}
// place here CMoveVDNode
NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: print before CMove vectorization"); print_loop(false);})
Node* bol = n->in(CMoveNode::Condition);
! if (!bol->is_Bool() && bol->Opcode() == Op_ExtractI && bol->req() > 1 ) {
NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: %d is not Bool node, trying its in(1) node %d", bol->_idx, bol->in(1)->_idx); bol->dump(); bol->in(1)->dump();})
bol = bol->in(1); //may be ExtractNode
}
assert(bol->is_Bool(), "should be BoolNode - too late to bail out!");
--- 2281,2291 ----
continue;
}
// place here CMoveVDNode
NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: print before CMove vectorization"); print_loop(false);})
Node* bol = n->in(CMoveNode::Condition);
! if (!bol->is_Bool() && bol->Opcode() == Opcodes::Op_ExtractI && bol->req() > 1 ) {
NOT_PRODUCT(if(is_trace_cmov()) {tty->print_cr("SWPointer::output: %d is not Bool node, trying its in(1) node %d", bol->_idx, bol->in(1)->_idx); bol->dump(); bol->in(1)->dump();})
bol = bol->in(1); //may be ExtractNode
}
assert(bol->is_Bool(), "should be BoolNode - too late to bail out!");
*** 2877,2887 ****
const Type* vt = vtn;
if (VectorNode::is_shift(in)) {
Node* load = in->in(1);
if (load->is_Load() && in_bb(load) && (velt_type(load)->basic_type() == T_INT)) {
vt = velt_type(load);
! } else if (in->Opcode() != Op_LShiftI) {
// Widen type to Int to avoid creation of right shift vector
// (align + data_size(s1) check in stmts_can_pack() will fail).
// Note, left shifts work regardless type.
vt = TypeInt::INT;
}
--- 2877,2887 ----
const Type* vt = vtn;
if (VectorNode::is_shift(in)) {
Node* load = in->in(1);
if (load->is_Load() && in_bb(load) && (velt_type(load)->basic_type() == T_INT)) {
vt = velt_type(load);
! } else if (in->Opcode() != Opcodes::Op_LShiftI) {
// Widen type to Int to avoid creation of right shift vector
// (align + data_size(s1) check in stmts_can_pack() will fail).
// Note, left shifts work regardless type.
vt = TypeInt::INT;
}
*** 2941,2951 ****
if (n->is_Store() && (bt == T_CHAR)) {
// Use T_SHORT type instead of T_CHAR for stored values because any
// preceding arithmetic operation extends values to signed Int.
bt = T_SHORT;
}
! if (n->Opcode() == Op_LoadUB) {
// Adjust type for unsigned byte loads, it is important for right shifts.
// T_BOOLEAN is used because there is no basic type representing type
// TypeInt::UBYTE. Use of T_BOOLEAN for vectors is fine because only
// size (one byte) and sign is important.
bt = T_BOOLEAN;
--- 2941,2951 ----
if (n->is_Store() && (bt == T_CHAR)) {
// Use T_SHORT type instead of T_CHAR for stored values because any
// preceding arithmetic operation extends values to signed Int.
bt = T_SHORT;
}
! if (n->Opcode() == Opcodes::Op_LoadUB) {
// Adjust type for unsigned byte loads, it is important for right shifts.
// T_BOOLEAN is used because there is no basic type representing type
// TypeInt::UBYTE. Use of T_BOOLEAN for vectors is fine because only
// size (one byte) and sign is important.
bt = T_BOOLEAN;
*** 3081,3091 ****
CountedLoopNode *main_head = lp()->as_CountedLoop();
assert(main_head->is_main_loop(), "");
CountedLoopEndNode* pre_end = get_pre_loop_end(main_head);
assert(pre_end != NULL, "we must have a correct pre-loop");
Node *pre_opaq1 = pre_end->limit();
! assert(pre_opaq1->Opcode() == Op_Opaque1, "");
Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
Node *lim0 = pre_opaq->in(1);
// Where we put new limit calculations
Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
--- 3081,3091 ----
CountedLoopNode *main_head = lp()->as_CountedLoop();
assert(main_head->is_main_loop(), "");
CountedLoopEndNode* pre_end = get_pre_loop_end(main_head);
assert(pre_end != NULL, "we must have a correct pre-loop");
Node *pre_opaq1 = pre_end->limit();
! assert(pre_opaq1->Opcode() == Opcodes::Op_Opaque1, "");
Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1;
Node *lim0 = pre_opaq->in(1);
// Where we put new limit calculations
Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl);
*** 3435,3455 ****
if (offset_plus_k(n)) {
NOT_PRODUCT(_tracer.scaled_iv_plus_offset_3(n);)
return true;
}
! int opc = n->Opcode();
! if (opc == Op_AddI) {
if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2))) {
NOT_PRODUCT(_tracer.scaled_iv_plus_offset_4(n);)
return true;
}
if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) {
NOT_PRODUCT(_tracer.scaled_iv_plus_offset_5(n);)
return true;
}
! } else if (opc == Op_SubI) {
if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2), true)) {
NOT_PRODUCT(_tracer.scaled_iv_plus_offset_6(n);)
return true;
}
if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) {
--- 3435,3455 ----
if (offset_plus_k(n)) {
NOT_PRODUCT(_tracer.scaled_iv_plus_offset_3(n);)
return true;
}
! Opcodes opc = n->Opcode();
! if (opc == Opcodes::Op_AddI) {
if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2))) {
NOT_PRODUCT(_tracer.scaled_iv_plus_offset_4(n);)
return true;
}
if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) {
NOT_PRODUCT(_tracer.scaled_iv_plus_offset_5(n);)
return true;
}
! } else if (opc == Opcodes::Op_SubI) {
if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2), true)) {
NOT_PRODUCT(_tracer.scaled_iv_plus_offset_6(n);)
return true;
}
if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) {
*** 3481,3518 ****
}
if (_analyze_only && (invariant(n) == false)) {
_nstack->push(n, _stack_idx++);
}
! int opc = n->Opcode();
! if (opc == Op_MulI) {
if (n->in(1) == iv() && n->in(2)->is_Con()) {
_scale = n->in(2)->get_int();
NOT_PRODUCT(_tracer.scaled_iv_4(n, _scale);)
return true;
} else if (n->in(2) == iv() && n->in(1)->is_Con()) {
_scale = n->in(1)->get_int();
NOT_PRODUCT(_tracer.scaled_iv_5(n, _scale);)
return true;
}
! } else if (opc == Op_LShiftI) {
if (n->in(1) == iv() && n->in(2)->is_Con()) {
_scale = 1 << n->in(2)->get_int();
NOT_PRODUCT(_tracer.scaled_iv_6(n, _scale);)
return true;
}
! } else if (opc == Op_ConvI2L) {
! if (n->in(1)->Opcode() == Op_CastII &&
n->in(1)->as_CastII()->has_range_check()) {
// Skip range check dependent CastII nodes
n = n->in(1);
}
if (scaled_iv_plus_offset(n->in(1))) {
NOT_PRODUCT(_tracer.scaled_iv_7(n);)
return true;
}
! } else if (opc == Op_LShiftL) {
if (!has_iv() && _invar == NULL) {
// Need to preserve the current _offset value, so
// create a temporary object for this expression subtree.
// Hacky, so should re-engineer the address pattern match.
NOT_PRODUCT(Tracer::Depth dddd;)
--- 3481,3518 ----
}
if (_analyze_only && (invariant(n) == false)) {
_nstack->push(n, _stack_idx++);
}
! Opcodes opc = n->Opcode();
! if (opc == Opcodes::Op_MulI) {
if (n->in(1) == iv() && n->in(2)->is_Con()) {
_scale = n->in(2)->get_int();
NOT_PRODUCT(_tracer.scaled_iv_4(n, _scale);)
return true;
} else if (n->in(2) == iv() && n->in(1)->is_Con()) {
_scale = n->in(1)->get_int();
NOT_PRODUCT(_tracer.scaled_iv_5(n, _scale);)
return true;
}
! } else if (opc == Opcodes::Op_LShiftI) {
if (n->in(1) == iv() && n->in(2)->is_Con()) {
_scale = 1 << n->in(2)->get_int();
NOT_PRODUCT(_tracer.scaled_iv_6(n, _scale);)
return true;
}
! } else if (opc == Opcodes::Op_ConvI2L) {
! if (n->in(1)->Opcode() == Opcodes::Op_CastII &&
n->in(1)->as_CastII()->has_range_check()) {
// Skip range check dependent CastII nodes
n = n->in(1);
}
if (scaled_iv_plus_offset(n->in(1))) {
NOT_PRODUCT(_tracer.scaled_iv_7(n);)
return true;
}
! } else if (opc == Opcodes::Op_LShiftL) {
if (!has_iv() && _invar == NULL) {
// Need to preserve the current _offset value, so
// create a temporary object for this expression subtree.
// Hacky, so should re-engineer the address pattern match.
NOT_PRODUCT(Tracer::Depth dddd;)
*** 3539,3554 ****
// where k maybe zero and invariant is optional, but not both.
bool SWPointer::offset_plus_k(Node* n, bool negate) {
NOT_PRODUCT(Tracer::Depth ddd;)
NOT_PRODUCT(_tracer.offset_plus_k_1(n);)
! int opc = n->Opcode();
! if (opc == Op_ConI) {
_offset += negate ? -(n->get_int()) : n->get_int();
NOT_PRODUCT(_tracer.offset_plus_k_2(n, _offset);)
return true;
! } else if (opc == Op_ConL) {
// Okay if value fits into an int
const TypeLong* t = n->find_long_type();
if (t->higher_equal(TypeLong::INT)) {
jlong loff = n->get_long();
jint off = (jint)loff;
--- 3539,3554 ----
// where k maybe zero and invariant is optional, but not both.
bool SWPointer::offset_plus_k(Node* n, bool negate) {
NOT_PRODUCT(Tracer::Depth ddd;)
NOT_PRODUCT(_tracer.offset_plus_k_1(n);)
! Opcodes opc = n->Opcode();
! if (opc == Opcodes::Op_ConI) {
_offset += negate ? -(n->get_int()) : n->get_int();
NOT_PRODUCT(_tracer.offset_plus_k_2(n, _offset);)
return true;
! } else if (opc == Opcodes::Op_ConL) {
// Okay if value fits into an int
const TypeLong* t = n->find_long_type();
if (t->higher_equal(TypeLong::INT)) {
jlong loff = n->get_long();
jint off = (jint)loff;
*** 3565,3575 ****
}
if (_analyze_only && (invariant(n) == false)) {
_nstack->push(n, _stack_idx++);
}
! if (opc == Op_AddI) {
if (n->in(2)->is_Con() && invariant(n->in(1))) {
_negate_invar = negate;
_invar = n->in(1);
_offset += negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
NOT_PRODUCT(_tracer.offset_plus_k_6(n, _invar, _negate_invar, _offset);)
--- 3565,3575 ----
}
if (_analyze_only && (invariant(n) == false)) {
_nstack->push(n, _stack_idx++);
}
! if (opc == Opcodes::Op_AddI) {
if (n->in(2)->is_Con() && invariant(n->in(1))) {
_negate_invar = negate;
_invar = n->in(1);
_offset += negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
NOT_PRODUCT(_tracer.offset_plus_k_6(n, _invar, _negate_invar, _offset);)
*** 3580,3590 ****
_invar = n->in(2);
NOT_PRODUCT(_tracer.offset_plus_k_7(n, _invar, _negate_invar, _offset);)
return true;
}
}
! if (opc == Op_SubI) {
if (n->in(2)->is_Con() && invariant(n->in(1))) {
_negate_invar = negate;
_invar = n->in(1);
_offset += !negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
NOT_PRODUCT(_tracer.offset_plus_k_8(n, _invar, _negate_invar, _offset);)
--- 3580,3590 ----
_invar = n->in(2);
NOT_PRODUCT(_tracer.offset_plus_k_7(n, _invar, _negate_invar, _offset);)
return true;
}
}
! if (opc == Opcodes::Op_SubI) {
if (n->in(2)->is_Con() && invariant(n->in(1))) {
_negate_invar = negate;
_invar = n->in(1);
_offset += !negate ? -(n->in(2)->get_int()) : n->in(2)->get_int();
NOT_PRODUCT(_tracer.offset_plus_k_8(n, _invar, _negate_invar, _offset);)
*** 3596,3608 ****
NOT_PRODUCT(_tracer.offset_plus_k_9(n, _invar, _negate_invar, _offset);)
return true;
}
}
if (invariant(n)) {
! if (opc == Op_ConvI2L) {
n = n->in(1);
! if (n->Opcode() == Op_CastII &&
n->as_CastII()->has_range_check()) {
// Skip range check dependent CastII nodes
assert(invariant(n), "sanity");
n = n->in(1);
}
--- 3596,3608 ----
NOT_PRODUCT(_tracer.offset_plus_k_9(n, _invar, _negate_invar, _offset);)
return true;
}
}
if (invariant(n)) {
! if (opc == Opcodes::Op_ConvI2L) {
n = n->in(1);
! if (n->Opcode() == Opcodes::Op_CastII &&
n->as_CastII()->has_range_check()) {
// Skip range check dependent CastII nodes
assert(invariant(n), "sanity");
n = n->in(1);
}
< prev index next >