154 Unique_Node_List worklist;
155 VectorSet visited(Thread::current()->resource_area());
156 worklist.push(xroot);
157 while (worklist.size() > 0) {
158 Node* n = worklist.pop();
159 visited <<= n->_idx;
160 assert(C->node_arena()->contains(n), "dead node");
161 for (uint j = 0; j < n->req(); j++) {
162 Node* in = n->in(j);
163 if (in != NULL) {
164 assert(C->node_arena()->contains(in), "dead node");
165 if (!visited.test(in->_idx)) {
166 worklist.push(in);
167 }
168 }
169 }
170 }
171 }
172 #endif
173
174
175 //---------------------------match---------------------------------------------
176 void Matcher::match( ) {
177 if( MaxLabelRootDepth < 100 ) { // Too small?
178 assert(false, "invalid MaxLabelRootDepth, increase it to 100 minimum");
179 MaxLabelRootDepth = 100;
180 }
181 // One-time initialization of some register masks.
182 init_spill_mask( C->root()->in(1) );
183 _return_addr_mask = return_addr();
184 #ifdef _LP64
185 // Pointers take 2 slots in 64-bit land
186 _return_addr_mask.Insert(OptoReg::add(return_addr(),1));
187 #endif
188
189 // Map a Java-signature return type into return register-value
190 // machine registers for 0, 1 and 2 returned values.
191 const TypeTuple *range = C->tf()->range();
192 if( range->cnt() > TypeFunc::Parms ) { // If not a void function
193 // Get ideal-register return type
194 uint ireg = range->field_at(TypeFunc::Parms)->ideal_reg();
195 // Get machine return register
196 uint sop = C->start()->Opcode();
197 OptoRegPair regs = return_value(ireg, false);
198
199 // And mask for same
200 _return_value_mask = RegMask(regs.first());
201 if( OptoReg::is_valid(regs.second()) )
202 _return_value_mask.Insert(regs.second());
203 }
204
205 // ---------------
206 // Frame Layout
207
208 // Need the method signature to determine the incoming argument types,
209 // because the types determine which registers the incoming arguments are
210 // in, and this affects the matched code.
211 const TypeTuple *domain = C->tf()->domain();
212 uint argcnt = domain->cnt() - TypeFunc::Parms;
213 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
214 VMRegPair *vm_parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
215 _parm_regs = NEW_RESOURCE_ARRAY( OptoRegPair, argcnt );
216 _calling_convention_mask = NEW_RESOURCE_ARRAY( RegMask, argcnt );
217 uint i;
218 for( i = 0; i<argcnt; i++ ) {
219 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
220 }
221
222 // Pass array of ideal registers and length to USER code (from the AD file)
223 // that will convert this to an array of register numbers.
224 const StartNode *start = C->start();
225 start->calling_convention( sig_bt, vm_parm_regs, argcnt );
226 #ifdef ASSERT
227 // Sanity check users' calling convention. Real handy while trying to
228 // get the initial port correct.
229 { for (uint i = 0; i<argcnt; i++) {
230 if( !vm_parm_regs[i].first()->is_valid() && !vm_parm_regs[i].second()->is_valid() ) {
231 assert(domain->field_at(i+TypeFunc::Parms)==Type::HALF, "only allowed on halve" );
445 idealreg2mhdebugmask[Op_RegF] = &rms[15];
446 idealreg2mhdebugmask[Op_RegD] = &rms[16];
447 idealreg2mhdebugmask[Op_RegP] = &rms[17];
448
449 idealreg2spillmask [Op_VecS] = &rms[18];
450 idealreg2spillmask [Op_VecD] = &rms[19];
451 idealreg2spillmask [Op_VecX] = &rms[20];
452 idealreg2spillmask [Op_VecY] = &rms[21];
453 idealreg2spillmask [Op_VecZ] = &rms[22];
454
455 OptoReg::Name i;
456
457 // At first, start with the empty mask
458 C->FIRST_STACK_mask().Clear();
459
460 // Add in the incoming argument area
461 OptoReg::Name init_in = OptoReg::add(_old_SP, C->out_preserve_stack_slots());
462 for (i = init_in; i < _in_arg_limit; i = OptoReg::add(i,1)) {
463 C->FIRST_STACK_mask().Insert(i);
464 }
465 // Add in all bits past the outgoing argument area
466 guarantee(RegMask::can_represent_arg(OptoReg::add(_out_arg_limit,-1)),
467 "must be able to represent all call arguments in reg mask");
468 OptoReg::Name init = _out_arg_limit;
469 for (i = init; RegMask::can_represent(i); i = OptoReg::add(i,1)) {
470 C->FIRST_STACK_mask().Insert(i);
471 }
472 // Finally, set the "infinite stack" bit.
473 C->FIRST_STACK_mask().set_AllStack();
474
475 // Make spill masks. Registers for their class, plus FIRST_STACK_mask.
476 RegMask aligned_stack_mask = C->FIRST_STACK_mask();
477 // Keep spill masks aligned.
478 aligned_stack_mask.clear_to_pairs();
479 assert(aligned_stack_mask.is_AllStack(), "should be infinite stack");
480
481 *idealreg2spillmask[Op_RegP] = *idealreg2regmask[Op_RegP];
482 #ifdef _LP64
483 *idealreg2spillmask[Op_RegN] = *idealreg2regmask[Op_RegN];
484 idealreg2spillmask[Op_RegN]->OR(C->FIRST_STACK_mask());
637 init_first_stack_mask();
638
639 Node *root = C->root(); // Short name for root
640 // Count number of save-on-entry registers.
641 uint soe_cnt = number_of_saved_registers();
642 uint i;
643
644 // Find the procedure Start Node
645 StartNode *start = C->start();
646 assert( start, "Expect a start node" );
647
648 // Save argument registers in the trampolining stubs
649 if( C->save_argument_registers() )
650 for( i = 0; i < _last_Mach_Reg; i++ )
651 if( is_spillable_arg(i) )
652 soe_cnt++;
653
654 // Input RegMask array shared by all Returns.
655 // The type for doubles and longs has a count of 2, but
656 // there is only 1 returned value
657 uint ret_edge_cnt = TypeFunc::Parms + ((C->tf()->range()->cnt() == TypeFunc::Parms) ? 0 : 1);
658 RegMask *ret_rms = init_input_masks( ret_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
659 // Returns have 0 or 1 returned values depending on call signature.
660 // Return register is specified by return_value in the AD file.
661 if (ret_edge_cnt > TypeFunc::Parms)
662 ret_rms[TypeFunc::Parms+0] = _return_value_mask;
663
664 // Input RegMask array shared by all Rethrows.
665 uint reth_edge_cnt = TypeFunc::Parms+1;
666 RegMask *reth_rms = init_input_masks( reth_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
667 // Rethrow takes exception oop only, but in the argument 0 slot.
668 OptoReg::Name reg = find_receiver(false);
669 if (reg >= 0) {
670 reth_rms[TypeFunc::Parms] = mreg2regmask[reg];
671 #ifdef _LP64
672 // Need two slots for ptrs in 64-bit land
673 reth_rms[TypeFunc::Parms].Insert(OptoReg::add(OptoReg::Name(reg), 1));
674 #endif
675 }
676
677 // Input RegMask array shared by all TailCalls
678 uint tail_call_edge_cnt = TypeFunc::Parms+2;
679 RegMask *tail_call_rms = init_input_masks( tail_call_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
680
681 // Input RegMask array shared by all TailJumps
682 uint tail_jump_edge_cnt = TypeFunc::Parms+2;
709 }
710
711 // Input RegMask array shared by all Halts
712 uint halt_edge_cnt = TypeFunc::Parms;
713 RegMask *halt_rms = init_input_masks( halt_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
714
715 // Capture the return input masks into each exit flavor
716 for( i=1; i < root->req(); i++ ) {
717 MachReturnNode *exit = root->in(i)->as_MachReturn();
718 switch( exit->ideal_Opcode() ) {
719 case Op_Return : exit->_in_rms = ret_rms; break;
720 case Op_Rethrow : exit->_in_rms = reth_rms; break;
721 case Op_TailCall : exit->_in_rms = tail_call_rms; break;
722 case Op_TailJump : exit->_in_rms = tail_jump_rms; break;
723 case Op_Halt : exit->_in_rms = halt_rms; break;
724 default : ShouldNotReachHere();
725 }
726 }
727
728 // Next unused projection number from Start.
729 int proj_cnt = C->tf()->domain()->cnt();
730
731 // Do all the save-on-entry registers. Make projections from Start for
732 // them, and give them a use at the exit points. To the allocator, they
733 // look like incoming register arguments.
734 for( i = 0; i < _last_Mach_Reg; i++ ) {
735 if( is_save_on_entry(i) ) {
736
737 // Add the save-on-entry to the mask array
738 ret_rms [ ret_edge_cnt] = mreg2regmask[i];
739 reth_rms [ reth_edge_cnt] = mreg2regmask[i];
740 tail_call_rms[tail_call_edge_cnt] = mreg2regmask[i];
741 tail_jump_rms[tail_jump_edge_cnt] = mreg2regmask[i];
742 // Halts need the SOE registers, but only in the stack as debug info.
743 // A just-prior uncommon-trap or deoptimization will use the SOE regs.
744 halt_rms [ halt_edge_cnt] = *idealreg2spillmask[_register_save_type[i]];
745
746 Node *mproj;
747
748 // Is this a RegF low half of a RegD? Double up 2 adjacent RegF's
749 // into a single RegD.
990 Node *oldn = n;
991 // Old-space or new-space check
992 if (!C->node_arena()->contains(n)) {
993 // Old space!
994 Node* m;
995 if (has_new_node(n)) { // Not yet Label/Reduced
996 m = new_node(n);
997 } else {
998 if (!is_dontcare(n)) { // Matcher can match this guy
999 // Calls match special. They match alone with no children.
1000 // Their children, the incoming arguments, match normally.
1001 m = n->is_SafePoint() ? match_sfpt(n->as_SafePoint()):match_tree(n);
1002 if (C->failing()) return NULL;
1003 if (m == NULL) { Matcher::soft_match_failure(); return NULL; }
1004 if (n->is_MemBar()) {
1005 m->as_MachMemBar()->set_adr_type(n->adr_type());
1006 }
1007 } else { // Nothing the matcher cares about
1008 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Multi()) { // Projections?
1009 // Convert to machine-dependent projection
1010 m = n->in(0)->as_Multi()->match( n->as_Proj(), this );
1011 #ifdef ASSERT
1012 _new2old_map.map(m->_idx, n);
1013 #endif
1014 if (m->in(0) != NULL) // m might be top
1015 collect_null_checks(m, n);
1016 } else { // Else just a regular 'ol guy
1017 m = n->clone(); // So just clone into new-space
1018 #ifdef ASSERT
1019 _new2old_map.map(m->_idx, n);
1020 #endif
1021 // Def-Use edges will be added incrementally as Uses
1022 // of this node are matched.
1023 assert(m->outcnt() == 0, "no Uses of this clone yet");
1024 }
1025 }
1026
1027 set_new_node(n, m); // Map old to new
1028 if (_old_node_note_array != NULL) {
1029 Node_Notes* nn = C->locate_node_notes(_old_node_note_array,
1030 n->_idx);
1135 }
1136 return OptoReg::as_OptoReg(reg);
1137 }
1138
1139
1140 //------------------------------match_sfpt-------------------------------------
1141 // Helper function to match call instructions. Calls match special.
1142 // They match alone with no children. Their children, the incoming
1143 // arguments, match normally.
1144 MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) {
1145 MachSafePointNode *msfpt = NULL;
1146 MachCallNode *mcall = NULL;
1147 uint cnt;
1148 // Split out case for SafePoint vs Call
1149 CallNode *call;
1150 const TypeTuple *domain;
1151 ciMethod* method = NULL;
1152 bool is_method_handle_invoke = false; // for special kill effects
1153 if( sfpt->is_Call() ) {
1154 call = sfpt->as_Call();
1155 domain = call->tf()->domain();
1156 cnt = domain->cnt();
1157
1158 // Match just the call, nothing else
1159 MachNode *m = match_tree(call);
1160 if (C->failing()) return NULL;
1161 if( m == NULL ) { Matcher::soft_match_failure(); return NULL; }
1162
1163 // Copy data from the Ideal SafePoint to the machine version
1164 mcall = m->as_MachCall();
1165
1166 mcall->set_tf( call->tf());
1167 mcall->set_entry_point(call->entry_point());
1168 mcall->set_cnt( call->cnt());
1169
1170 if( mcall->is_MachCallJava() ) {
1171 MachCallJavaNode *mcall_java = mcall->as_MachCallJava();
1172 const CallJavaNode *call_java = call->as_CallJava();
1173 assert(call_java->validate_symbolic_info(), "inconsistent info");
1174 method = call_java->method();
1175 mcall_java->_method = method;
1210 msfpt->_in_rms = NEW_RESOURCE_ARRAY( RegMask, cnt );
1211 // Empty them all.
1212 for (uint i = 0; i < cnt; i++) ::new (&(msfpt->_in_rms[i])) RegMask();
1213
1214 // Do all the pre-defined non-Empty register masks
1215 msfpt->_in_rms[TypeFunc::ReturnAdr] = _return_addr_mask;
1216 msfpt->_in_rms[TypeFunc::FramePtr ] = c_frame_ptr_mask;
1217
1218 // Place first outgoing argument can possibly be put.
1219 OptoReg::Name begin_out_arg_area = OptoReg::add(_new_SP, C->out_preserve_stack_slots());
1220 assert( is_even(begin_out_arg_area), "" );
1221 // Compute max outgoing register number per call site.
1222 OptoReg::Name out_arg_limit_per_call = begin_out_arg_area;
1223 // Calls to C may hammer extra stack slots above and beyond any arguments.
1224 // These are usually backing store for register arguments for varargs.
1225 if( call != NULL && call->is_CallRuntime() )
1226 out_arg_limit_per_call = OptoReg::add(out_arg_limit_per_call,C->varargs_C_out_slots_killed());
1227
1228
1229 // Do the normal argument list (parameters) register masks
1230 int argcnt = cnt - TypeFunc::Parms;
1231 if( argcnt > 0 ) { // Skip it all if we have no args
1232 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
1233 VMRegPair *parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
1234 int i;
1235 for( i = 0; i < argcnt; i++ ) {
1236 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
1237 }
1238 // V-call to pick proper calling convention
1239 call->calling_convention( sig_bt, parm_regs, argcnt );
1240
1241 #ifdef ASSERT
1242 // Sanity check users' calling convention. Really handy during
1243 // the initial porting effort. Fairly expensive otherwise.
1244 { for (int i = 0; i<argcnt; i++) {
1245 if( !parm_regs[i].first()->is_valid() &&
1246 !parm_regs[i].second()->is_valid() ) continue;
1247 VMReg reg1 = parm_regs[i].first();
1248 VMReg reg2 = parm_regs[i].second();
1249 for (int j = 0; j < i; j++) {
1250 if( !parm_regs[j].first()->is_valid() &&
1251 !parm_regs[j].second()->is_valid() ) continue;
1252 VMReg reg3 = parm_regs[j].first();
1253 VMReg reg4 = parm_regs[j].second();
1254 if( !reg1->is_valid() ) {
1255 assert( !reg2->is_valid(), "valid halvsies" );
1256 } else if( !reg3->is_valid() ) {
1257 assert( !reg4->is_valid(), "valid halvsies" );
1258 } else {
1259 assert( reg1 != reg2, "calling conv. must produce distinct regs");
1260 assert( reg1 != reg3, "calling conv. must produce distinct regs");
1261 assert( reg1 != reg4, "calling conv. must produce distinct regs");
1262 assert( reg2 != reg3, "calling conv. must produce distinct regs");
1263 assert( reg2 != reg4 || !reg2->is_valid(), "calling conv. must produce distinct regs");
1264 assert( reg3 != reg4, "calling conv. must produce distinct regs");
1265 }
1266 }
1267 }
1268 }
1269 #endif
1270
1271 // Visit each argument. Compute its outgoing register mask.
1272 // Return results now can have 2 bits returned.
1273 // Compute max over all outgoing arguments both per call-site
1274 // and over the entire method.
1275 for( i = 0; i < argcnt; i++ ) {
1276 // Address of incoming argument mask to fill in
1277 RegMask *rm = &mcall->_in_rms[i+TypeFunc::Parms];
1278 if( !parm_regs[i].first()->is_valid() &&
1279 !parm_regs[i].second()->is_valid() ) {
1280 continue; // Avoid Halves
1281 }
1282 // Grab first register, adjust stack slots and insert in mask.
1283 OptoReg::Name reg1 = warp_outgoing_stk_arg(parm_regs[i].first(), begin_out_arg_area, out_arg_limit_per_call );
1284 if (OptoReg::is_valid(reg1))
1285 rm->Insert( reg1 );
1286 // Grab second register (if any), adjust stack slots and insert in mask.
1287 OptoReg::Name reg2 = warp_outgoing_stk_arg(parm_regs[i].second(), begin_out_arg_area, out_arg_limit_per_call );
1288 if (OptoReg::is_valid(reg2))
1289 rm->Insert( reg2 );
1290 } // End of for all arguments
1291
1292 // Compute number of stack slots needed to restore stack in case of
1293 // Pascal-style argument popping.
1294 mcall->_argsize = out_arg_limit_per_call - begin_out_arg_area;
1295 }
1296
1297 // Compute the max stack slot killed by any call. These will not be
1298 // available for debug info, and will be used to adjust FIRST_STACK_mask
1299 // after all call sites have been visited.
1300 if( _out_arg_limit < out_arg_limit_per_call)
1301 _out_arg_limit = out_arg_limit_per_call;
1302
1303 if (mcall) {
1304 // Kill the outgoing argument area, including any non-argument holes and
1305 // any legacy C-killed slots. Use Fat-Projections to do the killing.
1306 // Since the max-per-method covers the max-per-call-site and debug info
1307 // is excluded on the max-per-method basis, debug info cannot land in
1308 // this killed area.
1309 uint r_cnt = mcall->tf()->range()->cnt();
1310 MachProjNode *proj = new MachProjNode( mcall, r_cnt+10000, RegMask::Empty, MachProjNode::fat_proj );
1311 if (!RegMask::can_represent_arg(OptoReg::Name(out_arg_limit_per_call-1))) {
1312 C->record_method_not_compilable("unsupported outgoing calling sequence");
1313 } else {
1314 for (int i = begin_out_arg_area; i < out_arg_limit_per_call; i++)
1315 proj->_rout.Insert(OptoReg::Name(i));
1316 }
1317 if (proj->_rout.is_NotEmpty()) {
1318 push_projection(proj);
1319 }
1320 }
1321 // Transfer the safepoint information from the call to the mcall
1322 // Move the JVMState list
1323 msfpt->set_jvms(sfpt->jvms());
1324 for (JVMState* jvms = msfpt->jvms(); jvms; jvms = jvms->caller()) {
1325 jvms->set_map(sfpt);
1326 }
1327
1328 // Debug inputs begin just after the last incoming parameter
1329 assert((mcall == NULL) || (mcall->jvms() == NULL) ||
1330 (mcall->jvms()->debug_start() + mcall->_jvmadj == mcall->tf()->domain()->cnt()), "");
1331
1332 // Move the OopMap
1333 msfpt->_oop_map = sfpt->_oop_map;
1334
1335 // Add additional edges.
1336 if (msfpt->mach_constant_base_node_input() != (uint)-1 && !msfpt->is_MachCallLeaf()) {
1337 // For these calls we can not add MachConstantBase in expand(), as the
1338 // ins are not complete then.
1339 msfpt->ins_req(msfpt->mach_constant_base_node_input(), C->mach_constant_base_node());
1340 if (msfpt->jvms() &&
1341 msfpt->mach_constant_base_node_input() <= msfpt->jvms()->debug_start() + msfpt->_jvmadj) {
1342 // We added an edge before jvms, so we must adapt the position of the ins.
1343 msfpt->jvms()->adapt_position(+1);
1344 }
1345 }
1346
1347 // Registers killed by the call are set in the local scheduling pass
1348 // of Global Code Motion.
1349 return msfpt;
1350 }
2343 break;
2344 }
2345 case Op_FmaD:
2346 case Op_FmaF:
2347 case Op_FmaVD:
2348 case Op_FmaVF: {
2349 // Restructure into a binary tree for Matching.
2350 Node* pair = new BinaryNode(n->in(1), n->in(2));
2351 n->set_req(2, pair);
2352 n->set_req(1, n->in(3));
2353 n->del_req(3);
2354 break;
2355 }
2356 case Op_MulAddS2I: {
2357 Node* pair1 = new BinaryNode(n->in(1), n->in(2));
2358 Node* pair2 = new BinaryNode(n->in(3), n->in(4));
2359 n->set_req(1, pair1);
2360 n->set_req(2, pair2);
2361 n->del_req(4);
2362 n->del_req(3);
2363 break;
2364 }
2365 default:
2366 break;
2367 }
2368 }
2369
2370 #ifdef ASSERT
2371 // machine-independent root to machine-dependent root
2372 void Matcher::dump_old2new_map() {
2373 _old2new_map.dump();
2374 }
2375 #endif
2376
2377 //---------------------------collect_null_checks-------------------------------
2378 // Find null checks in the ideal graph; write a machine-specific node for
2379 // it. Used by later implicit-null-check handling. Actually collects
2380 // either an IfTrue or IfFalse for the common NOT-null path, AND the ideal
2381 // value being tested.
2382 void Matcher::collect_null_checks( Node *proj, Node *orig_proj ) {
|
154 Unique_Node_List worklist;
155 VectorSet visited(Thread::current()->resource_area());
156 worklist.push(xroot);
157 while (worklist.size() > 0) {
158 Node* n = worklist.pop();
159 visited <<= n->_idx;
160 assert(C->node_arena()->contains(n), "dead node");
161 for (uint j = 0; j < n->req(); j++) {
162 Node* in = n->in(j);
163 if (in != NULL) {
164 assert(C->node_arena()->contains(in), "dead node");
165 if (!visited.test(in->_idx)) {
166 worklist.push(in);
167 }
168 }
169 }
170 }
171 }
172 #endif
173
174 // Array of RegMask, one per returned values (value type instances can
175 // be returned as multiple return values, one per field)
176 RegMask* Matcher::return_values_mask(const TypeTuple *range) {
177 uint cnt = range->cnt() - TypeFunc::Parms;
178 if (cnt == 0) {
179 return NULL;
180 }
181 RegMask* mask = NEW_RESOURCE_ARRAY(RegMask, cnt);
182
183 if (!ValueTypeReturnedAsFields) {
184 // Get ideal-register return type
185 uint ireg = range->field_at(TypeFunc::Parms)->ideal_reg();
186 // Get machine return register
187 OptoRegPair regs = return_value(ireg, false);
188
189 // And mask for same
190 mask[0].Clear();
191 mask[0].Insert(regs.first());
192 if (OptoReg::is_valid(regs.second())) {
193 mask[0].Insert(regs.second());
194 }
195 } else {
196 BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, cnt);
197 VMRegPair* vm_parm_regs = NEW_RESOURCE_ARRAY(VMRegPair, cnt);
198
199 for (uint i = 0; i < cnt; i++) {
200 sig_bt[i] = range->field_at(i+TypeFunc::Parms)->basic_type();
201 }
202
203 int regs = SharedRuntime::java_return_convention(sig_bt, vm_parm_regs, cnt);
204 assert(regs > 0, "should have been tested during graph construction");
205 for (uint i = 0; i < cnt; i++) {
206 mask[i].Clear();
207
208 OptoReg::Name reg1 = OptoReg::as_OptoReg(vm_parm_regs[i].first());
209 if (OptoReg::is_valid(reg1)) {
210 mask[i].Insert(reg1);
211 }
212 OptoReg::Name reg2 = OptoReg::as_OptoReg(vm_parm_regs[i].second());
213 if (OptoReg::is_valid(reg2)) {
214 mask[i].Insert(reg2);
215 }
216 }
217 }
218 return mask;
219 }
220
221 //---------------------------match---------------------------------------------
222 void Matcher::match( ) {
223 if( MaxLabelRootDepth < 100 ) { // Too small?
224 assert(false, "invalid MaxLabelRootDepth, increase it to 100 minimum");
225 MaxLabelRootDepth = 100;
226 }
227 // One-time initialization of some register masks.
228 init_spill_mask( C->root()->in(1) );
229 _return_addr_mask = return_addr();
230 #ifdef _LP64
231 // Pointers take 2 slots in 64-bit land
232 _return_addr_mask.Insert(OptoReg::add(return_addr(),1));
233 #endif
234
235 // Map Java-signature return types into return register-value
236 // machine registers.
237 const TypeTuple *range = C->tf()->range_cc();
238 _return_values_mask = return_values_mask(range);
239
240 // ---------------
241 // Frame Layout
242
243 // Need the method signature to determine the incoming argument types,
244 // because the types determine which registers the incoming arguments are
245 // in, and this affects the matched code.
246 const TypeTuple *domain = C->tf()->domain_cc();
247 uint argcnt = domain->cnt() - TypeFunc::Parms;
248 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
249 VMRegPair *vm_parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
250 _parm_regs = NEW_RESOURCE_ARRAY( OptoRegPair, argcnt );
251 _calling_convention_mask = NEW_RESOURCE_ARRAY( RegMask, argcnt );
252 uint i;
253 for( i = 0; i<argcnt; i++ ) {
254 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
255 }
256
257 // Pass array of ideal registers and length to USER code (from the AD file)
258 // that will convert this to an array of register numbers.
259 const StartNode *start = C->start();
260 start->calling_convention( sig_bt, vm_parm_regs, argcnt );
261 #ifdef ASSERT
262 // Sanity check users' calling convention. Real handy while trying to
263 // get the initial port correct.
264 { for (uint i = 0; i<argcnt; i++) {
265 if( !vm_parm_regs[i].first()->is_valid() && !vm_parm_regs[i].second()->is_valid() ) {
266 assert(domain->field_at(i+TypeFunc::Parms)==Type::HALF, "only allowed on halve" );
480 idealreg2mhdebugmask[Op_RegF] = &rms[15];
481 idealreg2mhdebugmask[Op_RegD] = &rms[16];
482 idealreg2mhdebugmask[Op_RegP] = &rms[17];
483
484 idealreg2spillmask [Op_VecS] = &rms[18];
485 idealreg2spillmask [Op_VecD] = &rms[19];
486 idealreg2spillmask [Op_VecX] = &rms[20];
487 idealreg2spillmask [Op_VecY] = &rms[21];
488 idealreg2spillmask [Op_VecZ] = &rms[22];
489
490 OptoReg::Name i;
491
492 // At first, start with the empty mask
493 C->FIRST_STACK_mask().Clear();
494
495 // Add in the incoming argument area
496 OptoReg::Name init_in = OptoReg::add(_old_SP, C->out_preserve_stack_slots());
497 for (i = init_in; i < _in_arg_limit; i = OptoReg::add(i,1)) {
498 C->FIRST_STACK_mask().Insert(i);
499 }
500
501 // Check if the method has a reserved entry in the argument stack area that
502 // should not be used for spilling because it may hold the return address.
503 if (C->method() != NULL && C->method()->has_scalarized_args()) {
504 ExtendedSignature sig_cc = ExtendedSignature(C->method()->get_sig_cc(), SigEntryFilter());
505 for (int off = 0; !sig_cc.at_end(); ) {
506 BasicType bt = (*sig_cc)._bt;
507 off += type2size[bt];
508 while (SigEntry::next_is_reserved(sig_cc, bt)) {
509 // Remove reserved stack slot from mask to avoid spilling
510 OptoRegPair reg = _parm_regs[off];
511 assert(OptoReg::is_valid(reg.first()), "invalid reserved register");
512 C->FIRST_STACK_mask().Remove(reg.first());
513 C->FIRST_STACK_mask().Remove(reg.first()+1); // Always occupies two stack slots
514 off += type2size[bt];
515 }
516 }
517 }
518
519 // Add in all bits past the outgoing argument area
520 guarantee(RegMask::can_represent_arg(OptoReg::add(_out_arg_limit,-1)),
521 "must be able to represent all call arguments in reg mask");
522 OptoReg::Name init = _out_arg_limit;
523 for (i = init; RegMask::can_represent(i); i = OptoReg::add(i,1)) {
524 C->FIRST_STACK_mask().Insert(i);
525 }
526 // Finally, set the "infinite stack" bit.
527 C->FIRST_STACK_mask().set_AllStack();
528
529 // Make spill masks. Registers for their class, plus FIRST_STACK_mask.
530 RegMask aligned_stack_mask = C->FIRST_STACK_mask();
531 // Keep spill masks aligned.
532 aligned_stack_mask.clear_to_pairs();
533 assert(aligned_stack_mask.is_AllStack(), "should be infinite stack");
534
535 *idealreg2spillmask[Op_RegP] = *idealreg2regmask[Op_RegP];
536 #ifdef _LP64
537 *idealreg2spillmask[Op_RegN] = *idealreg2regmask[Op_RegN];
538 idealreg2spillmask[Op_RegN]->OR(C->FIRST_STACK_mask());
691 init_first_stack_mask();
692
693 Node *root = C->root(); // Short name for root
694 // Count number of save-on-entry registers.
695 uint soe_cnt = number_of_saved_registers();
696 uint i;
697
698 // Find the procedure Start Node
699 StartNode *start = C->start();
700 assert( start, "Expect a start node" );
701
702 // Save argument registers in the trampolining stubs
703 if( C->save_argument_registers() )
704 for( i = 0; i < _last_Mach_Reg; i++ )
705 if( is_spillable_arg(i) )
706 soe_cnt++;
707
708 // Input RegMask array shared by all Returns.
709 // The type for doubles and longs has a count of 2, but
710 // there is only 1 returned value
711 uint ret_edge_cnt = C->tf()->range_cc()->cnt();
712 RegMask *ret_rms = init_input_masks( ret_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
713 for (i = TypeFunc::Parms; i < ret_edge_cnt; i++) {
714 ret_rms[i] = _return_values_mask[i-TypeFunc::Parms];
715 }
716
717 // Input RegMask array shared by all Rethrows.
718 uint reth_edge_cnt = TypeFunc::Parms+1;
719 RegMask *reth_rms = init_input_masks( reth_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
720 // Rethrow takes exception oop only, but in the argument 0 slot.
721 OptoReg::Name reg = find_receiver(false);
722 if (reg >= 0) {
723 reth_rms[TypeFunc::Parms] = mreg2regmask[reg];
724 #ifdef _LP64
725 // Need two slots for ptrs in 64-bit land
726 reth_rms[TypeFunc::Parms].Insert(OptoReg::add(OptoReg::Name(reg), 1));
727 #endif
728 }
729
730 // Input RegMask array shared by all TailCalls
731 uint tail_call_edge_cnt = TypeFunc::Parms+2;
732 RegMask *tail_call_rms = init_input_masks( tail_call_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
733
734 // Input RegMask array shared by all TailJumps
735 uint tail_jump_edge_cnt = TypeFunc::Parms+2;
762 }
763
764 // Input RegMask array shared by all Halts
765 uint halt_edge_cnt = TypeFunc::Parms;
766 RegMask *halt_rms = init_input_masks( halt_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
767
768 // Capture the return input masks into each exit flavor
769 for( i=1; i < root->req(); i++ ) {
770 MachReturnNode *exit = root->in(i)->as_MachReturn();
771 switch( exit->ideal_Opcode() ) {
772 case Op_Return : exit->_in_rms = ret_rms; break;
773 case Op_Rethrow : exit->_in_rms = reth_rms; break;
774 case Op_TailCall : exit->_in_rms = tail_call_rms; break;
775 case Op_TailJump : exit->_in_rms = tail_jump_rms; break;
776 case Op_Halt : exit->_in_rms = halt_rms; break;
777 default : ShouldNotReachHere();
778 }
779 }
780
781 // Next unused projection number from Start.
782 int proj_cnt = C->tf()->domain_cc()->cnt();
783
784 // Do all the save-on-entry registers. Make projections from Start for
785 // them, and give them a use at the exit points. To the allocator, they
786 // look like incoming register arguments.
787 for( i = 0; i < _last_Mach_Reg; i++ ) {
788 if( is_save_on_entry(i) ) {
789
790 // Add the save-on-entry to the mask array
791 ret_rms [ ret_edge_cnt] = mreg2regmask[i];
792 reth_rms [ reth_edge_cnt] = mreg2regmask[i];
793 tail_call_rms[tail_call_edge_cnt] = mreg2regmask[i];
794 tail_jump_rms[tail_jump_edge_cnt] = mreg2regmask[i];
795 // Halts need the SOE registers, but only in the stack as debug info.
796 // A just-prior uncommon-trap or deoptimization will use the SOE regs.
797 halt_rms [ halt_edge_cnt] = *idealreg2spillmask[_register_save_type[i]];
798
799 Node *mproj;
800
801 // Is this a RegF low half of a RegD? Double up 2 adjacent RegF's
802 // into a single RegD.
1043 Node *oldn = n;
1044 // Old-space or new-space check
1045 if (!C->node_arena()->contains(n)) {
1046 // Old space!
1047 Node* m;
1048 if (has_new_node(n)) { // Not yet Label/Reduced
1049 m = new_node(n);
1050 } else {
1051 if (!is_dontcare(n)) { // Matcher can match this guy
1052 // Calls match special. They match alone with no children.
1053 // Their children, the incoming arguments, match normally.
1054 m = n->is_SafePoint() ? match_sfpt(n->as_SafePoint()):match_tree(n);
1055 if (C->failing()) return NULL;
1056 if (m == NULL) { Matcher::soft_match_failure(); return NULL; }
1057 if (n->is_MemBar()) {
1058 m->as_MachMemBar()->set_adr_type(n->adr_type());
1059 }
1060 } else { // Nothing the matcher cares about
1061 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Multi()) { // Projections?
1062 // Convert to machine-dependent projection
1063 RegMask* mask = NULL;
1064 if (n->in(0)->is_Call()) {
1065 mask = return_values_mask(n->in(0)->as_Call()->tf()->range_cc());
1066 }
1067 m = n->in(0)->as_Multi()->match(n->as_Proj(), this, mask);
1068 #ifdef ASSERT
1069 _new2old_map.map(m->_idx, n);
1070 #endif
1071 if (m->in(0) != NULL) // m might be top
1072 collect_null_checks(m, n);
1073 } else { // Else just a regular 'ol guy
1074 m = n->clone(); // So just clone into new-space
1075 #ifdef ASSERT
1076 _new2old_map.map(m->_idx, n);
1077 #endif
1078 // Def-Use edges will be added incrementally as Uses
1079 // of this node are matched.
1080 assert(m->outcnt() == 0, "no Uses of this clone yet");
1081 }
1082 }
1083
1084 set_new_node(n, m); // Map old to new
1085 if (_old_node_note_array != NULL) {
1086 Node_Notes* nn = C->locate_node_notes(_old_node_note_array,
1087 n->_idx);
1192 }
1193 return OptoReg::as_OptoReg(reg);
1194 }
1195
1196
1197 //------------------------------match_sfpt-------------------------------------
1198 // Helper function to match call instructions. Calls match special.
1199 // They match alone with no children. Their children, the incoming
1200 // arguments, match normally.
1201 MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) {
1202 MachSafePointNode *msfpt = NULL;
1203 MachCallNode *mcall = NULL;
1204 uint cnt;
1205 // Split out case for SafePoint vs Call
1206 CallNode *call;
1207 const TypeTuple *domain;
1208 ciMethod* method = NULL;
1209 bool is_method_handle_invoke = false; // for special kill effects
1210 if( sfpt->is_Call() ) {
1211 call = sfpt->as_Call();
1212 domain = call->tf()->domain_cc();
1213 cnt = domain->cnt();
1214
1215 // Match just the call, nothing else
1216 MachNode *m = match_tree(call);
1217 if (C->failing()) return NULL;
1218 if( m == NULL ) { Matcher::soft_match_failure(); return NULL; }
1219
1220 // Copy data from the Ideal SafePoint to the machine version
1221 mcall = m->as_MachCall();
1222
1223 mcall->set_tf( call->tf());
1224 mcall->set_entry_point(call->entry_point());
1225 mcall->set_cnt( call->cnt());
1226
1227 if( mcall->is_MachCallJava() ) {
1228 MachCallJavaNode *mcall_java = mcall->as_MachCallJava();
1229 const CallJavaNode *call_java = call->as_CallJava();
1230 assert(call_java->validate_symbolic_info(), "inconsistent info");
1231 method = call_java->method();
1232 mcall_java->_method = method;
1267 msfpt->_in_rms = NEW_RESOURCE_ARRAY( RegMask, cnt );
1268 // Empty them all.
1269 for (uint i = 0; i < cnt; i++) ::new (&(msfpt->_in_rms[i])) RegMask();
1270
1271 // Do all the pre-defined non-Empty register masks
1272 msfpt->_in_rms[TypeFunc::ReturnAdr] = _return_addr_mask;
1273 msfpt->_in_rms[TypeFunc::FramePtr ] = c_frame_ptr_mask;
1274
1275 // Place first outgoing argument can possibly be put.
1276 OptoReg::Name begin_out_arg_area = OptoReg::add(_new_SP, C->out_preserve_stack_slots());
1277 assert( is_even(begin_out_arg_area), "" );
1278 // Compute max outgoing register number per call site.
1279 OptoReg::Name out_arg_limit_per_call = begin_out_arg_area;
1280 // Calls to C may hammer extra stack slots above and beyond any arguments.
1281 // These are usually backing store for register arguments for varargs.
1282 if( call != NULL && call->is_CallRuntime() )
1283 out_arg_limit_per_call = OptoReg::add(out_arg_limit_per_call,C->varargs_C_out_slots_killed());
1284
1285
1286 // Do the normal argument list (parameters) register masks
1287 // Null entry point is a special cast where the target of the call
1288 // is in a register.
1289 int adj = (call != NULL && call->entry_point() == NULL) ? 1 : 0;
1290 int argcnt = cnt - TypeFunc::Parms - adj;
1291 if( argcnt > 0 ) { // Skip it all if we have no args
1292 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
1293 VMRegPair *parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
1294 int i;
1295 for( i = 0; i < argcnt; i++ ) {
1296 sig_bt[i] = domain->field_at(i+TypeFunc::Parms+adj)->basic_type();
1297 }
1298 // V-call to pick proper calling convention
1299 call->calling_convention( sig_bt, parm_regs, argcnt );
1300
1301 #ifdef ASSERT
1302 // Sanity check users' calling convention. Really handy during
1303 // the initial porting effort. Fairly expensive otherwise.
1304 { for (int i = 0; i<argcnt; i++) {
1305 if( !parm_regs[i].first()->is_valid() &&
1306 !parm_regs[i].second()->is_valid() ) continue;
1307 VMReg reg1 = parm_regs[i].first();
1308 VMReg reg2 = parm_regs[i].second();
1309 for (int j = 0; j < i; j++) {
1310 if( !parm_regs[j].first()->is_valid() &&
1311 !parm_regs[j].second()->is_valid() ) continue;
1312 VMReg reg3 = parm_regs[j].first();
1313 VMReg reg4 = parm_regs[j].second();
1314 if( !reg1->is_valid() ) {
1315 assert( !reg2->is_valid(), "valid halvsies" );
1316 } else if( !reg3->is_valid() ) {
1317 assert( !reg4->is_valid(), "valid halvsies" );
1318 } else {
1319 assert( reg1 != reg2, "calling conv. must produce distinct regs");
1320 assert( reg1 != reg3, "calling conv. must produce distinct regs");
1321 assert( reg1 != reg4, "calling conv. must produce distinct regs");
1322 assert( reg2 != reg3, "calling conv. must produce distinct regs");
1323 assert( reg2 != reg4 || !reg2->is_valid(), "calling conv. must produce distinct regs");
1324 assert( reg3 != reg4, "calling conv. must produce distinct regs");
1325 }
1326 }
1327 }
1328 }
1329 #endif
1330
1331 // Visit each argument. Compute its outgoing register mask.
1332 // Return results now can have 2 bits returned.
1333 // Compute max over all outgoing arguments both per call-site
1334 // and over the entire method.
1335 for( i = 0; i < argcnt; i++ ) {
1336 // Address of incoming argument mask to fill in
1337 RegMask *rm = &mcall->_in_rms[i+TypeFunc::Parms+adj];
1338 if( !parm_regs[i].first()->is_valid() &&
1339 !parm_regs[i].second()->is_valid() ) {
1340 continue; // Avoid Halves
1341 }
1342 // Grab first register, adjust stack slots and insert in mask.
1343 OptoReg::Name reg1 = warp_outgoing_stk_arg(parm_regs[i].first(), begin_out_arg_area, out_arg_limit_per_call );
1344 if (OptoReg::is_valid(reg1)) {
1345 rm->Insert( reg1 );
1346 }
1347 // Grab second register (if any), adjust stack slots and insert in mask.
1348 OptoReg::Name reg2 = warp_outgoing_stk_arg(parm_regs[i].second(), begin_out_arg_area, out_arg_limit_per_call );
1349 if (OptoReg::is_valid(reg2)) {
1350 rm->Insert( reg2 );
1351 }
1352 } // End of for all arguments
1353
1354 // Compute number of stack slots needed to restore stack in case of
1355 // Pascal-style argument popping.
1356 mcall->_argsize = out_arg_limit_per_call - begin_out_arg_area;
1357 }
1358
1359 // Compute the max stack slot killed by any call. These will not be
1360 // available for debug info, and will be used to adjust FIRST_STACK_mask
1361 // after all call sites have been visited.
1362 if( _out_arg_limit < out_arg_limit_per_call)
1363 _out_arg_limit = out_arg_limit_per_call;
1364
1365 if (mcall) {
1366 // Kill the outgoing argument area, including any non-argument holes and
1367 // any legacy C-killed slots. Use Fat-Projections to do the killing.
1368 // Since the max-per-method covers the max-per-call-site and debug info
1369 // is excluded on the max-per-method basis, debug info cannot land in
1370 // this killed area.
1371 uint r_cnt = mcall->tf()->range_sig()->cnt();
1372 MachProjNode *proj = new MachProjNode( mcall, r_cnt+10000, RegMask::Empty, MachProjNode::fat_proj );
1373 if (!RegMask::can_represent_arg(OptoReg::Name(out_arg_limit_per_call-1))) {
1374 C->record_method_not_compilable("unsupported outgoing calling sequence");
1375 } else {
1376 for (int i = begin_out_arg_area; i < out_arg_limit_per_call; i++)
1377 proj->_rout.Insert(OptoReg::Name(i));
1378 }
1379 if (proj->_rout.is_NotEmpty()) {
1380 push_projection(proj);
1381 }
1382 }
1383 // Transfer the safepoint information from the call to the mcall
1384 // Move the JVMState list
1385 msfpt->set_jvms(sfpt->jvms());
1386 for (JVMState* jvms = msfpt->jvms(); jvms; jvms = jvms->caller()) {
1387 jvms->set_map(sfpt);
1388 }
1389
1390 // Debug inputs begin just after the last incoming parameter
1391 assert((mcall == NULL) || (mcall->jvms() == NULL) ||
1392 (mcall->jvms()->debug_start() + mcall->_jvmadj == mcall->tf()->domain_cc()->cnt()), "");
1393
1394 // Move the OopMap
1395 msfpt->_oop_map = sfpt->_oop_map;
1396
1397 // Add additional edges.
1398 if (msfpt->mach_constant_base_node_input() != (uint)-1 && !msfpt->is_MachCallLeaf()) {
1399 // For these calls we can not add MachConstantBase in expand(), as the
1400 // ins are not complete then.
1401 msfpt->ins_req(msfpt->mach_constant_base_node_input(), C->mach_constant_base_node());
1402 if (msfpt->jvms() &&
1403 msfpt->mach_constant_base_node_input() <= msfpt->jvms()->debug_start() + msfpt->_jvmadj) {
1404 // We added an edge before jvms, so we must adapt the position of the ins.
1405 msfpt->jvms()->adapt_position(+1);
1406 }
1407 }
1408
1409 // Registers killed by the call are set in the local scheduling pass
1410 // of Global Code Motion.
1411 return msfpt;
1412 }
2405 break;
2406 }
2407 case Op_FmaD:
2408 case Op_FmaF:
2409 case Op_FmaVD:
2410 case Op_FmaVF: {
2411 // Restructure into a binary tree for Matching.
2412 Node* pair = new BinaryNode(n->in(1), n->in(2));
2413 n->set_req(2, pair);
2414 n->set_req(1, n->in(3));
2415 n->del_req(3);
2416 break;
2417 }
2418 case Op_MulAddS2I: {
2419 Node* pair1 = new BinaryNode(n->in(1), n->in(2));
2420 Node* pair2 = new BinaryNode(n->in(3), n->in(4));
2421 n->set_req(1, pair1);
2422 n->set_req(2, pair2);
2423 n->del_req(4);
2424 n->del_req(3);
2425 break;
2426 }
2427 case Op_ClearArray: {
2428 Node* pair = new BinaryNode(n->in(2), n->in(3));
2429 n->set_req(2, pair);
2430 n->set_req(3, n->in(4));
2431 n->del_req(4);
2432 break;
2433 }
2434 default:
2435 break;
2436 }
2437 }
2438
2439 #ifdef ASSERT
2440 // machine-independent root to machine-dependent root
2441 void Matcher::dump_old2new_map() {
2442 _old2new_map.dump();
2443 }
2444 #endif
2445
2446 //---------------------------collect_null_checks-------------------------------
2447 // Find null checks in the ideal graph; write a machine-specific node for
2448 // it. Used by later implicit-null-check handling. Actually collects
2449 // either an IfTrue or IfFalse for the common NOT-null path, AND the ideal
2450 // value being tested.
2451 void Matcher::collect_null_checks( Node *proj, Node *orig_proj ) {
|